guix/ssh.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org>
   3 ;;;
   4 ;;; This file is part of GNU Guix.
   5 ;;;
   6 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   7 ;;; under the terms of the GNU General Public License as published by
   8 ;;; the Free Software Foundation; either version 3 of the License, or (at
   9 ;;; your option) any later version.
  10 ;;;
  11 ;;; GNU Guix is distributed in the hope that it will be useful, but
  12 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 ;;; GNU General Public License for more details.
  15 ;;;
  16 ;;; You should have received a copy of the GNU General Public License
  17 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 (define-module (guix ssh)
  20   #:use-module (guix store)
  21   #:use-module (guix inferior)
  22   #:use-module (guix i18n)
  23   #:use-module ((guix diagnostics) #:select (&fix-hint formatted-message))
  24   #:use-module (gcrypt pk-crypto)
  25   #:use-module (ssh session)
  26   #:use-module (ssh auth)
  27   #:use-module (ssh key)
  28   #:use-module (ssh channel)
  29   #:use-module (ssh popen)
  30   #:use-module (ssh session)
  31   #:use-module (srfi srfi-1)
  32   #:use-module (srfi srfi-11)
  33   #:use-module (srfi srfi-26)
  34   #:use-module (srfi srfi-34)
  35   #:use-module (srfi srfi-35)
  36   #:use-module (ice-9 match)
  37   #:use-module (ice-9 format)
  38   #:use-module (ice-9 binary-ports)
  39   #:export (open-ssh-session
  40             authenticate-server*
  41
  42             remote-inferior
  43             remote-daemon-channel
  44             connect-to-remote-daemon
  45             remote-system
  46             remote-authorize-signing-key
  47             send-files
  48             retrieve-files
  49             retrieve-files*
  50             remote-store-host
  51
  52             report-guile-error
  53             report-module-error))
  54
  55 ;;; Commentary:
  56 ;;;
  57 ;;; This module provides tools to support communication with remote stores
  58 ;;; over SSH, using Guile-SSH.
  59 ;;;
  60 ;;; Code:
  61
  62 (define %compression
  63   "zlib@openssh.com,zlib")
  64
  65 (define (host-key->type+key host-key)
  66   "Destructure HOST-KEY, an OpenSSH host key string, and return two values:
  67 its key type as a symbol, and the actual base64-encoded string."
  68   (define (type->symbol type)
  69     (and (string-prefix? "ssh-" type)
  70          (string->symbol (string-drop type 4))))
  71
  72   (match (string-tokenize host-key)
  73     ((type key x)
  74      (values (type->symbol type) key))
  75     ((type key)
  76      (values (type->symbol type) key))))
  77
  78 (define (authenticate-server* session key)
  79   "Make sure the server for SESSION has the given KEY, where KEY is a string
  80 such as \"ssh-ed25519 AAAAC3Nz… root@example.org\".  Raise an exception if the
  81 actual key does not match."
  82   (let-values (((server)   (get-server-public-key session))
  83                ((type key) (host-key->type+key key)))
  84     (unless (and (or (not (get-key-type server))
  85                      (eq? (get-key-type server) type))
  86                  (string=? (public-key->string server) key))
  87       ;; Key mismatch: something's wrong.  XXX: It could be that the server
  88       ;; provided its Ed25519 key when we where expecting its RSA key.  XXX:
  89       ;; Guile-SSH 0.10.1 doesn't know about ed25519 keys and 'get-key-type'
  90       ;; returns #f in that case.
  91       (raise (formatted-message (G_ "server at '~a' returned host key \
  92 '~a' of type '~a' instead of '~a' of type '~a'~%")
  93                                 (session-get session 'host)
  94                                 (public-key->string server)
  95                                 (get-key-type server)
  96                                 key type)))))
  97
  98 (define* (open-ssh-session host #:key user port identity
  99                            host-key
 100                            (compression %compression)
 101                            (timeout 3600))
 102   "Open an SSH session for HOST and return it.  IDENTITY specifies the file
 103 name of a private key to use for authenticating with the host.  When USER,
 104 PORT, or IDENTITY are #f, use default values or whatever '~/.ssh/config'
 105 specifies; otherwise use them.
 106
 107 When HOST-KEY is true, it must be a string like \"ssh-ed25519 AAAAC3Nz…
 108 root@example.org\"; the server is authenticated and an error is raised if its
 109 host key is different from HOST-KEY.
 110
 111 Install TIMEOUT as the maximum time in seconds after which a read or write
 112 operation on a channel of the returned session is considered as failing.
 113
 114 Throw an error on failure."
 115   (let ((session (make-session #:user user
 116                                #:identity identity
 117                                #:host host
 118                                #:port port
 119                                #:timeout 10       ;seconds
 120                                ;; #:log-verbosity 'protocol
 121
 122                                ;; Prevent libssh from reading
 123                                ;; ~/.ssh/known_hosts when the caller provides
 124                                ;; a HOST-KEY to match against.
 125                                #:knownhosts (and host-key "/dev/null")
 126
 127                                ;; We need lightweight compression when
 128                                ;; exchanging full archives.
 129                                #:compression compression
 130                                #:compression-level 3
 131
 132                                ;; Speed up RPCs by creating sockets with
 133                                ;; TCP_NODELAY.
 134                                #:nodelay #t)))
 135
 136     ;; Honor ~/.ssh/config.
 137     (session-parse-config! session)
 138
 139     (match (connect! session)
 140       ('ok
 141        (if host-key
 142            ;; Make sure the server's key is what we expect.
 143            (authenticate-server* session host-key)
 144
 145            ;; Authenticate against ~/.ssh/known_hosts.
 146            (match (authenticate-server session)
 147              ('ok #f)
 148              (reason
 149               (raise (formatted-message (G_ "failed to authenticate \
 150 server at '~a': ~a")
 151                                         (session-get session 'host)
 152                                         reason)))))
 153
 154        ;; Use public key authentication, via the SSH agent if it's available.
 155        (match (userauth-public-key/auto! session)
 156          ('success
 157           (session-set! session 'timeout timeout)
 158           session)
 159          (x
 160           (match (userauth-gssapi! session)
 161             ('success
 162              (session-set! session 'timeout timeout)
 163              session)
 164             (x
 165              (disconnect! session)
 166              (raise (condition
 167                      (&message
 168                       (message (format #f (G_ "SSH authentication failed for '~a': ~a~%")
 169                                        host (get-error session)))))))))))
 170       (x
 171        ;; Connection failed or timeout expired.
 172        (raise (formatted-message (G_ "SSH connection to '~a' failed: ~a~%")
 173                                  host (get-error session)))))))
 174
 175 (define* (remote-inferior session #:optional become-command)
 176   "Return a remote inferior for the given SESSION.  If BECOME-COMMAND is
 177 given, use that to invoke the remote Guile REPL."
 178   (let* ((repl-command (append (or become-command '())
 179                                '("guix" "repl" "-t" "machine")))
 180          (pipe (apply open-remote-pipe* session OPEN_BOTH repl-command)))
 181     (when (eof-object? (peek-char pipe))
 182       (let ((status (channel-get-exit-status pipe)))
 183         (close-port pipe)
 184         (raise (formatted-message (G_ "remote command '~{~a~^ ~}' failed \
 185 with status ~a")
 186                                   repl-command status))))
 187     (port->inferior pipe)))
 188
 189 (define* (inferior-remote-eval exp session #:optional become-command)
 190   "Evaluate EXP in a new inferior running in SESSION, and close the inferior
 191 right away.  If BECOME-COMMAND is given, use that to invoke the remote Guile
 192 REPL."
 193   (let ((inferior (remote-inferior session become-command)))
 194     (dynamic-wind
 195       (const #t)
 196       (lambda ()
 197         (inferior-eval exp inferior))
 198       (lambda ()
 199         ;; Close INFERIOR right away to prevent finalization from happening in
 200         ;; another thread at the wrong time (see
 201         ;; <https://bugs.gnu.org/26976>.)
 202         (close-inferior inferior)))))
 203
 204 (define* (remote-daemon-channel session
 205                                 #:optional
 206                                 (socket-name
 207                                  "/var/guix/daemon-socket/socket"))
 208   "Return an input/output port (an SSH channel) to the daemon at SESSION."
 209   (define redirect
 210     ;; Code run in SESSION to redirect the remote process' stdin/stdout to the
 211     ;; daemon's socket, à la socat.  The SSH protocol supports forwarding to
 212     ;; Unix-domain sockets but libssh doesn't have an API for that, hence this
 213     ;; hack.
 214     `(begin
 215        (use-modules (ice-9 match) (rnrs io ports)
 216                     (rnrs bytevectors))
 217
 218        (let ((sock    (socket AF_UNIX SOCK_STREAM 0))
 219              (stdin   (current-input-port))
 220              (stdout  (current-output-port))
 221              (select* (lambda (read write except)
 222                         ;; This is a workaround for
 223                         ;; <https://bugs.gnu.org/30365> in Guile < 2.2.4:
 224                         ;; since 'select' sometimes returns non-empty sets for
 225                         ;; no good reason, call 'select' a second time with a
 226                         ;; zero timeout to filter out incorrect replies.
 227                         (match (select read write except)
 228                           ((read write except)
 229                            (select read write except 0))))))
 230          (setvbuf stdout 'none)
 231
 232          ;; Use buffered ports so that 'get-bytevector-some' returns up to the
 233          ;; whole buffer like read(2) would--see <https://bugs.gnu.org/30066>.
 234          (setvbuf stdin 'block 65536)
 235          (setvbuf sock 'block 65536)
 236
 237          (connect sock AF_UNIX ,socket-name)
 238
 239          (let loop ()
 240            (match (select* (list stdin sock) '() '())
 241              ((reads () ())
 242               (when (memq stdin reads)
 243                 (match (get-bytevector-some stdin)
 244                   ((? eof-object?)
 245                    (primitive-exit 0))
 246                   (bv
 247                    (put-bytevector sock bv)
 248                    (force-output sock))))
 249               (when (memq sock reads)
 250                 (match (get-bytevector-some sock)
 251                   ((? eof-object?)
 252                    (primitive-exit 0))
 253                   (bv
 254                    (put-bytevector stdout bv))))
 255               (loop))
 256              (_
 257               (primitive-exit 1)))))))
 258
 259   (open-remote-pipe* session OPEN_BOTH
 260                      ;; Sort-of shell-quote REDIRECT.
 261                      "guile" "-c"
 262                      (object->string
 263                       (object->string redirect))))
 264
 265 (define* (connect-to-remote-daemon session
 266                                    #:optional
 267                                    (socket-name
 268                                     "/var/guix/daemon-socket/socket"))
 269   "Connect to the remote build daemon listening on SOCKET-NAME over SESSION,
 270 an SSH session.  Return a <store-connection> object."
 271   (open-connection #:port (remote-daemon-channel session socket-name)))
 272
 273
 274 (define (store-import-channel session)
 275   "Return an output port to which archives to be exported to SESSION's store
 276 can be written."
 277   ;; Using the 'import-paths' RPC on a remote store would be slow because it
 278   ;; makes a round trip every time 32 KiB have been transferred.  This
 279   ;; procedure instead opens a separate channel to use the remote
 280   ;; 'import-paths' procedure, which consumes all the data in a single round
 281   ;; trip.  This optimizes the successful case at the expense of error
 282   ;; conditions: errors can only be reported once all the input has been
 283   ;; consumed.
 284   (define import
 285     `(begin
 286        (eval-when (load expand eval)
 287          (unless (resolve-module '(guix) #:ensure #f)
 288            (write `(module-error))
 289            (exit 7)))
 290
 291        (use-modules (guix) (srfi srfi-34)
 292                     (rnrs io ports) (rnrs bytevectors))
 293
 294        (define (consume-input port)
 295          (let ((bv (make-bytevector 32768)))
 296            (let loop ()
 297              (let ((n (get-bytevector-n! port bv 0
 298                                          (bytevector-length bv))))
 299                (unless (eof-object? n)
 300                  (loop))))))
 301
 302        ;; Upon completion, write an sexp that denotes the status.
 303        (write
 304         (catch #t
 305           (lambda ()
 306             (guard (c ((nix-protocol-error? c)
 307                        ;; Consume all the input since the only time we can
 308                        ;; report the error is after everything has been
 309                        ;; consumed.
 310                        (consume-input (current-input-port))
 311                        (list 'protocol-error (nix-protocol-error-message c))))
 312               (with-store store
 313                 (write '(importing))              ;we're ready
 314                 (force-output)
 315
 316                 (setvbuf (current-input-port) 'none)
 317                 (import-paths store (current-input-port))
 318                 '(success))))
 319           (lambda args
 320             (cons 'error args))))))
 321
 322   (open-remote-pipe session
 323                     (string-join
 324                      `("guile" "-c"
 325                        ,(object->string (object->string import))))
 326                     OPEN_BOTH))
 327
 328 (define* (store-export-channel session files
 329                                #:key recursive?)
 330   "Return an input port from which an export of FILES from SESSION's store can
 331 be read.  When RECURSIVE? is true, the closure of FILES is exported."
 332   ;; Same as above: this is more efficient than calling 'export-paths' on a
 333   ;; remote store.
 334   (define export
 335     `(begin
 336        (eval-when (load expand eval)
 337          (unless (resolve-module '(guix) #:ensure #f)
 338            (write `(module-error))
 339            (exit 7)))
 340
 341        (use-modules (guix) (srfi srfi-1)
 342                     (srfi srfi-26) (srfi srfi-34))
 343
 344        (guard (c ((nix-connection-error? c)
 345                   (write `(connection-error ,(nix-connection-error-file c)
 346                                             ,(nix-connection-error-code c))))
 347                  ((nix-protocol-error? c)
 348                   (write `(protocol-error ,(nix-protocol-error-status c)
 349                                           ,(nix-protocol-error-message c))))
 350                  (else
 351                   (write `(exception))))
 352          (with-store store
 353            (let* ((files ',files)
 354                   (invalid (remove (cut valid-path? store <>)
 355                                    files)))
 356              (unless (null? invalid)
 357                (write `(invalid-items ,invalid))
 358                (exit 1))
 359
 360              ;; TODO: When RECURSIVE? is true, we could send the list of store
 361              ;; items in the closure so that the other end can filter out
 362              ;; those it already has.
 363
 364              (write '(exporting))                 ;we're ready
 365              (force-output)
 366
 367              (setvbuf (current-output-port) 'none)
 368              (export-paths store files (current-output-port)
 369                            #:recursive? ,recursive?))))))
 370
 371   (open-remote-input-pipe session
 372                           (string-join
 373                            `("guile" "-c"
 374                              ,(object->string
 375                                (object->string export))))))
 376
 377 (define (remote-system session)
 378   "Return the system type as expected by Nix, usually ARCHITECTURE-KERNEL, of
 379 the machine on the other end of SESSION."
 380   (inferior-remote-eval '(begin (use-modules (guix utils)) (%current-system))
 381                         session))
 382
 383 (define* (remote-authorize-signing-key key session #:optional become-command)
 384   "Send KEY, a canonical sexp containing a public key, over SESSION and add it
 385 to the system ACL file if it has not yet been authorized."
 386   (inferior-remote-eval
 387    `(begin
 388       (use-modules (guix build utils)
 389                    (guix pki)
 390                    (guix utils)
 391                    (gcrypt pk-crypto)
 392                    (srfi srfi-26))
 393
 394       (define acl (current-acl))
 395       (define key (string->canonical-sexp ,(canonical-sexp->string key)))
 396
 397       (unless (authorized-key? key)
 398         (let ((acl (public-keys->acl (cons key (acl->public-keys acl)))))
 399           (mkdir-p (dirname %acl-file))
 400           (with-atomic-file-output %acl-file
 401             (cut write-acl acl <>)))))
 402    session
 403    become-command))
 404
 405 (define* (send-files local files remote
 406                      #:key
 407                      recursive?
 408                      (log-port (current-error-port)))
 409   "Send the subset of FILES from LOCAL (a local store) that's missing to
 410 REMOTE, a remote store.  When RECURSIVE? is true, send the closure of FILES.
 411 Return the list of store items actually sent."
 412   ;; Compute the subset of FILES missing on SESSION and send them.
 413   (let* ((files   (if recursive? (requisites local files) files))
 414          (session (channel-get-session (store-connection-socket remote)))
 415          (missing (inferior-remote-eval
 416                    `(begin
 417                       (use-modules (guix)
 418                                    (srfi srfi-1) (srfi srfi-26))
 419
 420                       (with-store store
 421                         (remove (cut valid-path? store <>)
 422                                 ',files)))
 423                    session))
 424          (count   (length missing))
 425          (sizes   (map (lambda (item)
 426                          (path-info-nar-size (query-path-info local item)))
 427                        missing))
 428          (port    (store-import-channel session)))
 429     ;; Make sure everything alright on the remote side.
 430     (match (read port)
 431       (('importing)
 432        #t)
 433       (sexp
 434        (handle-import/export-channel-error sexp remote)))
 435
 436     (format log-port (N_ "sending ~a store item (~h MiB) to '~a'...~%"
 437                          "sending ~a store items (~h MiB) to '~a'...~%" count)
 438             count
 439             (inexact->exact (round (/ (reduce + 0 sizes) (expt 2. 20))))
 440             (session-get session 'host))
 441
 442     ;; Send MISSING in topological order.
 443     (export-paths local missing port)
 444
 445     ;; Tell the remote process that we're done.  (In theory the end-of-archive
 446     ;; mark of 'export-paths' would be enough, but in practice it's not.)
 447     (channel-send-eof port)
 448
 449     ;; Wait for completion of the remote process and read the status sexp from
 450     ;; PORT.  Wait for the exit status only when 'read' completed; otherwise,
 451     ;; we might wait forever if the other end is stuck.
 452     (let* ((result (false-if-exception (read port)))
 453            (status (and result
 454                         (zero? (channel-get-exit-status port)))))
 455       (close-port port)
 456       (match result
 457         (('success . _)
 458          missing)
 459         (('protocol-error message)
 460          (raise (condition
 461                  (&store-protocol-error (message message) (status 42)))))
 462         (('error key args ...)
 463          (raise (condition
 464                  (&store-protocol-error
 465                   (message (call-with-output-string
 466                              (lambda (port)
 467                                (print-exception port #f key args))))
 468                   (status 43)))))
 469         (_
 470          (raise (condition
 471                  (&store-protocol-error
 472                   (message "unknown error while sending files over SSH")
 473                   (status 44)))))))))
 474
 475 (define (remote-store-session remote)
 476   "Return the SSH channel beneath REMOTE, a remote store as returned by
 477 'connect-to-remote-daemon', or #f."
 478   (channel-get-session (store-connection-socket remote)))
 479
 480 (define (remote-store-host remote)
 481   "Return the name of the host REMOTE is connected to, where REMOTE is a
 482 remote store as returned by 'connect-to-remote-daemon'."
 483   (match (remote-store-session remote)
 484     (#f #f)
 485     ((? session? session)
 486      (session-get session 'host))))
 487
 488 (define* (file-retrieval-port files remote
 489                               #:key recursive?)
 490   "Return an input port from which to retrieve FILES (a list of store items)
 491 from REMOTE, along with the number of items to retrieve (lower than or equal
 492 to the length of FILES.)"
 493   (values (store-export-channel (remote-store-session remote) files
 494                                 #:recursive? recursive?)
 495           (length files)))            ;XXX: inaccurate when RECURSIVE? is true
 496
 497 (define-syntax raise-error
 498   (syntax-rules (=>)
 499     ((_ fmt args ... (=> hint-fmt hint-args ...))
 500      (raise (condition
 501              (&message
 502               (message (format #f fmt args ...)))
 503              (&fix-hint
 504               (hint (format #f hint-fmt hint-args ...))))))
 505     ((_ fmt args ...)
 506      (raise (condition
 507              (&message
 508               (message (format #f fmt args ...))))))))
 509
 510 (define (handle-import/export-channel-error sexp remote)
 511   "Report an error corresponding to SEXP, the EOF object or an sexp read from
 512 REMOTE."
 513   (match sexp
 514     ((? eof-object?)
 515      (report-guile-error (remote-store-host remote)))
 516     (('module-error . _)
 517      (report-module-error (remote-store-host remote)))
 518     (('connection-error file code . _)
 519      (raise-error (G_ "failed to connect to '~A' on remote host '~A': ~a")
 520                   file (remote-store-host remote) (strerror code)))
 521     (('invalid-items items . _)
 522      (raise-error (N_ "no such item on remote host '~A':~{ ~a~}"
 523                       "no such items on remote host '~A':~{ ~a~}"
 524                       (length items))
 525                   (remote-store-host remote) items))
 526     (('protocol-error status message . _)
 527      (raise-error (G_ "protocol error on remote host '~A': ~a")
 528                   (remote-store-host remote) message))
 529     (_
 530      (raise-error (G_ "failed to retrieve store items from '~a'")
 531                   (remote-store-host remote)))))
 532
 533 (define* (retrieve-files* files remote
 534                           #:key recursive? (log-port (current-error-port))
 535                           (import (const #f)))
 536   "Pass IMPORT an input port from which to read the sequence of FILES coming
 537 from REMOTE.  When RECURSIVE? is true, retrieve the closure of FILES."
 538   (let-values (((port count)
 539                 (file-retrieval-port files remote
 540                                      #:recursive? recursive?)))
 541     (match (read port)                            ;read the initial status
 542       (('exporting)
 543        (format #t (N_ "retrieving ~a store item from '~a'...~%"
 544                       "retrieving ~a store items from '~a'...~%" count)
 545                count (remote-store-host remote))
 546
 547        (dynamic-wind
 548          (const #t)
 549          (lambda ()
 550            (import port))
 551          (lambda ()
 552            (close-port port))))
 553       (sexp
 554        (handle-import/export-channel-error sexp remote)))))
 555
 556 (define* (retrieve-files local files remote
 557                          #:key recursive? (log-port (current-error-port)))
 558   "Retrieve FILES from REMOTE and import them using the 'import-paths' RPC on
 559 LOCAL.  When RECURSIVE? is true, retrieve the closure of FILES."
 560   (retrieve-files* (remove (cut valid-path? local <>) files)
 561                    remote
 562                    #:recursive? recursive?
 563                    #:log-port log-port
 564                    #:import (lambda (port)
 565                               (import-paths local port))))
 566
 567 \f
 568 ;;;
 569 ;;; Error reporting.
 570 ;;;
 571
 572 (define (report-guile-error host)
 573   (raise-error (G_ "failed to start Guile on remote host '~A'") host
 574                (=> (G_ "Make sure @command{guile} can be found in
 575 @code{$PATH} on the remote host.  Run @command{ssh ~A guile --version} to
 576 check.")
 577                    host)))
 578
 579 (define (report-module-error host)
 580   "Report an error about missing Guix modules on HOST."
 581   ;; TRANSLATORS: Leave "Guile" untranslated.
 582   (raise-error (G_ "Guile modules not found on remote host '~A'") host
 583                (=> (G_ "Make sure @code{GUILE_LOAD_PATH} includes Guix'
 584 own module directory.  Run @command{ssh ~A env | grep GUILE_LOAD_PATH} to
 585 check.")
 586                    host)))
 587
 588 (define (report-inferior-exception exception host)
 589   "Report EXCEPTION, an &inferior-exception that occurred on HOST."
 590   (raise-error (G_ "exception occurred on remote host '~A': ~s")
 591                host (inferior-exception-arguments exception)))
 592
 593 ;;; ssh.scm ends here