guix/ssh.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2016, 2017, 2018, 2019 Ludovic Courtès <ludo@gnu.org>
   3 ;;;
   4 ;;; This file is part of GNU Guix.
   5 ;;;
   6 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   7 ;;; under the terms of the GNU General Public License as published by
   8 ;;; the Free Software Foundation; either version 3 of the License, or (at
   9 ;;; your option) any later version.
  10 ;;;
  11 ;;; GNU Guix is distributed in the hope that it will be useful, but
  12 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 ;;; GNU General Public License for more details.
  15 ;;;
  16 ;;; You should have received a copy of the GNU General Public License
  17 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 (define-module (guix ssh)
  20   #:use-module (guix store)
  21   #:use-module (guix inferior)
  22   #:use-module (guix i18n)
  23   #:use-module ((guix utils) #:select (&fix-hint))
  24   #:use-module (gcrypt pk-crypto)
  25   #:use-module (ssh session)
  26   #:use-module (ssh auth)
  27   #:use-module (ssh key)
  28   #:use-module (ssh channel)
  29   #:use-module (ssh popen)
  30   #:use-module (ssh session)
  31   #:use-module (srfi srfi-1)
  32   #:use-module (srfi srfi-11)
  33   #:use-module (srfi srfi-26)
  34   #:use-module (srfi srfi-34)
  35   #:use-module (srfi srfi-35)
  36   #:use-module (ice-9 match)
  37   #:use-module (ice-9 format)
  38   #:use-module (ice-9 binary-ports)
  39   #:export (open-ssh-session
  40             remote-inferior
  41             remote-daemon-channel
  42             connect-to-remote-daemon
  43             remote-system
  44             remote-authorize-signing-key
  45             send-files
  46             retrieve-files
  47             retrieve-files*
  48             remote-store-host
  49
  50             report-guile-error
  51             report-module-error))
  52
  53 ;;; Commentary:
  54 ;;;
  55 ;;; This module provides tools to support communication with remote stores
  56 ;;; over SSH, using Guile-SSH.
  57 ;;;
  58 ;;; Code:
  59
  60 (define %compression
  61   "zlib@openssh.com,zlib")
  62
  63 (define* (open-ssh-session host #:key user port identity
  64                            (compression %compression))
  65   "Open an SSH session for HOST and return it.  IDENTITY specifies the file
  66 name of a private key to use for authenticating with the host.  When USER,
  67 PORT, or IDENTITY are #f, use default values or whatever '~/.ssh/config'
  68 specifies; otherwise use them.  Throw an error on failure."
  69   (let ((session (make-session #:user user
  70                                #:identity identity
  71                                #:host host
  72                                #:port port
  73                                #:timeout 10       ;seconds
  74                                ;; #:log-verbosity 'protocol
  75
  76                                ;; We need lightweight compression when
  77                                ;; exchanging full archives.
  78                                #:compression compression
  79                                #:compression-level 3)))
  80
  81     ;; Honor ~/.ssh/config.
  82     (session-parse-config! session)
  83
  84     (match (connect! session)
  85       ('ok
  86        ;; Use public key authentication, via the SSH agent if it's available.
  87        (match (userauth-public-key/auto! session)
  88          ('success
  89           session)
  90          (x
  91           (disconnect! session)
  92           (raise (condition
  93                   (&message
  94                    (message (format #f (G_ "SSH authentication failed for '~a': ~a~%")
  95                                     host (get-error session)))))))))
  96       (x
  97        ;; Connection failed or timeout expired.
  98        (raise (condition
  99                (&message
 100                 (message (format #f (G_ "SSH connection to '~a' failed: ~a~%")
 101                                  host (get-error session))))))))))
 102
 103 (define* (remote-inferior session #:optional become-command)
 104   "Return a remote inferior for the given SESSION.  If BECOME-COMMAND is
 105 given, use that to invoke the remote Guile REPL."
 106   (let* ((repl-command (append (or become-command '())
 107                                '("guix" "repl" "-t" "machine")))
 108          (pipe (apply open-remote-pipe* session OPEN_BOTH repl-command)))
 109     ;; XXX: 'channel-get-exit-status' would be better here, but hangs if the
 110     ;; process does succeed. This doesn't reflect the documentation, so it's
 111     ;; possible that it's a bug in guile-ssh.
 112     (when (eof-object? (peek-char pipe))
 113       (raise (condition
 114               (&message
 115                (message (format #f (G_ "failed to run '~{~a~^ ~}'")
 116                                 repl-command))))))
 117     (port->inferior pipe)))
 118
 119 (define* (inferior-remote-eval exp session #:optional become-command)
 120   "Evaluate EXP in a new inferior running in SESSION, and close the inferior
 121 right away.  If BECOME-COMMAND is given, use that to invoke the remote Guile
 122 REPL."
 123   (let ((inferior (remote-inferior session become-command)))
 124     (dynamic-wind
 125       (const #t)
 126       (lambda ()
 127         (inferior-eval exp inferior))
 128       (lambda ()
 129         ;; Close INFERIOR right away to prevent finalization from happening in
 130         ;; another thread at the wrong time (see
 131         ;; <https://bugs.gnu.org/26976>.)
 132         (close-inferior inferior)))))
 133
 134 (define* (remote-daemon-channel session
 135                                 #:optional
 136                                 (socket-name
 137                                  "/var/guix/daemon-socket/socket"))
 138   "Return an input/output port (an SSH channel) to the daemon at SESSION."
 139   (define redirect
 140     ;; Code run in SESSION to redirect the remote process' stdin/stdout to the
 141     ;; daemon's socket, à la socat.  The SSH protocol supports forwarding to
 142     ;; Unix-domain sockets but libssh doesn't have an API for that, hence this
 143     ;; hack.
 144     `(begin
 145        (use-modules (ice-9 match) (rnrs io ports)
 146                     (rnrs bytevectors))
 147
 148        (let ((sock    (socket AF_UNIX SOCK_STREAM 0))
 149              (stdin   (current-input-port))
 150              (stdout  (current-output-port))
 151              (select* (lambda (read write except)
 152                         ;; This is a workaround for
 153                         ;; <https://bugs.gnu.org/30365> in Guile < 2.2.4:
 154                         ;; since 'select' sometimes returns non-empty sets for
 155                         ;; no good reason, call 'select' a second time with a
 156                         ;; zero timeout to filter out incorrect replies.
 157                         (match (select read write except)
 158                           ((read write except)
 159                            (select read write except 0))))))
 160          (setvbuf stdout 'none)
 161
 162          ;; Use buffered ports so that 'get-bytevector-some' returns up to the
 163          ;; whole buffer like read(2) would--see <https://bugs.gnu.org/30066>.
 164          (setvbuf stdin 'block 65536)
 165          (setvbuf sock 'block 65536)
 166
 167          (connect sock AF_UNIX ,socket-name)
 168
 169          (let loop ()
 170            (match (select* (list stdin sock) '() '())
 171              ((reads () ())
 172               (when (memq stdin reads)
 173                 (match (get-bytevector-some stdin)
 174                   ((? eof-object?)
 175                    (primitive-exit 0))
 176                   (bv
 177                    (put-bytevector sock bv)
 178                    (force-output sock))))
 179               (when (memq sock reads)
 180                 (match (get-bytevector-some sock)
 181                   ((? eof-object?)
 182                    (primitive-exit 0))
 183                   (bv
 184                    (put-bytevector stdout bv))))
 185               (loop))
 186              (_
 187               (primitive-exit 1)))))))
 188
 189   (open-remote-pipe* session OPEN_BOTH
 190                      ;; Sort-of shell-quote REDIRECT.
 191                      "guile" "-c"
 192                      (object->string
 193                       (object->string redirect))))
 194
 195 (define* (connect-to-remote-daemon session
 196                                    #:optional
 197                                    (socket-name
 198                                     "/var/guix/daemon-socket/socket"))
 199   "Connect to the remote build daemon listening on SOCKET-NAME over SESSION,
 200 an SSH session.  Return a <store-connection> object."
 201   (open-connection #:port (remote-daemon-channel session socket-name)))
 202
 203
 204 (define (store-import-channel session)
 205   "Return an output port to which archives to be exported to SESSION's store
 206 can be written."
 207   ;; Using the 'import-paths' RPC on a remote store would be slow because it
 208   ;; makes a round trip every time 32 KiB have been transferred.  This
 209   ;; procedure instead opens a separate channel to use the remote
 210   ;; 'import-paths' procedure, which consumes all the data in a single round
 211   ;; trip.  This optimizes the successful case at the expense of error
 212   ;; conditions: errors can only be reported once all the input has been
 213   ;; consumed.
 214   (define import
 215     `(begin
 216        (use-modules (guix) (srfi srfi-34)
 217                     (rnrs io ports) (rnrs bytevectors))
 218
 219        (define (consume-input port)
 220          (let ((bv (make-bytevector 32768)))
 221            (let loop ()
 222              (let ((n (get-bytevector-n! port bv 0
 223                                          (bytevector-length bv))))
 224                (unless (eof-object? n)
 225                  (loop))))))
 226
 227        ;; Upon completion, write an sexp that denotes the status.
 228        (write
 229         (catch #t
 230           (lambda ()
 231             (guard (c ((nix-protocol-error? c)
 232                        ;; Consume all the input since the only time we can
 233                        ;; report the error is after everything has been
 234                        ;; consumed.
 235                        (consume-input (current-input-port))
 236                        (list 'protocol-error (nix-protocol-error-message c))))
 237               (with-store store
 238                 (setvbuf (current-input-port) 'none)
 239                 (import-paths store (current-input-port))
 240                 '(success))))
 241           (lambda args
 242             (cons 'error args))))))
 243
 244   (open-remote-pipe session
 245                     (string-join
 246                      `("guile" "-c"
 247                        ,(object->string (object->string import))))
 248                     OPEN_BOTH))
 249
 250 (define* (store-export-channel session files
 251                                #:key recursive?)
 252   "Return an input port from which an export of FILES from SESSION's store can
 253 be read.  When RECURSIVE? is true, the closure of FILES is exported."
 254   ;; Same as above: this is more efficient than calling 'export-paths' on a
 255   ;; remote store.
 256   (define export
 257     `(begin
 258        (eval-when (load expand eval)
 259          (unless (resolve-module '(guix) #:ensure #f)
 260            (write `(module-error))
 261            (exit 7)))
 262
 263        (use-modules (guix) (srfi srfi-1)
 264                     (srfi srfi-26) (srfi srfi-34))
 265
 266        (guard (c ((nix-connection-error? c)
 267                   (write `(connection-error ,(nix-connection-error-file c)
 268                                             ,(nix-connection-error-code c))))
 269                  ((nix-protocol-error? c)
 270                   (write `(protocol-error ,(nix-protocol-error-status c)
 271                                           ,(nix-protocol-error-message c))))
 272                  (else
 273                   (write `(exception))))
 274          (with-store store
 275            (let* ((files ',files)
 276                   (invalid (remove (cut valid-path? store <>)
 277                                    files)))
 278              (unless (null? invalid)
 279                (write `(invalid-items ,invalid))
 280                (exit 1))
 281
 282              ;; TODO: When RECURSIVE? is true, we could send the list of store
 283              ;; items in the closure so that the other end can filter out
 284              ;; those it already has.
 285
 286              (write '(exporting))                 ;we're ready
 287              (force-output)
 288
 289              (setvbuf (current-output-port) 'none)
 290              (export-paths store files (current-output-port)
 291                            #:recursive? ,recursive?))))))
 292
 293   (open-remote-input-pipe session
 294                           (string-join
 295                            `("guile" "-c"
 296                              ,(object->string
 297                                (object->string export))))))
 298
 299 (define (remote-system session)
 300   "Return the system type as expected by Nix, usually ARCHITECTURE-KERNEL, of
 301 the machine on the other end of SESSION."
 302   (inferior-remote-eval '(begin (use-modules (guix utils)) (%current-system))
 303                         session))
 304
 305 (define* (remote-authorize-signing-key key session #:optional become-command)
 306   "Send KEY, a canonical sexp containing a public key, over SESSION and add it
 307 to the system ACL file if it has not yet been authorized."
 308   (inferior-remote-eval
 309    `(begin
 310       (use-modules (guix build utils)
 311                    (guix pki)
 312                    (guix utils)
 313                    (gcrypt pk-crypto)
 314                    (srfi srfi-26))
 315
 316       (define acl (current-acl))
 317       (define key (string->canonical-sexp ,(canonical-sexp->string key)))
 318
 319       (unless (authorized-key? key)
 320         (let ((acl (public-keys->acl (cons key (acl->public-keys acl)))))
 321           (mkdir-p (dirname %acl-file))
 322           (with-atomic-file-output %acl-file
 323             (cut write-acl acl <>)))))
 324    session
 325    become-command))
 326
 327 (define* (send-files local files remote
 328                      #:key
 329                      recursive?
 330                      (log-port (current-error-port)))
 331   "Send the subset of FILES from LOCAL (a local store) that's missing to
 332 REMOTE, a remote store.  When RECURSIVE? is true, send the closure of FILES.
 333 Return the list of store items actually sent."
 334   ;; Compute the subset of FILES missing on SESSION and send them.
 335   (let* ((files   (if recursive? (requisites local files) files))
 336          (session (channel-get-session (store-connection-socket remote)))
 337          (missing (inferior-remote-eval
 338                    `(begin
 339                       (use-modules (guix)
 340                                    (srfi srfi-1) (srfi srfi-26))
 341
 342                       (with-store store
 343                         (remove (cut valid-path? store <>)
 344                                 ',files)))
 345                    session))
 346          (count   (length missing))
 347          (sizes   (map (lambda (item)
 348                          (path-info-nar-size (query-path-info local item)))
 349                        missing))
 350          (port    (store-import-channel session)))
 351     (format log-port (N_ "sending ~a store item (~h MiB) to '~a'...~%"
 352                          "sending ~a store items (~h MiB) to '~a'...~%" count)
 353             count
 354             (inexact->exact (round (/ (reduce + 0 sizes) (expt 2. 20))))
 355             (session-get session 'host))
 356
 357     ;; Send MISSING in topological order.
 358     (export-paths local missing port)
 359
 360     ;; Tell the remote process that we're done.  (In theory the end-of-archive
 361     ;; mark of 'export-paths' would be enough, but in practice it's not.)
 362     (channel-send-eof port)
 363
 364     ;; Wait for completion of the remote process and read the status sexp from
 365     ;; PORT.  Wait for the exit status only when 'read' completed; otherwise,
 366     ;; we might wait forever if the other end is stuck.
 367     (let* ((result (false-if-exception (read port)))
 368            (status (and result
 369                         (zero? (channel-get-exit-status port)))))
 370       (close-port port)
 371       (match result
 372         (('success . _)
 373          missing)
 374         (('protocol-error message)
 375          (raise (condition
 376                  (&store-protocol-error (message message) (status 42)))))
 377         (('error key args ...)
 378          (raise (condition
 379                  (&store-protocol-error
 380                   (message (call-with-output-string
 381                              (lambda (port)
 382                                (print-exception port #f key args))))
 383                   (status 43)))))
 384         (_
 385          (raise (condition
 386                  (&store-protocol-error
 387                   (message "unknown error while sending files over SSH")
 388                   (status 44)))))))))
 389
 390 (define (remote-store-session remote)
 391   "Return the SSH channel beneath REMOTE, a remote store as returned by
 392 'connect-to-remote-daemon', or #f."
 393   (channel-get-session (store-connection-socket remote)))
 394
 395 (define (remote-store-host remote)
 396   "Return the name of the host REMOTE is connected to, where REMOTE is a
 397 remote store as returned by 'connect-to-remote-daemon'."
 398   (match (remote-store-session remote)
 399     (#f #f)
 400     ((? session? session)
 401      (session-get session 'host))))
 402
 403 (define* (file-retrieval-port files remote
 404                               #:key recursive?)
 405   "Return an input port from which to retrieve FILES (a list of store items)
 406 from REMOTE, along with the number of items to retrieve (lower than or equal
 407 to the length of FILES.)"
 408   (values (store-export-channel (remote-store-session remote) files
 409                                 #:recursive? recursive?)
 410           (length files)))            ;XXX: inaccurate when RECURSIVE? is true
 411
 412 (define-syntax raise-error
 413   (syntax-rules (=>)
 414     ((_ fmt args ... (=> hint-fmt hint-args ...))
 415      (raise (condition
 416              (&message
 417               (message (format #f fmt args ...)))
 418              (&fix-hint
 419               (hint (format #f hint-fmt hint-args ...))))))
 420     ((_ fmt args ...)
 421      (raise (condition
 422              (&message
 423               (message (format #f fmt args ...))))))))
 424
 425 (define* (retrieve-files* files remote
 426                           #:key recursive? (log-port (current-error-port))
 427                           (import (const #f)))
 428   "Pass IMPORT an input port from which to read the sequence of FILES coming
 429 from REMOTE.  When RECURSIVE? is true, retrieve the closure of FILES."
 430   (let-values (((port count)
 431                 (file-retrieval-port files remote
 432                                      #:recursive? recursive?)))
 433     (match (read port)                            ;read the initial status
 434       (('exporting)
 435        (format #t (N_ "retrieving ~a store item from '~a'...~%"
 436                       "retrieving ~a store items from '~a'...~%" count)
 437                count (remote-store-host remote))
 438
 439        (dynamic-wind
 440          (const #t)
 441          (lambda ()
 442            (import port))
 443          (lambda ()
 444            (close-port port))))
 445       ((? eof-object?)
 446        (report-guile-error (remote-store-host remote)))
 447       (('module-error . _)
 448        (report-module-error (remote-store-host remote)))
 449       (('connection-error file code . _)
 450        (raise-error (G_ "failed to connect to '~A' on remote host '~A': ~a")
 451                     file (remote-store-host remote) (strerror code)))
 452       (('invalid-items items . _)
 453        (raise-error (N_ "no such item on remote host '~A':~{ ~a~}"
 454                         "no such items on remote host '~A':~{ ~a~}"
 455                         (length items))
 456                     (remote-store-host remote) items))
 457       (('protocol-error status message . _)
 458        (raise-error (G_ "protocol error on remote host '~A': ~a")
 459                     (remote-store-host remote) message))
 460       (_
 461        (raise-error (G_ "failed to retrieve store items from '~a'")
 462                     (remote-store-host remote))))))
 463
 464 (define* (retrieve-files local files remote
 465                          #:key recursive? (log-port (current-error-port)))
 466   "Retrieve FILES from REMOTE and import them using the 'import-paths' RPC on
 467 LOCAL.  When RECURSIVE? is true, retrieve the closure of FILES."
 468   (retrieve-files* (remove (cut valid-path? local <>) files)
 469                    remote
 470                    #:recursive? recursive?
 471                    #:log-port log-port
 472                    #:import (lambda (port)
 473                               (import-paths local port))))
 474
 475 \f
 476 ;;;
 477 ;;; Error reporting.
 478 ;;;
 479
 480 (define (report-guile-error host)
 481   (raise-error (G_ "failed to start Guile on remote host '~A'") host
 482                (=> (G_ "Make sure @command{guile} can be found in
 483 @code{$PATH} on the remote host.  Run @command{ssh ~A guile --version} to
 484 check.")
 485                    host)))
 486
 487 (define (report-module-error host)
 488   "Report an error about missing Guix modules on HOST."
 489   ;; TRANSLATORS: Leave "Guile" untranslated.
 490   (raise-error (G_ "Guile modules not found on remote host '~A'") host
 491                (=> (G_ "Make sure @code{GUILE_LOAD_PATH} includes Guix'
 492 own module directory.  Run @command{ssh ~A env | grep GUILE_LOAD_PATH} to
 493 check.")
 494                    host)))
 495
 496 ;;; ssh.scm ends here