guix/ssh.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2016, 2017, 2018, 2019 Ludovic Courtès <ludo@gnu.org>
   3 ;;;
   4 ;;; This file is part of GNU Guix.
   5 ;;;
   6 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   7 ;;; under the terms of the GNU General Public License as published by
   8 ;;; the Free Software Foundation; either version 3 of the License, or (at
   9 ;;; your option) any later version.
  10 ;;;
  11 ;;; GNU Guix is distributed in the hope that it will be useful, but
  12 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 ;;; GNU General Public License for more details.
  15 ;;;
  16 ;;; You should have received a copy of the GNU General Public License
  17 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 (define-module (guix ssh)
  20   #:use-module (guix store)
  21   #:use-module (guix inferior)
  22   #:use-module (guix i18n)
  23   #:use-module ((guix utils) #:select (&fix-hint))
  24   #:use-module (gcrypt pk-crypto)
  25   #:use-module (ssh session)
  26   #:use-module (ssh auth)
  27   #:use-module (ssh key)
  28   #:use-module (ssh channel)
  29   #:use-module (ssh popen)
  30   #:use-module (ssh session)
  31   #:use-module (srfi srfi-1)
  32   #:use-module (srfi srfi-11)
  33   #:use-module (srfi srfi-26)
  34   #:use-module (srfi srfi-34)
  35   #:use-module (srfi srfi-35)
  36   #:use-module (ice-9 match)
  37   #:use-module (ice-9 format)
  38   #:use-module (ice-9 binary-ports)
  39   #:export (open-ssh-session
  40             remote-inferior
  41             remote-daemon-channel
  42             connect-to-remote-daemon
  43             remote-system
  44             remote-authorize-signing-key
  45             send-files
  46             retrieve-files
  47             retrieve-files*
  48             remote-store-host
  49
  50             report-guile-error
  51             report-module-error))
  52
  53 ;;; Commentary:
  54 ;;;
  55 ;;; This module provides tools to support communication with remote stores
  56 ;;; over SSH, using Guile-SSH.
  57 ;;;
  58 ;;; Code:
  59
  60 (define %compression
  61   "zlib@openssh.com,zlib")
  62
  63 (define* (open-ssh-session host #:key user port identity
  64                            (compression %compression)
  65                            (timeout 3600))
  66   "Open an SSH session for HOST and return it.  IDENTITY specifies the file
  67 name of a private key to use for authenticating with the host.  When USER,
  68 PORT, or IDENTITY are #f, use default values or whatever '~/.ssh/config'
  69 specifies; otherwise use them.  Install TIMEOUT as the maximum time in seconds
  70 after which a read or write operation on a channel of the returned session is
  71 considered as failing.
  72
  73 Throw an error on failure."
  74   (let ((session (make-session #:user user
  75                                #:identity identity
  76                                #:host host
  77                                #:port port
  78                                #:timeout 10       ;seconds
  79                                ;; #:log-verbosity 'protocol
  80
  81                                ;; We need lightweight compression when
  82                                ;; exchanging full archives.
  83                                #:compression compression
  84                                #:compression-level 3)))
  85
  86     ;; Honor ~/.ssh/config.
  87     (session-parse-config! session)
  88
  89     (match (connect! session)
  90       ('ok
  91        ;; Use public key authentication, via the SSH agent if it's available.
  92        (match (userauth-public-key/auto! session)
  93          ('success
  94           (session-set! session 'timeout timeout)
  95           session)
  96          (x
  97           (disconnect! session)
  98           (raise (condition
  99                   (&message
 100                    (message (format #f (G_ "SSH authentication failed for '~a': ~a~%")
 101                                     host (get-error session)))))))))
 102       (x
 103        ;; Connection failed or timeout expired.
 104        (raise (condition
 105                (&message
 106                 (message (format #f (G_ "SSH connection to '~a' failed: ~a~%")
 107                                  host (get-error session))))))))))
 108
 109 (define* (remote-inferior session #:optional become-command)
 110   "Return a remote inferior for the given SESSION.  If BECOME-COMMAND is
 111 given, use that to invoke the remote Guile REPL."
 112   (let* ((repl-command (append (or become-command '())
 113                                '("guix" "repl" "-t" "machine")))
 114          (pipe (apply open-remote-pipe* session OPEN_BOTH repl-command)))
 115     (when (eof-object? (peek-char pipe))
 116       (let ((status (channel-get-exit-status pipe)))
 117         (close-port pipe)
 118         (raise (condition
 119                 (&message
 120                  (message (format #f (G_ "remote command '~{~a~^ ~}' failed \
 121 with status ~a")
 122                                   repl-command status)))))))
 123     (port->inferior pipe)))
 124
 125 (define* (inferior-remote-eval exp session #:optional become-command)
 126   "Evaluate EXP in a new inferior running in SESSION, and close the inferior
 127 right away.  If BECOME-COMMAND is given, use that to invoke the remote Guile
 128 REPL."
 129   (let ((inferior (remote-inferior session become-command)))
 130     (dynamic-wind
 131       (const #t)
 132       (lambda ()
 133         (inferior-eval exp inferior))
 134       (lambda ()
 135         ;; Close INFERIOR right away to prevent finalization from happening in
 136         ;; another thread at the wrong time (see
 137         ;; <https://bugs.gnu.org/26976>.)
 138         (close-inferior inferior)))))
 139
 140 (define* (remote-daemon-channel session
 141                                 #:optional
 142                                 (socket-name
 143                                  "/var/guix/daemon-socket/socket"))
 144   "Return an input/output port (an SSH channel) to the daemon at SESSION."
 145   (define redirect
 146     ;; Code run in SESSION to redirect the remote process' stdin/stdout to the
 147     ;; daemon's socket, à la socat.  The SSH protocol supports forwarding to
 148     ;; Unix-domain sockets but libssh doesn't have an API for that, hence this
 149     ;; hack.
 150     `(begin
 151        (use-modules (ice-9 match) (rnrs io ports)
 152                     (rnrs bytevectors))
 153
 154        (let ((sock    (socket AF_UNIX SOCK_STREAM 0))
 155              (stdin   (current-input-port))
 156              (stdout  (current-output-port))
 157              (select* (lambda (read write except)
 158                         ;; This is a workaround for
 159                         ;; <https://bugs.gnu.org/30365> in Guile < 2.2.4:
 160                         ;; since 'select' sometimes returns non-empty sets for
 161                         ;; no good reason, call 'select' a second time with a
 162                         ;; zero timeout to filter out incorrect replies.
 163                         (match (select read write except)
 164                           ((read write except)
 165                            (select read write except 0))))))
 166          (setvbuf stdout 'none)
 167
 168          ;; Use buffered ports so that 'get-bytevector-some' returns up to the
 169          ;; whole buffer like read(2) would--see <https://bugs.gnu.org/30066>.
 170          (setvbuf stdin 'block 65536)
 171          (setvbuf sock 'block 65536)
 172
 173          (connect sock AF_UNIX ,socket-name)
 174
 175          (let loop ()
 176            (match (select* (list stdin sock) '() '())
 177              ((reads () ())
 178               (when (memq stdin reads)
 179                 (match (get-bytevector-some stdin)
 180                   ((? eof-object?)
 181                    (primitive-exit 0))
 182                   (bv
 183                    (put-bytevector sock bv)
 184                    (force-output sock))))
 185               (when (memq sock reads)
 186                 (match (get-bytevector-some sock)
 187                   ((? eof-object?)
 188                    (primitive-exit 0))
 189                   (bv
 190                    (put-bytevector stdout bv))))
 191               (loop))
 192              (_
 193               (primitive-exit 1)))))))
 194
 195   (open-remote-pipe* session OPEN_BOTH
 196                      ;; Sort-of shell-quote REDIRECT.
 197                      "guile" "-c"
 198                      (object->string
 199                       (object->string redirect))))
 200
 201 (define* (connect-to-remote-daemon session
 202                                    #:optional
 203                                    (socket-name
 204                                     "/var/guix/daemon-socket/socket"))
 205   "Connect to the remote build daemon listening on SOCKET-NAME over SESSION,
 206 an SSH session.  Return a <store-connection> object."
 207   (open-connection #:port (remote-daemon-channel session socket-name)))
 208
 209
 210 (define (store-import-channel session)
 211   "Return an output port to which archives to be exported to SESSION's store
 212 can be written."
 213   ;; Using the 'import-paths' RPC on a remote store would be slow because it
 214   ;; makes a round trip every time 32 KiB have been transferred.  This
 215   ;; procedure instead opens a separate channel to use the remote
 216   ;; 'import-paths' procedure, which consumes all the data in a single round
 217   ;; trip.  This optimizes the successful case at the expense of error
 218   ;; conditions: errors can only be reported once all the input has been
 219   ;; consumed.
 220   (define import
 221     `(begin
 222        (use-modules (guix) (srfi srfi-34)
 223                     (rnrs io ports) (rnrs bytevectors))
 224
 225        (define (consume-input port)
 226          (let ((bv (make-bytevector 32768)))
 227            (let loop ()
 228              (let ((n (get-bytevector-n! port bv 0
 229                                          (bytevector-length bv))))
 230                (unless (eof-object? n)
 231                  (loop))))))
 232
 233        ;; Upon completion, write an sexp that denotes the status.
 234        (write
 235         (catch #t
 236           (lambda ()
 237             (guard (c ((nix-protocol-error? c)
 238                        ;; Consume all the input since the only time we can
 239                        ;; report the error is after everything has been
 240                        ;; consumed.
 241                        (consume-input (current-input-port))
 242                        (list 'protocol-error (nix-protocol-error-message c))))
 243               (with-store store
 244                 (setvbuf (current-input-port) 'none)
 245                 (import-paths store (current-input-port))
 246                 '(success))))
 247           (lambda args
 248             (cons 'error args))))))
 249
 250   (open-remote-pipe session
 251                     (string-join
 252                      `("guile" "-c"
 253                        ,(object->string (object->string import))))
 254                     OPEN_BOTH))
 255
 256 (define* (store-export-channel session files
 257                                #:key recursive?)
 258   "Return an input port from which an export of FILES from SESSION's store can
 259 be read.  When RECURSIVE? is true, the closure of FILES is exported."
 260   ;; Same as above: this is more efficient than calling 'export-paths' on a
 261   ;; remote store.
 262   (define export
 263     `(begin
 264        (eval-when (load expand eval)
 265          (unless (resolve-module '(guix) #:ensure #f)
 266            (write `(module-error))
 267            (exit 7)))
 268
 269        (use-modules (guix) (srfi srfi-1)
 270                     (srfi srfi-26) (srfi srfi-34))
 271
 272        (guard (c ((nix-connection-error? c)
 273                   (write `(connection-error ,(nix-connection-error-file c)
 274                                             ,(nix-connection-error-code c))))
 275                  ((nix-protocol-error? c)
 276                   (write `(protocol-error ,(nix-protocol-error-status c)
 277                                           ,(nix-protocol-error-message c))))
 278                  (else
 279                   (write `(exception))))
 280          (with-store store
 281            (let* ((files ',files)
 282                   (invalid (remove (cut valid-path? store <>)
 283                                    files)))
 284              (unless (null? invalid)
 285                (write `(invalid-items ,invalid))
 286                (exit 1))
 287
 288              ;; TODO: When RECURSIVE? is true, we could send the list of store
 289              ;; items in the closure so that the other end can filter out
 290              ;; those it already has.
 291
 292              (write '(exporting))                 ;we're ready
 293              (force-output)
 294
 295              (setvbuf (current-output-port) 'none)
 296              (export-paths store files (current-output-port)
 297                            #:recursive? ,recursive?))))))
 298
 299   (open-remote-input-pipe session
 300                           (string-join
 301                            `("guile" "-c"
 302                              ,(object->string
 303                                (object->string export))))))
 304
 305 (define (remote-system session)
 306   "Return the system type as expected by Nix, usually ARCHITECTURE-KERNEL, of
 307 the machine on the other end of SESSION."
 308   (inferior-remote-eval '(begin (use-modules (guix utils)) (%current-system))
 309                         session))
 310
 311 (define* (remote-authorize-signing-key key session #:optional become-command)
 312   "Send KEY, a canonical sexp containing a public key, over SESSION and add it
 313 to the system ACL file if it has not yet been authorized."
 314   (inferior-remote-eval
 315    `(begin
 316       (use-modules (guix build utils)
 317                    (guix pki)
 318                    (guix utils)
 319                    (gcrypt pk-crypto)
 320                    (srfi srfi-26))
 321
 322       (define acl (current-acl))
 323       (define key (string->canonical-sexp ,(canonical-sexp->string key)))
 324
 325       (unless (authorized-key? key)
 326         (let ((acl (public-keys->acl (cons key (acl->public-keys acl)))))
 327           (mkdir-p (dirname %acl-file))
 328           (with-atomic-file-output %acl-file
 329             (cut write-acl acl <>)))))
 330    session
 331    become-command))
 332
 333 (define* (send-files local files remote
 334                      #:key
 335                      recursive?
 336                      (log-port (current-error-port)))
 337   "Send the subset of FILES from LOCAL (a local store) that's missing to
 338 REMOTE, a remote store.  When RECURSIVE? is true, send the closure of FILES.
 339 Return the list of store items actually sent."
 340   ;; Compute the subset of FILES missing on SESSION and send them.
 341   (let* ((files   (if recursive? (requisites local files) files))
 342          (session (channel-get-session (store-connection-socket remote)))
 343          (missing (inferior-remote-eval
 344                    `(begin
 345                       (use-modules (guix)
 346                                    (srfi srfi-1) (srfi srfi-26))
 347
 348                       (with-store store
 349                         (remove (cut valid-path? store <>)
 350                                 ',files)))
 351                    session))
 352          (count   (length missing))
 353          (sizes   (map (lambda (item)
 354                          (path-info-nar-size (query-path-info local item)))
 355                        missing))
 356          (port    (store-import-channel session)))
 357     (format log-port (N_ "sending ~a store item (~h MiB) to '~a'...~%"
 358                          "sending ~a store items (~h MiB) to '~a'...~%" count)
 359             count
 360             (inexact->exact (round (/ (reduce + 0 sizes) (expt 2. 20))))
 361             (session-get session 'host))
 362
 363     ;; Send MISSING in topological order.
 364     (export-paths local missing port)
 365
 366     ;; Tell the remote process that we're done.  (In theory the end-of-archive
 367     ;; mark of 'export-paths' would be enough, but in practice it's not.)
 368     (channel-send-eof port)
 369
 370     ;; Wait for completion of the remote process and read the status sexp from
 371     ;; PORT.  Wait for the exit status only when 'read' completed; otherwise,
 372     ;; we might wait forever if the other end is stuck.
 373     (let* ((result (false-if-exception (read port)))
 374            (status (and result
 375                         (zero? (channel-get-exit-status port)))))
 376       (close-port port)
 377       (match result
 378         (('success . _)
 379          missing)
 380         (('protocol-error message)
 381          (raise (condition
 382                  (&store-protocol-error (message message) (status 42)))))
 383         (('error key args ...)
 384          (raise (condition
 385                  (&store-protocol-error
 386                   (message (call-with-output-string
 387                              (lambda (port)
 388                                (print-exception port #f key args))))
 389                   (status 43)))))
 390         (_
 391          (raise (condition
 392                  (&store-protocol-error
 393                   (message "unknown error while sending files over SSH")
 394                   (status 44)))))))))
 395
 396 (define (remote-store-session remote)
 397   "Return the SSH channel beneath REMOTE, a remote store as returned by
 398 'connect-to-remote-daemon', or #f."
 399   (channel-get-session (store-connection-socket remote)))
 400
 401 (define (remote-store-host remote)
 402   "Return the name of the host REMOTE is connected to, where REMOTE is a
 403 remote store as returned by 'connect-to-remote-daemon'."
 404   (match (remote-store-session remote)
 405     (#f #f)
 406     ((? session? session)
 407      (session-get session 'host))))
 408
 409 (define* (file-retrieval-port files remote
 410                               #:key recursive?)
 411   "Return an input port from which to retrieve FILES (a list of store items)
 412 from REMOTE, along with the number of items to retrieve (lower than or equal
 413 to the length of FILES.)"
 414   (values (store-export-channel (remote-store-session remote) files
 415                                 #:recursive? recursive?)
 416           (length files)))            ;XXX: inaccurate when RECURSIVE? is true
 417
 418 (define-syntax raise-error
 419   (syntax-rules (=>)
 420     ((_ fmt args ... (=> hint-fmt hint-args ...))
 421      (raise (condition
 422              (&message
 423               (message (format #f fmt args ...)))
 424              (&fix-hint
 425               (hint (format #f hint-fmt hint-args ...))))))
 426     ((_ fmt args ...)
 427      (raise (condition
 428              (&message
 429               (message (format #f fmt args ...))))))))
 430
 431 (define* (retrieve-files* files remote
 432                           #:key recursive? (log-port (current-error-port))
 433                           (import (const #f)))
 434   "Pass IMPORT an input port from which to read the sequence of FILES coming
 435 from REMOTE.  When RECURSIVE? is true, retrieve the closure of FILES."
 436   (let-values (((port count)
 437                 (file-retrieval-port files remote
 438                                      #:recursive? recursive?)))
 439     (match (read port)                            ;read the initial status
 440       (('exporting)
 441        (format #t (N_ "retrieving ~a store item from '~a'...~%"
 442                       "retrieving ~a store items from '~a'...~%" count)
 443                count (remote-store-host remote))
 444
 445        (dynamic-wind
 446          (const #t)
 447          (lambda ()
 448            (import port))
 449          (lambda ()
 450            (close-port port))))
 451       ((? eof-object?)
 452        (report-guile-error (remote-store-host remote)))
 453       (('module-error . _)
 454        (report-module-error (remote-store-host remote)))
 455       (('connection-error file code . _)
 456        (raise-error (G_ "failed to connect to '~A' on remote host '~A': ~a")
 457                     file (remote-store-host remote) (strerror code)))
 458       (('invalid-items items . _)
 459        (raise-error (N_ "no such item on remote host '~A':~{ ~a~}"
 460                         "no such items on remote host '~A':~{ ~a~}"
 461                         (length items))
 462                     (remote-store-host remote) items))
 463       (('protocol-error status message . _)
 464        (raise-error (G_ "protocol error on remote host '~A': ~a")
 465                     (remote-store-host remote) message))
 466       (_
 467        (raise-error (G_ "failed to retrieve store items from '~a'")
 468                     (remote-store-host remote))))))
 469
 470 (define* (retrieve-files local files remote
 471                          #:key recursive? (log-port (current-error-port)))
 472   "Retrieve FILES from REMOTE and import them using the 'import-paths' RPC on
 473 LOCAL.  When RECURSIVE? is true, retrieve the closure of FILES."
 474   (retrieve-files* (remove (cut valid-path? local <>) files)
 475                    remote
 476                    #:recursive? recursive?
 477                    #:log-port log-port
 478                    #:import (lambda (port)
 479                               (import-paths local port))))
 480
 481 \f
 482 ;;;
 483 ;;; Error reporting.
 484 ;;;
 485
 486 (define (report-guile-error host)
 487   (raise-error (G_ "failed to start Guile on remote host '~A'") host
 488                (=> (G_ "Make sure @command{guile} can be found in
 489 @code{$PATH} on the remote host.  Run @command{ssh ~A guile --version} to
 490 check.")
 491                    host)))
 492
 493 (define (report-module-error host)
 494   "Report an error about missing Guix modules on HOST."
 495   ;; TRANSLATORS: Leave "Guile" untranslated.
 496   (raise-error (G_ "Guile modules not found on remote host '~A'") host
 497                (=> (G_ "Make sure @code{GUILE_LOAD_PATH} includes Guix'
 498 own module directory.  Run @command{ssh ~A env | grep GUILE_LOAD_PATH} to
 499 check.")
 500                    host)))
 501
 502 ;;; ssh.scm ends here