guix/ssh.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
   3 ;;;
   4 ;;; This file is part of GNU Guix.
   5 ;;;
   6 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   7 ;;; under the terms of the GNU General Public License as published by
   8 ;;; the Free Software Foundation; either version 3 of the License, or (at
   9 ;;; your option) any later version.
  10 ;;;
  11 ;;; GNU Guix is distributed in the hope that it will be useful, but
  12 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 ;;; GNU General Public License for more details.
  15 ;;;
  16 ;;; You should have received a copy of the GNU General Public License
  17 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 (define-module (guix ssh)
  20   #:use-module (guix store)
  21   #:use-module (guix inferior)
  22   #:use-module (guix i18n)
  23   #:use-module ((guix utils) #:select (&fix-hint))
  24   #:use-module (ssh session)
  25   #:use-module (ssh auth)
  26   #:use-module (ssh key)
  27   #:use-module (ssh channel)
  28   #:use-module (ssh popen)
  29   #:use-module (ssh session)
  30   #:use-module (srfi srfi-1)
  31   #:use-module (srfi srfi-11)
  32   #:use-module (srfi srfi-26)
  33   #:use-module (srfi srfi-34)
  34   #:use-module (srfi srfi-35)
  35   #:use-module (ice-9 match)
  36   #:use-module (ice-9 binary-ports)
  37   #:export (open-ssh-session
  38             remote-inferior
  39             remote-daemon-channel
  40             connect-to-remote-daemon
  41             send-files
  42             retrieve-files
  43             retrieve-files*
  44             remote-store-host
  45
  46             report-guile-error
  47             report-module-error))
  48
  49 ;;; Commentary:
  50 ;;;
  51 ;;; This module provides tools to support communication with remote stores
  52 ;;; over SSH, using Guile-SSH.
  53 ;;;
  54 ;;; Code:
  55
  56 (define %compression
  57   "zlib@openssh.com,zlib")
  58
  59 (define* (open-ssh-session host #:key user port
  60                            (compression %compression))
  61   "Open an SSH session for HOST and return it.  When USER and PORT are #f, use
  62 default values or whatever '~/.ssh/config' specifies; otherwise use them.
  63 Throw an error on failure."
  64   (let ((session (make-session #:user user
  65                                #:host host
  66                                #:port port
  67                                #:timeout 10       ;seconds
  68                                ;; #:log-verbosity 'protocol
  69
  70                                ;; We need lightweight compression when
  71                                ;; exchanging full archives.
  72                                #:compression compression
  73                                #:compression-level 3)))
  74
  75     ;; Honor ~/.ssh/config.
  76     (session-parse-config! session)
  77
  78     (match (connect! session)
  79       ('ok
  80        ;; Use public key authentication, via the SSH agent if it's available.
  81        (match (userauth-public-key/auto! session)
  82          ('success
  83           session)
  84          (x
  85           (disconnect! session)
  86           (raise (condition
  87                   (&message
  88                    (message (format #f (G_ "SSH authentication failed for '~a': ~a~%")
  89                                     host (get-error session)))))))))
  90       (x
  91        ;; Connection failed or timeout expired.
  92        (raise (condition
  93                (&message
  94                 (message (format #f (G_ "SSH connection to '~a' failed: ~a~%")
  95                                  host (get-error session))))))))))
  96
  97 (define (remote-inferior session)
  98   "Return a remote inferior for the given SESSION."
  99   (let ((pipe (open-remote-pipe* session OPEN_BOTH
 100                                  "guix" "repl" "-t" "machine")))
 101     (port->inferior pipe)))
 102
 103 (define (inferior-remote-eval exp session)
 104   "Evaluate EXP in a new inferior running in SESSION, and close the inferior
 105 right away."
 106   (let ((inferior (remote-inferior session)))
 107     (dynamic-wind
 108       (const #t)
 109       (lambda ()
 110         (inferior-eval exp inferior))
 111       (lambda ()
 112         ;; Close INFERIOR right away to prevent finalization from happening in
 113         ;; another thread at the wrong time (see
 114         ;; <https://bugs.gnu.org/26976>.)
 115         (close-inferior inferior)))))
 116
 117 (define* (remote-daemon-channel session
 118                                 #:optional
 119                                 (socket-name
 120                                  "/var/guix/daemon-socket/socket"))
 121   "Return an input/output port (an SSH channel) to the daemon at SESSION."
 122   (define redirect
 123     ;; Code run in SESSION to redirect the remote process' stdin/stdout to the
 124     ;; daemon's socket, à la socat.  The SSH protocol supports forwarding to
 125     ;; Unix-domain sockets but libssh doesn't have an API for that, hence this
 126     ;; hack.
 127     `(begin
 128        (use-modules (ice-9 match) (rnrs io ports)
 129                     (rnrs bytevectors))
 130
 131        (let ((sock    (socket AF_UNIX SOCK_STREAM 0))
 132              (stdin   (current-input-port))
 133              (stdout  (current-output-port))
 134              (select* (lambda (read write except)
 135                         ;; This is a workaround for
 136                         ;; <https://bugs.gnu.org/30365> in Guile < 2.2.4:
 137                         ;; since 'select' sometimes returns non-empty sets for
 138                         ;; no good reason, call 'select' a second time with a
 139                         ;; zero timeout to filter out incorrect replies.
 140                         (match (select read write except)
 141                           ((read write except)
 142                            (select read write except 0))))))
 143          (setvbuf stdout _IONBF)
 144
 145          ;; Use buffered ports so that 'get-bytevector-some' returns up to the
 146          ;; whole buffer like read(2) would--see <https://bugs.gnu.org/30066>.
 147          (setvbuf stdin _IOFBF 65536)
 148          (setvbuf sock _IOFBF 65536)
 149
 150          (connect sock AF_UNIX ,socket-name)
 151
 152          (let loop ()
 153            (match (select* (list stdin sock) '() '())
 154              ((reads () ())
 155               (when (memq stdin reads)
 156                 (match (get-bytevector-some stdin)
 157                   ((? eof-object?)
 158                    (primitive-exit 0))
 159                   (bv
 160                    (put-bytevector sock bv)
 161                    (force-output sock))))
 162               (when (memq sock reads)
 163                 (match (get-bytevector-some sock)
 164                   ((? eof-object?)
 165                    (primitive-exit 0))
 166                   (bv
 167                    (put-bytevector stdout bv))))
 168               (loop))
 169              (_
 170               (primitive-exit 1)))))))
 171
 172   (open-remote-pipe* session OPEN_BOTH
 173                      ;; Sort-of shell-quote REDIRECT.
 174                      "guile" "-c"
 175                      (object->string
 176                       (object->string redirect))))
 177
 178 (define* (connect-to-remote-daemon session
 179                                    #:optional
 180                                    (socket-name
 181                                     "/var/guix/daemon-socket/socket"))
 182   "Connect to the remote build daemon listening on SOCKET-NAME over SESSION,
 183 an SSH session.  Return a <nix-server> object."
 184   (open-connection #:port (remote-daemon-channel session socket-name)))
 185
 186
 187 (define (store-import-channel session)
 188   "Return an output port to which archives to be exported to SESSION's store
 189 can be written."
 190   ;; Using the 'import-paths' RPC on a remote store would be slow because it
 191   ;; makes a round trip every time 32 KiB have been transferred.  This
 192   ;; procedure instead opens a separate channel to use the remote
 193   ;; 'import-paths' procedure, which consumes all the data in a single round
 194   ;; trip.  This optimizes the successful case at the expense of error
 195   ;; conditions: errors can only be reported once all the input has been
 196   ;; consumed.
 197   (define import
 198     `(begin
 199        (use-modules (guix) (srfi srfi-34)
 200                     (rnrs io ports) (rnrs bytevectors))
 201
 202        (define (consume-input port)
 203          (let ((bv (make-bytevector 32768)))
 204            (let loop ()
 205              (let ((n (get-bytevector-n! port bv 0
 206                                          (bytevector-length bv))))
 207                (unless (eof-object? n)
 208                  (loop))))))
 209
 210        ;; Upon completion, write an sexp that denotes the status.
 211        (write
 212         (catch #t
 213           (lambda ()
 214             (guard (c ((nix-protocol-error? c)
 215                        ;; Consume all the input since the only time we can
 216                        ;; report the error is after everything has been
 217                        ;; consumed.
 218                        (consume-input (current-input-port))
 219                        (list 'protocol-error (nix-protocol-error-message c))))
 220               (with-store store
 221                 (setvbuf (current-input-port) _IONBF)
 222                 (import-paths store (current-input-port))
 223                 '(success))))
 224           (lambda args
 225             (cons 'error args))))))
 226
 227   (open-remote-pipe session
 228                     (string-join
 229                      `("guile" "-c"
 230                        ,(object->string (object->string import))))
 231                     OPEN_BOTH))
 232
 233 (define* (store-export-channel session files
 234                                #:key recursive?)
 235   "Return an input port from which an export of FILES from SESSION's store can
 236 be read.  When RECURSIVE? is true, the closure of FILES is exported."
 237   ;; Same as above: this is more efficient than calling 'export-paths' on a
 238   ;; remote store.
 239   (define export
 240     `(begin
 241        (eval-when (load expand eval)
 242          (unless (resolve-module '(guix) #:ensure #f)
 243            (write `(module-error))
 244            (exit 7)))
 245
 246        (use-modules (guix) (srfi srfi-1)
 247                     (srfi srfi-26) (srfi srfi-34))
 248
 249        (guard (c ((nix-connection-error? c)
 250                   (write `(connection-error ,(nix-connection-error-file c)
 251                                             ,(nix-connection-error-code c))))
 252                  ((nix-protocol-error? c)
 253                   (write `(protocol-error ,(nix-protocol-error-status c)
 254                                           ,(nix-protocol-error-message c))))
 255                  (else
 256                   (write `(exception))))
 257          (with-store store
 258            (let* ((files ',files)
 259                   (invalid (remove (cut valid-path? store <>)
 260                                    files)))
 261              (unless (null? invalid)
 262                (write `(invalid-items ,invalid))
 263                (exit 1))
 264
 265              ;; TODO: When RECURSIVE? is true, we could send the list of store
 266              ;; items in the closure so that the other end can filter out
 267              ;; those it already has.
 268
 269              (write '(exporting))                 ;we're ready
 270              (force-output)
 271
 272              (setvbuf (current-output-port) _IONBF)
 273              (export-paths store files (current-output-port)
 274                            #:recursive? ,recursive?))))))
 275
 276   (open-remote-input-pipe session
 277                           (string-join
 278                            `("guile" "-c"
 279                              ,(object->string
 280                                (object->string export))))))
 281
 282 (define* (send-files local files remote
 283                      #:key
 284                      recursive?
 285                      (log-port (current-error-port)))
 286   "Send the subset of FILES from LOCAL (a local store) that's missing to
 287 REMOTE, a remote store.  When RECURSIVE? is true, send the closure of FILES.
 288 Return the list of store items actually sent."
 289   ;; Compute the subset of FILES missing on SESSION and send them.
 290   (let* ((files   (if recursive? (requisites local files) files))
 291          (session (channel-get-session (nix-server-socket remote)))
 292          (missing (inferior-remote-eval
 293                    `(begin
 294                       (use-modules (guix)
 295                                    (srfi srfi-1) (srfi srfi-26))
 296
 297                       (with-store store
 298                         (remove (cut valid-path? store <>)
 299                                 ',files)))
 300                    session))
 301          (count   (length missing))
 302          (sizes   (map (lambda (item)
 303                          (path-info-nar-size (query-path-info local item)))
 304                        missing))
 305          (port    (store-import-channel session)))
 306     (format log-port (N_ "sending ~a store item (~h MiB) to '~a'...~%"
 307                          "sending ~a store items (~h MiB) to '~a'...~%" count)
 308             count
 309             (inexact->exact (round (/ (reduce + 0 sizes) (expt 2. 20))))
 310             (session-get session 'host))
 311
 312     ;; Send MISSING in topological order.
 313     (export-paths local missing port)
 314
 315     ;; Tell the remote process that we're done.  (In theory the end-of-archive
 316     ;; mark of 'export-paths' would be enough, but in practice it's not.)
 317     (channel-send-eof port)
 318
 319     ;; Wait for completion of the remote process and read the status sexp from
 320     ;; PORT.  Wait for the exit status only when 'read' completed; otherwise,
 321     ;; we might wait forever if the other end is stuck.
 322     (let* ((result (false-if-exception (read port)))
 323            (status (and result
 324                         (zero? (channel-get-exit-status port)))))
 325       (close-port port)
 326       (match result
 327         (('success . _)
 328          missing)
 329         (('protocol-error message)
 330          (raise (condition
 331                  (&nix-protocol-error (message message) (status 42)))))
 332         (('error key args ...)
 333          (raise (condition
 334                  (&nix-protocol-error
 335                   (message (call-with-output-string
 336                              (lambda (port)
 337                                (print-exception port #f key args))))
 338                   (status 43)))))
 339         (_
 340          (raise (condition
 341                  (&nix-protocol-error
 342                   (message "unknown error while sending files over SSH")
 343                   (status 44)))))))))
 344
 345 (define (remote-store-session remote)
 346   "Return the SSH channel beneath REMOTE, a remote store as returned by
 347 'connect-to-remote-daemon', or #f."
 348   (channel-get-session (nix-server-socket remote)))
 349
 350 (define (remote-store-host remote)
 351   "Return the name of the host REMOTE is connected to, where REMOTE is a
 352 remote store as returned by 'connect-to-remote-daemon'."
 353   (match (remote-store-session remote)
 354     (#f #f)
 355     ((? session? session)
 356      (session-get session 'host))))
 357
 358 (define* (file-retrieval-port files remote
 359                               #:key recursive?)
 360   "Return an input port from which to retrieve FILES (a list of store items)
 361 from REMOTE, along with the number of items to retrieve (lower than or equal
 362 to the length of FILES.)"
 363   (values (store-export-channel (remote-store-session remote) files
 364                                 #:recursive? recursive?)
 365           (length files)))            ;XXX: inaccurate when RECURSIVE? is true
 366
 367 (define-syntax raise-error
 368   (syntax-rules (=>)
 369     ((_ fmt args ... (=> hint-fmt hint-args ...))
 370      (raise (condition
 371              (&message
 372               (message (format #f fmt args ...)))
 373              (&fix-hint
 374               (hint (format #f hint-fmt hint-args ...))))))
 375     ((_ fmt args ...)
 376      (raise (condition
 377              (&message
 378               (message (format #f fmt args ...))))))))
 379
 380 (define* (retrieve-files* files remote
 381                           #:key recursive? (log-port (current-error-port))
 382                           (import (const #f)))
 383   "Pass IMPORT an input port from which to read the sequence of FILES coming
 384 from REMOTE.  When RECURSIVE? is true, retrieve the closure of FILES."
 385   (let-values (((port count)
 386                 (file-retrieval-port files remote
 387                                      #:recursive? recursive?)))
 388     (match (read port)                            ;read the initial status
 389       (('exporting)
 390        (format #t (N_ "retrieving ~a store item from '~a'...~%"
 391                       "retrieving ~a store items from '~a'...~%" count)
 392                count (remote-store-host remote))
 393
 394        (dynamic-wind
 395          (const #t)
 396          (lambda ()
 397            (import port))
 398          (lambda ()
 399            (close-port port))))
 400       ((? eof-object?)
 401        (report-guile-error (remote-store-host remote)))
 402       (('module-error . _)
 403        (report-module-error (remote-store-host remote)))
 404       (('connection-error file code . _)
 405        (raise-error (G_ "failed to connect to '~A' on remote host '~A': ~a")
 406                     file (remote-store-host remote) (strerror code)))
 407       (('invalid-items items . _)
 408        (raise-error (N_ "no such item on remote host '~A':~{ ~a~}"
 409                         "no such items on remote host '~A':~{ ~a~}"
 410                         (length items))
 411                     (remote-store-host remote) items))
 412       (('protocol-error status message . _)
 413        (raise-error (G_ "protocol error on remote host '~A': ~a")
 414                     (remote-store-host remote) message))
 415       (_
 416        (raise-error (G_ "failed to retrieve store items from '~a'")
 417                     (remote-store-host remote))))))
 418
 419 (define* (retrieve-files local files remote
 420                          #:key recursive? (log-port (current-error-port)))
 421   "Retrieve FILES from REMOTE and import them using the 'import-paths' RPC on
 422 LOCAL.  When RECURSIVE? is true, retrieve the closure of FILES."
 423   (retrieve-files* (remove (cut valid-path? local <>) files)
 424                    remote
 425                    #:recursive? recursive?
 426                    #:log-port log-port
 427                    #:import (lambda (port)
 428                               (import-paths local port))))
 429
 430 \f
 431 ;;;
 432 ;;; Error reporting.
 433 ;;;
 434
 435 (define (report-guile-error host)
 436   (raise-error (G_ "failed to start Guile on remote host '~A'") host
 437                (=> (G_ "Make sure @command{guile} can be found in
 438 @code{$PATH} on the remote host.  Run @command{ssh ~A guile --version} to
 439 check.")
 440                    host)))
 441
 442 (define (report-module-error host)
 443   "Report an error about missing Guix modules on HOST."
 444   ;; TRANSLATORS: Leave "Guile" untranslated.
 445   (raise-error (G_ "Guile modules not found on remote host '~A'") host
 446                (=> (G_ "Make sure @code{GUILE_LOAD_PATH} includes Guix'
 447 own module directory.  Run @command{ssh ~A env | grep GUILE_LOAD_PATH} to
 448 check.")
 449                    host)))
 450
 451 ;;; ssh.scm ends here