guix/ssh.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2016, 2017, 2018, 2019 Ludovic Courtès <ludo@gnu.org>
   3 ;;;
   4 ;;; This file is part of GNU Guix.
   5 ;;;
   6 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   7 ;;; under the terms of the GNU General Public License as published by
   8 ;;; the Free Software Foundation; either version 3 of the License, or (at
   9 ;;; your option) any later version.
  10 ;;;
  11 ;;; GNU Guix is distributed in the hope that it will be useful, but
  12 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 ;;; GNU General Public License for more details.
  15 ;;;
  16 ;;; You should have received a copy of the GNU General Public License
  17 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 (define-module (guix ssh)
  20   #:use-module (guix store)
  21   #:use-module (guix inferior)
  22   #:use-module (guix i18n)
  23   #:use-module ((guix utils) #:select (&fix-hint))
  24   #:use-module (ssh session)
  25   #:use-module (ssh auth)
  26   #:use-module (ssh key)
  27   #:use-module (ssh channel)
  28   #:use-module (ssh popen)
  29   #:use-module (ssh session)
  30   #:use-module (srfi srfi-1)
  31   #:use-module (srfi srfi-11)
  32   #:use-module (srfi srfi-26)
  33   #:use-module (srfi srfi-34)
  34   #:use-module (srfi srfi-35)
  35   #:use-module (ice-9 match)
  36   #:use-module (ice-9 format)
  37   #:use-module (ice-9 binary-ports)
  38   #:export (open-ssh-session
  39             remote-inferior
  40             remote-daemon-channel
  41             connect-to-remote-daemon
  42             send-files
  43             retrieve-files
  44             retrieve-files*
  45             remote-store-host
  46
  47             report-guile-error
  48             report-module-error))
  49
  50 ;;; Commentary:
  51 ;;;
  52 ;;; This module provides tools to support communication with remote stores
  53 ;;; over SSH, using Guile-SSH.
  54 ;;;
  55 ;;; Code:
  56
  57 (define %compression
  58   "zlib@openssh.com,zlib")
  59
  60 (define* (open-ssh-session host #:key user port
  61                            (compression %compression))
  62   "Open an SSH session for HOST and return it.  When USER and PORT are #f, use
  63 default values or whatever '~/.ssh/config' specifies; otherwise use them.
  64 Throw an error on failure."
  65   (let ((session (make-session #:user user
  66                                #:host host
  67                                #:port port
  68                                #:timeout 10       ;seconds
  69                                ;; #:log-verbosity 'protocol
  70
  71                                ;; We need lightweight compression when
  72                                ;; exchanging full archives.
  73                                #:compression compression
  74                                #:compression-level 3)))
  75
  76     ;; Honor ~/.ssh/config.
  77     (session-parse-config! session)
  78
  79     (match (connect! session)
  80       ('ok
  81        ;; Use public key authentication, via the SSH agent if it's available.
  82        (match (userauth-public-key/auto! session)
  83          ('success
  84           session)
  85          (x
  86           (disconnect! session)
  87           (raise (condition
  88                   (&message
  89                    (message (format #f (G_ "SSH authentication failed for '~a': ~a~%")
  90                                     host (get-error session)))))))))
  91       (x
  92        ;; Connection failed or timeout expired.
  93        (raise (condition
  94                (&message
  95                 (message (format #f (G_ "SSH connection to '~a' failed: ~a~%")
  96                                  host (get-error session))))))))))
  97
  98 (define (remote-inferior session)
  99   "Return a remote inferior for the given SESSION."
 100   (let ((pipe (open-remote-pipe* session OPEN_BOTH
 101                                  "guix" "repl" "-t" "machine")))
 102     (port->inferior pipe)))
 103
 104 (define (inferior-remote-eval exp session)
 105   "Evaluate EXP in a new inferior running in SESSION, and close the inferior
 106 right away."
 107   (let ((inferior (remote-inferior session)))
 108     (dynamic-wind
 109       (const #t)
 110       (lambda ()
 111         (inferior-eval exp inferior))
 112       (lambda ()
 113         ;; Close INFERIOR right away to prevent finalization from happening in
 114         ;; another thread at the wrong time (see
 115         ;; <https://bugs.gnu.org/26976>.)
 116         (close-inferior inferior)))))
 117
 118 (define* (remote-daemon-channel session
 119                                 #:optional
 120                                 (socket-name
 121                                  "/var/guix/daemon-socket/socket"))
 122   "Return an input/output port (an SSH channel) to the daemon at SESSION."
 123   (define redirect
 124     ;; Code run in SESSION to redirect the remote process' stdin/stdout to the
 125     ;; daemon's socket, à la socat.  The SSH protocol supports forwarding to
 126     ;; Unix-domain sockets but libssh doesn't have an API for that, hence this
 127     ;; hack.
 128     `(begin
 129        (use-modules (ice-9 match) (rnrs io ports)
 130                     (rnrs bytevectors))
 131
 132        (let ((sock    (socket AF_UNIX SOCK_STREAM 0))
 133              (stdin   (current-input-port))
 134              (stdout  (current-output-port))
 135              (select* (lambda (read write except)
 136                         ;; This is a workaround for
 137                         ;; <https://bugs.gnu.org/30365> in Guile < 2.2.4:
 138                         ;; since 'select' sometimes returns non-empty sets for
 139                         ;; no good reason, call 'select' a second time with a
 140                         ;; zero timeout to filter out incorrect replies.
 141                         (match (select read write except)
 142                           ((read write except)
 143                            (select read write except 0))))))
 144          (setvbuf stdout 'none)
 145
 146          ;; Use buffered ports so that 'get-bytevector-some' returns up to the
 147          ;; whole buffer like read(2) would--see <https://bugs.gnu.org/30066>.
 148          (setvbuf stdin 'block 65536)
 149          (setvbuf sock 'block 65536)
 150
 151          (connect sock AF_UNIX ,socket-name)
 152
 153          (let loop ()
 154            (match (select* (list stdin sock) '() '())
 155              ((reads () ())
 156               (when (memq stdin reads)
 157                 (match (get-bytevector-some stdin)
 158                   ((? eof-object?)
 159                    (primitive-exit 0))
 160                   (bv
 161                    (put-bytevector sock bv)
 162                    (force-output sock))))
 163               (when (memq sock reads)
 164                 (match (get-bytevector-some sock)
 165                   ((? eof-object?)
 166                    (primitive-exit 0))
 167                   (bv
 168                    (put-bytevector stdout bv))))
 169               (loop))
 170              (_
 171               (primitive-exit 1)))))))
 172
 173   (open-remote-pipe* session OPEN_BOTH
 174                      ;; Sort-of shell-quote REDIRECT.
 175                      "guile" "-c"
 176                      (object->string
 177                       (object->string redirect))))
 178
 179 (define* (connect-to-remote-daemon session
 180                                    #:optional
 181                                    (socket-name
 182                                     "/var/guix/daemon-socket/socket"))
 183   "Connect to the remote build daemon listening on SOCKET-NAME over SESSION,
 184 an SSH session.  Return a <store-connection> object."
 185   (open-connection #:port (remote-daemon-channel session socket-name)))
 186
 187
 188 (define (store-import-channel session)
 189   "Return an output port to which archives to be exported to SESSION's store
 190 can be written."
 191   ;; Using the 'import-paths' RPC on a remote store would be slow because it
 192   ;; makes a round trip every time 32 KiB have been transferred.  This
 193   ;; procedure instead opens a separate channel to use the remote
 194   ;; 'import-paths' procedure, which consumes all the data in a single round
 195   ;; trip.  This optimizes the successful case at the expense of error
 196   ;; conditions: errors can only be reported once all the input has been
 197   ;; consumed.
 198   (define import
 199     `(begin
 200        (use-modules (guix) (srfi srfi-34)
 201                     (rnrs io ports) (rnrs bytevectors))
 202
 203        (define (consume-input port)
 204          (let ((bv (make-bytevector 32768)))
 205            (let loop ()
 206              (let ((n (get-bytevector-n! port bv 0
 207                                          (bytevector-length bv))))
 208                (unless (eof-object? n)
 209                  (loop))))))
 210
 211        ;; Upon completion, write an sexp that denotes the status.
 212        (write
 213         (catch #t
 214           (lambda ()
 215             (guard (c ((nix-protocol-error? c)
 216                        ;; Consume all the input since the only time we can
 217                        ;; report the error is after everything has been
 218                        ;; consumed.
 219                        (consume-input (current-input-port))
 220                        (list 'protocol-error (nix-protocol-error-message c))))
 221               (with-store store
 222                 (setvbuf (current-input-port) 'none)
 223                 (import-paths store (current-input-port))
 224                 '(success))))
 225           (lambda args
 226             (cons 'error args))))))
 227
 228   (open-remote-pipe session
 229                     (string-join
 230                      `("guile" "-c"
 231                        ,(object->string (object->string import))))
 232                     OPEN_BOTH))
 233
 234 (define* (store-export-channel session files
 235                                #:key recursive?)
 236   "Return an input port from which an export of FILES from SESSION's store can
 237 be read.  When RECURSIVE? is true, the closure of FILES is exported."
 238   ;; Same as above: this is more efficient than calling 'export-paths' on a
 239   ;; remote store.
 240   (define export
 241     `(begin
 242        (eval-when (load expand eval)
 243          (unless (resolve-module '(guix) #:ensure #f)
 244            (write `(module-error))
 245            (exit 7)))
 246
 247        (use-modules (guix) (srfi srfi-1)
 248                     (srfi srfi-26) (srfi srfi-34))
 249
 250        (guard (c ((nix-connection-error? c)
 251                   (write `(connection-error ,(nix-connection-error-file c)
 252                                             ,(nix-connection-error-code c))))
 253                  ((nix-protocol-error? c)
 254                   (write `(protocol-error ,(nix-protocol-error-status c)
 255                                           ,(nix-protocol-error-message c))))
 256                  (else
 257                   (write `(exception))))
 258          (with-store store
 259            (let* ((files ',files)
 260                   (invalid (remove (cut valid-path? store <>)
 261                                    files)))
 262              (unless (null? invalid)
 263                (write `(invalid-items ,invalid))
 264                (exit 1))
 265
 266              ;; TODO: When RECURSIVE? is true, we could send the list of store
 267              ;; items in the closure so that the other end can filter out
 268              ;; those it already has.
 269
 270              (write '(exporting))                 ;we're ready
 271              (force-output)
 272
 273              (setvbuf (current-output-port) 'none)
 274              (export-paths store files (current-output-port)
 275                            #:recursive? ,recursive?))))))
 276
 277   (open-remote-input-pipe session
 278                           (string-join
 279                            `("guile" "-c"
 280                              ,(object->string
 281                                (object->string export))))))
 282
 283 (define* (send-files local files remote
 284                      #:key
 285                      recursive?
 286                      (log-port (current-error-port)))
 287   "Send the subset of FILES from LOCAL (a local store) that's missing to
 288 REMOTE, a remote store.  When RECURSIVE? is true, send the closure of FILES.
 289 Return the list of store items actually sent."
 290   ;; Compute the subset of FILES missing on SESSION and send them.
 291   (let* ((files   (if recursive? (requisites local files) files))
 292          (session (channel-get-session (store-connection-socket remote)))
 293          (missing (inferior-remote-eval
 294                    `(begin
 295                       (use-modules (guix)
 296                                    (srfi srfi-1) (srfi srfi-26))
 297
 298                       (with-store store
 299                         (remove (cut valid-path? store <>)
 300                                 ',files)))
 301                    session))
 302          (count   (length missing))
 303          (sizes   (map (lambda (item)
 304                          (path-info-nar-size (query-path-info local item)))
 305                        missing))
 306          (port    (store-import-channel session)))
 307     (format log-port (N_ "sending ~a store item (~h MiB) to '~a'...~%"
 308                          "sending ~a store items (~h MiB) to '~a'...~%" count)
 309             count
 310             (inexact->exact (round (/ (reduce + 0 sizes) (expt 2. 20))))
 311             (session-get session 'host))
 312
 313     ;; Send MISSING in topological order.
 314     (export-paths local missing port)
 315
 316     ;; Tell the remote process that we're done.  (In theory the end-of-archive
 317     ;; mark of 'export-paths' would be enough, but in practice it's not.)
 318     (channel-send-eof port)
 319
 320     ;; Wait for completion of the remote process and read the status sexp from
 321     ;; PORT.  Wait for the exit status only when 'read' completed; otherwise,
 322     ;; we might wait forever if the other end is stuck.
 323     (let* ((result (false-if-exception (read port)))
 324            (status (and result
 325                         (zero? (channel-get-exit-status port)))))
 326       (close-port port)
 327       (match result
 328         (('success . _)
 329          missing)
 330         (('protocol-error message)
 331          (raise (condition
 332                  (&store-protocol-error (message message) (status 42)))))
 333         (('error key args ...)
 334          (raise (condition
 335                  (&store-protocol-error
 336                   (message (call-with-output-string
 337                              (lambda (port)
 338                                (print-exception port #f key args))))
 339                   (status 43)))))
 340         (_
 341          (raise (condition
 342                  (&store-protocol-error
 343                   (message "unknown error while sending files over SSH")
 344                   (status 44)))))))))
 345
 346 (define (remote-store-session remote)
 347   "Return the SSH channel beneath REMOTE, a remote store as returned by
 348 'connect-to-remote-daemon', or #f."
 349   (channel-get-session (store-connection-socket remote)))
 350
 351 (define (remote-store-host remote)
 352   "Return the name of the host REMOTE is connected to, where REMOTE is a
 353 remote store as returned by 'connect-to-remote-daemon'."
 354   (match (remote-store-session remote)
 355     (#f #f)
 356     ((? session? session)
 357      (session-get session 'host))))
 358
 359 (define* (file-retrieval-port files remote
 360                               #:key recursive?)
 361   "Return an input port from which to retrieve FILES (a list of store items)
 362 from REMOTE, along with the number of items to retrieve (lower than or equal
 363 to the length of FILES.)"
 364   (values (store-export-channel (remote-store-session remote) files
 365                                 #:recursive? recursive?)
 366           (length files)))            ;XXX: inaccurate when RECURSIVE? is true
 367
 368 (define-syntax raise-error
 369   (syntax-rules (=>)
 370     ((_ fmt args ... (=> hint-fmt hint-args ...))
 371      (raise (condition
 372              (&message
 373               (message (format #f fmt args ...)))
 374              (&fix-hint
 375               (hint (format #f hint-fmt hint-args ...))))))
 376     ((_ fmt args ...)
 377      (raise (condition
 378              (&message
 379               (message (format #f fmt args ...))))))))
 380
 381 (define* (retrieve-files* files remote
 382                           #:key recursive? (log-port (current-error-port))
 383                           (import (const #f)))
 384   "Pass IMPORT an input port from which to read the sequence of FILES coming
 385 from REMOTE.  When RECURSIVE? is true, retrieve the closure of FILES."
 386   (let-values (((port count)
 387                 (file-retrieval-port files remote
 388                                      #:recursive? recursive?)))
 389     (match (read port)                            ;read the initial status
 390       (('exporting)
 391        (format #t (N_ "retrieving ~a store item from '~a'...~%"
 392                       "retrieving ~a store items from '~a'...~%" count)
 393                count (remote-store-host remote))
 394
 395        (dynamic-wind
 396          (const #t)
 397          (lambda ()
 398            (import port))
 399          (lambda ()
 400            (close-port port))))
 401       ((? eof-object?)
 402        (report-guile-error (remote-store-host remote)))
 403       (('module-error . _)
 404        (report-module-error (remote-store-host remote)))
 405       (('connection-error file code . _)
 406        (raise-error (G_ "failed to connect to '~A' on remote host '~A': ~a")
 407                     file (remote-store-host remote) (strerror code)))
 408       (('invalid-items items . _)
 409        (raise-error (N_ "no such item on remote host '~A':~{ ~a~}"
 410                         "no such items on remote host '~A':~{ ~a~}"
 411                         (length items))
 412                     (remote-store-host remote) items))
 413       (('protocol-error status message . _)
 414        (raise-error (G_ "protocol error on remote host '~A': ~a")
 415                     (remote-store-host remote) message))
 416       (_
 417        (raise-error (G_ "failed to retrieve store items from '~a'")
 418                     (remote-store-host remote))))))
 419
 420 (define* (retrieve-files local files remote
 421                          #:key recursive? (log-port (current-error-port)))
 422   "Retrieve FILES from REMOTE and import them using the 'import-paths' RPC on
 423 LOCAL.  When RECURSIVE? is true, retrieve the closure of FILES."
 424   (retrieve-files* (remove (cut valid-path? local <>) files)
 425                    remote
 426                    #:recursive? recursive?
 427                    #:log-port log-port
 428                    #:import (lambda (port)
 429                               (import-paths local port))))
 430
 431 \f
 432 ;;;
 433 ;;; Error reporting.
 434 ;;;
 435
 436 (define (report-guile-error host)
 437   (raise-error (G_ "failed to start Guile on remote host '~A'") host
 438                (=> (G_ "Make sure @command{guile} can be found in
 439 @code{$PATH} on the remote host.  Run @command{ssh ~A guile --version} to
 440 check.")
 441                    host)))
 442
 443 (define (report-module-error host)
 444   "Report an error about missing Guix modules on HOST."
 445   ;; TRANSLATORS: Leave "Guile" untranslated.
 446   (raise-error (G_ "Guile modules not found on remote host '~A'") host
 447                (=> (G_ "Make sure @code{GUILE_LOAD_PATH} includes Guix'
 448 own module directory.  Run @command{ssh ~A env | grep GUILE_LOAD_PATH} to
 449 check.")
 450                    host)))
 451
 452 ;;; ssh.scm ends here