guix/ssh.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
   3 ;;;
   4 ;;; This file is part of GNU Guix.
   5 ;;;
   6 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   7 ;;; under the terms of the GNU General Public License as published by
   8 ;;; the Free Software Foundation; either version 3 of the License, or (at
   9 ;;; your option) any later version.
  10 ;;;
  11 ;;; GNU Guix is distributed in the hope that it will be useful, but
  12 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 ;;; GNU General Public License for more details.
  15 ;;;
  16 ;;; You should have received a copy of the GNU General Public License
  17 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 (define-module (guix ssh)
  20   #:use-module (guix store)
  21   #:use-module (guix inferior)
  22   #:use-module (guix i18n)
  23   #:use-module ((guix utils) #:select (&fix-hint))
  24   #:use-module (ssh session)
  25   #:use-module (ssh auth)
  26   #:use-module (ssh key)
  27   #:use-module (ssh channel)
  28   #:use-module (ssh popen)
  29   #:use-module (ssh session)
  30   #:use-module (ssh dist)
  31   #:use-module (ssh dist node)
  32   #:use-module (srfi srfi-1)
  33   #:use-module (srfi srfi-11)
  34   #:use-module (srfi srfi-26)
  35   #:use-module (srfi srfi-34)
  36   #:use-module (srfi srfi-35)
  37   #:use-module (ice-9 match)
  38   #:use-module (ice-9 binary-ports)
  39   #:export (open-ssh-session
  40             remote-inferior
  41             remote-daemon-channel
  42             connect-to-remote-daemon
  43             send-files
  44             retrieve-files
  45             retrieve-files*
  46             remote-store-host
  47
  48             report-guile-error
  49             report-module-error))
  50
  51 ;;; Commentary:
  52 ;;;
  53 ;;; This module provides tools to support communication with remote stores
  54 ;;; over SSH, using Guile-SSH.
  55 ;;;
  56 ;;; Code:
  57
  58 (define %compression
  59   "zlib@openssh.com,zlib")
  60
  61 (define* (open-ssh-session host #:key user port
  62                            (compression %compression))
  63   "Open an SSH session for HOST and return it.  When USER and PORT are #f, use
  64 default values or whatever '~/.ssh/config' specifies; otherwise use them.
  65 Throw an error on failure."
  66   (let ((session (make-session #:user user
  67                                #:host host
  68                                #:port port
  69                                #:timeout 10       ;seconds
  70                                ;; #:log-verbosity 'protocol
  71
  72                                ;; We need lightweight compression when
  73                                ;; exchanging full archives.
  74                                #:compression compression
  75                                #:compression-level 3)))
  76
  77     ;; Honor ~/.ssh/config.
  78     (session-parse-config! session)
  79
  80     (match (connect! session)
  81       ('ok
  82        ;; Use public key authentication, via the SSH agent if it's available.
  83        (match (userauth-public-key/auto! session)
  84          ('success
  85           session)
  86          (x
  87           (disconnect! session)
  88           (raise (condition
  89                   (&message
  90                    (message (format #f (G_ "SSH authentication failed for '~a': ~a~%")
  91                                     host (get-error session)))))))))
  92       (x
  93        ;; Connection failed or timeout expired.
  94        (raise (condition
  95                (&message
  96                 (message (format #f (G_ "SSH connection to '~a' failed: ~a~%")
  97                                  host (get-error session))))))))))
  98
  99 (define (remote-inferior session)
 100   "Return a remote inferior for the given SESSION."
 101   (let ((pipe (open-remote-pipe* session OPEN_BOTH
 102                                  "guix" "repl" "-t" "machine")))
 103     (port->inferior pipe)))
 104
 105 (define* (remote-daemon-channel session
 106                                 #:optional
 107                                 (socket-name
 108                                  "/var/guix/daemon-socket/socket"))
 109   "Return an input/output port (an SSH channel) to the daemon at SESSION."
 110   (define redirect
 111     ;; Code run in SESSION to redirect the remote process' stdin/stdout to the
 112     ;; daemon's socket, à la socat.  The SSH protocol supports forwarding to
 113     ;; Unix-domain sockets but libssh doesn't have an API for that, hence this
 114     ;; hack.
 115     `(begin
 116        (use-modules (ice-9 match) (rnrs io ports)
 117                     (rnrs bytevectors))
 118
 119        (let ((sock    (socket AF_UNIX SOCK_STREAM 0))
 120              (stdin   (current-input-port))
 121              (stdout  (current-output-port))
 122              (select* (lambda (read write except)
 123                         ;; This is a workaround for
 124                         ;; <https://bugs.gnu.org/30365> in Guile < 2.2.4:
 125                         ;; since 'select' sometimes returns non-empty sets for
 126                         ;; no good reason, call 'select' a second time with a
 127                         ;; zero timeout to filter out incorrect replies.
 128                         (match (select read write except)
 129                           ((read write except)
 130                            (select read write except 0))))))
 131          (setvbuf stdout _IONBF)
 132
 133          ;; Use buffered ports so that 'get-bytevector-some' returns up to the
 134          ;; whole buffer like read(2) would--see <https://bugs.gnu.org/30066>.
 135          (setvbuf stdin _IOFBF 65536)
 136          (setvbuf sock _IOFBF 65536)
 137
 138          (connect sock AF_UNIX ,socket-name)
 139
 140          (let loop ()
 141            (match (select* (list stdin sock) '() '())
 142              ((reads () ())
 143               (when (memq stdin reads)
 144                 (match (get-bytevector-some stdin)
 145                   ((? eof-object?)
 146                    (primitive-exit 0))
 147                   (bv
 148                    (put-bytevector sock bv)
 149                    (force-output sock))))
 150               (when (memq sock reads)
 151                 (match (get-bytevector-some sock)
 152                   ((? eof-object?)
 153                    (primitive-exit 0))
 154                   (bv
 155                    (put-bytevector stdout bv))))
 156               (loop))
 157              (_
 158               (primitive-exit 1)))))))
 159
 160   (open-remote-pipe* session OPEN_BOTH
 161                      ;; Sort-of shell-quote REDIRECT.
 162                      "guile" "-c"
 163                      (object->string
 164                       (object->string redirect))))
 165
 166 (define* (connect-to-remote-daemon session
 167                                    #:optional
 168                                    (socket-name
 169                                     "/var/guix/daemon-socket/socket"))
 170   "Connect to the remote build daemon listening on SOCKET-NAME over SESSION,
 171 an SSH session.  Return a <nix-server> object."
 172   (open-connection #:port (remote-daemon-channel session socket-name)))
 173
 174
 175 (define (store-import-channel session)
 176   "Return an output port to which archives to be exported to SESSION's store
 177 can be written."
 178   ;; Using the 'import-paths' RPC on a remote store would be slow because it
 179   ;; makes a round trip every time 32 KiB have been transferred.  This
 180   ;; procedure instead opens a separate channel to use the remote
 181   ;; 'import-paths' procedure, which consumes all the data in a single round
 182   ;; trip.  This optimizes the successful case at the expense of error
 183   ;; conditions: errors can only be reported once all the input has been
 184   ;; consumed.
 185   (define import
 186     `(begin
 187        (use-modules (guix) (srfi srfi-34)
 188                     (rnrs io ports) (rnrs bytevectors))
 189
 190        (define (consume-input port)
 191          (let ((bv (make-bytevector 32768)))
 192            (let loop ()
 193              (let ((n (get-bytevector-n! port bv 0
 194                                          (bytevector-length bv))))
 195                (unless (eof-object? n)
 196                  (loop))))))
 197
 198        ;; Upon completion, write an sexp that denotes the status.
 199        (write
 200         (catch #t
 201           (lambda ()
 202             (guard (c ((nix-protocol-error? c)
 203                        ;; Consume all the input since the only time we can
 204                        ;; report the error is after everything has been
 205                        ;; consumed.
 206                        (consume-input (current-input-port))
 207                        (list 'protocol-error (nix-protocol-error-message c))))
 208               (with-store store
 209                 (setvbuf (current-input-port) _IONBF)
 210                 (import-paths store (current-input-port))
 211                 '(success))))
 212           (lambda args
 213             (cons 'error args))))))
 214
 215   (open-remote-pipe session
 216                     (string-join
 217                      `("guile" "-c"
 218                        ,(object->string (object->string import))))
 219                     OPEN_BOTH))
 220
 221 (define* (store-export-channel session files
 222                                #:key recursive?)
 223   "Return an input port from which an export of FILES from SESSION's store can
 224 be read.  When RECURSIVE? is true, the closure of FILES is exported."
 225   ;; Same as above: this is more efficient than calling 'export-paths' on a
 226   ;; remote store.
 227   (define export
 228     `(begin
 229        (eval-when (load expand eval)
 230          (unless (resolve-module '(guix) #:ensure #f)
 231            (write `(module-error))
 232            (exit 7)))
 233
 234        (use-modules (guix) (srfi srfi-1)
 235                     (srfi srfi-26) (srfi srfi-34))
 236
 237        (guard (c ((nix-connection-error? c)
 238                   (write `(connection-error ,(nix-connection-error-file c)
 239                                             ,(nix-connection-error-code c))))
 240                  ((nix-protocol-error? c)
 241                   (write `(protocol-error ,(nix-protocol-error-status c)
 242                                           ,(nix-protocol-error-message c))))
 243                  (else
 244                   (write `(exception))))
 245          (with-store store
 246            (let* ((files ',files)
 247                   (invalid (remove (cut valid-path? store <>)
 248                                    files)))
 249              (unless (null? invalid)
 250                (write `(invalid-items ,invalid))
 251                (exit 1))
 252
 253              ;; TODO: When RECURSIVE? is true, we could send the list of store
 254              ;; items in the closure so that the other end can filter out
 255              ;; those it already has.
 256
 257              (write '(exporting))                 ;we're ready
 258              (force-output)
 259
 260              (setvbuf (current-output-port) _IONBF)
 261              (export-paths store files (current-output-port)
 262                            #:recursive? ,recursive?))))))
 263
 264   (open-remote-input-pipe session
 265                           (string-join
 266                            `("guile" "-c"
 267                              ,(object->string
 268                                (object->string export))))))
 269
 270 (define* (send-files local files remote
 271                      #:key
 272                      recursive?
 273                      (log-port (current-error-port)))
 274   "Send the subset of FILES from LOCAL (a local store) that's missing to
 275 REMOTE, a remote store.  When RECURSIVE? is true, send the closure of FILES.
 276 Return the list of store items actually sent."
 277   ;; Compute the subset of FILES missing on SESSION and send them.
 278   (let* ((files   (if recursive? (requisites local files) files))
 279          (session (channel-get-session (nix-server-socket remote)))
 280          (node    (make-node session))
 281          (missing (node-eval node
 282                              `(begin
 283                                 (use-modules (guix)
 284                                              (srfi srfi-1) (srfi srfi-26))
 285
 286                                 (with-store store
 287                                   (remove (cut valid-path? store <>)
 288                                           ',files)))))
 289          (count   (length missing))
 290          (sizes   (map (lambda (item)
 291                          (path-info-nar-size (query-path-info local item)))
 292                        missing))
 293          (port    (store-import-channel session)))
 294     (format log-port (N_ "sending ~a store item (~h MiB) to '~a'...~%"
 295                          "sending ~a store items (~h MiB) to '~a'...~%" count)
 296             count
 297             (inexact->exact (round (/ (reduce + 0 sizes) (expt 2. 20))))
 298             (session-get session 'host))
 299
 300     ;; Send MISSING in topological order.
 301     (export-paths local missing port)
 302
 303     ;; Tell the remote process that we're done.  (In theory the end-of-archive
 304     ;; mark of 'export-paths' would be enough, but in practice it's not.)
 305     (channel-send-eof port)
 306
 307     ;; Wait for completion of the remote process and read the status sexp from
 308     ;; PORT.  Wait for the exit status only when 'read' completed; otherwise,
 309     ;; we might wait forever if the other end is stuck.
 310     (let* ((result (false-if-exception (read port)))
 311            (status (and result
 312                         (zero? (channel-get-exit-status port)))))
 313       (close-port port)
 314       (match result
 315         (('success . _)
 316          missing)
 317         (('protocol-error message)
 318          (raise (condition
 319                  (&nix-protocol-error (message message) (status 42)))))
 320         (('error key args ...)
 321          (raise (condition
 322                  (&nix-protocol-error
 323                   (message (call-with-output-string
 324                              (lambda (port)
 325                                (print-exception port #f key args))))
 326                   (status 43)))))
 327         (_
 328          (raise (condition
 329                  (&nix-protocol-error
 330                   (message "unknown error while sending files over SSH")
 331                   (status 44)))))))))
 332
 333 (define (remote-store-session remote)
 334   "Return the SSH channel beneath REMOTE, a remote store as returned by
 335 'connect-to-remote-daemon', or #f."
 336   (channel-get-session (nix-server-socket remote)))
 337
 338 (define (remote-store-host remote)
 339   "Return the name of the host REMOTE is connected to, where REMOTE is a
 340 remote store as returned by 'connect-to-remote-daemon'."
 341   (match (remote-store-session remote)
 342     (#f #f)
 343     ((? session? session)
 344      (session-get session 'host))))
 345
 346 (define* (file-retrieval-port files remote
 347                               #:key recursive?)
 348   "Return an input port from which to retrieve FILES (a list of store items)
 349 from REMOTE, along with the number of items to retrieve (lower than or equal
 350 to the length of FILES.)"
 351   (values (store-export-channel (remote-store-session remote) files
 352                                 #:recursive? recursive?)
 353           (length files)))            ;XXX: inaccurate when RECURSIVE? is true
 354
 355 (define-syntax raise-error
 356   (syntax-rules (=>)
 357     ((_ fmt args ... (=> hint-fmt hint-args ...))
 358      (raise (condition
 359              (&message
 360               (message (format #f fmt args ...)))
 361              (&fix-hint
 362               (hint (format #f hint-fmt hint-args ...))))))
 363     ((_ fmt args ...)
 364      (raise (condition
 365              (&message
 366               (message (format #f fmt args ...))))))))
 367
 368 (define* (retrieve-files* files remote
 369                           #:key recursive? (log-port (current-error-port))
 370                           (import (const #f)))
 371   "Pass IMPORT an input port from which to read the sequence of FILES coming
 372 from REMOTE.  When RECURSIVE? is true, retrieve the closure of FILES."
 373   (let-values (((port count)
 374                 (file-retrieval-port files remote
 375                                      #:recursive? recursive?)))
 376     (match (read port)                            ;read the initial status
 377       (('exporting)
 378        (format #t (N_ "retrieving ~a store item from '~a'...~%"
 379                       "retrieving ~a store items from '~a'...~%" count)
 380                count (remote-store-host remote))
 381
 382        (dynamic-wind
 383          (const #t)
 384          (lambda ()
 385            (import port))
 386          (lambda ()
 387            (close-port port))))
 388       ((? eof-object?)
 389        (report-guile-error (remote-store-host remote)))
 390       (('module-error . _)
 391        (report-module-error (remote-store-host remote)))
 392       (('connection-error file code . _)
 393        (raise-error (G_ "failed to connect to '~A' on remote host '~A': ~a")
 394                     file (remote-store-host remote) (strerror code)))
 395       (('invalid-items items . _)
 396        (raise-error (N_ "no such item on remote host '~A':~{ ~a~}"
 397                         "no such items on remote host '~A':~{ ~a~}"
 398                         (length items))
 399                     (remote-store-host remote) items))
 400       (('protocol-error status message . _)
 401        (raise-error (G_ "protocol error on remote host '~A': ~a")
 402                     (remote-store-host remote) message))
 403       (_
 404        (raise-error (G_ "failed to retrieve store items from '~a'")
 405                     (remote-store-host remote))))))
 406
 407 (define* (retrieve-files local files remote
 408                          #:key recursive? (log-port (current-error-port)))
 409   "Retrieve FILES from REMOTE and import them using the 'import-paths' RPC on
 410 LOCAL.  When RECURSIVE? is true, retrieve the closure of FILES."
 411   (retrieve-files* (remove (cut valid-path? local <>) files)
 412                    remote
 413                    #:recursive? recursive?
 414                    #:log-port log-port
 415                    #:import (lambda (port)
 416                               (import-paths local port))))
 417
 418 \f
 419 ;;;
 420 ;;; Error reporting.
 421 ;;;
 422
 423 (define (report-guile-error host)
 424   (raise-error (G_ "failed to start Guile on remote host '~A'") host
 425                (=> (G_ "Make sure @command{guile} can be found in
 426 @code{$PATH} on the remote host.  Run @command{ssh ~A guile --version} to
 427 check.")
 428                    host)))
 429
 430 (define (report-module-error host)
 431   "Report an error about missing Guix modules on HOST."
 432   ;; TRANSLATORS: Leave "Guile" untranslated.
 433   (raise-error (G_ "Guile modules not found on remote host '~A'") host
 434                (=> (G_ "Make sure @code{GUILE_LOAD_PATH} includes Guix'
 435 own module directory.  Run @command{ssh ~A env | grep GUILE_LOAD_PATH} to
 436 check.")
 437                    host)))
 438
 439 ;;; ssh.scm ends here