guix/ssh.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
   3 ;;;
   4 ;;; This file is part of GNU Guix.
   5 ;;;
   6 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   7 ;;; under the terms of the GNU General Public License as published by
   8 ;;; the Free Software Foundation; either version 3 of the License, or (at
   9 ;;; your option) any later version.
  10 ;;;
  11 ;;; GNU Guix is distributed in the hope that it will be useful, but
  12 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 ;;; GNU General Public License for more details.
  15 ;;;
  16 ;;; You should have received a copy of the GNU General Public License
  17 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 (define-module (guix ssh)
  20   #:use-module (guix store)
  21   #:use-module (guix i18n)
  22   #:use-module ((guix utils) #:select (&fix-hint))
  23   #:use-module (ssh session)
  24   #:use-module (ssh auth)
  25   #:use-module (ssh key)
  26   #:use-module (ssh channel)
  27   #:use-module (ssh popen)
  28   #:use-module (ssh session)
  29   #:use-module (ssh dist)
  30   #:use-module (ssh dist node)
  31   #:use-module (srfi srfi-1)
  32   #:use-module (srfi srfi-11)
  33   #:use-module (srfi srfi-26)
  34   #:use-module (srfi srfi-34)
  35   #:use-module (srfi srfi-35)
  36   #:use-module (ice-9 match)
  37   #:use-module (ice-9 binary-ports)
  38   #:export (open-ssh-session
  39             remote-daemon-channel
  40             connect-to-remote-daemon
  41             send-files
  42             retrieve-files
  43             retrieve-files*
  44             remote-store-host
  45
  46             report-guile-error
  47             report-module-error))
  48
  49 ;;; Commentary:
  50 ;;;
  51 ;;; This module provides tools to support communication with remote stores
  52 ;;; over SSH, using Guile-SSH.
  53 ;;;
  54 ;;; Code:
  55
  56 (define %compression
  57   "zlib@openssh.com,zlib")
  58
  59 (define* (open-ssh-session host #:key user port
  60                            (compression %compression))
  61   "Open an SSH session for HOST and return it.  When USER and PORT are #f, use
  62 default values or whatever '~/.ssh/config' specifies; otherwise use them.
  63 Throw an error on failure."
  64   (let ((session (make-session #:user user
  65                                #:host host
  66                                #:port port
  67                                #:timeout 10       ;seconds
  68                                ;; #:log-verbosity 'protocol
  69
  70                                ;; We need lightweight compression when
  71                                ;; exchanging full archives.
  72                                #:compression compression
  73                                #:compression-level 3)))
  74
  75     ;; Honor ~/.ssh/config.
  76     (session-parse-config! session)
  77
  78     (match (connect! session)
  79       ('ok
  80        ;; Use public key authentication, via the SSH agent if it's available.
  81        (match (userauth-public-key/auto! session)
  82          ('success
  83           session)
  84          (x
  85           (disconnect! session)
  86           (raise (condition
  87                   (&message
  88                    (message (format #f (G_ "SSH authentication failed for '~a': ~a~%")
  89                                     host (get-error session)))))))))
  90       (x
  91        ;; Connection failed or timeout expired.
  92        (raise (condition
  93                (&message
  94                 (message (format #f (G_ "SSH connection to '~a' failed: ~a~%")
  95                                  host (get-error session))))))))))
  96
  97 (define* (remote-daemon-channel session
  98                                 #:optional
  99                                 (socket-name
 100                                  "/var/guix/daemon-socket/socket"))
 101   "Return an input/output port (an SSH channel) to the daemon at SESSION."
 102   (define redirect
 103     ;; Code run in SESSION to redirect the remote process' stdin/stdout to the
 104     ;; daemon's socket, à la socat.  The SSH protocol supports forwarding to
 105     ;; Unix-domain sockets but libssh doesn't have an API for that, hence this
 106     ;; hack.
 107     `(begin
 108        (use-modules (ice-9 match) (rnrs io ports)
 109                     (rnrs bytevectors))
 110
 111        (let ((sock    (socket AF_UNIX SOCK_STREAM 0))
 112              (stdin   (current-input-port))
 113              (stdout  (current-output-port))
 114              (select* (lambda (read write except)
 115                         ;; This is a workaround for
 116                         ;; <https://bugs.gnu.org/30365> in Guile < 2.2.4:
 117                         ;; since 'select' sometimes returns non-empty sets for
 118                         ;; no good reason, call 'select' a second time with a
 119                         ;; zero timeout to filter out incorrect replies.
 120                         (match (select read write except)
 121                           ((read write except)
 122                            (select read write except 0))))))
 123          (setvbuf stdout _IONBF)
 124
 125          ;; Use buffered ports so that 'get-bytevector-some' returns up to the
 126          ;; whole buffer like read(2) would--see <https://bugs.gnu.org/30066>.
 127          (setvbuf stdin _IOFBF 65536)
 128          (setvbuf sock _IOFBF 65536)
 129
 130          (connect sock AF_UNIX ,socket-name)
 131
 132          (let loop ()
 133            (match (select* (list stdin sock) '() '())
 134              ((reads () ())
 135               (when (memq stdin reads)
 136                 (match (get-bytevector-some stdin)
 137                   ((? eof-object?)
 138                    (primitive-exit 0))
 139                   (bv
 140                    (put-bytevector sock bv)
 141                    (force-output sock))))
 142               (when (memq sock reads)
 143                 (match (get-bytevector-some sock)
 144                   ((? eof-object?)
 145                    (primitive-exit 0))
 146                   (bv
 147                    (put-bytevector stdout bv))))
 148               (loop))
 149              (_
 150               (primitive-exit 1)))))))
 151
 152   (open-remote-pipe* session OPEN_BOTH
 153                      ;; Sort-of shell-quote REDIRECT.
 154                      "guile" "-c"
 155                      (object->string
 156                       (object->string redirect))))
 157
 158 (define* (connect-to-remote-daemon session
 159                                    #:optional
 160                                    (socket-name
 161                                     "/var/guix/daemon-socket/socket"))
 162   "Connect to the remote build daemon listening on SOCKET-NAME over SESSION,
 163 an SSH session.  Return a <nix-server> object."
 164   (open-connection #:port (remote-daemon-channel session socket-name)))
 165
 166
 167 (define (store-import-channel session)
 168   "Return an output port to which archives to be exported to SESSION's store
 169 can be written."
 170   ;; Using the 'import-paths' RPC on a remote store would be slow because it
 171   ;; makes a round trip every time 32 KiB have been transferred.  This
 172   ;; procedure instead opens a separate channel to use the remote
 173   ;; 'import-paths' procedure, which consumes all the data in a single round
 174   ;; trip.  This optimizes the successful case at the expense of error
 175   ;; conditions: errors can only be reported once all the input has been
 176   ;; consumed.
 177   (define import
 178     `(begin
 179        (use-modules (guix) (srfi srfi-34)
 180                     (rnrs io ports) (rnrs bytevectors))
 181
 182        (define (consume-input port)
 183          (let ((bv (make-bytevector 32768)))
 184            (let loop ()
 185              (let ((n (get-bytevector-n! port bv 0
 186                                          (bytevector-length bv))))
 187                (unless (eof-object? n)
 188                  (loop))))))
 189
 190        ;; Upon completion, write an sexp that denotes the status.
 191        (write
 192         (catch #t
 193           (lambda ()
 194             (guard (c ((nix-protocol-error? c)
 195                        ;; Consume all the input since the only time we can
 196                        ;; report the error is after everything has been
 197                        ;; consumed.
 198                        (consume-input (current-input-port))
 199                        (list 'protocol-error (nix-protocol-error-message c))))
 200               (with-store store
 201                 (setvbuf (current-input-port) _IONBF)
 202                 (import-paths store (current-input-port))
 203                 '(success))))
 204           (lambda args
 205             (cons 'error args))))))
 206
 207   (open-remote-pipe session
 208                     (string-join
 209                      `("guile" "-c"
 210                        ,(object->string (object->string import))))
 211                     OPEN_BOTH))
 212
 213 (define* (store-export-channel session files
 214                                #:key recursive?)
 215   "Return an input port from which an export of FILES from SESSION's store can
 216 be read.  When RECURSIVE? is true, the closure of FILES is exported."
 217   ;; Same as above: this is more efficient than calling 'export-paths' on a
 218   ;; remote store.
 219   (define export
 220     `(begin
 221        (eval-when (load expand eval)
 222          (unless (resolve-module '(guix) #:ensure #f)
 223            (write `(module-error))
 224            (exit 7)))
 225
 226        (use-modules (guix) (srfi srfi-1)
 227                     (srfi srfi-26) (srfi srfi-34))
 228
 229        (guard (c ((nix-connection-error? c)
 230                   (write `(connection-error ,(nix-connection-error-file c)
 231                                             ,(nix-connection-error-code c))))
 232                  ((nix-protocol-error? c)
 233                   (write `(protocol-error ,(nix-protocol-error-status c)
 234                                           ,(nix-protocol-error-message c))))
 235                  (else
 236                   (write `(exception))))
 237          (with-store store
 238            (let* ((files ',files)
 239                   (invalid (remove (cut valid-path? store <>)
 240                                    files)))
 241              (unless (null? invalid)
 242                (write `(invalid-items ,invalid))
 243                (exit 1))
 244
 245              ;; TODO: When RECURSIVE? is true, we could send the list of store
 246              ;; items in the closure so that the other end can filter out
 247              ;; those it already has.
 248
 249              (write '(exporting))                 ;we're ready
 250              (force-output)
 251
 252              (setvbuf (current-output-port) _IONBF)
 253              (export-paths store files (current-output-port)
 254                            #:recursive? ,recursive?))))))
 255
 256   (open-remote-input-pipe session
 257                           (string-join
 258                            `("guile" "-c"
 259                              ,(object->string
 260                                (object->string export))))))
 261
 262 (define* (send-files local files remote
 263                      #:key
 264                      recursive?
 265                      (log-port (current-error-port)))
 266   "Send the subset of FILES from LOCAL (a local store) that's missing to
 267 REMOTE, a remote store.  When RECURSIVE? is true, send the closure of FILES.
 268 Return the list of store items actually sent."
 269   ;; Compute the subset of FILES missing on SESSION and send them.
 270   (let* ((files   (if recursive? (requisites local files) files))
 271          (session (channel-get-session (nix-server-socket remote)))
 272          (node    (make-node session))
 273          (missing (node-eval node
 274                              `(begin
 275                                 (use-modules (guix)
 276                                              (srfi srfi-1) (srfi srfi-26))
 277
 278                                 (with-store store
 279                                   (remove (cut valid-path? store <>)
 280                                           ',files)))))
 281          (count   (length missing))
 282          (sizes   (map (lambda (item)
 283                          (path-info-nar-size (query-path-info local item)))
 284                        missing))
 285          (port    (store-import-channel session)))
 286     (format log-port (N_ "sending ~a store item (~h MiB) to '~a'...~%"
 287                          "sending ~a store items (~h MiB) to '~a'...~%" count)
 288             count
 289             (inexact->exact (round (/ (reduce + 0 sizes) (expt 2. 20))))
 290             (session-get session 'host))
 291
 292     ;; Send MISSING in topological order.
 293     (export-paths local missing port)
 294
 295     ;; Tell the remote process that we're done.  (In theory the end-of-archive
 296     ;; mark of 'export-paths' would be enough, but in practice it's not.)
 297     (channel-send-eof port)
 298
 299     ;; Wait for completion of the remote process and read the status sexp from
 300     ;; PORT.  Wait for the exit status only when 'read' completed; otherwise,
 301     ;; we might wait forever if the other end is stuck.
 302     (let* ((result (false-if-exception (read port)))
 303            (status (and result
 304                         (zero? (channel-get-exit-status port)))))
 305       (close-port port)
 306       (match result
 307         (('success . _)
 308          missing)
 309         (('protocol-error message)
 310          (raise (condition
 311                  (&nix-protocol-error (message message) (status 42)))))
 312         (('error key args ...)
 313          (raise (condition
 314                  (&nix-protocol-error
 315                   (message (call-with-output-string
 316                              (lambda (port)
 317                                (print-exception port #f key args))))
 318                   (status 43)))))
 319         (_
 320          (raise (condition
 321                  (&nix-protocol-error
 322                   (message "unknown error while sending files over SSH")
 323                   (status 44)))))))))
 324
 325 (define (remote-store-session remote)
 326   "Return the SSH channel beneath REMOTE, a remote store as returned by
 327 'connect-to-remote-daemon', or #f."
 328   (channel-get-session (nix-server-socket remote)))
 329
 330 (define (remote-store-host remote)
 331   "Return the name of the host REMOTE is connected to, where REMOTE is a
 332 remote store as returned by 'connect-to-remote-daemon'."
 333   (match (remote-store-session remote)
 334     (#f #f)
 335     ((? session? session)
 336      (session-get session 'host))))
 337
 338 (define* (file-retrieval-port files remote
 339                               #:key recursive?)
 340   "Return an input port from which to retrieve FILES (a list of store items)
 341 from REMOTE, along with the number of items to retrieve (lower than or equal
 342 to the length of FILES.)"
 343   (values (store-export-channel (remote-store-session remote) files
 344                                 #:recursive? recursive?)
 345           (length files)))            ;XXX: inaccurate when RECURSIVE? is true
 346
 347 (define-syntax raise-error
 348   (syntax-rules (=>)
 349     ((_ fmt args ... (=> hint-fmt hint-args ...))
 350      (raise (condition
 351              (&message
 352               (message (format #f fmt args ...)))
 353              (&fix-hint
 354               (hint (format #f hint-fmt hint-args ...))))))
 355     ((_ fmt args ...)
 356      (raise (condition
 357              (&message
 358               (message (format #f fmt args ...))))))))
 359
 360 (define* (retrieve-files* files remote
 361                           #:key recursive? (log-port (current-error-port))
 362                           (import (const #f)))
 363   "Pass IMPORT an input port from which to read the sequence of FILES coming
 364 from REMOTE.  When RECURSIVE? is true, retrieve the closure of FILES."
 365   (let-values (((port count)
 366                 (file-retrieval-port files remote
 367                                      #:recursive? recursive?)))
 368     (match (read port)                            ;read the initial status
 369       (('exporting)
 370        (format #t (N_ "retrieving ~a store item from '~a'...~%"
 371                       "retrieving ~a store items from '~a'...~%" count)
 372                count (remote-store-host remote))
 373
 374        (dynamic-wind
 375          (const #t)
 376          (lambda ()
 377            (import port))
 378          (lambda ()
 379            (close-port port))))
 380       ((? eof-object?)
 381        (report-guile-error (remote-store-host remote)))
 382       (('module-error . _)
 383        (report-module-error (remote-store-host remote)))
 384       (('connection-error file code . _)
 385        (raise-error (G_ "failed to connect to '~A' on remote host '~A': ~a")
 386                     file (remote-store-host remote) (strerror code)))
 387       (('invalid-items items . _)
 388        (raise-error (N_ "no such item on remote host '~A':~{ ~a~}"
 389                         "no such items on remote host '~A':~{ ~a~}"
 390                         (length items))
 391                     (remote-store-host remote) items))
 392       (('protocol-error status message . _)
 393        (raise-error (G_ "protocol error on remote host '~A': ~a")
 394                     (remote-store-host remote) message))
 395       (_
 396        (raise-error (G_ "failed to retrieve store items from '~a'")
 397                     (remote-store-host remote))))))
 398
 399 (define* (retrieve-files local files remote
 400                          #:key recursive? (log-port (current-error-port)))
 401   "Retrieve FILES from REMOTE and import them using the 'import-paths' RPC on
 402 LOCAL.  When RECURSIVE? is true, retrieve the closure of FILES."
 403   (retrieve-files* (remove (cut valid-path? local <>) files)
 404                    remote
 405                    #:recursive? recursive?
 406                    #:log-port log-port
 407                    #:import (lambda (port)
 408                               (import-paths local port))))
 409
 410 \f
 411 ;;;
 412 ;;; Error reporting.
 413 ;;;
 414
 415 (define (report-guile-error host)
 416   (raise-error (G_ "failed to start Guile on remote host '~A'") host
 417                (=> (G_ "Make sure @command{guile} can be found in
 418 @code{$PATH} on the remote host.  Run @command{ssh ~A guile --version} to
 419 check.")
 420                    host)))
 421
 422 (define (report-module-error host)
 423   "Report an error about missing Guix modules on HOST."
 424   ;; TRANSLATORS: Leave "Guile" untranslated.
 425   (raise-error (G_ "Guile modules not found on remote host '~A'") host
 426                (=> (G_ "Make sure @code{GUILE_LOAD_PATH} includes Guix'
 427 own module directory.  Run @command{ssh ~A env | grep GUILE_LOAD_PATH} to
 428 check.")
 429                    host)))
 430
 431 ;;; ssh.scm ends here