guix/ssh.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
   3 ;;;
   4 ;;; This file is part of GNU Guix.
   5 ;;;
   6 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   7 ;;; under the terms of the GNU General Public License as published by
   8 ;;; the Free Software Foundation; either version 3 of the License, or (at
   9 ;;; your option) any later version.
  10 ;;;
  11 ;;; GNU Guix is distributed in the hope that it will be useful, but
  12 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 ;;; GNU General Public License for more details.
  15 ;;;
  16 ;;; You should have received a copy of the GNU General Public License
  17 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 (define-module (guix ssh)
  20   #:use-module (guix store)
  21   #:use-module (guix i18n)
  22   #:use-module ((guix utils) #:select (&fix-hint))
  23   #:use-module (ssh session)
  24   #:use-module (ssh auth)
  25   #:use-module (ssh key)
  26   #:use-module (ssh channel)
  27   #:use-module (ssh popen)
  28   #:use-module (ssh session)
  29   #:use-module (ssh dist)
  30   #:use-module (ssh dist node)
  31   #:use-module (srfi srfi-11)
  32   #:use-module (srfi srfi-26)
  33   #:use-module (srfi srfi-34)
  34   #:use-module (srfi srfi-35)
  35   #:use-module (ice-9 match)
  36   #:use-module (ice-9 binary-ports)
  37   #:export (open-ssh-session
  38             remote-daemon-channel
  39             connect-to-remote-daemon
  40             send-files
  41             retrieve-files
  42             retrieve-files*
  43             remote-store-host))
  44
  45 ;;; Commentary:
  46 ;;;
  47 ;;; This module provides tools to support communication with remote stores
  48 ;;; over SSH, using Guile-SSH.
  49 ;;;
  50 ;;; Code:
  51
  52 (define %compression
  53   "zlib@openssh.com,zlib")
  54
  55 (define* (open-ssh-session host #:key user port
  56                            (compression %compression))
  57   "Open an SSH session for HOST and return it.  When USER and PORT are #f, use
  58 default values or whatever '~/.ssh/config' specifies; otherwise use them.
  59 Throw an error on failure."
  60   (let ((session (make-session #:user user
  61                                #:host host
  62                                #:port port
  63                                #:timeout 10       ;seconds
  64                                ;; #:log-verbosity 'protocol
  65
  66                                ;; We need lightweight compression when
  67                                ;; exchanging full archives.
  68                                #:compression compression
  69                                #:compression-level 3)))
  70
  71     ;; Honor ~/.ssh/config.
  72     (session-parse-config! session)
  73
  74     (match (connect! session)
  75       ('ok
  76        ;; Use public key authentication, via the SSH agent if it's available.
  77        (match (userauth-public-key/auto! session)
  78          ('success
  79           session)
  80          (x
  81           (disconnect! session)
  82           (raise (condition
  83                   (&message
  84                    (message (format #f (G_ "SSH authentication failed for '~a': ~a~%")
  85                                     host (get-error session)))))))))
  86       (x
  87        ;; Connection failed or timeout expired.
  88        (raise (condition
  89                (&message
  90                 (message (format #f (G_ "SSH connection to '~a' failed: ~a~%")
  91                                  host (get-error session))))))))))
  92
  93 (define* (remote-daemon-channel session
  94                                 #:optional
  95                                 (socket-name
  96                                  "/var/guix/daemon-socket/socket"))
  97   "Return an input/output port (an SSH channel) to the daemon at SESSION."
  98   (define redirect
  99     ;; Code run in SESSION to redirect the remote process' stdin/stdout to the
 100     ;; daemon's socket, à la socat.  The SSH protocol supports forwarding to
 101     ;; Unix-domain sockets but libssh doesn't have an API for that, hence this
 102     ;; hack.
 103     `(begin
 104        (use-modules (ice-9 match) (rnrs io ports)
 105                     (rnrs bytevectors) (system foreign))
 106
 107        (define read!
 108          ;; XXX: We would use 'get-bytevector-some' but it always returns a
 109          ;; single byte in Guile <= 2.2.3---see <https://bugs.gnu.org/30066>.
 110          ;; This procedure works around it.
 111          (let ((proc (pointer->procedure int
 112                                          (dynamic-func "read" (dynamic-link))
 113                                          (list int '* size_t))))
 114            (lambda (port bv)
 115              (proc (fileno port) (bytevector->pointer bv)
 116                    (bytevector-length bv)))))
 117
 118        (let ((sock   (socket AF_UNIX SOCK_STREAM 0))
 119              (stdin  (current-input-port))
 120              (stdout (current-output-port))
 121              (buffer (make-bytevector 65536)))
 122          (setvbuf stdin _IONBF)
 123          (setvbuf stdout _IONBF)
 124          (connect sock AF_UNIX ,socket-name)
 125
 126          (let loop ()
 127            (match (select (list stdin sock) '() '())
 128              ((reads () ())
 129               (when (memq stdin reads)
 130                 (match (read! stdin buffer)
 131                   ((? zero?)                      ;EOF
 132                    (primitive-exit 0))
 133                   (count
 134                    (put-bytevector sock buffer 0 count))))
 135               (when (memq sock reads)
 136                 (match (read! sock buffer)
 137                   ((? zero?)                      ;EOF
 138                    (primitive-exit 0))
 139                   (count
 140                    (put-bytevector stdout buffer 0 count))))
 141               (loop))
 142              (_
 143               (primitive-exit 1)))))))
 144
 145   (open-remote-pipe* session OPEN_BOTH
 146                      ;; Sort-of shell-quote REDIRECT.
 147                      "guile" "-c"
 148                      (object->string
 149                       (object->string redirect))))
 150
 151 (define* (connect-to-remote-daemon session
 152                                    #:optional
 153                                    (socket-name
 154                                     "/var/guix/daemon-socket/socket"))
 155   "Connect to the remote build daemon listening on SOCKET-NAME over SESSION,
 156 an SSH session.  Return a <nix-server> object."
 157   (open-connection #:port (remote-daemon-channel session)))
 158
 159
 160 (define (store-import-channel session)
 161   "Return an output port to which archives to be exported to SESSION's store
 162 can be written."
 163   ;; Using the 'import-paths' RPC on a remote store would be slow because it
 164   ;; makes a round trip every time 32 KiB have been transferred.  This
 165   ;; procedure instead opens a separate channel to use the remote
 166   ;; 'import-paths' procedure, which consumes all the data in a single round
 167   ;; trip.  This optimizes the successful case at the expense of error
 168   ;; conditions: errors can only be reported once all the input has been
 169   ;; consumed.
 170   (define import
 171     `(begin
 172        (use-modules (guix) (srfi srfi-34)
 173                     (rnrs io ports) (rnrs bytevectors))
 174
 175        (define (consume-input port)
 176          (let ((bv (make-bytevector 32768)))
 177            (let loop ()
 178              (let ((n (get-bytevector-n! port bv 0
 179                                          (bytevector-length bv))))
 180                (unless (eof-object? n)
 181                  (loop))))))
 182
 183        ;; Upon completion, write an sexp that denotes the status.
 184        (write
 185         (catch #t
 186           (lambda ()
 187             (guard (c ((nix-protocol-error? c)
 188                        ;; Consume all the input since the only time we can
 189                        ;; report the error is after everything has been
 190                        ;; consumed.
 191                        (consume-input (current-input-port))
 192                        (list 'protocol-error (nix-protocol-error-message c))))
 193               (with-store store
 194                 (setvbuf (current-input-port) _IONBF)
 195                 (import-paths store (current-input-port))
 196                 '(success))))
 197           (lambda args
 198             (cons 'error args))))))
 199
 200   (open-remote-pipe session
 201                     (string-join
 202                      `("guile" "-c"
 203                        ,(object->string (object->string import))))
 204                     OPEN_BOTH))
 205
 206 (define* (store-export-channel session files
 207                                #:key recursive?)
 208   "Return an input port from which an export of FILES from SESSION's store can
 209 be read.  When RECURSIVE? is true, the closure of FILES is exported."
 210   ;; Same as above: this is more efficient than calling 'export-paths' on a
 211   ;; remote store.
 212   (define export
 213     `(begin
 214        (eval-when (load expand eval)
 215          (unless (resolve-module '(guix) #:ensure #f)
 216            (write `(module-error))
 217            (exit 7)))
 218
 219        (use-modules (guix) (srfi srfi-1)
 220                     (srfi srfi-26) (srfi srfi-34))
 221
 222        (guard (c ((nix-connection-error? c)
 223                   (write `(connection-error ,(nix-connection-error-file c)
 224                                             ,(nix-connection-error-code c))))
 225                  ((nix-protocol-error? c)
 226                   (write `(protocol-error ,(nix-protocol-error-status c)
 227                                           ,(nix-protocol-error-message c))))
 228                  (else
 229                   (write `(exception))))
 230          (with-store store
 231            (let* ((files ',files)
 232                   (invalid (remove (cut valid-path? store <>)
 233                                    files)))
 234              (unless (null? invalid)
 235                (write `(invalid-items ,invalid))
 236                (exit 1))
 237
 238              (write '(exporting))                 ;we're ready
 239              (force-output)
 240
 241              (setvbuf (current-output-port) _IONBF)
 242              (export-paths store files (current-output-port)
 243                            #:recursive? ,recursive?))))))
 244
 245   (open-remote-input-pipe session
 246                           (string-join
 247                            `("guile" "-c"
 248                              ,(object->string
 249                                (object->string export))))))
 250
 251 (define* (send-files local files remote
 252                      #:key
 253                      recursive?
 254                      (log-port (current-error-port)))
 255   "Send the subset of FILES from LOCAL (a local store) that's missing to
 256 REMOTE, a remote store.  When RECURSIVE? is true, send the closure of FILES.
 257 Return the list of store items actually sent."
 258   ;; Compute the subset of FILES missing on SESSION and send them.
 259   (let* ((files   (if recursive? (requisites local files) files))
 260          (session (channel-get-session (nix-server-socket remote)))
 261          (node    (make-node session))
 262          (missing (node-eval node
 263                              `(begin
 264                                 (use-modules (guix)
 265                                              (srfi srfi-1) (srfi srfi-26))
 266
 267                                 (with-store store
 268                                   (remove (cut valid-path? store <>)
 269                                           ',files)))))
 270          (count   (length missing))
 271          (port    (store-import-channel session)))
 272     (format log-port (N_ "sending ~a store item to '~a'...~%"
 273                          "sending ~a store items to '~a'...~%" count)
 274             count (session-get session 'host))
 275
 276     ;; Send MISSING in topological order.
 277     (export-paths local missing port)
 278
 279     ;; Tell the remote process that we're done.  (In theory the end-of-archive
 280     ;; mark of 'export-paths' would be enough, but in practice it's not.)
 281     (channel-send-eof port)
 282
 283     ;; Wait for completion of the remote process and read the status sexp from
 284     ;; PORT.
 285     (let* ((result (false-if-exception (read port)))
 286            (status (zero? (channel-get-exit-status port))))
 287       (close-port port)
 288       (match result
 289         (('success . _)
 290          missing)
 291         (('protocol-error message)
 292          (raise (condition
 293                  (&nix-protocol-error (message message) (status 42)))))
 294         (('error key args ...)
 295          (raise (condition
 296                  (&nix-protocol-error
 297                   (message (call-with-output-string
 298                              (lambda (port)
 299                                (print-exception port #f key args))))
 300                   (status 43)))))
 301         (_
 302          (raise (condition
 303                  (&nix-protocol-error
 304                   (message "unknown error while sending files over SSH")
 305                   (status 44)))))))))
 306
 307 (define (remote-store-session remote)
 308   "Return the SSH channel beneath REMOTE, a remote store as returned by
 309 'connect-to-remote-daemon', or #f."
 310   (channel-get-session (nix-server-socket remote)))
 311
 312 (define (remote-store-host remote)
 313   "Return the name of the host REMOTE is connected to, where REMOTE is a
 314 remote store as returned by 'connect-to-remote-daemon'."
 315   (match (remote-store-session remote)
 316     (#f #f)
 317     ((? session? session)
 318      (session-get session 'host))))
 319
 320 (define* (file-retrieval-port files remote
 321                               #:key recursive?)
 322   "Return an input port from which to retrieve FILES (a list of store items)
 323 from REMOTE, along with the number of items to retrieve (lower than or equal
 324 to the length of FILES.)"
 325   (values (store-export-channel (remote-store-session remote) files
 326                                 #:recursive? recursive?)
 327           (length files)))            ;XXX: inaccurate when RECURSIVE? is true
 328
 329 (define-syntax raise-error
 330   (syntax-rules (=>)
 331     ((_ fmt args ... (=> hint-fmt hint-args ...))
 332      (raise (condition
 333              (&message
 334               (message (format #f fmt args ...)))
 335              (&fix-hint
 336               (hint (format #f hint-fmt hint-args ...))))))
 337     ((_ fmt args ...)
 338      (raise (condition
 339              (&message
 340               (message (format #f fmt args ...))))))))
 341
 342 (define* (retrieve-files* files remote
 343                           #:key recursive? (log-port (current-error-port))
 344                           (import (const #f)))
 345   "Pass IMPORT an input port from which to read the sequence of FILES coming
 346 from REMOTE.  When RECURSIVE? is true, retrieve the closure of FILES."
 347   (let-values (((port count)
 348                 (file-retrieval-port files remote
 349                                      #:recursive? recursive?)))
 350     (match (read port)                            ;read the initial status
 351       (('exporting)
 352        (format #t (N_ "retrieving ~a store item from '~a'...~%"
 353                       "retrieving ~a store items from '~a'...~%" count)
 354                count (remote-store-host remote))
 355
 356        (dynamic-wind
 357          (const #t)
 358          (lambda ()
 359            (import port))
 360          (lambda ()
 361            (close-port port))))
 362       ((? eof-object?)
 363        (raise-error (G_ "failed to start Guile on remote host '~A': exit code ~A")
 364                     (remote-store-host remote)
 365                     (channel-get-exit-status port)
 366                     (=> (G_ "Make sure @command{guile} can be found in
 367 @code{$PATH} on the remote host.  Run @command{ssh ~A guile --version} to
 368 check.")
 369                         (remote-store-host remote))))
 370       (('module-error . _)
 371        ;; TRANSLATORS: Leave "Guile" untranslated.
 372        (raise-error (G_ "Guile modules not found on remote host '~A'")
 373                     (remote-store-host remote)
 374                     (=> (G_ "Make sure @code{GUILE_LOAD_PATH} includes Guix'
 375 own module directory.  Run @command{ssh ~A env | grep GUILE_LOAD_PATH} to
 376 check.")
 377                         (remote-store-host remote))))
 378       (('connection-error file code . _)
 379        (raise-error (G_ "failed to connect to '~A' on remote host '~A': ~a")
 380                     file (remote-store-host remote) (strerror code)))
 381       (('invalid-items items . _)
 382        (raise-error (N_ "no such item on remote host '~A':~{ ~a~}"
 383                         "no such items on remote host '~A':~{ ~a~}"
 384                         (length items))
 385                     (remote-store-host remote) items))
 386       (('protocol-error status message . _)
 387        (raise-error (G_ "protocol error on remote host '~A': ~a")
 388                     (remote-store-host remote) message))
 389       (_
 390        (raise-error (G_ "failed to retrieve store items from '~a'")
 391                     (remote-store-host remote))))))
 392
 393 (define* (retrieve-files local files remote
 394                          #:key recursive? (log-port (current-error-port)))
 395   "Retrieve FILES from REMOTE and import them using the 'import-paths' RPC on
 396 LOCAL.  When RECURSIVE? is true, retrieve the closure of FILES."
 397   (retrieve-files* files remote
 398                    #:recursive? recursive?
 399                    #:log-port log-port
 400                    #:import (lambda (port)
 401                               (import-paths local port))))
 402
 403 ;;; ssh.scm ends here