news: Add entry for "guix system image".
[jackhill/guix/guix.git] / guix / ssh.scm
1 ;;; GNU Guix --- Functional package management for GNU
2 ;;; Copyright © 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org>
3 ;;;
4 ;;; This file is part of GNU Guix.
5 ;;;
6 ;;; GNU Guix is free software; you can redistribute it and/or modify it
7 ;;; under the terms of the GNU General Public License as published by
8 ;;; the Free Software Foundation; either version 3 of the License, or (at
9 ;;; your option) any later version.
10 ;;;
11 ;;; GNU Guix is distributed in the hope that it will be useful, but
12 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 ;;; GNU General Public License for more details.
15 ;;;
16 ;;; You should have received a copy of the GNU General Public License
17 ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
18
19 (define-module (guix ssh)
20 #:use-module (guix store)
21 #:use-module (guix inferior)
22 #:use-module (guix i18n)
23 #:use-module ((guix diagnostics)
24 #:select (info &fix-hint formatted-message))
25 #:use-module ((guix progress)
26 #:select (progress-bar
27 erase-current-line current-terminal-columns))
28 #:use-module (gcrypt pk-crypto)
29 #:use-module (ssh session)
30 #:use-module (ssh auth)
31 #:use-module (ssh key)
32 #:use-module (ssh channel)
33 #:use-module (ssh popen)
34 #:use-module (ssh session)
35 #:use-module (srfi srfi-1)
36 #:use-module (srfi srfi-11)
37 #:use-module (srfi srfi-26)
38 #:use-module (srfi srfi-34)
39 #:use-module (srfi srfi-35)
40 #:use-module (ice-9 match)
41 #:use-module (ice-9 format)
42 #:use-module (ice-9 binary-ports)
43 #:use-module (ice-9 vlist)
44 #:export (open-ssh-session
45 authenticate-server*
46
47 remote-inferior
48 remote-daemon-channel
49 connect-to-remote-daemon
50 remote-system
51 remote-authorize-signing-key
52 send-files
53 retrieve-files
54 retrieve-files*
55 remote-store-host
56
57 report-guile-error))
58
59 ;;; Commentary:
60 ;;;
61 ;;; This module provides tools to support communication with remote stores
62 ;;; over SSH, using Guile-SSH.
63 ;;;
64 ;;; Code:
65
66 (define %compression
67 "zlib@openssh.com,zlib")
68
69 (define (host-key->type+key host-key)
70 "Destructure HOST-KEY, an OpenSSH host key string, and return two values:
71 its key type as a symbol, and the actual base64-encoded string."
72 (define (type->symbol type)
73 (and (string-prefix? "ssh-" type)
74 (string->symbol (string-drop type 4))))
75
76 (match (string-tokenize host-key)
77 ((type key x)
78 (values (type->symbol type) key))
79 ((type key)
80 (values (type->symbol type) key))))
81
82 (define (authenticate-server* session key)
83 "Make sure the server for SESSION has the given KEY, where KEY is a string
84 such as \"ssh-ed25519 AAAAC3Nz… root@example.org\". Raise an exception if the
85 actual key does not match."
86 (let-values (((server) (get-server-public-key session))
87 ((type key) (host-key->type+key key)))
88 (unless (and (or (not (get-key-type server))
89 (eq? (get-key-type server) type))
90 (string=? (public-key->string server) key))
91 ;; Key mismatch: something's wrong. XXX: It could be that the server
92 ;; provided its Ed25519 key when we where expecting its RSA key. XXX:
93 ;; Guile-SSH 0.10.1 doesn't know about ed25519 keys and 'get-key-type'
94 ;; returns #f in that case.
95 (raise (formatted-message (G_ "server at '~a' returned host key \
96 '~a' of type '~a' instead of '~a' of type '~a'~%")
97 (session-get session 'host)
98 (public-key->string server)
99 (get-key-type server)
100 key type)))))
101
102 (define* (open-ssh-session host #:key user port identity
103 host-key
104 (compression %compression)
105 (timeout 3600))
106 "Open an SSH session for HOST and return it. IDENTITY specifies the file
107 name of a private key to use for authenticating with the host. When USER,
108 PORT, or IDENTITY are #f, use default values or whatever '~/.ssh/config'
109 specifies; otherwise use them.
110
111 When HOST-KEY is true, it must be a string like \"ssh-ed25519 AAAAC3Nz…
112 root@example.org\"; the server is authenticated and an error is raised if its
113 host key is different from HOST-KEY.
114
115 Install TIMEOUT as the maximum time in seconds after which a read or write
116 operation on a channel of the returned session is considered as failing.
117
118 Throw an error on failure."
119 (let ((session (make-session #:user user
120 #:identity identity
121 #:host host
122 #:port port
123 #:timeout 10 ;seconds
124 ;; #:log-verbosity 'protocol
125
126 ;; Prevent libssh from reading
127 ;; ~/.ssh/known_hosts when the caller provides
128 ;; a HOST-KEY to match against.
129 #:knownhosts (and host-key "/dev/null")
130
131 ;; We need lightweight compression when
132 ;; exchanging full archives.
133 #:compression compression
134 #:compression-level 3
135
136 ;; Speed up RPCs by creating sockets with
137 ;; TCP_NODELAY.
138 #:nodelay #t)))
139
140 ;; Honor ~/.ssh/config.
141 (session-parse-config! session)
142
143 (match (connect! session)
144 ('ok
145 (if host-key
146 ;; Make sure the server's key is what we expect.
147 (authenticate-server* session host-key)
148
149 ;; Authenticate against ~/.ssh/known_hosts.
150 (match (authenticate-server session)
151 ('ok #f)
152 (reason
153 (raise (formatted-message (G_ "failed to authenticate \
154 server at '~a': ~a")
155 (session-get session 'host)
156 reason)))))
157
158 ;; Use public key authentication, via the SSH agent if it's available.
159 (match (userauth-public-key/auto! session)
160 ('success
161 (session-set! session 'timeout timeout)
162 session)
163 (x
164 (match (userauth-gssapi! session)
165 ('success
166 (session-set! session 'timeout timeout)
167 session)
168 (x
169 (disconnect! session)
170 (raise (condition
171 (&message
172 (message (format #f (G_ "SSH authentication failed for '~a': ~a~%")
173 host (get-error session)))))))))))
174 (x
175 ;; Connection failed or timeout expired.
176 (raise (formatted-message (G_ "SSH connection to '~a' failed: ~a~%")
177 host (get-error session)))))))
178
179 (define* (remote-inferior session #:optional become-command)
180 "Return a remote inferior for the given SESSION. If BECOME-COMMAND is
181 given, use that to invoke the remote Guile REPL."
182 (let* ((repl-command (append (or become-command '())
183 '("guix" "repl" "-t" "machine")))
184 (pipe (apply open-remote-pipe* session OPEN_BOTH repl-command)))
185 (when (eof-object? (peek-char pipe))
186 (let ((status (channel-get-exit-status pipe)))
187 (close-port pipe)
188 (raise (formatted-message (G_ "remote command '~{~a~^ ~}' failed \
189 with status ~a")
190 repl-command status))))
191 (port->inferior pipe)))
192
193 (define* (inferior-remote-eval exp session #:optional become-command)
194 "Evaluate EXP in a new inferior running in SESSION, and close the inferior
195 right away. If BECOME-COMMAND is given, use that to invoke the remote Guile
196 REPL."
197 (let ((inferior (remote-inferior session become-command)))
198 (dynamic-wind
199 (const #t)
200 (lambda ()
201 (inferior-eval exp inferior))
202 (lambda ()
203 ;; Close INFERIOR right away to prevent finalization from happening in
204 ;; another thread at the wrong time (see
205 ;; <https://bugs.gnu.org/26976>.)
206 (close-inferior inferior)))))
207
208 (define (remote-run exp session)
209 "Run EXP in a new process in SESSION and return a remote pipe.
210
211 Unlike 'inferior-remote-eval', this is used for side effects and may
212 communicate over stdout/stdin as it sees fit. EXP is typically a loop that
213 processes data from stdin and/or sends data to stdout. The assumption is that
214 EXP never returns or calls 'primitive-exit' when it's done."
215 (define pipe
216 (open-remote-pipe* session OPEN_BOTH
217 "guix" "repl" "-t" "machine"))
218
219 (match (read pipe)
220 (('repl-version _ ...)
221 #t)
222 ((? eof-object?)
223 (close-port pipe)
224 (raise (formatted-message
225 (G_ "failed to start 'guix repl' on '~a'")
226 (session-get session 'host)))))
227
228 ;; Disable buffering so 'guix repl' does not read more than what's really
229 ;; sent to itself.
230 (write '(setvbuf (current-input-port) 'none) pipe)
231 (force-output pipe)
232
233 ;; Read the reply and subsequent newline.
234 (read pipe) (get-u8 pipe)
235
236 (write exp pipe)
237 (force-output pipe)
238
239 ;; From now on, we stop following the inferior protocol.
240 pipe)
241
242 (define* (remote-daemon-channel session
243 #:optional
244 (socket-name
245 "/var/guix/daemon-socket/socket"))
246 "Return an input/output port (an SSH channel) to the daemon at SESSION."
247 (define redirect
248 ;; Code run in SESSION to redirect the remote process' stdin/stdout to the
249 ;; daemon's socket, à la socat. The SSH protocol supports forwarding to
250 ;; Unix-domain sockets but libssh doesn't have an API for that, hence this
251 ;; hack.
252 `(begin
253 (use-modules (ice-9 match) (rnrs io ports)
254 (rnrs bytevectors))
255
256 (let ((sock (socket AF_UNIX SOCK_STREAM 0))
257 (stdin (current-input-port))
258 (stdout (current-output-port))
259 (select* (lambda (read write except)
260 ;; This is a workaround for
261 ;; <https://bugs.gnu.org/30365> in Guile < 2.2.4:
262 ;; since 'select' sometimes returns non-empty sets for
263 ;; no good reason, call 'select' a second time with a
264 ;; zero timeout to filter out incorrect replies.
265 (match (select read write except)
266 ((read write except)
267 (select read write except 0))))))
268 (setvbuf stdout 'none)
269
270 ;; Use buffered ports so that 'get-bytevector-some' returns up to the
271 ;; whole buffer like read(2) would--see <https://bugs.gnu.org/30066>.
272 (setvbuf stdin 'block 65536)
273 (setvbuf sock 'block 65536)
274
275 (connect sock AF_UNIX ,socket-name)
276
277 (let loop ()
278 (match (select* (list stdin sock) '() '())
279 ((reads () ())
280 (when (memq stdin reads)
281 (match (get-bytevector-some stdin)
282 ((? eof-object?)
283 (primitive-exit 0))
284 (bv
285 (put-bytevector sock bv)
286 (force-output sock))))
287 (when (memq sock reads)
288 (match (get-bytevector-some sock)
289 ((? eof-object?)
290 (primitive-exit 0))
291 (bv
292 (put-bytevector stdout bv))))
293 (loop))
294 (_
295 (primitive-exit 1)))))))
296
297 (remote-run redirect session))
298
299 (define* (connect-to-remote-daemon session
300 #:optional
301 (socket-name
302 "/var/guix/daemon-socket/socket"))
303 "Connect to the remote build daemon listening on SOCKET-NAME over SESSION,
304 an SSH session. Return a <store-connection> object."
305 (open-connection #:port (remote-daemon-channel session socket-name)))
306
307
308 (define (store-import-channel session)
309 "Return an output port to which archives to be exported to SESSION's store
310 can be written."
311 ;; Using the 'import-paths' RPC on a remote store would be slow because it
312 ;; makes a round trip every time 32 KiB have been transferred. This
313 ;; procedure instead opens a separate channel to use the remote
314 ;; 'import-paths' procedure, which consumes all the data in a single round
315 ;; trip. This optimizes the successful case at the expense of error
316 ;; conditions: errors can only be reported once all the input has been
317 ;; consumed.
318 (define import
319 `(begin
320 (use-modules (guix) (srfi srfi-34)
321 (rnrs io ports) (rnrs bytevectors))
322
323 (define (consume-input port)
324 (let ((bv (make-bytevector 32768)))
325 (let loop ()
326 (let ((n (get-bytevector-n! port bv 0
327 (bytevector-length bv))))
328 (unless (eof-object? n)
329 (loop))))))
330
331 ;; Upon completion, write an sexp that denotes the status.
332 (write
333 (catch #t
334 (lambda ()
335 (guard (c ((nix-protocol-error? c)
336 ;; Consume all the input since the only time we can
337 ;; report the error is after everything has been
338 ;; consumed.
339 (consume-input (current-input-port))
340 (list 'protocol-error (nix-protocol-error-message c))))
341 (with-store store
342 (write '(importing)) ;we're ready
343 (force-output)
344
345 (setvbuf (current-input-port) 'none)
346 (import-paths store (current-input-port))
347 '(success))))
348 (lambda args
349 (cons 'error args))))
350 (primitive-exit 0)))
351
352 (remote-run import session))
353
354 (define* (store-export-channel session files
355 #:key recursive?)
356 "Return an input port from which an export of FILES from SESSION's store can
357 be read. When RECURSIVE? is true, the closure of FILES is exported."
358 ;; Same as above: this is more efficient than calling 'export-paths' on a
359 ;; remote store.
360 (define export
361 `(begin
362 (use-modules (guix) (srfi srfi-1)
363 (srfi srfi-26) (srfi srfi-34))
364
365 (guard (c ((nix-connection-error? c)
366 (write `(connection-error ,(nix-connection-error-file c)
367 ,(nix-connection-error-code c)))
368 (primitive-exit 1))
369 ((nix-protocol-error? c)
370 (write `(protocol-error ,(nix-protocol-error-status c)
371 ,(nix-protocol-error-message c)))
372 (primitive-exit 2))
373 (else
374 (write `(exception))
375 (primitive-exit 3)))
376 (with-store store
377 (let* ((files ',files)
378 (invalid (remove (cut valid-path? store <>)
379 files)))
380 (unless (null? invalid)
381 (write `(invalid-items ,invalid))
382 (exit 1))
383
384 ;; TODO: When RECURSIVE? is true, we could send the list of store
385 ;; items in the closure so that the other end can filter out
386 ;; those it already has.
387
388 (write '(exporting)) ;we're ready
389 (force-output)
390
391 (setvbuf (current-output-port) 'none)
392 (export-paths store files (current-output-port)
393 #:recursive? ,recursive?)
394 (primitive-exit 0))))))
395
396 (remote-run export session))
397
398 (define (remote-system session)
399 "Return the system type as expected by Nix, usually ARCHITECTURE-KERNEL, of
400 the machine on the other end of SESSION."
401 (inferior-remote-eval '(begin (use-modules (guix utils)) (%current-system))
402 session))
403
404 (define* (remote-authorize-signing-key key session #:optional become-command)
405 "Send KEY, a canonical sexp containing a public key, over SESSION and add it
406 to the system ACL file if it has not yet been authorized."
407 (inferior-remote-eval
408 `(begin
409 (use-modules (guix build utils)
410 (guix pki)
411 (guix utils)
412 (gcrypt pk-crypto)
413 (srfi srfi-26))
414
415 (define acl (current-acl))
416 (define key (string->canonical-sexp ,(canonical-sexp->string key)))
417
418 (unless (authorized-key? key)
419 (let ((acl (public-keys->acl (cons key (acl->public-keys acl)))))
420 (mkdir-p (dirname %acl-file))
421 (with-atomic-file-output %acl-file
422 (cut write-acl acl <>)))))
423 session
424 become-command))
425
426 (define (prepare-to-send store host log-port items)
427 "Notify the user that we're about to send ITEMS to HOST. Return three
428 values allowing 'notify-send-progress' to track the state of this transfer."
429 (let* ((count (length items))
430 (sizes (fold (lambda (item result)
431 (vhash-cons item
432 (path-info-nar-size
433 (query-path-info store item))
434 result))
435 vlist-null
436 items))
437 (total (vlist-fold (lambda (pair result)
438 (match pair
439 ((_ . size) (+ size result))))
440 0
441 sizes)))
442 (info (N_ "sending ~a store item (~h MiB) to '~a'...~%"
443 "sending ~a store items (~h MiB) to '~a'...~%" count)
444 count
445 (inexact->exact (round (/ total (expt 2. 20))))
446 host)
447
448 (values log-port sizes total 0)))
449
450 (define (notify-transfer-progress item port sizes total sent)
451 "Notify the user that we've already transferred SENT bytes out of TOTAL.
452 Use SIZES to determine the size of ITEM, which is about to be sent."
453 (define (display-bar %)
454 (erase-current-line port)
455 (format port "~3@a% ~a"
456 (inexact->exact (round (* 100. (/ sent total))))
457 (progress-bar % (- (max (current-terminal-columns) 5) 5)))
458 (force-output port))
459
460 (unless (zero? total)
461 (let ((% (* 100. (/ sent total))))
462 (match (vhash-assoc item sizes)
463 (#f
464 (display-bar %)
465 (values port sizes total sent))
466 ((_ . size)
467 (display-bar %)
468 (values port sizes total (+ sent size)))))))
469
470 (define (notify-transfer-completion port . args)
471 "Notify the user that the transfer has completed."
472 (apply notify-transfer-progress "" port args) ;display the 100% progress bar
473 (erase-current-line port)
474 (force-output port))
475
476 (define* (send-files local files remote
477 #:key
478 recursive?
479 (log-port (current-error-port)))
480 "Send the subset of FILES from LOCAL (a local store) that's missing to
481 REMOTE, a remote store. When RECURSIVE? is true, send the closure of FILES.
482 Return the list of store items actually sent."
483 ;; Compute the subset of FILES missing on SESSION and send them.
484 (let* ((files (if recursive? (requisites local files) files))
485 (session (channel-get-session (store-connection-socket remote)))
486 (missing (inferior-remote-eval
487 `(begin
488 (use-modules (guix)
489 (srfi srfi-1) (srfi srfi-26))
490
491 (with-store store
492 (remove (cut valid-path? store <>)
493 ',files)))
494 session))
495 (port (store-import-channel session))
496 (host (session-get session 'host)))
497 ;; Make sure everything alright on the remote side.
498 (match (read port)
499 (('importing)
500 #t)
501 (sexp
502 (handle-import/export-channel-error sexp remote)))
503
504 ;; Send MISSING in topological order.
505 (let ((tty? (isatty? log-port)))
506 (export-paths local missing port
507 #:start (cut prepare-to-send local host log-port <>)
508 #:progress (if tty? notify-transfer-progress (const #f))
509 #:finish (if tty? notify-transfer-completion (const #f))))
510
511 ;; Tell the remote process that we're done. (In theory the end-of-archive
512 ;; mark of 'export-paths' would be enough, but in practice it's not.)
513 (channel-send-eof port)
514
515 ;; Wait for completion of the remote process and read the status sexp from
516 ;; PORT. Wait for the exit status only when 'read' completed; otherwise,
517 ;; we might wait forever if the other end is stuck.
518 (let* ((result (false-if-exception (read port)))
519 (status (and result
520 (zero? (channel-get-exit-status port)))))
521 (close-port port)
522 (match result
523 (('success . _)
524 missing)
525 (('protocol-error message)
526 (raise (condition
527 (&store-protocol-error (message message) (status 42)))))
528 (('error key args ...)
529 (raise (condition
530 (&store-protocol-error
531 (message (call-with-output-string
532 (lambda (port)
533 (print-exception port #f key args))))
534 (status 43)))))
535 (_
536 (raise (condition
537 (&store-protocol-error
538 (message "unknown error while sending files over SSH")
539 (status 44)))))))))
540
541 (define (remote-store-session remote)
542 "Return the SSH channel beneath REMOTE, a remote store as returned by
543 'connect-to-remote-daemon', or #f."
544 (channel-get-session (store-connection-socket remote)))
545
546 (define (remote-store-host remote)
547 "Return the name of the host REMOTE is connected to, where REMOTE is a
548 remote store as returned by 'connect-to-remote-daemon'."
549 (match (remote-store-session remote)
550 (#f #f)
551 ((? session? session)
552 (session-get session 'host))))
553
554 (define* (file-retrieval-port files remote
555 #:key recursive?)
556 "Return an input port from which to retrieve FILES (a list of store items)
557 from REMOTE, along with the number of items to retrieve (lower than or equal
558 to the length of FILES.)"
559 (values (store-export-channel (remote-store-session remote) files
560 #:recursive? recursive?)
561 (length files))) ;XXX: inaccurate when RECURSIVE? is true
562
563 (define-syntax raise-error
564 (syntax-rules (=>)
565 ((_ fmt args ... (=> hint-fmt hint-args ...))
566 (raise (condition
567 (&message
568 (message (format #f fmt args ...)))
569 (&fix-hint
570 (hint (format #f hint-fmt hint-args ...))))))
571 ((_ fmt args ...)
572 (raise (condition
573 (&message
574 (message (format #f fmt args ...))))))))
575
576 (define (handle-import/export-channel-error sexp remote)
577 "Report an error corresponding to SEXP, the EOF object or an sexp read from
578 REMOTE."
579 (match sexp
580 ((? eof-object?)
581 (report-guile-error (remote-store-host remote)))
582 (('connection-error file code . _)
583 (raise-error (G_ "failed to connect to '~A' on remote host '~A': ~a")
584 file (remote-store-host remote) (strerror code)))
585 (('invalid-items items . _)
586 (raise-error (N_ "no such item on remote host '~A':~{ ~a~}"
587 "no such items on remote host '~A':~{ ~a~}"
588 (length items))
589 (remote-store-host remote) items))
590 (('protocol-error status message . _)
591 (raise-error (G_ "protocol error on remote host '~A': ~a")
592 (remote-store-host remote) message))
593 (_
594 (raise-error (G_ "failed to retrieve store items from '~a'")
595 (remote-store-host remote)))))
596
597 (define* (retrieve-files* files remote
598 #:key recursive? (log-port (current-error-port))
599 (import (const #f)))
600 "Pass IMPORT an input port from which to read the sequence of FILES coming
601 from REMOTE. When RECURSIVE? is true, retrieve the closure of FILES."
602 (let-values (((port count)
603 (file-retrieval-port files remote
604 #:recursive? recursive?)))
605 (match (read port) ;read the initial status
606 (('exporting)
607 (format #t (N_ "retrieving ~a store item from '~a'...~%"
608 "retrieving ~a store items from '~a'...~%" count)
609 count (remote-store-host remote))
610
611 (dynamic-wind
612 (const #t)
613 (lambda ()
614 (import port))
615 (lambda ()
616 (close-port port))))
617 (sexp
618 (handle-import/export-channel-error sexp remote)))))
619
620 (define* (retrieve-files local files remote
621 #:key recursive? (log-port (current-error-port)))
622 "Retrieve FILES from REMOTE and import them using the 'import-paths' RPC on
623 LOCAL. When RECURSIVE? is true, retrieve the closure of FILES."
624 (retrieve-files* (remove (cut valid-path? local <>) files)
625 remote
626 #:recursive? recursive?
627 #:log-port log-port
628 #:import (lambda (port)
629 (import-paths local port))))
630
631 \f
632 ;;;
633 ;;; Error reporting.
634 ;;;
635
636 (define (report-guile-error host)
637 (raise-error (G_ "failed to start Guile on remote host '~A'") host
638 (=> (G_ "Make sure @command{guile} can be found in
639 @code{$PATH} on the remote host. Run @command{ssh ~A guile --version} to
640 check.")
641 host)))
642
643 (define (report-inferior-exception exception host)
644 "Report EXCEPTION, an &inferior-exception that occurred on HOST."
645 (raise-error (G_ "exception occurred on remote host '~A': ~s")
646 host (inferior-exception-arguments exception)))
647
648 ;;; ssh.scm ends here