gnu: easyrpg-player: Update to 0.6.2.2.
[jackhill/guix/guix.git] / guix / ssh.scm
1 ;;; GNU Guix --- Functional package management for GNU
2 ;;; Copyright © 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org>
3 ;;;
4 ;;; This file is part of GNU Guix.
5 ;;;
6 ;;; GNU Guix is free software; you can redistribute it and/or modify it
7 ;;; under the terms of the GNU General Public License as published by
8 ;;; the Free Software Foundation; either version 3 of the License, or (at
9 ;;; your option) any later version.
10 ;;;
11 ;;; GNU Guix is distributed in the hope that it will be useful, but
12 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 ;;; GNU General Public License for more details.
15 ;;;
16 ;;; You should have received a copy of the GNU General Public License
17 ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
18
19 (define-module (guix ssh)
20 #:use-module (guix store)
21 #:use-module (guix inferior)
22 #:use-module (guix i18n)
23 #:use-module ((guix diagnostics)
24 #:select (info &fix-hint formatted-message))
25 #:use-module ((guix progress)
26 #:select (progress-bar
27 erase-current-line current-terminal-columns))
28 #:use-module (gcrypt pk-crypto)
29 #:use-module (ssh session)
30 #:use-module (ssh auth)
31 #:use-module (ssh key)
32 #:use-module (ssh channel)
33 #:use-module (ssh popen)
34 #:use-module (ssh session)
35 #:use-module (srfi srfi-1)
36 #:use-module (srfi srfi-11)
37 #:use-module (srfi srfi-26)
38 #:use-module (srfi srfi-34)
39 #:use-module (srfi srfi-35)
40 #:use-module (ice-9 match)
41 #:use-module (ice-9 format)
42 #:use-module (ice-9 binary-ports)
43 #:use-module (ice-9 vlist)
44 #:export (open-ssh-session
45 authenticate-server*
46
47 remote-inferior
48 remote-daemon-channel
49 connect-to-remote-daemon
50 remote-system
51 remote-authorize-signing-key
52 send-files
53 retrieve-files
54 retrieve-files*
55 remote-store-host
56
57 report-guile-error
58 report-module-error))
59
60 ;;; Commentary:
61 ;;;
62 ;;; This module provides tools to support communication with remote stores
63 ;;; over SSH, using Guile-SSH.
64 ;;;
65 ;;; Code:
66
67 (define %compression
68 "zlib@openssh.com,zlib")
69
70 (define (host-key->type+key host-key)
71 "Destructure HOST-KEY, an OpenSSH host key string, and return two values:
72 its key type as a symbol, and the actual base64-encoded string."
73 (define (type->symbol type)
74 (and (string-prefix? "ssh-" type)
75 (string->symbol (string-drop type 4))))
76
77 (match (string-tokenize host-key)
78 ((type key x)
79 (values (type->symbol type) key))
80 ((type key)
81 (values (type->symbol type) key))))
82
83 (define (authenticate-server* session key)
84 "Make sure the server for SESSION has the given KEY, where KEY is a string
85 such as \"ssh-ed25519 AAAAC3Nz… root@example.org\". Raise an exception if the
86 actual key does not match."
87 (let-values (((server) (get-server-public-key session))
88 ((type key) (host-key->type+key key)))
89 (unless (and (or (not (get-key-type server))
90 (eq? (get-key-type server) type))
91 (string=? (public-key->string server) key))
92 ;; Key mismatch: something's wrong. XXX: It could be that the server
93 ;; provided its Ed25519 key when we where expecting its RSA key. XXX:
94 ;; Guile-SSH 0.10.1 doesn't know about ed25519 keys and 'get-key-type'
95 ;; returns #f in that case.
96 (raise (formatted-message (G_ "server at '~a' returned host key \
97 '~a' of type '~a' instead of '~a' of type '~a'~%")
98 (session-get session 'host)
99 (public-key->string server)
100 (get-key-type server)
101 key type)))))
102
103 (define* (open-ssh-session host #:key user port identity
104 host-key
105 (compression %compression)
106 (timeout 3600))
107 "Open an SSH session for HOST and return it. IDENTITY specifies the file
108 name of a private key to use for authenticating with the host. When USER,
109 PORT, or IDENTITY are #f, use default values or whatever '~/.ssh/config'
110 specifies; otherwise use them.
111
112 When HOST-KEY is true, it must be a string like \"ssh-ed25519 AAAAC3Nz…
113 root@example.org\"; the server is authenticated and an error is raised if its
114 host key is different from HOST-KEY.
115
116 Install TIMEOUT as the maximum time in seconds after which a read or write
117 operation on a channel of the returned session is considered as failing.
118
119 Throw an error on failure."
120 (let ((session (make-session #:user user
121 #:identity identity
122 #:host host
123 #:port port
124 #:timeout 10 ;seconds
125 ;; #:log-verbosity 'protocol
126
127 ;; Prevent libssh from reading
128 ;; ~/.ssh/known_hosts when the caller provides
129 ;; a HOST-KEY to match against.
130 #:knownhosts (and host-key "/dev/null")
131
132 ;; We need lightweight compression when
133 ;; exchanging full archives.
134 #:compression compression
135 #:compression-level 3
136
137 ;; Speed up RPCs by creating sockets with
138 ;; TCP_NODELAY.
139 #:nodelay #t)))
140
141 ;; Honor ~/.ssh/config.
142 (session-parse-config! session)
143
144 (match (connect! session)
145 ('ok
146 (if host-key
147 ;; Make sure the server's key is what we expect.
148 (authenticate-server* session host-key)
149
150 ;; Authenticate against ~/.ssh/known_hosts.
151 (match (authenticate-server session)
152 ('ok #f)
153 (reason
154 (raise (formatted-message (G_ "failed to authenticate \
155 server at '~a': ~a")
156 (session-get session 'host)
157 reason)))))
158
159 ;; Use public key authentication, via the SSH agent if it's available.
160 (match (userauth-public-key/auto! session)
161 ('success
162 (session-set! session 'timeout timeout)
163 session)
164 (x
165 (match (userauth-gssapi! session)
166 ('success
167 (session-set! session 'timeout timeout)
168 session)
169 (x
170 (disconnect! session)
171 (raise (condition
172 (&message
173 (message (format #f (G_ "SSH authentication failed for '~a': ~a~%")
174 host (get-error session)))))))))))
175 (x
176 ;; Connection failed or timeout expired.
177 (raise (formatted-message (G_ "SSH connection to '~a' failed: ~a~%")
178 host (get-error session)))))))
179
180 (define* (remote-inferior session #:optional become-command)
181 "Return a remote inferior for the given SESSION. If BECOME-COMMAND is
182 given, use that to invoke the remote Guile REPL."
183 (let* ((repl-command (append (or become-command '())
184 '("guix" "repl" "-t" "machine")))
185 (pipe (apply open-remote-pipe* session OPEN_BOTH repl-command)))
186 (when (eof-object? (peek-char pipe))
187 (let ((status (channel-get-exit-status pipe)))
188 (close-port pipe)
189 (raise (formatted-message (G_ "remote command '~{~a~^ ~}' failed \
190 with status ~a")
191 repl-command status))))
192 (port->inferior pipe)))
193
194 (define* (inferior-remote-eval exp session #:optional become-command)
195 "Evaluate EXP in a new inferior running in SESSION, and close the inferior
196 right away. If BECOME-COMMAND is given, use that to invoke the remote Guile
197 REPL."
198 (let ((inferior (remote-inferior session become-command)))
199 (dynamic-wind
200 (const #t)
201 (lambda ()
202 (inferior-eval exp inferior))
203 (lambda ()
204 ;; Close INFERIOR right away to prevent finalization from happening in
205 ;; another thread at the wrong time (see
206 ;; <https://bugs.gnu.org/26976>.)
207 (close-inferior inferior)))))
208
209 (define* (remote-daemon-channel session
210 #:optional
211 (socket-name
212 "/var/guix/daemon-socket/socket"))
213 "Return an input/output port (an SSH channel) to the daemon at SESSION."
214 (define redirect
215 ;; Code run in SESSION to redirect the remote process' stdin/stdout to the
216 ;; daemon's socket, à la socat. The SSH protocol supports forwarding to
217 ;; Unix-domain sockets but libssh doesn't have an API for that, hence this
218 ;; hack.
219 `(begin
220 (use-modules (ice-9 match) (rnrs io ports)
221 (rnrs bytevectors))
222
223 (let ((sock (socket AF_UNIX SOCK_STREAM 0))
224 (stdin (current-input-port))
225 (stdout (current-output-port))
226 (select* (lambda (read write except)
227 ;; This is a workaround for
228 ;; <https://bugs.gnu.org/30365> in Guile < 2.2.4:
229 ;; since 'select' sometimes returns non-empty sets for
230 ;; no good reason, call 'select' a second time with a
231 ;; zero timeout to filter out incorrect replies.
232 (match (select read write except)
233 ((read write except)
234 (select read write except 0))))))
235 (setvbuf stdout 'none)
236
237 ;; Use buffered ports so that 'get-bytevector-some' returns up to the
238 ;; whole buffer like read(2) would--see <https://bugs.gnu.org/30066>.
239 (setvbuf stdin 'block 65536)
240 (setvbuf sock 'block 65536)
241
242 (connect sock AF_UNIX ,socket-name)
243
244 (let loop ()
245 (match (select* (list stdin sock) '() '())
246 ((reads () ())
247 (when (memq stdin reads)
248 (match (get-bytevector-some stdin)
249 ((? eof-object?)
250 (primitive-exit 0))
251 (bv
252 (put-bytevector sock bv)
253 (force-output sock))))
254 (when (memq sock reads)
255 (match (get-bytevector-some sock)
256 ((? eof-object?)
257 (primitive-exit 0))
258 (bv
259 (put-bytevector stdout bv))))
260 (loop))
261 (_
262 (primitive-exit 1)))))))
263
264 (open-remote-pipe* session OPEN_BOTH
265 ;; Sort-of shell-quote REDIRECT.
266 "guile" "-c"
267 (object->string
268 (object->string redirect))))
269
270 (define* (connect-to-remote-daemon session
271 #:optional
272 (socket-name
273 "/var/guix/daemon-socket/socket"))
274 "Connect to the remote build daemon listening on SOCKET-NAME over SESSION,
275 an SSH session. Return a <store-connection> object."
276 (open-connection #:port (remote-daemon-channel session socket-name)))
277
278
279 (define (store-import-channel session)
280 "Return an output port to which archives to be exported to SESSION's store
281 can be written."
282 ;; Using the 'import-paths' RPC on a remote store would be slow because it
283 ;; makes a round trip every time 32 KiB have been transferred. This
284 ;; procedure instead opens a separate channel to use the remote
285 ;; 'import-paths' procedure, which consumes all the data in a single round
286 ;; trip. This optimizes the successful case at the expense of error
287 ;; conditions: errors can only be reported once all the input has been
288 ;; consumed.
289 (define import
290 `(begin
291 (eval-when (load expand eval)
292 (unless (resolve-module '(guix) #:ensure #f)
293 (write `(module-error))
294 (exit 7)))
295
296 (use-modules (guix) (srfi srfi-34)
297 (rnrs io ports) (rnrs bytevectors))
298
299 (define (consume-input port)
300 (let ((bv (make-bytevector 32768)))
301 (let loop ()
302 (let ((n (get-bytevector-n! port bv 0
303 (bytevector-length bv))))
304 (unless (eof-object? n)
305 (loop))))))
306
307 ;; Upon completion, write an sexp that denotes the status.
308 (write
309 (catch #t
310 (lambda ()
311 (guard (c ((nix-protocol-error? c)
312 ;; Consume all the input since the only time we can
313 ;; report the error is after everything has been
314 ;; consumed.
315 (consume-input (current-input-port))
316 (list 'protocol-error (nix-protocol-error-message c))))
317 (with-store store
318 (write '(importing)) ;we're ready
319 (force-output)
320
321 (setvbuf (current-input-port) 'none)
322 (import-paths store (current-input-port))
323 '(success))))
324 (lambda args
325 (cons 'error args))))))
326
327 (open-remote-pipe session
328 (string-join
329 `("guile" "-c"
330 ,(object->string (object->string import))))
331 OPEN_BOTH))
332
333 (define* (store-export-channel session files
334 #:key recursive?)
335 "Return an input port from which an export of FILES from SESSION's store can
336 be read. When RECURSIVE? is true, the closure of FILES is exported."
337 ;; Same as above: this is more efficient than calling 'export-paths' on a
338 ;; remote store.
339 (define export
340 `(begin
341 (eval-when (load expand eval)
342 (unless (resolve-module '(guix) #:ensure #f)
343 (write `(module-error))
344 (exit 7)))
345
346 (use-modules (guix) (srfi srfi-1)
347 (srfi srfi-26) (srfi srfi-34))
348
349 (guard (c ((nix-connection-error? c)
350 (write `(connection-error ,(nix-connection-error-file c)
351 ,(nix-connection-error-code c))))
352 ((nix-protocol-error? c)
353 (write `(protocol-error ,(nix-protocol-error-status c)
354 ,(nix-protocol-error-message c))))
355 (else
356 (write `(exception))))
357 (with-store store
358 (let* ((files ',files)
359 (invalid (remove (cut valid-path? store <>)
360 files)))
361 (unless (null? invalid)
362 (write `(invalid-items ,invalid))
363 (exit 1))
364
365 ;; TODO: When RECURSIVE? is true, we could send the list of store
366 ;; items in the closure so that the other end can filter out
367 ;; those it already has.
368
369 (write '(exporting)) ;we're ready
370 (force-output)
371
372 (setvbuf (current-output-port) 'none)
373 (export-paths store files (current-output-port)
374 #:recursive? ,recursive?))))))
375
376 (open-remote-input-pipe session
377 (string-join
378 `("guile" "-c"
379 ,(object->string
380 (object->string export))))))
381
382 (define (remote-system session)
383 "Return the system type as expected by Nix, usually ARCHITECTURE-KERNEL, of
384 the machine on the other end of SESSION."
385 (inferior-remote-eval '(begin (use-modules (guix utils)) (%current-system))
386 session))
387
388 (define* (remote-authorize-signing-key key session #:optional become-command)
389 "Send KEY, a canonical sexp containing a public key, over SESSION and add it
390 to the system ACL file if it has not yet been authorized."
391 (inferior-remote-eval
392 `(begin
393 (use-modules (guix build utils)
394 (guix pki)
395 (guix utils)
396 (gcrypt pk-crypto)
397 (srfi srfi-26))
398
399 (define acl (current-acl))
400 (define key (string->canonical-sexp ,(canonical-sexp->string key)))
401
402 (unless (authorized-key? key)
403 (let ((acl (public-keys->acl (cons key (acl->public-keys acl)))))
404 (mkdir-p (dirname %acl-file))
405 (with-atomic-file-output %acl-file
406 (cut write-acl acl <>)))))
407 session
408 become-command))
409
410 (define (prepare-to-send store host log-port items)
411 "Notify the user that we're about to send ITEMS to HOST. Return three
412 values allowing 'notify-send-progress' to track the state of this transfer."
413 (let* ((count (length items))
414 (sizes (fold (lambda (item result)
415 (vhash-cons item
416 (path-info-nar-size
417 (query-path-info store item))
418 result))
419 vlist-null
420 items))
421 (total (vlist-fold (lambda (pair result)
422 (match pair
423 ((_ . size) (+ size result))))
424 0
425 sizes)))
426 (info (N_ "sending ~a store item (~h MiB) to '~a'...~%"
427 "sending ~a store items (~h MiB) to '~a'...~%" count)
428 count
429 (inexact->exact (round (/ total (expt 2. 20))))
430 host)
431
432 (values log-port sizes total 0)))
433
434 (define (notify-transfer-progress item port sizes total sent)
435 "Notify the user that we've already transferred SENT bytes out of TOTAL.
436 Use SIZES to determine the size of ITEM, which is about to be sent."
437 (define (display-bar %)
438 (erase-current-line port)
439 (format port "~3@a% ~a"
440 (inexact->exact (round (* 100. (/ sent total))))
441 (progress-bar % (- (max (current-terminal-columns) 5) 5)))
442 (force-output port))
443
444 (unless (zero? total)
445 (let ((% (* 100. (/ sent total))))
446 (match (vhash-assoc item sizes)
447 (#f
448 (display-bar %)
449 (values port sizes total sent))
450 ((_ . size)
451 (display-bar %)
452 (values port sizes total (+ sent size)))))))
453
454 (define (notify-transfer-completion port . args)
455 "Notify the user that the transfer has completed."
456 (apply notify-transfer-progress "" port args) ;display the 100% progress bar
457 (erase-current-line port)
458 (force-output port))
459
460 (define* (send-files local files remote
461 #:key
462 recursive?
463 (log-port (current-error-port)))
464 "Send the subset of FILES from LOCAL (a local store) that's missing to
465 REMOTE, a remote store. When RECURSIVE? is true, send the closure of FILES.
466 Return the list of store items actually sent."
467 ;; Compute the subset of FILES missing on SESSION and send them.
468 (let* ((files (if recursive? (requisites local files) files))
469 (session (channel-get-session (store-connection-socket remote)))
470 (missing (inferior-remote-eval
471 `(begin
472 (use-modules (guix)
473 (srfi srfi-1) (srfi srfi-26))
474
475 (with-store store
476 (remove (cut valid-path? store <>)
477 ',files)))
478 session))
479 (port (store-import-channel session))
480 (host (session-get session 'host)))
481 ;; Make sure everything alright on the remote side.
482 (match (read port)
483 (('importing)
484 #t)
485 (sexp
486 (handle-import/export-channel-error sexp remote)))
487
488 ;; Send MISSING in topological order.
489 (let ((tty? (isatty? log-port)))
490 (export-paths local missing port
491 #:start (cut prepare-to-send local host log-port <>)
492 #:progress (if tty? notify-transfer-progress (const #f))
493 #:finish (if tty? notify-transfer-completion (const #f))))
494
495 ;; Tell the remote process that we're done. (In theory the end-of-archive
496 ;; mark of 'export-paths' would be enough, but in practice it's not.)
497 (channel-send-eof port)
498
499 ;; Wait for completion of the remote process and read the status sexp from
500 ;; PORT. Wait for the exit status only when 'read' completed; otherwise,
501 ;; we might wait forever if the other end is stuck.
502 (let* ((result (false-if-exception (read port)))
503 (status (and result
504 (zero? (channel-get-exit-status port)))))
505 (close-port port)
506 (match result
507 (('success . _)
508 missing)
509 (('protocol-error message)
510 (raise (condition
511 (&store-protocol-error (message message) (status 42)))))
512 (('error key args ...)
513 (raise (condition
514 (&store-protocol-error
515 (message (call-with-output-string
516 (lambda (port)
517 (print-exception port #f key args))))
518 (status 43)))))
519 (_
520 (raise (condition
521 (&store-protocol-error
522 (message "unknown error while sending files over SSH")
523 (status 44)))))))))
524
525 (define (remote-store-session remote)
526 "Return the SSH channel beneath REMOTE, a remote store as returned by
527 'connect-to-remote-daemon', or #f."
528 (channel-get-session (store-connection-socket remote)))
529
530 (define (remote-store-host remote)
531 "Return the name of the host REMOTE is connected to, where REMOTE is a
532 remote store as returned by 'connect-to-remote-daemon'."
533 (match (remote-store-session remote)
534 (#f #f)
535 ((? session? session)
536 (session-get session 'host))))
537
538 (define* (file-retrieval-port files remote
539 #:key recursive?)
540 "Return an input port from which to retrieve FILES (a list of store items)
541 from REMOTE, along with the number of items to retrieve (lower than or equal
542 to the length of FILES.)"
543 (values (store-export-channel (remote-store-session remote) files
544 #:recursive? recursive?)
545 (length files))) ;XXX: inaccurate when RECURSIVE? is true
546
547 (define-syntax raise-error
548 (syntax-rules (=>)
549 ((_ fmt args ... (=> hint-fmt hint-args ...))
550 (raise (condition
551 (&message
552 (message (format #f fmt args ...)))
553 (&fix-hint
554 (hint (format #f hint-fmt hint-args ...))))))
555 ((_ fmt args ...)
556 (raise (condition
557 (&message
558 (message (format #f fmt args ...))))))))
559
560 (define (handle-import/export-channel-error sexp remote)
561 "Report an error corresponding to SEXP, the EOF object or an sexp read from
562 REMOTE."
563 (match sexp
564 ((? eof-object?)
565 (report-guile-error (remote-store-host remote)))
566 (('module-error . _)
567 (report-module-error (remote-store-host remote)))
568 (('connection-error file code . _)
569 (raise-error (G_ "failed to connect to '~A' on remote host '~A': ~a")
570 file (remote-store-host remote) (strerror code)))
571 (('invalid-items items . _)
572 (raise-error (N_ "no such item on remote host '~A':~{ ~a~}"
573 "no such items on remote host '~A':~{ ~a~}"
574 (length items))
575 (remote-store-host remote) items))
576 (('protocol-error status message . _)
577 (raise-error (G_ "protocol error on remote host '~A': ~a")
578 (remote-store-host remote) message))
579 (_
580 (raise-error (G_ "failed to retrieve store items from '~a'")
581 (remote-store-host remote)))))
582
583 (define* (retrieve-files* files remote
584 #:key recursive? (log-port (current-error-port))
585 (import (const #f)))
586 "Pass IMPORT an input port from which to read the sequence of FILES coming
587 from REMOTE. When RECURSIVE? is true, retrieve the closure of FILES."
588 (let-values (((port count)
589 (file-retrieval-port files remote
590 #:recursive? recursive?)))
591 (match (read port) ;read the initial status
592 (('exporting)
593 (format #t (N_ "retrieving ~a store item from '~a'...~%"
594 "retrieving ~a store items from '~a'...~%" count)
595 count (remote-store-host remote))
596
597 (dynamic-wind
598 (const #t)
599 (lambda ()
600 (import port))
601 (lambda ()
602 (close-port port))))
603 (sexp
604 (handle-import/export-channel-error sexp remote)))))
605
606 (define* (retrieve-files local files remote
607 #:key recursive? (log-port (current-error-port)))
608 "Retrieve FILES from REMOTE and import them using the 'import-paths' RPC on
609 LOCAL. When RECURSIVE? is true, retrieve the closure of FILES."
610 (retrieve-files* (remove (cut valid-path? local <>) files)
611 remote
612 #:recursive? recursive?
613 #:log-port log-port
614 #:import (lambda (port)
615 (import-paths local port))))
616
617 \f
618 ;;;
619 ;;; Error reporting.
620 ;;;
621
622 (define (report-guile-error host)
623 (raise-error (G_ "failed to start Guile on remote host '~A'") host
624 (=> (G_ "Make sure @command{guile} can be found in
625 @code{$PATH} on the remote host. Run @command{ssh ~A guile --version} to
626 check.")
627 host)))
628
629 (define (report-module-error host)
630 "Report an error about missing Guix modules on HOST."
631 ;; TRANSLATORS: Leave "Guile" untranslated.
632 (raise-error (G_ "Guile modules not found on remote host '~A'") host
633 (=> (G_ "Make sure @code{GUILE_LOAD_PATH} includes Guix'
634 own module directory. Run @command{ssh ~A env | grep GUILE_LOAD_PATH} to
635 check.")
636 host)))
637
638 (define (report-inferior-exception exception host)
639 "Report EXCEPTION, an &inferior-exception that occurred on HOST."
640 (raise-error (G_ "exception occurred on remote host '~A': ~s")
641 host (inferior-exception-arguments exception)))
642
643 ;;; ssh.scm ends here