1 ;;; GNU Guix --- Functional package management for GNU
2 ;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org>
3 ;;; Copyright © 2018 Jan Nieuwenhuizen <janneke@gnu.org>
4 ;;; Copyright © 2019, 2020 Mathieu Othacehe <m.othacehe@gmail.com>
5 ;;; Copyright © 2020 Florian Pelz <pelzflorian@pelzflorian.de>
6 ;;; Copyright © 2020 Lars-Dominik Braun <ldb@leibniz-psychology.org>
8 ;;; This file is part of GNU Guix.
10 ;;; GNU Guix is free software; you can redistribute it and/or modify it
11 ;;; under the terms of the GNU General Public License as published by
12 ;;; the Free Software Foundation; either version 3 of the License, or (at
13 ;;; your option) any later version.
15 ;;; GNU Guix is distributed in the hope that it will be useful, but
16 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
17 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 ;;; GNU General Public License for more details.
20 ;;; You should have received a copy of the GNU General Public License
21 ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
23 (define-module (guix store)
24 #:use-module (guix utils)
25 #:use-module (guix config)
26 #:use-module (guix deprecation)
27 #:use-module (guix memoization)
28 #:use-module (guix serialization)
29 #:use-module (guix monads)
30 #:use-module (guix records)
31 #:use-module (guix base16)
32 #:use-module (guix base32)
33 #:use-module (gcrypt hash)
34 #:use-module (guix profiling)
35 #:autoload (guix build syscalls) (terminal-columns)
36 #:use-module (rnrs bytevectors)
37 #:use-module (ice-9 binary-ports)
38 #:use-module ((ice-9 control) #:select (let/ec))
39 #:use-module (srfi srfi-1)
40 #:use-module (srfi srfi-9)
41 #:use-module (srfi srfi-9 gnu)
42 #:use-module (srfi srfi-11)
43 #:use-module (srfi srfi-26)
44 #:use-module (srfi srfi-34)
45 #:use-module (srfi srfi-35)
46 #:use-module (srfi srfi-39)
47 #:use-module (ice-9 match)
48 #:use-module (ice-9 vlist)
49 #:use-module (ice-9 popen)
50 #:use-module (ice-9 threads)
51 #:use-module (ice-9 format)
52 #:use-module (web uri)
53 #:export (%daemon-socket-uri
55 %default-substitute-urls
58 store-connection-version
59 store-connection-major-version
60 store-connection-minor-version
61 store-connection-socket
63 ;; Deprecated forms for 'store-connection'.
66 nix-server-major-version
67 nix-server-minor-version
70 current-store-protocol-version ;for internal use
73 &store-error store-error?
74 &store-connection-error store-connection-error?
75 store-connection-error-file
76 store-connection-error-code
77 &store-protocol-error store-protocol-error?
78 store-protocol-error-message
79 store-protocol-error-status
81 ;; Deprecated forms for '&store-error' et al.
83 &nix-connection-error nix-connection-error?
84 nix-connection-error-file
85 nix-connection-error-code
86 &nix-protocol-error nix-protocol-error?
87 nix-protocol-error-message
88 nix-protocol-error-status
107 add-file-tree-to-store
111 map/accumulate-builds
112 mapm/accumulate-builds
122 remove-permanent-root
126 substitutable-deriver
127 substitutable-references
128 substitutable-download-size
129 substitutable-nar-size
132 substitutable-path-info
138 path-info-registration-time
144 references/substitutes
153 query-derivation-outputs
161 current-build-output-port
172 current-target-system
186 store-path-package-name
192 (define %protocol-version #x163)
194 (define %worker-magic-1 #x6e697863) ; "nixc"
195 (define %worker-magic-2 #x6478696f) ; "dxio"
197 (define (protocol-major magic)
198 (logand magic #xff00))
199 (define (protocol-minor magic)
200 (logand magic #x00ff))
201 (define (protocol-version major minor)
202 (logior major minor))
204 (define-syntax define-enumerate-type
206 ((_ name->int (name id) ...)
207 (define-syntax name->int
208 (syntax-rules (name ...)
209 ((_ name) id) ...)))))
211 (define-enumerate-type operation-id
212 ;; operation numbers from worker-protocol.hh
220 (add-text-to-store 8)
224 (add-indirect-root 12)
231 ;;(query-substitutable-path-info 21) ; obsolete as of #x10c
232 (query-derivation-outputs 22)
233 (query-all-valid-paths 23)
234 (query-failed-paths 24)
235 (clear-failed-paths 25)
238 (query-derivation-output-names 28)
239 (query-path-from-hash-part 29)
240 (query-substitutable-path-infos 30)
241 (query-valid-paths 31)
242 (query-substitutable-paths 32)
243 (query-valid-derivers 33)
246 (built-in-builders 80))
248 (define-enumerate-type hash-algo
254 (define-enumerate-type build-mode
260 (define-enumerate-type gc-action
267 (define %default-socket-path
268 (string-append %state-directory "/daemon-socket/socket"))
270 (define %daemon-socket-uri
271 ;; URI or file name of the socket the daemon listens too.
272 (make-parameter (or (getenv "GUIX_DAEMON_SOCKET")
273 %default-socket-path)))
277 ;; Information about a substitutable store path.
278 (define-record-type <substitutable>
279 (substitutable path deriver refs dl-size nar-size)
281 (path substitutable-path)
282 (deriver substitutable-deriver)
283 (refs substitutable-references)
284 (dl-size substitutable-download-size)
285 (nar-size substitutable-nar-size))
287 (define (read-substitutable-path-list p)
288 (let loop ((len (read-int p))
292 (let ((path (read-store-path p))
293 (deriver (read-store-path p))
294 (refs (read-store-path-list p))
295 (dl-size (read-long-long p))
296 (nar-size (read-long-long p)))
298 (cons (substitutable path deriver refs dl-size nar-size)
301 ;; Information about a store path.
302 (define-record-type <path-info>
303 (path-info deriver hash references registration-time nar-size)
305 (deriver path-info-deriver) ;string | #f
306 (hash path-info-hash)
307 (references path-info-references)
308 (registration-time path-info-registration-time)
309 (nar-size path-info-nar-size))
311 (define (read-path-info p)
312 (let ((deriver (match (read-store-path p)
315 (hash (base16-string->bytevector (read-string p)))
316 (refs (read-store-path-list p))
317 (registration-time (read-int p))
318 (nar-size (read-long-long p)))
319 (path-info deriver hash refs registration-time nar-size)))
321 (define-syntax write-arg
322 (syntax-rules (integer boolean bytevector
323 string string-list string-pairs
324 store-path store-path-list base16)
328 (write-int (if arg 1 0) p))
329 ((_ bytevector arg p)
330 (write-bytevector arg p))
332 (write-string arg p))
333 ((_ string-list arg p)
334 (write-string-list arg p))
335 ((_ string-pairs arg p)
336 (write-string-pairs arg p))
337 ((_ store-path arg p)
338 (write-store-path arg p))
339 ((_ store-path-list arg p)
340 (write-store-path-list arg p))
342 (write-string (bytevector->base16-string arg) p))))
344 (define-syntax read-arg
345 (syntax-rules (integer boolean string store-path
346 store-path-list string-list string-pairs
347 substitutable-path-list path-info base16)
351 (not (zero? (read-int p))))
356 ((_ store-path-list p)
357 (read-store-path-list p))
359 (read-string-list p))
361 (read-string-pairs p))
362 ((_ substitutable-path-list p)
363 (read-substitutable-path-list p))
367 (base16-string->bytevector (read-string p)))))
372 (define-record-type* <store-connection> store-connection %make-store-connection
374 (socket store-connection-socket)
375 (major store-connection-major-version)
376 (minor store-connection-minor-version)
378 (buffer store-connection-output-port) ;output port
379 (flush store-connection-flush-output) ;thunk
381 ;; Caches. We keep them per-connection, because store paths build
382 ;; during the session are temporary GC roots kept for the duration of
384 (ats-cache store-connection-add-to-store-cache)
385 (atts-cache store-connection-add-text-to-store-cache)
386 (object-cache store-connection-object-cache
387 (default vlist-null)) ;vhash
388 (built-in-builders store-connection-built-in-builders
389 (default (delay '())))) ;promise
391 (set-record-type-printer! <store-connection>
393 (format port "#<store-connection ~a.~a ~a>"
394 (store-connection-major-version obj)
395 (store-connection-minor-version obj)
396 (number->string (object-address obj)
399 (define-deprecated/alias nix-server? store-connection?)
400 (define-deprecated/alias nix-server-major-version
401 store-connection-major-version)
402 (define-deprecated/alias nix-server-minor-version
403 store-connection-minor-version)
404 (define-deprecated/alias nix-server-socket store-connection-socket)
407 (define-condition-type &store-error &error
410 (define-condition-type &store-connection-error &store-error
411 store-connection-error?
412 (file store-connection-error-file)
413 (errno store-connection-error-code))
415 (define-condition-type &store-protocol-error &store-error
416 store-protocol-error?
417 (message store-protocol-error-message)
418 (status store-protocol-error-status))
420 (define-deprecated/alias &nix-error &store-error)
421 (define-deprecated/alias nix-error? store-error?)
422 (define-deprecated/alias &nix-connection-error &store-connection-error)
423 (define-deprecated/alias nix-connection-error? store-connection-error?)
424 (define-deprecated/alias nix-connection-error-file
425 store-connection-error-file)
426 (define-deprecated/alias nix-connection-error-code
427 store-connection-error-code)
428 (define-deprecated/alias &nix-protocol-error &store-protocol-error)
429 (define-deprecated/alias nix-protocol-error? store-protocol-error?)
430 (define-deprecated/alias nix-protocol-error-message
431 store-protocol-error-message)
432 (define-deprecated/alias nix-protocol-error-status
433 store-protocol-error-status)
436 (define-syntax-rule (system-error-to-connection-error file exp ...)
437 "Catch 'system-error' exceptions and translate them to
438 '&store-connection-error'."
443 (let ((errno (system-error-errno args)))
444 (raise (condition (&store-connection-error
448 (define (open-unix-domain-socket file)
449 "Connect to the Unix-domain socket at FILE and return it. Raise a
450 '&store-connection-error' upon error."
451 (let ((s (with-fluids ((%default-port-encoding #f))
452 ;; This trick allows use of the `scm_c_read' optimization.
453 (socket PF_UNIX SOCK_STREAM 0)))
454 (a (make-socket-address PF_UNIX file)))
456 (system-error-to-connection-error file
460 (define %default-guix-port
461 ;; Default port when connecting to a daemon over TCP/IP.
464 (define (open-inet-socket host port)
465 "Connect to the Unix-domain socket at HOST:PORT and return it. Raise a
466 '&store-connection-error' upon error."
467 (let ((sock (with-fluids ((%default-port-encoding #f))
468 ;; This trick allows use of the `scm_c_read' optimization.
469 (socket PF_UNIX SOCK_STREAM 0))))
472 (if (number? port) (number->string port) port)
474 (logior AI_ADDRCONFIG AI_NUMERICSERV)
476 0 ;any address family
477 SOCK_STREAM)) ;TCP only
479 (let loop ((addresses addresses))
482 (let ((s (socket (addrinfo:fam ai)
484 SOCK_STREAM IPPROTO_IP)))
488 (connect s (addrinfo:addr ai))
490 ;; Setting this option makes a dramatic difference because it
491 ;; avoids the "ACK delay" on our RPC messages.
492 (setsockopt s IPPROTO_TCP TCP_NODELAY 1)
495 ;; Connection failed, so try one of the other addresses.
498 (raise (condition (&store-connection-error
500 (errno (system-error-errno args)))))
503 (define (connect-to-daemon uri)
504 "Connect to the daemon at URI, a string that may be an actual URI or a file
505 name, and return an input/output port.
507 This is a low-level procedure that does not perform the initial handshake with
508 the daemon. Use 'open-connection' for that."
509 (define (not-supported)
510 (raise (condition (&store-connection-error
515 (match (string->uri uri)
516 (#f ;URI is a file name
517 open-unix-domain-socket)
519 (match (uri-scheme uri)
522 (open-unix-domain-socket (uri-path uri))))
525 (open-inet-socket (uri-host uri)
526 (or (uri-port uri) %default-guix-port))))
528 ;; Try to dynamically load a module for SCHEME.
529 ;; XXX: Errors are swallowed.
530 (match (false-if-exception
531 (resolve-interface `(guix store ,scheme)))
533 (match (false-if-exception
534 (module-ref module 'connect-to-daemon))
535 ((? procedure? connect)
538 (x (not-supported))))
539 (#f (not-supported))))
545 (define* (open-connection #:optional (uri (%daemon-socket-uri))
546 #:key port (reserve-space? #t) cpu-affinity)
547 "Connect to the daemon at URI (a string), or, if PORT is not #f, use it as
548 the I/O port over which to communicate to a build daemon.
550 When RESERVE-SPACE? is true, instruct it to reserve a little bit of extra
551 space on the file system so that the garbage collector can still operate,
552 should the disk become full. When CPU-AFFINITY is true, it must be an integer
553 corresponding to an OS-level CPU number to which the daemon's worker process
554 for this connection will be pinned. Return a server object."
555 (define (handshake-error)
557 (&store-connection-error (file (or port uri))
559 (&message (message "build daemon handshake failed")))))
561 (guard (c ((nar-error? c)
562 ;; One of the 'write-' or 'read-' calls below failed, but this is
563 ;; really a connection error.
565 (let*-values (((port)
566 (or port (connect-to-daemon uri)))
568 (buffering-output-port port
569 (make-bytevector 8192))))
570 (write-int %worker-magic-1 port)
571 (let ((r (read-int port)))
572 (unless (= r %worker-magic-2)
575 (let ((v (read-int port)))
576 (unless (= (protocol-major %protocol-version)
580 (write-int %protocol-version port)
581 (when (>= (protocol-minor v) 14)
582 (write-int (if cpu-affinity 1 0) port)
584 (write-int cpu-affinity port)))
585 (when (>= (protocol-minor v) 11)
586 (write-int (if reserve-space? 1 0) port))
587 (letrec* ((built-in-builders
588 (delay (%built-in-builders conn)))
590 (%make-store-connection port
594 (make-hash-table 100)
595 (make-hash-table 100)
598 (let loop ((done? (process-stderr conn)))
599 (or done? (process-stderr conn)))
602 (define* (port->connection port
603 #:key (version %protocol-version))
604 "Assimilate PORT, an input/output port, and return a connection to the
605 daemon, assuming the given protocol VERSION.
607 Warning: this procedure assumes that the initial handshake with the daemon has
608 already taken place on PORT and that we're just continuing on this established
609 connection. Use with care."
610 (let-values (((output flush)
611 (buffering-output-port port (make-bytevector 8192))))
613 (%make-store-connection port
614 (protocol-major version)
615 (protocol-minor version)
617 (make-hash-table 100)
618 (make-hash-table 100)
620 (delay (%built-in-builders connection))))
624 (define (store-connection-version store)
625 "Return the protocol version of STORE as an integer."
626 (protocol-version (store-connection-major-version store)
627 (store-connection-minor-version store)))
629 (define-deprecated/alias nix-server-version store-connection-version)
631 (define (write-buffered-output server)
632 "Flush SERVER's output port."
633 (force-output (store-connection-output-port server))
634 ((store-connection-flush-output server)))
636 (define (close-connection server)
637 "Close the connection to SERVER."
638 (close (store-connection-socket server)))
640 (define (call-with-store proc)
641 "Call PROC with an open store connection."
642 (let ((store (open-connection)))
644 (parameterize ((current-store-protocol-version
645 (store-connection-version store)))
646 (call-with-values (lambda () (proc store))
648 (close-connection store)
649 (apply values results)))))
653 (with-exception-handler (lambda (exception)
654 (close-connection store)
655 (raise-exception exception))
661 (close-connection store)
662 (apply throw key args)))))))
664 (define-syntax-rule (with-store store exp ...)
665 "Bind STORE to an open connection to the store and evaluate EXPs;
666 automatically close the store when the dynamic extent of EXP is left."
667 (call-with-store (lambda (store) exp ...)))
669 (define current-store-protocol-version
670 ;; Protocol version of the store currently used. XXX: This is a hack to
671 ;; communicate the protocol version to the build output port. It's a hack
672 ;; because it could be inaccurrate, for instance if there's code that
673 ;; manipulates several store connections at once; it works well for the
674 ;; purposes of (guix status) though.
677 (define current-build-output-port
678 ;; The port where build output is sent.
679 (make-parameter (current-error-port)))
681 (define* (dump-port in out
683 #:key (buffer-size 16384))
684 "Read LEN bytes from IN (or as much as possible if LEN is #f) and write it
685 to OUT, using chunks of BUFFER-SIZE bytes."
687 (make-bytevector buffer-size))
690 (bytes (get-bytevector-n! in buffer 0
692 (min len buffer-size)
694 (or (eof-object? bytes)
695 (and len (= total len))
696 (let ((total (+ total bytes)))
697 (put-bytevector out buffer 0 bytes)
699 (get-bytevector-n! in buffer 0
701 (min (- len total) buffer-size)
705 ;; Newline characters triggering a flush of 'current-build-output-port'.
706 ;; Unlike Guile's 'line, we flush upon #\return so that progress reports
707 ;; that use that trick are correctly displayed.
708 (char-set #\newline #\return))
710 (define* (process-stderr server #:optional user-port)
711 "Read standard output and standard error from SERVER, writing it to
712 CURRENT-BUILD-OUTPUT-PORT. Return #t when SERVER is done sending data, and
713 #f otherwise; in the latter case, the caller should call `process-stderr'
714 again until #t is returned or an error is raised.
716 Since the build process's output cannot be assumed to be UTF-8, we
717 conservatively consider it to be Latin-1, thereby avoiding possible
718 encoding conversion errors."
720 (store-connection-socket server))
722 ;; magic cookies from worker-protocol.hh
723 (define %stderr-next #x6f6c6d67) ; "olmg", build log
724 (define %stderr-read #x64617461) ; "data", data needed from source
725 (define %stderr-write #x64617416) ; "dat\x16", data for sink
726 (define %stderr-last #x616c7473) ; "alts", we're done
727 (define %stderr-error #x63787470) ; "cxtp", error reporting
729 (let ((k (read-int p)))
730 (cond ((= k %stderr-write)
731 ;; Write a byte stream to USER-PORT.
732 (let* ((len (read-int p))
734 (dump-port p user-port len
735 #:buffer-size (if (<= len 16384) 16384 65536))
737 ;; Consume padding, as for strings.
738 (get-bytevector-n p (- 8 m))))
741 ;; Read a byte stream from USER-PORT.
742 ;; Note: Avoid 'get-bytevector-n' to work around
743 ;; <http://bugs.gnu.org/17591> in Guile up to 2.0.11.
744 (let* ((max-len (read-int p))
745 (data (make-bytevector max-len))
746 (len (get-bytevector-n! user-port data 0 max-len)))
747 (write-bytevector data p len)
750 ;; Log a string. Build logs are usually UTF-8-encoded, but they
751 ;; may also contain arbitrary byte sequences that should not cause
752 ;; this to fail. Thus, use the permissive
753 ;; 'read-maybe-utf8-string'.
754 (let ((s (read-maybe-utf8-string p)))
755 (display s (current-build-output-port))
756 (when (string-any %newlines s)
757 (force-output (current-build-output-port)))
761 (let ((error (read-maybe-utf8-string p))
762 ;; Currently the daemon fails to send a status code for early
763 ;; errors like DB schema version mismatches, so check for EOF.
764 (status (if (and (>= (store-connection-minor-version server) 8)
765 (not (eof-object? (lookahead-u8 p))))
768 (raise (condition (&store-protocol-error
772 ;; The daemon is done (see `stopWork' in `nix-worker.cc'.)
775 (raise (condition (&store-protocol-error
776 (message "invalid error code")
779 (define %default-substitute-urls
780 ;; Default list of substituters. This is *not* the list baked in
781 ;; 'guix-daemon', but it is used by 'guix-service-type' and and a couple of
782 ;; clients ('guix build --log-file' uses it.)
783 (map (if (false-if-exception (resolve-interface '(gnutls)))
784 (cut string-append "https://" <>)
785 (cut string-append "http://" <>))
786 '("ci.guix.gnu.org")))
788 (define (current-user-name)
789 "Return the name of the calling user."
792 (passwd:name (getpwuid (getuid))))
796 (define* (set-build-options server
797 #:key keep-failed? keep-going? fallback?
799 rounds ;number of build rounds
804 (use-build-hook? *unspecified*) ;deprecated
807 (print-build-trace #t)
808 (user-name (current-user-name))
810 ;; When true, provide machine-readable "build
811 ;; traces" for use by (guix status). Old clients
812 ;; are unable to make sense, which is why it's
813 ;; disabled by default.
814 print-extended-build-trace?
816 ;; When true, the daemon prefixes builder output
817 ;; with "@ build-log" traces so we can
818 ;; distinguish it from daemon output, and we can
819 ;; distinguish each builder's output
820 ;; (PRINT-BUILD-TRACE must be true as well.) The
821 ;; latter is particularly useful when
822 ;; MAX-BUILD-JOBS > 1.
823 multiplexed-build-output?
826 (use-substitutes? #t)
828 ;; Client-provided substitute URLs. If it is #f,
829 ;; the daemon's settings are used. Otherwise, it
830 ;; overrides the daemons settings; see 'guix
834 ;; Number of columns in the client's terminal.
835 (terminal-columns (terminal-columns))
837 ;; Locale of the client.
838 (locale (false-if-exception (setlocale LC_MESSAGES))))
839 ;; Must be called after `open-connection'.
842 (store-connection-output-port server))
844 (unless (unspecified? use-build-hook?)
845 (warn-about-deprecation #:use-build-hook? #f
846 #:replacement #:offload?))
848 (let-syntax ((send (syntax-rules ()
849 ((_ (type option) ...)
851 (write-arg type option buffered)
853 (write-int (operation-id set-options) buffered)
854 (send (boolean keep-failed?) (boolean keep-going?)
855 (boolean fallback?) (integer verbosity))
856 (when (< (store-connection-minor-version server) #x61)
857 (let ((max-build-jobs (or max-build-jobs 1))
858 (max-silent-time (or max-silent-time 3600)))
859 (send (integer max-build-jobs) (integer max-silent-time))))
860 (when (>= (store-connection-minor-version server) 2)
861 (send (boolean (if (unspecified? use-build-hook?)
864 (when (>= (store-connection-minor-version server) 4)
865 (send (integer build-verbosity) (integer log-type)
866 (boolean print-build-trace)))
867 (when (and (>= (store-connection-minor-version server) 6)
868 (< (store-connection-minor-version server) #x61))
869 (let ((build-cores (or build-cores (current-processor-count))))
870 (send (integer build-cores))))
871 (when (>= (store-connection-minor-version server) 10)
872 (send (boolean use-substitutes?)))
873 (when (>= (store-connection-minor-version server) 12)
874 (let ((pairs `(;; This option is honored by 'guix substitute' et al.
875 ,@(if print-build-trace
876 `(("print-extended-build-trace"
877 . ,(if print-extended-build-trace? "1" "0")))
879 ,@(if multiplexed-build-output?
880 `(("multiplexed-build-output"
881 . ,(if multiplexed-build-output? "true" "false")))
884 `(("build-timeout" . ,(number->string timeout)))
886 ,@(if max-silent-time
887 `(("build-max-silent-time"
888 . ,(number->string max-silent-time)))
892 . ,(number->string max-build-jobs)))
895 `(("build-cores" . ,(number->string build-cores)))
897 ,@(if substitute-urls
899 . ,(string-join substitute-urls)))
903 . ,(number->string (max 0 (1- rounds)))))
906 `(("user-name" . ,user-name))
908 ,@(if terminal-columns
909 `(("terminal-columns"
910 . ,(number->string terminal-columns)))
913 `(("locale" . ,locale))
915 (send (string-pairs pairs))))
916 (write-buffered-output server)
917 (let loop ((done? (process-stderr server)))
918 (or done? (process-stderr server)))))
920 (define (buffering-output-port port buffer)
921 "Return two value: an output port wrapped around PORT that uses BUFFER (a
922 bytevector) as its internal buffer, and a thunk to flush this output port."
923 ;; Note: In Guile 2.2.2, custom binary output ports already have their own
924 ;; 4K internal buffer.
926 (bytevector-length buffer))
931 (put-bytevector port buffer 0 total)
935 (define (write bv offset count)
936 (if (zero? count) ;end of file
938 (let loop ((offset offset)
941 (cond ((= total size)
943 (loop offset count written))
947 (let ((to-copy (min count (- size total))))
948 (bytevector-copy! bv offset buffer total to-copy)
949 (set! total (+ total to-copy))
950 (loop (+ offset to-copy) (- count to-copy)
951 (+ written to-copy))))))))
953 ;; Note: We need to return FLUSH because the custom binary port has no way
954 ;; to be notified of a 'force-output' call on itself.
955 (values (make-custom-binary-output-port "buffering-output-port"
961 (or (and=> (getenv "GUIX_PROFILING") string-tokenize)
964 "Return true if COMPONENT profiling is active."
965 (member component profiled))))
968 ;; Mapping from RPC names (symbols) to invocation counts.
971 (define* (show-rpc-profile #:optional (port (current-error-port)))
972 "Write to PORT a summary of the RPCs that have been made."
973 (let ((profile (sort (hash-fold alist-cons '() %rpc-calls)
975 (< (cdr rpc1) (cdr rpc2))))))
976 (format port "Remote procedure call summary: ~a RPCs~%"
978 (((names . counts) ...)
979 (reduce + 0 counts))))
980 (for-each (match-lambda
982 (format port " ~30a ... ~5@a~%" rpc count)))
985 (define record-operation
986 ;; Optionally, increment the number of calls of the given RPC.
987 (if (profiled? "rpc")
989 (register-profiling-hook! "rpc" show-rpc-profile)
991 (let ((count (or (hashq-ref %rpc-calls name) 0)))
992 (hashq-set! %rpc-calls name (+ count 1)))))
996 (define-syntax operation
998 "Define a client-side RPC stub for the given operation."
999 ((_ (name (type arg) ...) docstring return ...)
1000 (lambda (server arg ...)
1002 (let* ((s (store-connection-socket server))
1003 (buffered (store-connection-output-port server)))
1004 (record-operation 'name)
1005 (write-int (operation-id name) buffered)
1006 (write-arg type arg buffered)
1008 (write-buffered-output server)
1010 ;; Loop until the server is done sending error output.
1011 (let loop ((done? (process-stderr server)))
1012 (or done? (loop (process-stderr server))))
1013 (values (read-arg return s) ...))))))
1015 (define-syntax-rule (define-operation (name args ...)
1016 docstring return ...)
1018 (operation (name args ...) docstring return ...)))
1020 (define-operation (valid-path? (string path))
1021 "Return #t when PATH designates a valid store item and #f otherwise (an
1022 invalid item may exist on disk but still be invalid, for instance because it
1023 is the result of an aborted or failed build.)
1025 A '&store-protocol-error' condition is raised if PATH is not prefixed by the
1026 store directory (/gnu/store)."
1029 (define-operation (query-path-hash (store-path path))
1030 "Return the SHA256 hash of the nar serialization of PATH as a bytevector."
1033 (define hash-part->path
1034 (let ((query-path-from-hash-part
1035 (operation (query-path-from-hash-part (string hash))
1038 (lambda (server hash-part)
1039 "Return the store path whose hash part is HASH-PART (a nix-base32
1040 string). Return the empty string if no such path exists."
1041 ;; This RPC is primarily used by Hydra to reply to HTTP GETs of
1043 (query-path-from-hash-part server hash-part))))
1045 (define-operation (query-path-info (store-path path))
1046 "Return the info (hash, references, etc.) for PATH."
1049 (define add-data-to-store
1050 ;; A memoizing version of `add-to-store', to avoid repeated RPCs with
1051 ;; the very same arguments during a given session.
1052 (let ((add-text-to-store
1053 (operation (add-text-to-store (string name) (bytevector text)
1054 (string-list references))
1057 (lookup (if (profiled? "add-data-to-store-cache")
1062 (define (show-stats)
1066 (* 100. (/ n lookups))))
1068 (format (current-error-port) "
1069 'add-data-to-store' cache:
1072 .drv files: ~5@a (~,1f%)
1073 Scheme files: ~5@a (~,1f%)~%"
1074 lookups hits (% hits)
1078 (register-profiling-hook! "add-data-to-store-cache"
1080 (lambda (cache args)
1081 (let ((result (hash-ref cache args)))
1082 (set! lookups (+ 1 lookups))
1084 (set! hits (+ 1 hits)))
1087 (cond ((string-suffix? ".drv" name)
1088 (set! drv (+ drv 1)))
1089 ((string-suffix? "-builder" name)
1090 (set! scheme (+ scheme 1)))
1091 ((string-suffix? ".scm" name)
1092 (set! scheme (+ scheme 1))))))
1095 (lambda* (server name bytes #:optional (references '()))
1096 "Add BYTES under file NAME in the store, and return its store path.
1097 REFERENCES is the list of store paths referred to by the resulting store
1099 (let* ((args `(,bytes ,name ,references))
1100 (cache (store-connection-add-text-to-store-cache server)))
1101 (or (lookup cache args)
1102 (let ((path (add-text-to-store server name bytes references)))
1103 (hash-set! cache args path)
1106 (define* (add-text-to-store store name text #:optional (references '()))
1107 "Add TEXT under file NAME in the store, and return its store path.
1108 REFERENCES is the list of store paths referred to by the resulting store
1110 (add-data-to-store store name (string->utf8 text) references))
1113 ;; Define it once and for all since we use it as a default value for
1114 ;; 'add-to-store' and want to make sure two default values are 'eq?' for the
1115 ;; purposes or memoization.
1119 (define add-to-store
1120 ;; A memoizing version of `add-to-store'. This is important because
1121 ;; `add-to-store' leads to huge data transfers to the server, and
1122 ;; because it's often called many times with the very same argument.
1124 (lambda* (server basename recursive? hash-algo file-name
1125 #:key (select? true))
1126 ;; We don't use the 'operation' macro so we can pass SELECT? to
1128 (record-operation 'add-to-store)
1129 (let ((port (store-connection-socket server))
1130 (buffered (store-connection-output-port server)))
1131 (write-int (operation-id add-to-store) buffered)
1132 (write-string basename buffered)
1133 (write-int 1 buffered) ;obsolete, must be #t
1134 (write-int (if recursive? 1 0) buffered)
1135 (write-string hash-algo buffered)
1136 (write-file file-name buffered #:select? select?)
1137 (write-buffered-output server)
1138 (let loop ((done? (process-stderr server)))
1139 (or done? (loop (process-stderr server))))
1140 (read-store-path port)))))
1141 (lambda* (server basename recursive? hash-algo file-name
1142 #:key (select? true))
1143 "Add the contents of FILE-NAME under BASENAME to the store. When
1144 RECURSIVE? is false, FILE-NAME must designate a regular file--not a directory
1145 nor a symlink. When RECURSIVE? is true and FILE-NAME designates a directory,
1146 the contents of FILE-NAME are added recursively; if FILE-NAME designates a
1147 flat file and RECURSIVE? is true, its contents are added, and its permission
1148 bits are kept. HASH-ALGO must be a string such as \"sha256\".
1150 When RECURSIVE? is true, call (SELECT? FILE STAT) for each directory entry,
1151 where FILE is the entry's absolute file name and STAT is the result of
1152 'lstat'; exclude entries for which SELECT? does not return true."
1153 ;; Note: We don't stat FILE-NAME at each call, and thus we assume that
1154 ;; the file remains unchanged for the lifetime of SERVER.
1155 (let* ((args `(,file-name ,basename ,recursive? ,hash-algo ,select?))
1156 (cache (store-connection-add-to-store-cache server)))
1157 (or (hash-ref cache args)
1158 (let ((path (add-to-store server basename recursive?
1160 #:select? select?)))
1161 (hash-set! cache args path)
1165 (char-set-complement (char-set #\/)))
1167 (define* (add-file-tree-to-store server tree
1169 (hash-algo "sha256")
1171 "Add the given TREE to the store on SERVER. TREE must be an entry such as:
1173 (\"my-tree\" directory
1174 (\"a\" regular (data \"hello\"))
1175 (\"b\" symlink \"a\")
1177 (\"d\" executable (file \"/bin/sh\"))))
1179 This is a generalized version of 'add-to-store'. It allows you to reproduce
1180 an arbitrary directory layout in the store without creating a derivation."
1182 ;; Note: The format of TREE was chosen to allow trees to be compared with
1183 ;; 'equal?', which in turn allows us to memoize things.
1186 ;; TREE is a single entry.
1193 (define (lookup file)
1194 (let loop ((components (string-tokenize file %not-slash))
1198 (assoc basename tree))
1201 (match (assoc-ref tree head)
1202 (('directory . entries) entries)))))))
1204 (define (file-type+size file)
1205 (match (lookup file)
1206 ((_ (and type (or 'directory 'symlink)) . _)
1208 ((_ type ('file file))
1209 (values type (stat:size (stat file))))
1210 ((_ type ('data (? string? data)))
1211 (values type (string-length data)))
1212 ((_ type ('data (? bytevector? data)))
1213 (values type (bytevector-length data)))))
1215 (define (file-port file)
1216 (match (lookup file)
1217 ((_ (or 'regular 'executable) content)
1219 (('file (? string? file))
1220 (open-file file "r0b"))
1221 (('data (? string? str))
1222 (open-input-string str))
1223 (('data (? bytevector? bv))
1224 (open-bytevector-input-port bv))))))
1226 (define (symlink-target file)
1227 (match (lookup file)
1228 ((_ 'symlink target) target)))
1230 (define (directory-entries directory)
1231 (match (lookup directory)
1232 ((_ 'directory (names . _) ...) names)))
1235 (store-connection-add-to-store-cache server))
1237 (or (hash-ref cache tree)
1239 ;; We don't use the 'operation' macro so we can use 'write-file-tree'
1240 ;; instead of 'write-file'.
1241 (record-operation 'add-to-store/tree)
1242 (let ((port (store-connection-socket server))
1243 (buffered (store-connection-output-port server)))
1244 (write-int (operation-id add-to-store) buffered)
1245 (write-string basename buffered)
1246 (write-int 1 buffered) ;obsolete, must be #t
1247 (write-int (if recursive? 1 0) buffered)
1248 (write-string hash-algo buffered)
1249 (write-file-tree basename buffered
1250 #:file-type+size file-type+size
1251 #:file-port file-port
1252 #:symlink-target symlink-target
1253 #:directory-entries directory-entries)
1254 (write-buffered-output server)
1255 (let loop ((done? (process-stderr server)))
1256 (or done? (loop (process-stderr server))))
1257 (let ((result (read-store-path port)))
1258 (hash-set! cache tree result)
1261 (define (file-mapping->tree mapping)
1262 "Convert MAPPING, an alist like:
1264 ((\"guix/build/utils.scm\" . \"…/utils.scm\"))
1266 to a tree suitable for 'add-file-tree-to-store' and 'interned-file-tree'."
1267 (let ((mapping (map (match-lambda
1268 ((destination . source)
1269 (cons (string-tokenize destination %not-slash)
1272 (fold (lambda (pair result)
1274 ((destination . source)
1275 (let loop ((destination destination)
1279 (let* ((mode (stat:mode (stat source)))
1280 (type (if (zero? (logand mode #o100))
1284 `(,type (file ,source))
1287 (let ((directory (assoc-ref result file)))
1292 (('directory . entries) entries)
1295 (alist-delete file result)
1300 (define current-build-prompt
1301 ;; When true, this is the prompt to abort to when 'build-things' is called.
1302 (make-parameter #f))
1304 (define (call-with-build-handler handler thunk)
1305 "Register HANDLER as a \"build handler\" and invoke THUNK."
1307 (make-prompt-tag "build handler"))
1309 (parameterize ((current-build-prompt tag))
1310 (call-with-prompt tag
1313 ;; Since HANDLER may call K, which in turn may call 'build-things'
1314 ;; again, reinstate a prompt (thus, it's not a tail call.)
1315 (call-with-build-handler handler
1317 (apply handler k args)))))))
1319 (define (invoke-build-handler store things mode)
1320 "Abort to 'current-build-prompt' if it is set."
1321 (or (not (current-build-prompt))
1322 (abort-to-prompt (current-build-prompt) store things mode)))
1324 (define-syntax-rule (with-build-handler handler exp ...)
1325 "Register HANDLER as a \"build handler\" and invoke THUNK. When
1326 'build-things' is called within the dynamic extent of the call to THUNK,
1327 HANDLER is invoked like so:
1329 (HANDLER CONTINUE STORE THINGS MODE)
1331 where CONTINUE is the continuation, and the remaining arguments are those that
1332 were passed to 'build-things'.
1334 Build handlers are useful to announce a build plan with 'show-what-to-build'
1335 and to implement dry runs (by not invoking CONTINUE) in a way that gracefully
1336 deals with \"dynamic dependencies\" such as grafts---derivations that depend
1337 on the build output of a previous derivation."
1338 (call-with-build-handler handler (lambda () exp ...)))
1340 ;; Unresolved dynamic dependency.
1341 (define-record-type <unresolved>
1342 (unresolved things continuation)
1344 (things unresolved-things)
1345 (continuation unresolved-continuation))
1347 (define (build-accumulator continue store things mode)
1348 "This build handler accumulates THINGS and returns an <unresolved> object."
1349 (if (= mode (build-mode normal))
1350 (unresolved things continue)
1353 (define (map/accumulate-builds store proc lst)
1354 "Apply PROC over each element of LST, accumulating 'build-things' calls and
1355 coalescing them into a single call."
1358 (with-build-handler build-accumulator
1362 (match (append-map (lambda (obj)
1363 (if (unresolved? obj)
1364 (unresolved-things obj)
1370 ;; We've accumulated things TO-BUILD. Actually build them and resume the
1371 ;; corresponding continuations.
1372 (build-things store (delete-duplicates to-build))
1373 (map/accumulate-builds store
1375 (if (unresolved? obj)
1376 ;; Pass #f because 'build-things' is now
1378 ((unresolved-continuation obj) #f)
1382 (define build-things
1383 (let ((build (operation (build-things (string-list things)
1387 (build/old (operation (build-things (string-list things))
1390 (lambda* (store things #:optional (mode (build-mode normal)))
1391 "Build THINGS, a list of store items which may be either '.drv' files or
1392 outputs, and return when the worker is done building them. Elements of THINGS
1393 that are not derivations can only be substituted and not built locally.
1394 Alternately, an element of THING can be a derivation/output name pair, in
1395 which case the daemon will attempt to substitute just the requested output of
1396 the derivation. Return #t on success.
1398 When a handler is installed with 'with-build-handler', it is called any time
1399 'build-things' is called."
1400 (or (not (invoke-build-handler store things mode))
1401 (let ((things (map (match-lambda
1402 ((drv . output) (string-append drv "!" output))
1405 (parameterize ((current-store-protocol-version
1406 (store-connection-version store)))
1407 (if (>= (store-connection-minor-version store) 15)
1408 (build store things mode)
1409 (if (= mode (build-mode normal))
1410 (build/old store things)
1411 (raise (condition (&store-protocol-error
1412 (message "unsupported build mode")
1413 (status 1))))))))))))
1415 (define-operation (ensure-path (store-path path))
1416 "Ensure that a path is valid. If it is not valid, it may be made valid by
1417 running a substitute. As a GC root is not created by the daemon, you may want
1418 to call ADD-TEMP-ROOT on that store path."
1421 (define-operation (find-roots)
1422 "Return a list of root/target pairs: for each pair, the first element is the
1423 GC root file name and the second element is its target in the store.
1425 When talking to a local daemon, this operation is equivalent to the 'gc-roots'
1426 procedure in (guix store roots), except that the 'find-roots' excludes
1427 potential roots that do not point to store items."
1430 (define-operation (add-temp-root (store-path path))
1431 "Make PATH a temporary root for the duration of the current session.
1435 (define-operation (add-indirect-root (string file-name))
1436 "Make the symlink FILE-NAME an indirect root for the garbage collector:
1437 whatever store item FILE-NAME points to will not be collected. Return #t on
1440 FILE-NAME can be anywhere on the file system, but it must be an absolute file
1441 name--it is the caller's responsibility to ensure that it is an absolute file
1445 (define %gc-roots-directory
1446 ;; The place where garbage collector roots (symlinks) are kept.
1447 (string-append %state-directory "/gcroots"))
1449 (define (add-permanent-root target)
1450 "Add a garbage collector root pointing to TARGET, an element of the store,
1451 preventing TARGET from even being collected. This can also be used if TARGET
1454 Raise an error if the caller does not have write access to the GC root
1456 (let* ((root (string-append %gc-roots-directory "/" (basename target))))
1457 (catch 'system-error
1459 (symlink target root))
1461 ;; If ROOT already exists, this is fine; otherwise, re-throw.
1462 (unless (= EEXIST (system-error-errno args))
1463 (apply throw args))))))
1465 (define (remove-permanent-root target)
1466 "Remove the permanent garbage collector root pointing to TARGET. Raise an
1467 error if there is no such root."
1468 (delete-file (string-append %gc-roots-directory "/" (basename target))))
1471 (operation (query-references (store-path path))
1472 "Return the list of references of PATH."
1475 (define %reference-cache
1476 ;; Brute-force cache mapping store items to their list of references.
1477 ;; Caching matters because when building a profile in the presence of
1478 ;; grafts, we keep calling 'graft-derivation', which in turn calls
1479 ;; 'references/substitutes' many times with the same arguments. Ideally we
1480 ;; would use a cache associated with the daemon connection instead (XXX).
1481 (make-hash-table 100))
1483 (define (references/cached store item)
1484 "Like 'references', but cache results."
1485 (or (hash-ref %reference-cache item)
1486 (let ((references (references store item)))
1487 (hash-set! %reference-cache item references)
1490 (define (references/substitutes store items)
1491 "Return the list of list of references of ITEMS; the result has the same
1492 length as ITEMS. Query substitute information for any item missing from the
1493 store at once. Raise a '&store-protocol-error' exception if reference
1494 information for one of ITEMS is missing."
1495 (let* ((requested items)
1496 (local-refs (map (lambda (item)
1497 (or (hash-ref %reference-cache item)
1498 (guard (c ((store-protocol-error? c) #f))
1499 (references store item))))
1501 (missing (fold-right (lambda (item local-ref result)
1504 (cons item result)))
1508 ;; Query all the substitutes at once to minimize the cost of
1509 ;; launching 'guix substitute' and making HTTP requests.
1510 (substs (if (null? missing)
1512 (substitutable-path-info store missing))))
1513 (when (< (length substs) (length missing))
1514 (raise (condition (&store-protocol-error
1515 (message "cannot determine \
1516 the list of references")
1519 ;; Intersperse SUBSTS and LOCAL-REFS.
1520 (let loop ((items items)
1521 (local-refs local-refs)
1525 (let ((result (reverse result)))
1526 (for-each (cut hash-set! %reference-cache <> <>)
1533 (cons (any (lambda (subst)
1534 (and (string=? (substitutable-path subst) item)
1535 (substitutable-references subst)))
1540 (cons head result)))))))))
1542 (define* (fold-path store proc seed paths
1543 #:optional (relatives (cut references store <>)))
1544 "Call PROC for each of the RELATIVES of PATHS, exactly once, and return the
1545 result formed from the successive calls to PROC, the first of which is passed
1547 (let loop ((paths paths)
1552 (if (vhash-assoc path seen)
1553 (loop rest result seen)
1554 (let ((seen (vhash-cons path #t seen))
1555 (rest (append rest (relatives path)))
1556 (result (proc path result)))
1557 (loop rest result seen))))
1561 (define (requisites store paths)
1562 "Return the requisites of PATHS, including PATHS---i.e., their closures (all
1563 its references, recursively)."
1564 (fold-path store cons '() paths))
1566 (define (topologically-sorted store paths)
1567 "Return a list containing PATHS and all their references sorted in
1570 ;; Do a simple depth-first traversal of all of PATHS.
1571 (let loop ((paths paths)
1572 (visited vlist-null)
1575 (vhash-cons n #t visited))
1577 (define (visited? n)
1578 (vhash-assoc n visited))
1583 (loop tail visited result)
1586 (loop (references store head)
1589 (lambda (visited result)
1592 (cons head result))))))
1594 (values visited result)))))
1596 (call-with-values traverse
1601 (operation (query-referrers (store-path path))
1602 "Return the list of path that refer to PATH."
1605 (define valid-derivers
1606 (operation (query-valid-derivers (store-path path))
1607 "Return the list of valid \"derivers\" of PATH---i.e., all the
1608 .drv present in the store that have PATH among their outputs."
1611 (define query-derivation-outputs ; avoid name clash with `derivation-outputs'
1612 (operation (query-derivation-outputs (store-path path))
1613 "Return the list of outputs of PATH, a .drv file."
1616 (define-operation (has-substitutes? (store-path path))
1617 "Return #t if binary substitutes are available for PATH, and #f otherwise."
1620 (define substitutable-paths
1621 (operation (query-substitutable-paths (store-path-list paths))
1622 "Return the subset of PATHS that is substitutable."
1625 (define substitutable-path-info
1626 (operation (query-substitutable-path-infos (store-path-list paths))
1627 "Return information about the subset of PATHS that is
1628 substitutable. For each substitutable path, a `substitutable?' object is
1629 returned; thus, the resulting list can be shorter than PATHS. Furthermore,
1630 that there is no guarantee that the order of the resulting list matches the
1632 substitutable-path-list))
1634 (define %built-in-builders
1635 (let ((builders (operation (built-in-builders)
1636 "Return the built-in builders."
1639 "Return the names of the supported built-in derivation builders
1640 supported by STORE. The result is memoized for STORE."
1641 ;; Check whether STORE's version supports this RPC and built-in
1642 ;; derivation builders in general, which appeared in Guix > 0.11.0.
1643 ;; Return the empty list if it doesn't. Note that this RPC does not
1644 ;; exist in 'nix-daemon'.
1645 (if (or (> (store-connection-major-version store) #x100)
1646 (and (= (store-connection-major-version store) #x100)
1647 (>= (store-connection-minor-version store) #x60)))
1651 (define (built-in-builders store)
1652 "Return the names of the supported built-in derivation builders
1653 supported by STORE."
1654 (force (store-connection-built-in-builders store)))
1656 (define-operation (optimize-store)
1657 "Optimize the store by hard-linking identical files (\"deduplication\".)
1658 Return #t on success."
1659 ;; Note: the daemon in Guix <= 0.8.2 does not implement this RPC.
1662 (define verify-store
1663 (let ((verify (operation (verify-store (boolean check-contents?)
1667 (lambda* (store #:key check-contents? repair?)
1668 "Verify the integrity of the store and return false if errors remain,
1669 and true otherwise. When REPAIR? is true, repair any missing or altered store
1670 items by substituting them (this typically requires root privileges because it
1671 is not an atomic operation.) When CHECK-CONTENTS? is true, check the contents
1672 of store items; this can take a lot of time."
1673 (not (verify store check-contents? repair?)))))
1675 (define (run-gc server action to-delete min-freed)
1676 "Perform the garbage-collector operation ACTION, one of the
1677 `gc-action' values. When ACTION is `delete-specific', the TO-DELETE is
1678 the list of store paths to delete. IGNORE-LIVENESS? should always be
1679 #f. MIN-FREED is the minimum amount of disk space to be freed, in
1680 bytes, before the GC can stop. Return the list of store paths delete,
1681 and the number of bytes freed."
1682 (let ((s (store-connection-socket server))
1683 (buffered (store-connection-output-port server)))
1684 (write-int (operation-id collect-garbage) buffered)
1685 (write-int action buffered)
1686 (write-store-path-list to-delete buffered)
1687 (write-arg boolean #f buffered) ; ignore-liveness?
1688 (write-long-long min-freed buffered)
1689 (write-int 0 buffered) ; obsolete
1690 (when (>= (store-connection-minor-version server) 5)
1691 ;; Obsolete `use-atime' and `max-atime' parameters.
1692 (write-int 0 buffered)
1693 (write-int 0 buffered))
1694 (write-buffered-output server)
1696 ;; Loop until the server is done sending error output.
1697 (let loop ((done? (process-stderr server)))
1698 (or done? (loop (process-stderr server))))
1700 (let ((paths (read-store-path-list s))
1701 (freed (read-long-long s))
1702 (obsolete (read-long-long s)))
1703 (unless (null? paths)
1704 ;; To be on the safe side, completely invalidate both caches.
1705 ;; Otherwise we could end up returning store paths that are no longer
1707 (hash-clear! (store-connection-add-to-store-cache server))
1708 (hash-clear! (store-connection-add-text-to-store-cache server)))
1710 (values paths freed))))
1712 (define-syntax-rule (%long-long-max)
1713 ;; Maximum unsigned 64-bit integer.
1716 (define (live-paths server)
1717 "Return the list of live store paths---i.e., store paths still
1718 referenced, and thus not subject to being garbage-collected."
1719 (run-gc server (gc-action return-live) '() (%long-long-max)))
1721 (define (dead-paths server)
1722 "Return the list of dead store paths---i.e., store paths no longer
1723 referenced, and thus subject to being garbage-collected."
1724 (run-gc server (gc-action return-dead) '() (%long-long-max)))
1726 (define* (collect-garbage server #:optional (min-freed (%long-long-max)))
1727 "Collect garbage from the store at SERVER. If MIN-FREED is non-zero,
1728 then collect at least MIN-FREED bytes. Return the paths that were
1729 collected, and the number of bytes freed."
1730 (run-gc server (gc-action delete-dead) '() min-freed))
1732 (define* (delete-paths server paths #:optional (min-freed (%long-long-max)))
1733 "Delete PATHS from the store at SERVER, if they are no longer
1734 referenced. If MIN-FREED is non-zero, then stop after at least
1735 MIN-FREED bytes have been collected. Return the paths that were
1736 collected, and the number of bytes freed."
1737 (run-gc server (gc-action delete-specific) paths min-freed))
1739 (define (import-paths server port)
1740 "Import the set of store paths read from PORT into SERVER's store. An error
1741 is raised if the set of paths read from PORT is not signed (as per
1742 'export-path #:sign? #t'.) Return the list of store paths imported."
1743 (let ((s (store-connection-socket server)))
1744 (write-int (operation-id import-paths) s)
1745 (let loop ((done? (process-stderr server port)))
1746 (or done? (loop (process-stderr server port))))
1747 (read-store-path-list s)))
1749 (define* (export-path server path port #:key (sign? #t))
1750 "Export PATH to PORT. When SIGN? is true, sign it."
1751 (let ((s (store-connection-socket server))
1752 (buffered (store-connection-output-port server)))
1753 (write-int (operation-id export-path) buffered)
1754 (write-store-path path buffered)
1755 (write-arg boolean sign? buffered)
1756 (write-buffered-output server)
1757 (let loop ((done? (process-stderr server port)))
1758 (or done? (loop (process-stderr server port))))
1759 (= 1 (read-int s))))
1761 (define* (export-paths server paths port #:key (sign? #t) recursive?
1763 (progress (const #f))
1764 (finish (const #f)))
1765 "Export the store paths listed in PATHS to PORT, in topological order,
1766 signing them if SIGN? is true. When RECURSIVE? is true, export the closure of
1767 PATHS---i.e., PATHS and all their dependencies.
1769 START, PROGRESS, and FINISH are used to track progress of the data transfer.
1770 START is a one-argument that is passed the list of store items that will be
1771 transferred; it returns values that are then used as the initial state
1772 threaded through PROGRESS calls. PROGRESS is passed the store item about to
1773 be sent, along with the values previously return by START or by PROGRESS
1774 itself. FINISH is called when the last store item has been called."
1776 (let ((sorted (topologically-sorted server paths)))
1777 ;; When RECURSIVE? is #f, filter out the references of PATHS.
1780 (filter (cut member <> paths) sorted))))
1782 (let loop ((paths ordered)
1783 (state (call-with-values (lambda () (start ordered))
1787 (apply finish state)
1791 (and (export-path server head port #:sign? sign?)
1794 (lambda () (apply progress head state))
1797 (define-operation (query-failed-paths)
1798 "Return the list of store items for which a build failure is cached.
1800 The result is always the empty list unless the daemon was started with
1801 '--cache-failures'."
1804 (define-operation (clear-failed-paths (store-path-list items))
1805 "Remove ITEMS from the list of cached build failures.
1807 This makes sense only when the daemon was started with '--cache-failures'."
1815 (define-syntax-rule (define-alias new old)
1816 (define-syntax new (identifier-syntax old)))
1818 ;; The store monad allows us to (1) build sequences of operations in the
1819 ;; store, and (2) make the store an implicit part of the execution context,
1820 ;; rather than a parameter of every single function.
1821 (define-alias %store-monad %state-monad)
1822 (define-alias store-return state-return)
1823 (define-alias store-bind state-bind)
1825 ;; Instantiate templates for %STORE-MONAD since it's syntactically different
1826 ;; from %STATE-MONAD.
1827 (template-directory instantiations %store-monad)
1829 (define* (cache-object-mapping object keys result
1830 #:key (vhash-cons vhash-consq))
1831 "Augment the store's object cache with a mapping from OBJECT/KEYS to RESULT.
1832 KEYS is a list of additional keys to match against, for instance a (SYSTEM
1833 TARGET) tuple. Use VHASH-CONS to insert OBJECT into the cache.
1835 OBJECT is typically a high-level object such as a <package> or an <origin>,
1836 and RESULT is typically its derivation."
1841 (object-cache (vhash-cons object (cons result keys)
1842 (store-connection-object-cache store)))))))
1844 (define record-cache-lookup!
1845 (if (profiled? "object-cache")
1850 (register-profiling-hook!
1853 (format (current-error-port) "Store object cache:
1857 cache size: ~5@a entries~%"
1861 (* 100. (/ hits lookups)))
1864 (lambda (hit? cache)
1866 (if (eq? cache vlist-null)
1869 (set! lookups (+ 1 lookups))
1870 (set! hits (if hit? (+ hits 1) hits))
1871 (set! size (+ (if hit? 0 1)
1872 (vlist-length cache)))))
1876 (define-inlinable (lookup-cached-object object keys vhash-fold*)
1877 "Return the cached object in the store connection corresponding to OBJECT
1878 and KEYS; use VHASH-FOLD* to look for OBJECT in the cache. KEYS is a list of
1879 additional keys to match against, and which are compared with 'equal?'.
1880 Return #f on failure and the cached result otherwise."
1882 (let* ((cache (store-connection-object-cache store))
1884 ;; Escape as soon as we find the result. This avoids traversing
1885 ;; the whole vlist chain and significantly reduces the number of
1887 (value (let/ec return
1888 (vhash-fold* (lambda (item result)
1891 (if (equal? keys keys*)
1896 (record-cache-lookup! value cache)
1897 (values value store))))
1899 (define* (%mcached mthunk object #:optional (keys '())
1901 (vhash-cons vhash-consq)
1902 (vhash-fold* vhash-foldq*))
1903 "Bind the monadic value returned by MTHUNK, which supposedly corresponds to
1904 OBJECT/KEYS, or return its cached value. Use VHASH-CONS to insert OBJECT into
1905 the cache, and VHASH-FOLD* to look it up."
1906 (mlet %store-monad ((cached (lookup-cached-object object keys
1912 (cache-object-mapping object keys result
1913 #:vhash-cons vhash-cons))))))
1915 (define-syntax mcached
1916 (syntax-rules (eq? equal?)
1917 "Run MVALUE, which corresponds to OBJECT/KEYS, and cache it; or return the
1918 value associated with OBJECT/KEYS in the store's object cache if there is
1920 ((_ eq? mvalue object keys ...)
1921 (%mcached (lambda () mvalue)
1922 object (list keys ...)
1923 #:vhash-cons vhash-consq
1924 #:vhash-fold* vhash-foldq*))
1925 ((_ equal? mvalue object keys ...)
1926 (%mcached (lambda () mvalue)
1927 object (list keys ...)
1928 #:vhash-cons vhash-cons
1929 #:vhash-fold* vhash-fold*))
1930 ((_ mvalue object keys ...)
1931 (mcached eq? mvalue object keys ...))))
1933 (define (preserve-documentation original proc)
1934 "Return PROC with documentation taken from ORIGINAL."
1935 (set-object-property! proc 'documentation
1936 (procedure-property original 'documentation))
1939 (define (store-lift proc)
1940 "Lift PROC, a procedure whose first argument is a connection to the store,
1941 in the store monad."
1942 (preserve-documentation proc
1945 (values (apply proc store args) store)))))
1947 (define (store-lower proc)
1948 "Lower PROC, a monadic procedure in %STORE-MONAD, to a \"normal\" procedure
1949 taking the store as its first argument."
1950 (preserve-documentation proc
1951 (lambda (store . args)
1952 (run-with-store store (apply proc args)))))
1954 (define (mapm/accumulate-builds mproc lst)
1955 "Like 'mapm' in %STORE-MONAD, but accumulate 'build-things' calls and
1956 coalesce them into a single call."
1958 (values (map/accumulate-builds store
1960 (run-with-store store
1962 #:system (%current-system)
1963 #:target (%current-target-system)))
1969 ;; Store monad operators.
1972 (define* (binary-file name
1974 #:optional (references '()))
1975 "Return as a monadic value the absolute file name in the store of the file
1976 containing DATA, a bytevector. REFERENCES is a list of store items that the
1977 resulting text file refers to; it defaults to the empty list."
1979 (values (add-data-to-store store name data references)
1982 (define* (text-file name
1984 #:optional (references '()))
1985 "Return as a monadic value the absolute file name in the store of the file
1986 containing TEXT, a string. REFERENCES is a list of store items that the
1987 resulting text file refers to; it defaults to the empty list."
1989 (values (add-text-to-store store name text references)
1992 (define* (interned-file file #:optional name
1993 #:key (recursive? #t) (select? true))
1994 "Return the name of FILE once interned in the store. Use NAME as its store
1995 name, or the basename of FILE if NAME is omitted.
1997 When RECURSIVE? is true, the contents of FILE are added recursively; if FILE
1998 designates a flat file and RECURSIVE? is true, its contents are added, and its
1999 permission bits are kept.
2001 When RECURSIVE? is true, call (SELECT? FILE STAT) for each directory entry,
2002 where FILE is the entry's absolute file name and STAT is the result of
2003 'lstat'; exclude entries for which SELECT? does not return true."
2005 (values (add-to-store store (or name (basename file))
2006 recursive? "sha256" file
2010 (define interned-file-tree
2011 (store-lift add-file-tree-to-store))
2014 ;; Monadic variant of 'build-things'.
2015 (store-lift build-things))
2017 (define set-build-options*
2018 (store-lift set-build-options))
2021 (store-lift references))
2023 (define (query-path-info* item)
2024 "Monadic version of 'query-path-info' that returns #f when ITEM is not in
2027 (guard (c ((store-protocol-error? c)
2028 ;; ITEM is not in the store; return #f.
2030 (values (query-path-info store item) store))))
2032 (define-inlinable (current-system)
2033 ;; Consult the %CURRENT-SYSTEM fluid at bind time. This is equivalent to
2034 ;; (lift0 %current-system %store-monad), but inlinable, thus avoiding
2035 ;; closure allocation in some cases.
2037 (values (%current-system) state)))
2039 (define-inlinable (set-current-system system)
2040 ;; Set the %CURRENT-SYSTEM fluid at bind time.
2042 (values (%current-system system) state)))
2044 (define-inlinable (current-target-system)
2045 ;; Consult the %CURRENT-TARGET-SYSTEM fluid at bind time.
2047 (values (%current-target-system) state)))
2049 (define-inlinable (set-current-target target)
2050 ;; Set the %CURRENT-TARGET-SYSTEM fluid at bind time.
2052 (values (%current-target-system target) state)))
2054 (define %guile-for-build
2055 ;; The derivation of the Guile to be used within the build environment,
2056 ;; when using 'gexp->derivation' and co.
2057 (make-parameter #f))
2059 (define set-store-connection-object-cache!
2060 (record-modifier <store-connection> 'object-cache))
2062 (define* (run-with-store store mval
2064 (guile-for-build (%guile-for-build))
2065 (system (%current-system))
2067 "Run MVAL, a monadic value in the store monad, in STORE, an open store
2068 connection, and return the result."
2069 ;; Initialize the dynamic bindings here to avoid bad surprises. The
2070 ;; difficulty lies in the fact that dynamic bindings are resolved at
2071 ;; bind-time and not at call time, which can be disconcerting.
2072 (parameterize ((%guile-for-build guile-for-build)
2073 (%current-system system)
2074 (%current-target-system target))
2075 (call-with-values (lambda ()
2076 (run-with-state mval store))
2077 (lambda (result new-store)
2078 (when (and store new-store)
2079 ;; Copy the object cache from NEW-STORE so we don't fully discard
2081 (let ((cache (store-connection-object-cache new-store)))
2082 (set-store-connection-object-cache! store cache)))
2090 (define %store-prefix
2091 ;; Absolute path to the Nix store.
2092 (make-parameter %store-directory))
2094 (define (compressed-hash bv size) ; `compressHash'
2095 "Given the hash stored in BV, return a compressed version thereof that fits
2097 (define new (make-bytevector size 0))
2098 (define old-size (bytevector-length bv))
2102 (let* ((j (modulo i size))
2103 (o (bytevector-u8-ref new j)))
2104 (bytevector-u8-set! new j
2105 (logxor o (bytevector-u8-ref bv i)))
2108 (define (store-path type hash name) ; makeStorePath
2109 "Return the store path for NAME/HASH/TYPE."
2110 (let* ((s (string-append type ":sha256:"
2111 (bytevector->base16-string hash) ":"
2112 (%store-prefix) ":" name))
2113 (h (sha256 (string->utf8 s)))
2114 (c (compressed-hash h 20)))
2115 (string-append (%store-prefix) "/"
2116 (bytevector->nix-base32-string c) "-"
2119 (define (output-path output hash name) ; makeOutputPath
2120 "Return an output path for OUTPUT (the name of the output as a string) of
2121 the derivation called NAME with hash HASH."
2122 (store-path (string-append "output:" output) hash
2123 (if (string=? output "out")
2125 (string-append name "-" output))))
2127 (define* (fixed-output-path name hash
2132 "Return an output path for the fixed output OUTPUT defined by HASH of type
2133 HASH-ALGO, of the derivation NAME. RECURSIVE? has the same meaning as for
2135 (if (and recursive? (eq? hash-algo 'sha256))
2136 (store-path "source" hash name)
2137 (let ((tag (string-append "fixed:" output ":"
2138 (if recursive? "r:" "")
2139 (symbol->string hash-algo) ":"
2140 (bytevector->base16-string hash) ":")))
2141 (store-path (string-append "output:" output)
2142 (sha256 (string->utf8 tag))
2145 (define (store-path? path)
2146 "Return #t if PATH is a store path."
2147 ;; This is a lightweight check, compared to using a regexp, but this has to
2148 ;; be fast as it's called often in `derivation', for instance.
2149 ;; `isStorePath' in Nix does something similar.
2150 (string-prefix? (%store-prefix) path))
2152 (define (direct-store-path? path)
2153 "Return #t if PATH is a store path, and not a sub-directory of a store path.
2154 This predicate is sometimes needed because files *under* a store path are not
2156 (and (store-path? path)
2157 (not (string=? path (%store-prefix)))
2158 (let ((len (+ 1 (string-length (%store-prefix)))))
2159 (not (string-index (substring path len) #\/)))))
2161 (define (direct-store-path path)
2162 "Return the direct store path part of PATH, stripping components after
2163 '/gnu/store/xxxx-foo'."
2164 (let ((prefix-length (+ (string-length (%store-prefix)) 35)))
2165 (if (> (string-length path) prefix-length)
2166 (let ((slash (string-index path #\/ prefix-length)))
2167 (if slash (string-take path slash) path))
2170 (define (derivation-path? path)
2171 "Return #t if PATH is a derivation path."
2172 (and (store-path? path) (string-suffix? ".drv" path)))
2174 (define (store-path-base path)
2175 "Return the base path of a path in the store."
2176 (and (string-prefix? (%store-prefix) path)
2177 (let ((base (string-drop path (+ 1 (string-length (%store-prefix))))))
2178 (and (> (string-length base) 33)
2179 (not (string-index base #\/))
2182 (define (store-path-package-name path)
2183 "Return the package name part of PATH, a file name in the store."
2184 (let ((base (store-path-base path)))
2185 (string-drop base (+ 32 1)))) ;32 hash part + 1 hyphen
2187 (define (store-path-hash-part path)
2188 "Return the hash part of PATH as a base32 string, or #f if PATH is not a
2189 syntactically valid store path."
2190 (match (store-path-base path)
2193 (let ((hash (string-take base 32)))
2194 (and (string-every %nix-base32-charset hash)
2197 (define (derivation-log-file drv)
2198 "Return the build log file for DRV, a derivation file name, or #f if it
2199 could not be found."
2200 (let* ((base (basename drv))
2201 (log (string-append (or (getenv "GUIX_LOG_DIRECTORY")
2202 (string-append %localstatedir "/log/guix"))
2204 (string-take base 2) "/"
2205 (string-drop base 2)))
2206 (log.gz (string-append log ".gz"))
2207 (log.bz2 (string-append log ".bz2")))
2208 (cond ((file-exists? log.gz) log.gz)
2209 ((file-exists? log.bz2) log.bz2)
2210 ((file-exists? log) log)
2213 (define (log-file store file)
2214 "Return the build log file for FILE, or #f if none could be found. FILE
2215 must be an absolute store file name, or a derivation file name."
2216 (cond ((derivation-path? file)
2217 (derivation-log-file file))
2219 (match (valid-derivers store file)
2221 ;; Return the first that works.
2222 (any (cut log-file store <>) derivers))
2225 ;;; Local Variables:
2226 ;;; eval: (put 'system-error-to-connection-error 'scheme-indent-function 1)