guix/store.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2012, 2013, 2014, 2015 Ludovic Courtès <ludo@gnu.org>
   3 ;;;
   4 ;;; This file is part of GNU Guix.
   5 ;;;
   6 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   7 ;;; under the terms of the GNU General Public License as published by
   8 ;;; the Free Software Foundation; either version 3 of the License, or (at
   9 ;;; your option) any later version.
  10 ;;;
  11 ;;; GNU Guix is distributed in the hope that it will be useful, but
  12 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 ;;; GNU General Public License for more details.
  15 ;;;
  16 ;;; You should have received a copy of the GNU General Public License
  17 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 (define-module (guix store)
  20   #:use-module (guix utils)
  21   #:use-module (guix config)
  22   #:use-module (guix serialization)
  23   #:use-module (guix monads)
  24   #:autoload   (guix base32) (bytevector->base32-string)
  25   #:use-module (rnrs bytevectors)
  26   #:use-module (rnrs io ports)
  27   #:use-module (srfi srfi-1)
  28   #:use-module (srfi srfi-9)
  29   #:use-module (srfi srfi-9 gnu)
  30   #:use-module (srfi srfi-26)
  31   #:use-module (srfi srfi-34)
  32   #:use-module (srfi srfi-35)
  33   #:use-module (srfi srfi-39)
  34   #:use-module (ice-9 match)
  35   #:use-module (ice-9 regex)
  36   #:use-module (ice-9 vlist)
  37   #:use-module (ice-9 popen)
  38   #:export (%daemon-socket-file
  39             %gc-roots-directory
  40
  41             nix-server?
  42             nix-server-major-version
  43             nix-server-minor-version
  44             nix-server-socket
  45
  46             &nix-error nix-error?
  47             &nix-connection-error nix-connection-error?
  48             nix-connection-error-file
  49             nix-connection-error-code
  50             &nix-protocol-error nix-protocol-error?
  51             nix-protocol-error-message
  52             nix-protocol-error-status
  53
  54             hash-algo
  55
  56             open-connection
  57             close-connection
  58             with-store
  59             set-build-options
  60             valid-path?
  61             query-path-hash
  62             hash-part->path
  63             query-path-info
  64             add-text-to-store
  65             add-to-store
  66             build-things
  67             build
  68             add-temp-root
  69             add-indirect-root
  70             add-permanent-root
  71             remove-permanent-root
  72
  73             substitutable?
  74             substitutable-path
  75             substitutable-deriver
  76             substitutable-references
  77             substitutable-download-size
  78             substitutable-nar-size
  79             has-substitutes?
  80             substitutable-paths
  81             substitutable-path-info
  82
  83             path-info?
  84             path-info-deriver
  85             path-info-hash
  86             path-info-references
  87             path-info-registration-time
  88             path-info-nar-size
  89
  90             references
  91             requisites
  92             referrers
  93             topologically-sorted
  94             valid-derivers
  95             query-derivation-outputs
  96             live-paths
  97             dead-paths
  98             collect-garbage
  99             delete-paths
 100             import-paths
 101             export-paths
 102
 103             current-build-output-port
 104
 105             register-path
 106
 107             %store-monad
 108             store-bind
 109             store-return
 110             store-lift
 111             store-lower
 112             run-with-store
 113             %guile-for-build
 114             text-file
 115             interned-file
 116
 117             %store-prefix
 118             store-path?
 119             direct-store-path?
 120             derivation-path?
 121             store-path-package-name
 122             store-path-hash-part
 123             log-file))
 124
 125 (define %protocol-version #x10c)
 126
 127 (define %worker-magic-1 #x6e697863)               ; "nixc"
 128 (define %worker-magic-2 #x6478696f)               ; "dxio"
 129
 130 (define (protocol-major magic)
 131   (logand magic #xff00))
 132 (define (protocol-minor magic)
 133   (logand magic #x00ff))
 134
 135 (define-syntax define-enumerate-type
 136   (syntax-rules ()
 137     ((_ name->int (name id) ...)
 138      (define-syntax name->int
 139        (syntax-rules (name ...)
 140          ((_ name) id) ...)))))
 141
 142 (define-enumerate-type operation-id
 143   ;; operation numbers from worker-protocol.hh
 144   (quit 0)
 145   (valid-path? 1)
 146   (has-substitutes? 3)
 147   (query-path-hash 4)
 148   (query-references 5)
 149   (query-referrers 6)
 150   (add-to-store 7)
 151   (add-text-to-store 8)
 152   (build-things 9)
 153   (ensure-path 10)
 154   (add-temp-root 11)
 155   (add-indirect-root 12)
 156   (sync-with-gc 13)
 157   (find-roots 14)
 158   (export-path 16)
 159   (query-deriver 18)
 160   (set-options 19)
 161   (collect-garbage 20)
 162   ;;(query-substitutable-path-info 21)  ; obsolete as of #x10c
 163   (query-derivation-outputs 22)
 164   (query-all-valid-paths 23)
 165   (query-failed-paths 24)
 166   (clear-failed-paths 25)
 167   (query-path-info 26)
 168   (import-paths 27)
 169   (query-derivation-output-names 28)
 170   (query-path-from-hash-part 29)
 171   (query-substitutable-path-infos 30)
 172   (query-valid-paths 31)
 173   (query-substitutable-paths 32)
 174   (query-valid-derivers 33))
 175
 176 (define-enumerate-type hash-algo
 177   ;; hash.hh
 178   (md5 1)
 179   (sha1 2)
 180   (sha256 3))
 181
 182 (define-enumerate-type gc-action
 183   ;; store-api.hh
 184   (return-live 0)
 185   (return-dead 1)
 186   (delete-dead 2)
 187   (delete-specific 3))
 188
 189 (define %default-socket-path
 190   (string-append %state-directory "/daemon-socket/socket"))
 191
 192 (define %daemon-socket-file
 193   ;; File name of the socket the daemon listens too.
 194   (make-parameter (or (getenv "GUIX_DAEMON_SOCKET")
 195                       %default-socket-path)))
 196
 197
 198 \f
 199 ;; Information about a substitutable store path.
 200 (define-record-type <substitutable>
 201   (substitutable path deriver refs dl-size nar-size)
 202   substitutable?
 203   (path      substitutable-path)
 204   (deriver   substitutable-deriver)
 205   (refs      substitutable-references)
 206   (dl-size   substitutable-download-size)
 207   (nar-size  substitutable-nar-size))
 208
 209 (define (read-substitutable-path-list p)
 210   (let loop ((len    (read-int p))
 211              (result '()))
 212     (if (zero? len)
 213         (reverse result)
 214         (let ((path     (read-store-path p))
 215               (deriver  (read-store-path p))
 216               (refs     (read-store-path-list p))
 217               (dl-size  (read-long-long p))
 218               (nar-size (read-long-long p)))
 219           (loop (- len 1)
 220                 (cons (substitutable path deriver refs dl-size nar-size)
 221                       result))))))
 222
 223 ;; Information about a store path.
 224 (define-record-type <path-info>
 225   (path-info deriver hash references registration-time nar-size)
 226   path-info?
 227   (deriver path-info-deriver)
 228   (hash path-info-hash)
 229   (references path-info-references)
 230   (registration-time path-info-registration-time)
 231   (nar-size path-info-nar-size))
 232
 233 (define (read-path-info p)
 234   (let ((deriver  (read-store-path p))
 235         (hash     (base16-string->bytevector (read-string p)))
 236         (refs     (read-store-path-list p))
 237         (registration-time (read-int p))
 238         (nar-size (read-long-long p)))
 239     (path-info deriver hash refs registration-time nar-size)))
 240
 241 (define-syntax write-arg
 242   (syntax-rules (integer boolean file string string-list string-pairs
 243                  store-path store-path-list base16)
 244     ((_ integer arg p)
 245      (write-int arg p))
 246     ((_ boolean arg p)
 247      (write-int (if arg 1 0) p))
 248     ((_ file arg p)
 249      (write-file arg p))
 250     ((_ string arg p)
 251      (write-string arg p))
 252     ((_ string-list arg p)
 253      (write-string-list arg p))
 254     ((_ string-pairs arg p)
 255      (write-string-pairs arg p))
 256     ((_ store-path arg p)
 257      (write-store-path arg p))
 258     ((_ store-path-list arg p)
 259      (write-store-path-list arg p))
 260     ((_ base16 arg p)
 261      (write-string (bytevector->base16-string arg) p))))
 262
 263 (define-syntax read-arg
 264   (syntax-rules (integer boolean string store-path store-path-list
 265                  substitutable-path-list path-info base16)
 266     ((_ integer p)
 267      (read-int p))
 268     ((_ boolean p)
 269      (not (zero? (read-int p))))
 270     ((_ string p)
 271      (read-string p))
 272     ((_ store-path p)
 273      (read-store-path p))
 274     ((_ store-path-list p)
 275      (read-store-path-list p))
 276     ((_ substitutable-path-list p)
 277      (read-substitutable-path-list p))
 278     ((_ path-info p)
 279      (read-path-info p))
 280     ((_ base16 p)
 281      (base16-string->bytevector (read-string p)))))
 282
 283 \f
 284 ;; remote-store.cc
 285
 286 (define-record-type <nix-server>
 287   (%make-nix-server socket major minor
 288                     ats-cache atts-cache)
 289   nix-server?
 290   (socket nix-server-socket)
 291   (major  nix-server-major-version)
 292   (minor  nix-server-minor-version)
 293
 294   ;; Caches.  We keep them per-connection, because store paths build
 295   ;; during the session are temporary GC roots kept for the duration of
 296   ;; the session.
 297   (ats-cache  nix-server-add-to-store-cache)
 298   (atts-cache nix-server-add-text-to-store-cache))
 299
 300 (set-record-type-printer! <nix-server>
 301                           (lambda (obj port)
 302                             (format port "#<build-daemon ~a.~a ~a>"
 303                                     (nix-server-major-version obj)
 304                                     (nix-server-minor-version obj)
 305                                     (number->string (object-address obj)
 306                                                     16))))
 307
 308 (define-condition-type &nix-error &error
 309   nix-error?)
 310
 311 (define-condition-type &nix-connection-error &nix-error
 312   nix-connection-error?
 313   (file   nix-connection-error-file)
 314   (errno  nix-connection-error-code))
 315
 316 (define-condition-type &nix-protocol-error &nix-error
 317   nix-protocol-error?
 318   (message nix-protocol-error-message)
 319   (status  nix-protocol-error-status))
 320
 321 (define* (open-connection #:optional (file (%daemon-socket-file))
 322                           #:key (reserve-space? #t))
 323   "Connect to the daemon over the Unix-domain socket at FILE.  When
 324 RESERVE-SPACE? is true, instruct it to reserve a little bit of extra
 325 space on the file system so that the garbage collector can still
 326 operate, should the disk become full.  Return a server object."
 327   (let ((s (with-fluids ((%default-port-encoding #f))
 328              ;; This trick allows use of the `scm_c_read' optimization.
 329              (socket PF_UNIX SOCK_STREAM 0)))
 330         (a (make-socket-address PF_UNIX file)))
 331
 332     (catch 'system-error
 333       (cut connect s a)
 334       (lambda args
 335         ;; Translate the error to something user-friendly.
 336         (let ((errno (system-error-errno args)))
 337           (raise (condition (&nix-connection-error
 338                              (file file)
 339                              (errno errno)))))))
 340
 341     (write-int %worker-magic-1 s)
 342     (let ((r (read-int s)))
 343       (and (eqv? r %worker-magic-2)
 344            (let ((v (read-int s)))
 345              (and (eqv? (protocol-major %protocol-version)
 346                         (protocol-major v))
 347                   (begin
 348                     (write-int %protocol-version s)
 349                     (if (>= (protocol-minor v) 11)
 350                         (write-int (if reserve-space? 1 0) s))
 351                     (let ((s (%make-nix-server s
 352                                                (protocol-major v)
 353                                                (protocol-minor v)
 354                                                (make-hash-table 100)
 355                                                (make-hash-table 100))))
 356                       (let loop ((done? (process-stderr s)))
 357                         (or done? (process-stderr s)))
 358                       s))))))))
 359
 360 (define (close-connection server)
 361   "Close the connection to SERVER."
 362   (close (nix-server-socket server)))
 363
 364 (define-syntax-rule (with-store store exp ...)
 365   "Bind STORE to an open connection to the store and evaluate EXPs;
 366 automatically close the store when the dynamic extent of EXP is left."
 367   (let ((store (open-connection)))
 368     (dynamic-wind
 369       (const #f)
 370       (lambda ()
 371         exp ...)
 372       (lambda ()
 373         (false-if-exception (close-connection store))))))
 374
 375 (define current-build-output-port
 376   ;; The port where build output is sent.
 377   (make-parameter (current-error-port)))
 378
 379 (define* (dump-port in out
 380                     #:optional len
 381                     #:key (buffer-size 16384))
 382   "Read LEN bytes from IN (or as much as possible if LEN is #f) and write it
 383 to OUT, using chunks of BUFFER-SIZE bytes."
 384   (define buffer
 385     (make-bytevector buffer-size))
 386
 387   (let loop ((total 0)
 388              (bytes (get-bytevector-n! in buffer 0
 389                                        (if len
 390                                            (min len buffer-size)
 391                                            buffer-size))))
 392     (or (eof-object? bytes)
 393         (and len (= total len))
 394         (let ((total (+ total bytes)))
 395           (put-bytevector out buffer 0 bytes)
 396           (loop total
 397                 (get-bytevector-n! in buffer 0
 398                                    (if len
 399                                        (min (- len total) buffer-size)
 400                                        buffer-size)))))))
 401
 402 (define %newlines
 403   ;; Newline characters triggering a flush of 'current-build-output-port'.
 404   ;; Unlike Guile's _IOLBF, we flush upon #\return so that progress reports
 405   ;; that use that trick are correctly displayed.
 406   (char-set #\newline #\return))
 407
 408 (define* (process-stderr server #:optional user-port)
 409   "Read standard output and standard error from SERVER, writing it to
 410 CURRENT-BUILD-OUTPUT-PORT.  Return #t when SERVER is done sending data, and
 411 #f otherwise; in the latter case, the caller should call `process-stderr'
 412 again until #t is returned or an error is raised.
 413
 414 Since the build process's output cannot be assumed to be UTF-8, we
 415 conservatively consider it to be Latin-1, thereby avoiding possible
 416 encoding conversion errors."
 417   (define p
 418     (nix-server-socket server))
 419
 420   ;; magic cookies from worker-protocol.hh
 421   (define %stderr-next  #x6f6c6d67)          ; "olmg", build log
 422   (define %stderr-read  #x64617461)          ; "data", data needed from source
 423   (define %stderr-write #x64617416)          ; "dat\x16", data for sink
 424   (define %stderr-last  #x616c7473)          ; "alts", we're done
 425   (define %stderr-error #x63787470)          ; "cxtp", error reporting
 426
 427   (let ((k (read-int p)))
 428     (cond ((= k %stderr-write)
 429            ;; Write a byte stream to USER-PORT.
 430            (let* ((len (read-int p))
 431                   (m   (modulo len 8)))
 432              (dump-port p user-port len)
 433              (unless (zero? m)
 434                ;; Consume padding, as for strings.
 435                (get-bytevector-n p (- 8 m))))
 436            #f)
 437           ((= k %stderr-read)
 438            ;; Read a byte stream from USER-PORT.
 439            ;; Note: Avoid 'get-bytevector-n' to work around
 440            ;; <http://bugs.gnu.org/17591> in Guile up to 2.0.11.
 441            (let* ((max-len (read-int p))
 442                   (data    (make-bytevector max-len))
 443                   (len     (get-bytevector-n! user-port data 0 max-len)))
 444              (write-int len p)
 445              (put-bytevector p data 0 len)
 446              (write-padding len p)
 447              #f))
 448           ((= k %stderr-next)
 449            ;; Log a string.  Build logs are usually UTF-8-encoded, but they
 450            ;; may also contain arbitrary byte sequences that should not cause
 451            ;; this to fail.  Thus, use the permissive
 452            ;; 'read-maybe-utf8-string'.
 453            (let ((s (read-maybe-utf8-string p)))
 454              (display s (current-build-output-port))
 455              (when (string-any %newlines s)
 456                (flush-output-port (current-build-output-port)))
 457              #f))
 458           ((= k %stderr-error)
 459            ;; Report an error.
 460            (let ((error  (read-maybe-utf8-string p))
 461                  ;; Currently the daemon fails to send a status code for early
 462                  ;; errors like DB schema version mismatches, so check for EOF.
 463                  (status (if (and (>= (nix-server-minor-version server) 8)
 464                                   (not (eof-object? (lookahead-u8 p))))
 465                              (read-int p)
 466                              1)))
 467              (raise (condition (&nix-protocol-error
 468                                 (message error)
 469                                 (status  status))))))
 470           ((= k %stderr-last)
 471            ;; The daemon is done (see `stopWork' in `nix-worker.cc'.)
 472            #t)
 473           (else
 474            (raise (condition (&nix-protocol-error
 475                               (message "invalid error code")
 476                               (status   k))))))))
 477
 478 (define %default-substitute-urls
 479   ;; Default list of substituters.
 480   '("http://hydra.gnu.org"))
 481
 482 (define* (set-build-options server
 483                             #:key keep-failed? keep-going? fallback?
 484                             (verbosity 0)
 485                             (max-build-jobs 1)
 486                             timeout
 487                             (max-silent-time 3600)
 488                             (use-build-hook? #t)
 489                             (build-verbosity 0)
 490                             (log-type 0)
 491                             (print-build-trace #t)
 492                             (build-cores (current-processor-count))
 493                             (use-substitutes? #t)
 494
 495                             ;; Client-provided substitute URLs.  For
 496                             ;; unprivileged clients, these are considered
 497                             ;; "untrusted"; for root, they override the
 498                             ;; daemon's settings.
 499                             (substitute-urls %default-substitute-urls))
 500   ;; Must be called after `open-connection'.
 501
 502   (define socket
 503     (nix-server-socket server))
 504
 505   (let-syntax ((send (syntax-rules ()
 506                        ((_ (type option) ...)
 507                         (begin
 508                           (write-arg type option socket)
 509                           ...)))))
 510     (write-int (operation-id set-options) socket)
 511     (send (boolean keep-failed?) (boolean keep-going?)
 512           (boolean fallback?) (integer verbosity)
 513           (integer max-build-jobs) (integer max-silent-time))
 514     (when (>= (nix-server-minor-version server) 2)
 515       (send (boolean use-build-hook?)))
 516     (when (>= (nix-server-minor-version server) 4)
 517       (send (integer build-verbosity) (integer log-type)
 518             (boolean print-build-trace)))
 519     (when (>= (nix-server-minor-version server) 6)
 520       (send (integer build-cores)))
 521     (when (>= (nix-server-minor-version server) 10)
 522       (send (boolean use-substitutes?)))
 523     (when (>= (nix-server-minor-version server) 12)
 524       (let ((pairs `(,@(if timeout
 525                            `(("build-timeout" . ,(number->string timeout)))
 526                            '())
 527                      ("substitute-urls" . ,(string-join substitute-urls)))))
 528         (send (string-pairs pairs))))
 529     (let loop ((done? (process-stderr server)))
 530       (or done? (process-stderr server)))))
 531
 532 (define-syntax operation
 533   (syntax-rules ()
 534     "Define a client-side RPC stub for the given operation."
 535     ((_ (name (type arg) ...) docstring return ...)
 536      (lambda (server arg ...)
 537        docstring
 538        (let ((s (nix-server-socket server)))
 539          (write-int (operation-id name) s)
 540          (write-arg type arg s)
 541          ...
 542          ;; Loop until the server is done sending error output.
 543          (let loop ((done? (process-stderr server)))
 544            (or done? (loop (process-stderr server))))
 545          (values (read-arg return s) ...))))))
 546
 547 (define-syntax-rule (define-operation (name args ...)
 548                       docstring return ...)
 549   (define name
 550     (operation (name args ...) docstring return ...)))
 551
 552 (define-operation (valid-path? (string path))
 553   "Return #t when PATH is a valid store path."
 554   boolean)
 555
 556 (define-operation (query-path-hash (store-path path))
 557   "Return the SHA256 hash of PATH as a bytevector."
 558   base16)
 559
 560 (define hash-part->path
 561   (let ((query-path-from-hash-part
 562          (operation (query-path-from-hash-part (string hash))
 563                     #f
 564                     store-path)))
 565    (lambda (server hash-part)
 566      "Return the store path whose hash part is HASH-PART (a nix-base32
 567 string).  Raise an error if no such path exists."
 568      ;; This RPC is primarily used by Hydra to reply to HTTP GETs of
 569      ;; /HASH.narinfo.
 570      (query-path-from-hash-part server hash-part))))
 571
 572 (define-operation (query-path-info (store-path path))
 573   "Return the info (hash, references, etc.) for PATH."
 574   path-info)
 575
 576 (define add-text-to-store
 577   ;; A memoizing version of `add-to-store', to avoid repeated RPCs with
 578   ;; the very same arguments during a given session.
 579   (let ((add-text-to-store
 580          (operation (add-text-to-store (string name) (string text)
 581                                        (string-list references))
 582                     #f
 583                     store-path)))
 584     (lambda* (server name text #:optional (references '()))
 585       "Add TEXT under file NAME in the store, and return its store path.
 586 REFERENCES is the list of store paths referred to by the resulting store
 587 path."
 588       (let ((args  `(,text ,name ,references))
 589             (cache (nix-server-add-text-to-store-cache server)))
 590         (or (hash-ref cache args)
 591             (let ((path (add-text-to-store server name text references)))
 592               (hash-set! cache args path)
 593               path))))))
 594
 595 (define add-to-store
 596   ;; A memoizing version of `add-to-store'.  This is important because
 597   ;; `add-to-store' leads to huge data transfers to the server, and
 598   ;; because it's often called many times with the very same argument.
 599   (let ((add-to-store (operation (add-to-store (string basename)
 600                                                (boolean fixed?) ; obsolete, must be #t
 601                                                (boolean recursive?)
 602                                                (string hash-algo)
 603                                                (file file-name))
 604                                  #f
 605                                  store-path)))
 606     (lambda (server basename recursive? hash-algo file-name)
 607       "Add the contents of FILE-NAME under BASENAME to the store.  When
 608 RECURSIVE? is true and FILE-NAME designates a directory, the contents of
 609 FILE-NAME are added recursively; if FILE-NAME designates a flat file and
 610 RECURSIVE? is true, its contents are added, and its permission bits are
 611 kept.  HASH-ALGO must be a string such as \"sha256\"."
 612       (let* ((st    (stat file-name #f))
 613              (args  `(,st ,basename ,recursive? ,hash-algo))
 614              (cache (nix-server-add-to-store-cache server)))
 615         (or (and st (hash-ref cache args))
 616             (let ((path (add-to-store server basename #t recursive?
 617                                       hash-algo file-name)))
 618               (hash-set! cache args path)
 619               path))))))
 620
 621 (define-operation (build-things (string-list things))
 622   "Build THINGS, a list of store items which may be either '.drv' files or
 623 outputs, and return when the worker is done building them.  Elements of THINGS
 624 that are not derivations can only be substituted and not built locally.
 625 Return #t on success."
 626   boolean)
 627
 628 (define-operation (add-temp-root (store-path path))
 629   "Make PATH a temporary root for the duration of the current session.
 630 Return #t."
 631   boolean)
 632
 633 (define-operation (add-indirect-root (string file-name))
 634   "Make the symlink FILE-NAME an indirect root for the garbage collector:
 635 whatever store item FILE-NAME points to will not be collected.  Return #t on
 636 success.
 637
 638 FILE-NAME can be anywhere on the file system, but it must be an absolute file
 639 name--it is the caller's responsibility to ensure that it is an absolute file
 640 name."
 641   boolean)
 642
 643 (define %gc-roots-directory
 644   ;; The place where garbage collector roots (symlinks) are kept.
 645   (string-append %state-directory "/gcroots"))
 646
 647 (define (add-permanent-root target)
 648   "Add a garbage collector root pointing to TARGET, an element of the store,
 649 preventing TARGET from even being collected.  This can also be used if TARGET
 650 does not exist yet.
 651
 652 Raise an error if the caller does not have write access to the GC root
 653 directory."
 654   (let* ((root (string-append %gc-roots-directory "/" (basename target))))
 655     (catch 'system-error
 656       (lambda ()
 657         (symlink target root))
 658       (lambda args
 659         ;; If ROOT already exists, this is fine; otherwise, re-throw.
 660         (unless (= EEXIST (system-error-errno args))
 661           (apply throw args))))))
 662
 663 (define (remove-permanent-root target)
 664   "Remove the permanent garbage collector root pointing to TARGET.  Raise an
 665 error if there is no such root."
 666   (delete-file (string-append %gc-roots-directory "/" (basename target))))
 667
 668 (define references
 669   (operation (query-references (store-path path))
 670              "Return the list of references of PATH."
 671              store-path-list))
 672
 673 (define* (fold-path store proc seed path
 674                     #:optional (relatives (cut references store <>)))
 675   "Call PROC for each of the RELATIVES of PATH, exactly once, and return the
 676 result formed from the successive calls to PROC, the first of which is passed
 677 SEED."
 678   (let loop ((paths  (list path))
 679              (result seed)
 680              (seen   vlist-null))
 681     (match paths
 682       ((path rest ...)
 683        (if (vhash-assoc path seen)
 684            (loop rest result seen)
 685            (let ((seen   (vhash-cons path #t seen))
 686                  (rest   (append rest (relatives path)))
 687                  (result (proc path result)))
 688              (loop rest result seen))))
 689       (()
 690        result))))
 691
 692 (define (requisites store path)
 693   "Return the requisites of PATH, including PATH---i.e., its closure (all its
 694 references, recursively)."
 695   (fold-path store cons '() path))
 696
 697 (define (topologically-sorted store paths)
 698   "Return a list containing PATHS and all their references sorted in
 699 topological order."
 700   (define (traverse)
 701     ;; Do a simple depth-first traversal of all of PATHS.
 702     (let loop ((paths   paths)
 703                (visited vlist-null)
 704                (result  '()))
 705       (define (visit n)
 706         (vhash-cons n #t visited))
 707
 708       (define (visited? n)
 709         (vhash-assoc n visited))
 710
 711       (match paths
 712         ((head tail ...)
 713          (if (visited? head)
 714              (loop tail visited result)
 715              (call-with-values
 716                  (lambda ()
 717                    (loop (references store head)
 718                          (visit head)
 719                          result))
 720                (lambda (visited result)
 721                  (loop tail
 722                        visited
 723                        (cons head result))))))
 724         (()
 725          (values visited result)))))
 726
 727   (call-with-values traverse
 728     (lambda (_ result)
 729       (reverse result))))
 730
 731 (define referrers
 732   (operation (query-referrers (store-path path))
 733              "Return the list of path that refer to PATH."
 734              store-path-list))
 735
 736 (define valid-derivers
 737   (operation (query-valid-derivers (store-path path))
 738              "Return the list of valid \"derivers\" of PATH---i.e., all the
 739 .drv present in the store that have PATH among their outputs."
 740              store-path-list))
 741
 742 (define query-derivation-outputs  ; avoid name clash with `derivation-outputs'
 743   (operation (query-derivation-outputs (store-path path))
 744              "Return the list of outputs of PATH, a .drv file."
 745              store-path-list))
 746
 747 (define-operation (has-substitutes? (store-path path))
 748   "Return #t if binary substitutes are available for PATH, and #f otherwise."
 749   boolean)
 750
 751 (define substitutable-paths
 752   (operation (query-substitutable-paths (store-path-list paths))
 753              "Return the subset of PATHS that is substitutable."
 754              store-path-list))
 755
 756 (define substitutable-path-info
 757   (operation (query-substitutable-path-infos (store-path-list paths))
 758              "Return information about the subset of PATHS that is
 759 substitutable.  For each substitutable path, a `substitutable?' object is
 760 returned."
 761              substitutable-path-list))
 762
 763 (define (run-gc server action to-delete min-freed)
 764   "Perform the garbage-collector operation ACTION, one of the
 765 `gc-action' values.  When ACTION is `delete-specific', the TO-DELETE is
 766 the list of store paths to delete.  IGNORE-LIVENESS? should always be
 767 #f.  MIN-FREED is the minimum amount of disk space to be freed, in
 768 bytes, before the GC can stop.  Return the list of store paths delete,
 769 and the number of bytes freed."
 770   (let ((s (nix-server-socket server)))
 771     (write-int (operation-id collect-garbage) s)
 772     (write-int action s)
 773     (write-store-path-list to-delete s)
 774     (write-arg boolean #f s)                      ; ignore-liveness?
 775     (write-long-long min-freed s)
 776     (write-int 0 s)                               ; obsolete
 777     (when (>= (nix-server-minor-version server) 5)
 778       ;; Obsolete `use-atime' and `max-atime' parameters.
 779       (write-int 0 s)
 780       (write-int 0 s))
 781
 782     ;; Loop until the server is done sending error output.
 783     (let loop ((done? (process-stderr server)))
 784       (or done? (loop (process-stderr server))))
 785
 786     (let ((paths    (read-store-path-list s))
 787           (freed    (read-long-long s))
 788           (obsolete (read-long-long s)))
 789       (unless (null? paths)
 790         ;; To be on the safe side, completely invalidate both caches.
 791         ;; Otherwise we could end up returning store paths that are no longer
 792         ;; valid.
 793         (hash-clear! (nix-server-add-to-store-cache server))
 794         (hash-clear! (nix-server-add-text-to-store-cache server)))
 795
 796      (values paths freed))))
 797
 798 (define-syntax-rule (%long-long-max)
 799   ;; Maximum unsigned 64-bit integer.
 800   (- (expt 2 64) 1))
 801
 802 (define (live-paths server)
 803   "Return the list of live store paths---i.e., store paths still
 804 referenced, and thus not subject to being garbage-collected."
 805   (run-gc server (gc-action return-live) '() (%long-long-max)))
 806
 807 (define (dead-paths server)
 808   "Return the list of dead store paths---i.e., store paths no longer
 809 referenced, and thus subject to being garbage-collected."
 810   (run-gc server (gc-action return-dead) '() (%long-long-max)))
 811
 812 (define* (collect-garbage server #:optional (min-freed (%long-long-max)))
 813   "Collect garbage from the store at SERVER.  If MIN-FREED is non-zero,
 814 then collect at least MIN-FREED bytes.  Return the paths that were
 815 collected, and the number of bytes freed."
 816   (run-gc server (gc-action delete-dead) '() min-freed))
 817
 818 (define* (delete-paths server paths #:optional (min-freed (%long-long-max)))
 819   "Delete PATHS from the store at SERVER, if they are no longer
 820 referenced.  If MIN-FREED is non-zero, then stop after at least
 821 MIN-FREED bytes have been collected.  Return the paths that were
 822 collected, and the number of bytes freed."
 823   (run-gc server (gc-action delete-specific) paths min-freed))
 824
 825 (define (import-paths server port)
 826   "Import the set of store paths read from PORT into SERVER's store.  An error
 827 is raised if the set of paths read from PORT is not signed (as per
 828 'export-path #:sign? #t'.)  Return the list of store paths imported."
 829   (let ((s (nix-server-socket server)))
 830     (write-int (operation-id import-paths) s)
 831     (let loop ((done? (process-stderr server port)))
 832       (or done? (loop (process-stderr server port))))
 833     (read-store-path-list s)))
 834
 835 (define* (export-path server path port #:key (sign? #t))
 836   "Export PATH to PORT.  When SIGN? is true, sign it."
 837   (let ((s (nix-server-socket server)))
 838     (write-int (operation-id export-path) s)
 839     (write-store-path path s)
 840     (write-arg boolean sign? s)
 841     (let loop ((done? (process-stderr server port)))
 842       (or done? (loop (process-stderr server port))))
 843     (= 1 (read-int s))))
 844
 845 (define* (export-paths server paths port #:key (sign? #t) recursive?)
 846   "Export the store paths listed in PATHS to PORT, in topological order,
 847 signing them if SIGN? is true.  When RECURSIVE? is true, export the closure of
 848 PATHS---i.e., PATHS and all their dependencies."
 849   (define ordered
 850     (let ((sorted (topologically-sorted server paths)))
 851       ;; When RECURSIVE? is #f, filter out the references of PATHS.
 852       (if recursive?
 853           sorted
 854           (filter (cut member <> paths) sorted))))
 855
 856   (let loop ((paths ordered))
 857     (match paths
 858       (()
 859        (write-int 0 port))
 860       ((head tail ...)
 861        (write-int 1 port)
 862        (and (export-path server head port #:sign? sign?)
 863             (loop tail))))))
 864
 865 (define* (register-path path
 866                         #:key (references '()) deriver prefix
 867                         state-directory)
 868   "Register PATH as a valid store file, with REFERENCES as its list of
 869 references, and DERIVER as its deriver (.drv that led to it.)  If PREFIX is
 870 not #f, it must be the name of the directory containing the new store to
 871 initialize; if STATE-DIRECTORY is not #f, it must be a string containing the
 872 absolute file name to the state directory of the store being initialized.
 873 Return #t on success.
 874
 875 Use with care as it directly modifies the store!  This is primarily meant to
 876 be used internally by the daemon's build hook."
 877   ;; Currently this is implemented by calling out to the fine C++ blob.
 878   (catch 'system-error
 879     (lambda ()
 880       (let ((pipe (apply open-pipe* OPEN_WRITE %guix-register-program
 881                          `(,@(if prefix
 882                                  `("--prefix" ,prefix)
 883                                  '())
 884                            ,@(if state-directory
 885                                  `("--state-directory" ,state-directory)
 886                                  '())))))
 887         (and pipe
 888              (begin
 889                (format pipe "~a~%~a~%~a~%"
 890                        path (or deriver "") (length references))
 891                (for-each (cut format pipe "~a~%" <>) references)
 892                (zero? (close-pipe pipe))))))
 893     (lambda args
 894       ;; Failed to run %GUIX-REGISTER-PROGRAM.
 895       #f)))
 896
 897 \f
 898 ;;;
 899 ;;; Store monad.
 900 ;;;
 901
 902 (define-syntax-rule (define-alias new old)
 903   (define-syntax new (identifier-syntax old)))
 904
 905 ;; The store monad allows us to (1) build sequences of operations in the
 906 ;; store, and (2) make the store an implicit part of the execution context,
 907 ;; rather than a parameter of every single function.
 908 (define-alias %store-monad %state-monad)
 909 (define-alias store-return state-return)
 910 (define-alias store-bind state-bind)
 911
 912 (define (preserve-documentation original proc)
 913   "Return PROC with documentation taken from ORIGINAL."
 914   (set-object-property! proc 'documentation
 915                         (procedure-property original 'documentation))
 916   proc)
 917
 918 (define (store-lift proc)
 919   "Lift PROC, a procedure whose first argument is a connection to the store,
 920 in the store monad."
 921   (preserve-documentation proc
 922                           (lambda args
 923                             (lambda (store)
 924                               (values (apply proc store args) store)))))
 925
 926 (define (store-lower proc)
 927   "Lower PROC, a monadic procedure in %STORE-MONAD, to a \"normal\" procedure
 928 taking the store as its first argument."
 929   (preserve-documentation proc
 930                           (lambda (store . args)
 931                             (run-with-store store (apply proc args)))))
 932
 933 ;;
 934 ;; Store monad operators.
 935 ;;
 936
 937 (define* (text-file name text
 938                     #:optional (references '()))
 939   "Return as a monadic value the absolute file name in the store of the file
 940 containing TEXT, a string.  REFERENCES is a list of store items that the
 941 resulting text file refers to; it defaults to the empty list."
 942   (lambda (store)
 943     (values (add-text-to-store store name text references)
 944             store)))
 945
 946 (define* (interned-file file #:optional name
 947                         #:key (recursive? #t))
 948   "Return the name of FILE once interned in the store.  Use NAME as its store
 949 name, or the basename of FILE if NAME is omitted.
 950
 951 When RECURSIVE? is true, the contents of FILE are added recursively; if FILE
 952 designates a flat file and RECURSIVE? is true, its contents are added, and its
 953 permission bits are kept."
 954   (lambda (store)
 955     (values (add-to-store store (or name (basename file))
 956                           recursive? "sha256" file)
 957             store)))
 958
 959 (define build
 960   ;; Monadic variant of 'build-things'.
 961   (store-lift build-things))
 962
 963 (define %guile-for-build
 964   ;; The derivation of the Guile to be used within the build environment,
 965   ;; when using 'gexp->derivation' and co.
 966   (make-parameter #f))
 967
 968 (define* (run-with-store store mval
 969                          #:key
 970                          (guile-for-build (%guile-for-build))
 971                          (system (%current-system)))
 972   "Run MVAL, a monadic value in the store monad, in STORE, an open store
 973 connection, and return the result."
 974   (parameterize ((%guile-for-build guile-for-build)
 975                  (%current-system system))
 976     (call-with-values (lambda ()
 977                         (run-with-state mval store))
 978       (lambda (result store)
 979         ;; Discard the state.
 980         result))))
 981
 982 \f
 983 ;;;
 984 ;;; Store paths.
 985 ;;;
 986
 987 (define %store-prefix
 988   ;; Absolute path to the Nix store.
 989   (make-parameter %store-directory))
 990
 991 (define (store-path? path)
 992   "Return #t if PATH is a store path."
 993   ;; This is a lightweight check, compared to using a regexp, but this has to
 994   ;; be fast as it's called often in `derivation', for instance.
 995   ;; `isStorePath' in Nix does something similar.
 996   (string-prefix? (%store-prefix) path))
 997
 998 (define (direct-store-path? path)
 999   "Return #t if PATH is a store path, and not a sub-directory of a store path.
1000 This predicate is sometimes needed because files *under* a store path are not
1001 valid inputs."
1002   (and (store-path? path)
1003        (not (string=? path (%store-prefix)))
1004        (let ((len (+ 1 (string-length (%store-prefix)))))
1005          (not (string-index (substring path len) #\/)))))
1006
1007 (define (derivation-path? path)
1008   "Return #t if PATH is a derivation path."
1009   (and (store-path? path) (string-suffix? ".drv" path)))
1010
1011 (define store-regexp*
1012   ;; The substituter makes repeated calls to 'store-path-hash-part', hence
1013   ;; this optimization.
1014   (memoize
1015    (lambda (store)
1016      "Return a regexp matching a file in STORE."
1017      (make-regexp (string-append "^" (regexp-quote store)
1018                                  "/([0-9a-df-np-sv-z]{32})-([^/]+)$")))))
1019
1020 (define (store-path-package-name path)
1021   "Return the package name part of PATH, a file name in the store."
1022   (let ((path-rx (store-regexp* (%store-prefix))))
1023     (and=> (regexp-exec path-rx path)
1024            (cut match:substring <> 2))))
1025
1026 (define (store-path-hash-part path)
1027   "Return the hash part of PATH as a base32 string, or #f if PATH is not a
1028 syntactically valid store path."
1029   (let ((path-rx (store-regexp* (%store-prefix))))
1030     (and=> (regexp-exec path-rx path)
1031            (cut match:substring <> 1))))
1032
1033 (define (log-file store file)
1034   "Return the build log file for FILE, or #f if none could be found.  FILE
1035 must be an absolute store file name, or a derivation file name."
1036   (cond ((derivation-path? file)
1037          (let* ((base    (basename file))
1038                 (log     (string-append (dirname %state-directory) ; XXX
1039                                         "/log/guix/drvs/"
1040                                         (string-take base 2) "/"
1041                                         (string-drop base 2)))
1042                 (log.bz2 (string-append log ".bz2")))
1043            (cond ((file-exists? log.bz2) log.bz2)
1044                  ((file-exists? log) log)
1045                  (else #f))))
1046         (else
1047          (match (valid-derivers store file)
1048            ((derivers ...)
1049             ;; Return the first that works.
1050             (any (cut log-file store <>) derivers))
1051            (_ #f)))))