guix/store.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2012, 2013, 2014, 2015 Ludovic Courtès <ludo@gnu.org>
   3 ;;;
   4 ;;; This file is part of GNU Guix.
   5 ;;;
   6 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   7 ;;; under the terms of the GNU General Public License as published by
   8 ;;; the Free Software Foundation; either version 3 of the License, or (at
   9 ;;; your option) any later version.
  10 ;;;
  11 ;;; GNU Guix is distributed in the hope that it will be useful, but
  12 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 ;;; GNU General Public License for more details.
  15 ;;;
  16 ;;; You should have received a copy of the GNU General Public License
  17 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 (define-module (guix store)
  20   #:use-module (guix utils)
  21   #:use-module (guix config)
  22   #:use-module (guix serialization)
  23   #:use-module (guix monads)
  24   #:autoload   (guix base32) (bytevector->base32-string)
  25   #:use-module (rnrs bytevectors)
  26   #:use-module (rnrs io ports)
  27   #:use-module (srfi srfi-1)
  28   #:use-module (srfi srfi-9)
  29   #:use-module (srfi srfi-9 gnu)
  30   #:use-module (srfi srfi-26)
  31   #:use-module (srfi srfi-34)
  32   #:use-module (srfi srfi-35)
  33   #:use-module (srfi srfi-39)
  34   #:use-module (ice-9 match)
  35   #:use-module (ice-9 regex)
  36   #:use-module (ice-9 vlist)
  37   #:use-module (ice-9 popen)
  38   #:export (%daemon-socket-file
  39             %gc-roots-directory
  40
  41             nix-server?
  42             nix-server-major-version
  43             nix-server-minor-version
  44             nix-server-socket
  45
  46             &nix-error nix-error?
  47             &nix-connection-error nix-connection-error?
  48             nix-connection-error-file
  49             nix-connection-error-code
  50             &nix-protocol-error nix-protocol-error?
  51             nix-protocol-error-message
  52             nix-protocol-error-status
  53
  54             hash-algo
  55
  56             open-connection
  57             close-connection
  58             with-store
  59             set-build-options
  60             valid-path?
  61             query-path-hash
  62             hash-part->path
  63             query-path-info
  64             add-text-to-store
  65             add-to-store
  66             build-things
  67             build
  68             add-temp-root
  69             add-indirect-root
  70             add-permanent-root
  71             remove-permanent-root
  72
  73             substitutable?
  74             substitutable-path
  75             substitutable-deriver
  76             substitutable-references
  77             substitutable-download-size
  78             substitutable-nar-size
  79             has-substitutes?
  80             substitutable-paths
  81             substitutable-path-info
  82
  83             path-info?
  84             path-info-deriver
  85             path-info-hash
  86             path-info-references
  87             path-info-registration-time
  88             path-info-nar-size
  89
  90             references
  91             requisites
  92             referrers
  93             optimize-store
  94             topologically-sorted
  95             valid-derivers
  96             query-derivation-outputs
  97             live-paths
  98             dead-paths
  99             collect-garbage
 100             delete-paths
 101             import-paths
 102             export-paths
 103
 104             current-build-output-port
 105
 106             register-path
 107
 108             %store-monad
 109             store-bind
 110             store-return
 111             store-lift
 112             store-lower
 113             run-with-store
 114             %guile-for-build
 115             text-file
 116             interned-file
 117
 118             %store-prefix
 119             store-path?
 120             direct-store-path?
 121             derivation-path?
 122             store-path-package-name
 123             store-path-hash-part
 124             direct-store-path
 125             log-file))
 126
 127 (define %protocol-version #x10c)
 128
 129 (define %worker-magic-1 #x6e697863)               ; "nixc"
 130 (define %worker-magic-2 #x6478696f)               ; "dxio"
 131
 132 (define (protocol-major magic)
 133   (logand magic #xff00))
 134 (define (protocol-minor magic)
 135   (logand magic #x00ff))
 136
 137 (define-syntax define-enumerate-type
 138   (syntax-rules ()
 139     ((_ name->int (name id) ...)
 140      (define-syntax name->int
 141        (syntax-rules (name ...)
 142          ((_ name) id) ...)))))
 143
 144 (define-enumerate-type operation-id
 145   ;; operation numbers from worker-protocol.hh
 146   (quit 0)
 147   (valid-path? 1)
 148   (has-substitutes? 3)
 149   (query-path-hash 4)
 150   (query-references 5)
 151   (query-referrers 6)
 152   (add-to-store 7)
 153   (add-text-to-store 8)
 154   (build-things 9)
 155   (ensure-path 10)
 156   (add-temp-root 11)
 157   (add-indirect-root 12)
 158   (sync-with-gc 13)
 159   (find-roots 14)
 160   (export-path 16)
 161   (query-deriver 18)
 162   (set-options 19)
 163   (collect-garbage 20)
 164   ;;(query-substitutable-path-info 21)  ; obsolete as of #x10c
 165   (query-derivation-outputs 22)
 166   (query-all-valid-paths 23)
 167   (query-failed-paths 24)
 168   (clear-failed-paths 25)
 169   (query-path-info 26)
 170   (import-paths 27)
 171   (query-derivation-output-names 28)
 172   (query-path-from-hash-part 29)
 173   (query-substitutable-path-infos 30)
 174   (query-valid-paths 31)
 175   (query-substitutable-paths 32)
 176   (query-valid-derivers 33)
 177   (optimize-store 34))
 178
 179 (define-enumerate-type hash-algo
 180   ;; hash.hh
 181   (md5 1)
 182   (sha1 2)
 183   (sha256 3))
 184
 185 (define-enumerate-type gc-action
 186   ;; store-api.hh
 187   (return-live 0)
 188   (return-dead 1)
 189   (delete-dead 2)
 190   (delete-specific 3))
 191
 192 (define %default-socket-path
 193   (string-append %state-directory "/daemon-socket/socket"))
 194
 195 (define %daemon-socket-file
 196   ;; File name of the socket the daemon listens too.
 197   (make-parameter (or (getenv "GUIX_DAEMON_SOCKET")
 198                       %default-socket-path)))
 199
 200
 201 \f
 202 ;; Information about a substitutable store path.
 203 (define-record-type <substitutable>
 204   (substitutable path deriver refs dl-size nar-size)
 205   substitutable?
 206   (path      substitutable-path)
 207   (deriver   substitutable-deriver)
 208   (refs      substitutable-references)
 209   (dl-size   substitutable-download-size)
 210   (nar-size  substitutable-nar-size))
 211
 212 (define (read-substitutable-path-list p)
 213   (let loop ((len    (read-int p))
 214              (result '()))
 215     (if (zero? len)
 216         (reverse result)
 217         (let ((path     (read-store-path p))
 218               (deriver  (read-store-path p))
 219               (refs     (read-store-path-list p))
 220               (dl-size  (read-long-long p))
 221               (nar-size (read-long-long p)))
 222           (loop (- len 1)
 223                 (cons (substitutable path deriver refs dl-size nar-size)
 224                       result))))))
 225
 226 ;; Information about a store path.
 227 (define-record-type <path-info>
 228   (path-info deriver hash references registration-time nar-size)
 229   path-info?
 230   (deriver path-info-deriver)
 231   (hash path-info-hash)
 232   (references path-info-references)
 233   (registration-time path-info-registration-time)
 234   (nar-size path-info-nar-size))
 235
 236 (define (read-path-info p)
 237   (let ((deriver  (read-store-path p))
 238         (hash     (base16-string->bytevector (read-string p)))
 239         (refs     (read-store-path-list p))
 240         (registration-time (read-int p))
 241         (nar-size (read-long-long p)))
 242     (path-info deriver hash refs registration-time nar-size)))
 243
 244 (define-syntax write-arg
 245   (syntax-rules (integer boolean file string string-list string-pairs
 246                  store-path store-path-list base16)
 247     ((_ integer arg p)
 248      (write-int arg p))
 249     ((_ boolean arg p)
 250      (write-int (if arg 1 0) p))
 251     ((_ file arg p)
 252      (write-file arg p))
 253     ((_ string arg p)
 254      (write-string arg p))
 255     ((_ string-list arg p)
 256      (write-string-list arg p))
 257     ((_ string-pairs arg p)
 258      (write-string-pairs arg p))
 259     ((_ store-path arg p)
 260      (write-store-path arg p))
 261     ((_ store-path-list arg p)
 262      (write-store-path-list arg p))
 263     ((_ base16 arg p)
 264      (write-string (bytevector->base16-string arg) p))))
 265
 266 (define-syntax read-arg
 267   (syntax-rules (integer boolean string store-path store-path-list
 268                  substitutable-path-list path-info base16)
 269     ((_ integer p)
 270      (read-int p))
 271     ((_ boolean p)
 272      (not (zero? (read-int p))))
 273     ((_ string p)
 274      (read-string p))
 275     ((_ store-path p)
 276      (read-store-path p))
 277     ((_ store-path-list p)
 278      (read-store-path-list p))
 279     ((_ substitutable-path-list p)
 280      (read-substitutable-path-list p))
 281     ((_ path-info p)
 282      (read-path-info p))
 283     ((_ base16 p)
 284      (base16-string->bytevector (read-string p)))))
 285
 286 \f
 287 ;; remote-store.cc
 288
 289 (define-record-type <nix-server>
 290   (%make-nix-server socket major minor
 291                     ats-cache atts-cache)
 292   nix-server?
 293   (socket nix-server-socket)
 294   (major  nix-server-major-version)
 295   (minor  nix-server-minor-version)
 296
 297   ;; Caches.  We keep them per-connection, because store paths build
 298   ;; during the session are temporary GC roots kept for the duration of
 299   ;; the session.
 300   (ats-cache  nix-server-add-to-store-cache)
 301   (atts-cache nix-server-add-text-to-store-cache))
 302
 303 (set-record-type-printer! <nix-server>
 304                           (lambda (obj port)
 305                             (format port "#<build-daemon ~a.~a ~a>"
 306                                     (nix-server-major-version obj)
 307                                     (nix-server-minor-version obj)
 308                                     (number->string (object-address obj)
 309                                                     16))))
 310
 311 (define-condition-type &nix-error &error
 312   nix-error?)
 313
 314 (define-condition-type &nix-connection-error &nix-error
 315   nix-connection-error?
 316   (file   nix-connection-error-file)
 317   (errno  nix-connection-error-code))
 318
 319 (define-condition-type &nix-protocol-error &nix-error
 320   nix-protocol-error?
 321   (message nix-protocol-error-message)
 322   (status  nix-protocol-error-status))
 323
 324 (define* (open-connection #:optional (file (%daemon-socket-file))
 325                           #:key (reserve-space? #t))
 326   "Connect to the daemon over the Unix-domain socket at FILE.  When
 327 RESERVE-SPACE? is true, instruct it to reserve a little bit of extra
 328 space on the file system so that the garbage collector can still
 329 operate, should the disk become full.  Return a server object."
 330   (let ((s (with-fluids ((%default-port-encoding #f))
 331              ;; This trick allows use of the `scm_c_read' optimization.
 332              (socket PF_UNIX SOCK_STREAM 0)))
 333         (a (make-socket-address PF_UNIX file)))
 334
 335     (catch 'system-error
 336       (cut connect s a)
 337       (lambda args
 338         ;; Translate the error to something user-friendly.
 339         (let ((errno (system-error-errno args)))
 340           (raise (condition (&nix-connection-error
 341                              (file file)
 342                              (errno errno)))))))
 343
 344     (write-int %worker-magic-1 s)
 345     (let ((r (read-int s)))
 346       (and (eqv? r %worker-magic-2)
 347            (let ((v (read-int s)))
 348              (and (eqv? (protocol-major %protocol-version)
 349                         (protocol-major v))
 350                   (begin
 351                     (write-int %protocol-version s)
 352                     (if (>= (protocol-minor v) 11)
 353                         (write-int (if reserve-space? 1 0) s))
 354                     (let ((s (%make-nix-server s
 355                                                (protocol-major v)
 356                                                (protocol-minor v)
 357                                                (make-hash-table 100)
 358                                                (make-hash-table 100))))
 359                       (let loop ((done? (process-stderr s)))
 360                         (or done? (process-stderr s)))
 361                       s))))))))
 362
 363 (define (close-connection server)
 364   "Close the connection to SERVER."
 365   (close (nix-server-socket server)))
 366
 367 (define-syntax-rule (with-store store exp ...)
 368   "Bind STORE to an open connection to the store and evaluate EXPs;
 369 automatically close the store when the dynamic extent of EXP is left."
 370   (let ((store (open-connection)))
 371     (dynamic-wind
 372       (const #f)
 373       (lambda ()
 374         exp ...)
 375       (lambda ()
 376         (false-if-exception (close-connection store))))))
 377
 378 (define current-build-output-port
 379   ;; The port where build output is sent.
 380   (make-parameter (current-error-port)))
 381
 382 (define* (dump-port in out
 383                     #:optional len
 384                     #:key (buffer-size 16384))
 385   "Read LEN bytes from IN (or as much as possible if LEN is #f) and write it
 386 to OUT, using chunks of BUFFER-SIZE bytes."
 387   (define buffer
 388     (make-bytevector buffer-size))
 389
 390   (let loop ((total 0)
 391              (bytes (get-bytevector-n! in buffer 0
 392                                        (if len
 393                                            (min len buffer-size)
 394                                            buffer-size))))
 395     (or (eof-object? bytes)
 396         (and len (= total len))
 397         (let ((total (+ total bytes)))
 398           (put-bytevector out buffer 0 bytes)
 399           (loop total
 400                 (get-bytevector-n! in buffer 0
 401                                    (if len
 402                                        (min (- len total) buffer-size)
 403                                        buffer-size)))))))
 404
 405 (define %newlines
 406   ;; Newline characters triggering a flush of 'current-build-output-port'.
 407   ;; Unlike Guile's _IOLBF, we flush upon #\return so that progress reports
 408   ;; that use that trick are correctly displayed.
 409   (char-set #\newline #\return))
 410
 411 (define* (process-stderr server #:optional user-port)
 412   "Read standard output and standard error from SERVER, writing it to
 413 CURRENT-BUILD-OUTPUT-PORT.  Return #t when SERVER is done sending data, and
 414 #f otherwise; in the latter case, the caller should call `process-stderr'
 415 again until #t is returned or an error is raised.
 416
 417 Since the build process's output cannot be assumed to be UTF-8, we
 418 conservatively consider it to be Latin-1, thereby avoiding possible
 419 encoding conversion errors."
 420   (define p
 421     (nix-server-socket server))
 422
 423   ;; magic cookies from worker-protocol.hh
 424   (define %stderr-next  #x6f6c6d67)          ; "olmg", build log
 425   (define %stderr-read  #x64617461)          ; "data", data needed from source
 426   (define %stderr-write #x64617416)          ; "dat\x16", data for sink
 427   (define %stderr-last  #x616c7473)          ; "alts", we're done
 428   (define %stderr-error #x63787470)          ; "cxtp", error reporting
 429
 430   (let ((k (read-int p)))
 431     (cond ((= k %stderr-write)
 432            ;; Write a byte stream to USER-PORT.
 433            (let* ((len (read-int p))
 434                   (m   (modulo len 8)))
 435              (dump-port p user-port len)
 436              (unless (zero? m)
 437                ;; Consume padding, as for strings.
 438                (get-bytevector-n p (- 8 m))))
 439            #f)
 440           ((= k %stderr-read)
 441            ;; Read a byte stream from USER-PORT.
 442            ;; Note: Avoid 'get-bytevector-n' to work around
 443            ;; <http://bugs.gnu.org/17591> in Guile up to 2.0.11.
 444            (let* ((max-len (read-int p))
 445                   (data    (make-bytevector max-len))
 446                   (len     (get-bytevector-n! user-port data 0 max-len)))
 447              (write-int len p)
 448              (put-bytevector p data 0 len)
 449              (write-padding len p)
 450              #f))
 451           ((= k %stderr-next)
 452            ;; Log a string.  Build logs are usually UTF-8-encoded, but they
 453            ;; may also contain arbitrary byte sequences that should not cause
 454            ;; this to fail.  Thus, use the permissive
 455            ;; 'read-maybe-utf8-string'.
 456            (let ((s (read-maybe-utf8-string p)))
 457              (display s (current-build-output-port))
 458              (when (string-any %newlines s)
 459                (flush-output-port (current-build-output-port)))
 460              #f))
 461           ((= k %stderr-error)
 462            ;; Report an error.
 463            (let ((error  (read-maybe-utf8-string p))
 464                  ;; Currently the daemon fails to send a status code for early
 465                  ;; errors like DB schema version mismatches, so check for EOF.
 466                  (status (if (and (>= (nix-server-minor-version server) 8)
 467                                   (not (eof-object? (lookahead-u8 p))))
 468                              (read-int p)
 469                              1)))
 470              (raise (condition (&nix-protocol-error
 471                                 (message error)
 472                                 (status  status))))))
 473           ((= k %stderr-last)
 474            ;; The daemon is done (see `stopWork' in `nix-worker.cc'.)
 475            #t)
 476           (else
 477            (raise (condition (&nix-protocol-error
 478                               (message "invalid error code")
 479                               (status   k))))))))
 480
 481 (define %default-substitute-urls
 482   ;; Default list of substituters.
 483   '("http://hydra.gnu.org"))
 484
 485 (define* (set-build-options server
 486                             #:key keep-failed? keep-going? fallback?
 487                             (verbosity 0)
 488                             (max-build-jobs 1)
 489                             timeout
 490                             (max-silent-time 3600)
 491                             (use-build-hook? #t)
 492                             (build-verbosity 0)
 493                             (log-type 0)
 494                             (print-build-trace #t)
 495                             (build-cores (current-processor-count))
 496                             (use-substitutes? #t)
 497
 498                             ;; Client-provided substitute URLs.  For
 499                             ;; unprivileged clients, these are considered
 500                             ;; "untrusted"; for root, they override the
 501                             ;; daemon's settings.
 502                             (substitute-urls %default-substitute-urls))
 503   ;; Must be called after `open-connection'.
 504
 505   (define socket
 506     (nix-server-socket server))
 507
 508   (let-syntax ((send (syntax-rules ()
 509                        ((_ (type option) ...)
 510                         (begin
 511                           (write-arg type option socket)
 512                           ...)))))
 513     (write-int (operation-id set-options) socket)
 514     (send (boolean keep-failed?) (boolean keep-going?)
 515           (boolean fallback?) (integer verbosity)
 516           (integer max-build-jobs) (integer max-silent-time))
 517     (when (>= (nix-server-minor-version server) 2)
 518       (send (boolean use-build-hook?)))
 519     (when (>= (nix-server-minor-version server) 4)
 520       (send (integer build-verbosity) (integer log-type)
 521             (boolean print-build-trace)))
 522     (when (>= (nix-server-minor-version server) 6)
 523       (send (integer build-cores)))
 524     (when (>= (nix-server-minor-version server) 10)
 525       (send (boolean use-substitutes?)))
 526     (when (>= (nix-server-minor-version server) 12)
 527       (let ((pairs `(,@(if timeout
 528                            `(("build-timeout" . ,(number->string timeout)))
 529                            '())
 530                      ("substitute-urls" . ,(string-join substitute-urls)))))
 531         (send (string-pairs pairs))))
 532     (let loop ((done? (process-stderr server)))
 533       (or done? (process-stderr server)))))
 534
 535 (define-syntax operation
 536   (syntax-rules ()
 537     "Define a client-side RPC stub for the given operation."
 538     ((_ (name (type arg) ...) docstring return ...)
 539      (lambda (server arg ...)
 540        docstring
 541        (let ((s (nix-server-socket server)))
 542          (write-int (operation-id name) s)
 543          (write-arg type arg s)
 544          ...
 545          ;; Loop until the server is done sending error output.
 546          (let loop ((done? (process-stderr server)))
 547            (or done? (loop (process-stderr server))))
 548          (values (read-arg return s) ...))))))
 549
 550 (define-syntax-rule (define-operation (name args ...)
 551                       docstring return ...)
 552   (define name
 553     (operation (name args ...) docstring return ...)))
 554
 555 (define-operation (valid-path? (string path))
 556   "Return #t when PATH is a valid store path."
 557   boolean)
 558
 559 (define-operation (query-path-hash (store-path path))
 560   "Return the SHA256 hash of PATH as a bytevector."
 561   base16)
 562
 563 (define hash-part->path
 564   (let ((query-path-from-hash-part
 565          (operation (query-path-from-hash-part (string hash))
 566                     #f
 567                     store-path)))
 568    (lambda (server hash-part)
 569      "Return the store path whose hash part is HASH-PART (a nix-base32
 570 string).  Raise an error if no such path exists."
 571      ;; This RPC is primarily used by Hydra to reply to HTTP GETs of
 572      ;; /HASH.narinfo.
 573      (query-path-from-hash-part server hash-part))))
 574
 575 (define-operation (query-path-info (store-path path))
 576   "Return the info (hash, references, etc.) for PATH."
 577   path-info)
 578
 579 (define add-text-to-store
 580   ;; A memoizing version of `add-to-store', to avoid repeated RPCs with
 581   ;; the very same arguments during a given session.
 582   (let ((add-text-to-store
 583          (operation (add-text-to-store (string name) (string text)
 584                                        (string-list references))
 585                     #f
 586                     store-path)))
 587     (lambda* (server name text #:optional (references '()))
 588       "Add TEXT under file NAME in the store, and return its store path.
 589 REFERENCES is the list of store paths referred to by the resulting store
 590 path."
 591       (let ((args  `(,text ,name ,references))
 592             (cache (nix-server-add-text-to-store-cache server)))
 593         (or (hash-ref cache args)
 594             (let ((path (add-text-to-store server name text references)))
 595               (hash-set! cache args path)
 596               path))))))
 597
 598 (define add-to-store
 599   ;; A memoizing version of `add-to-store'.  This is important because
 600   ;; `add-to-store' leads to huge data transfers to the server, and
 601   ;; because it's often called many times with the very same argument.
 602   (let ((add-to-store (operation (add-to-store (string basename)
 603                                                (boolean fixed?) ; obsolete, must be #t
 604                                                (boolean recursive?)
 605                                                (string hash-algo)
 606                                                (file file-name))
 607                                  #f
 608                                  store-path)))
 609     (lambda (server basename recursive? hash-algo file-name)
 610       "Add the contents of FILE-NAME under BASENAME to the store.  When
 611 RECURSIVE? is true and FILE-NAME designates a directory, the contents of
 612 FILE-NAME are added recursively; if FILE-NAME designates a flat file and
 613 RECURSIVE? is true, its contents are added, and its permission bits are
 614 kept.  HASH-ALGO must be a string such as \"sha256\"."
 615       (let* ((st    (stat file-name #f))
 616              (args  `(,st ,basename ,recursive? ,hash-algo))
 617              (cache (nix-server-add-to-store-cache server)))
 618         (or (and st (hash-ref cache args))
 619             (let ((path (add-to-store server basename #t recursive?
 620                                       hash-algo file-name)))
 621               (hash-set! cache args path)
 622               path))))))
 623
 624 (define-operation (build-things (string-list things))
 625   "Build THINGS, a list of store items which may be either '.drv' files or
 626 outputs, and return when the worker is done building them.  Elements of THINGS
 627 that are not derivations can only be substituted and not built locally.
 628 Return #t on success."
 629   boolean)
 630
 631 (define-operation (add-temp-root (store-path path))
 632   "Make PATH a temporary root for the duration of the current session.
 633 Return #t."
 634   boolean)
 635
 636 (define-operation (add-indirect-root (string file-name))
 637   "Make the symlink FILE-NAME an indirect root for the garbage collector:
 638 whatever store item FILE-NAME points to will not be collected.  Return #t on
 639 success.
 640
 641 FILE-NAME can be anywhere on the file system, but it must be an absolute file
 642 name--it is the caller's responsibility to ensure that it is an absolute file
 643 name."
 644   boolean)
 645
 646 (define %gc-roots-directory
 647   ;; The place where garbage collector roots (symlinks) are kept.
 648   (string-append %state-directory "/gcroots"))
 649
 650 (define (add-permanent-root target)
 651   "Add a garbage collector root pointing to TARGET, an element of the store,
 652 preventing TARGET from even being collected.  This can also be used if TARGET
 653 does not exist yet.
 654
 655 Raise an error if the caller does not have write access to the GC root
 656 directory."
 657   (let* ((root (string-append %gc-roots-directory "/" (basename target))))
 658     (catch 'system-error
 659       (lambda ()
 660         (symlink target root))
 661       (lambda args
 662         ;; If ROOT already exists, this is fine; otherwise, re-throw.
 663         (unless (= EEXIST (system-error-errno args))
 664           (apply throw args))))))
 665
 666 (define (remove-permanent-root target)
 667   "Remove the permanent garbage collector root pointing to TARGET.  Raise an
 668 error if there is no such root."
 669   (delete-file (string-append %gc-roots-directory "/" (basename target))))
 670
 671 (define references
 672   (operation (query-references (store-path path))
 673              "Return the list of references of PATH."
 674              store-path-list))
 675
 676 (define* (fold-path store proc seed path
 677                     #:optional (relatives (cut references store <>)))
 678   "Call PROC for each of the RELATIVES of PATH, exactly once, and return the
 679 result formed from the successive calls to PROC, the first of which is passed
 680 SEED."
 681   (let loop ((paths  (list path))
 682              (result seed)
 683              (seen   vlist-null))
 684     (match paths
 685       ((path rest ...)
 686        (if (vhash-assoc path seen)
 687            (loop rest result seen)
 688            (let ((seen   (vhash-cons path #t seen))
 689                  (rest   (append rest (relatives path)))
 690                  (result (proc path result)))
 691              (loop rest result seen))))
 692       (()
 693        result))))
 694
 695 (define (requisites store path)
 696   "Return the requisites of PATH, including PATH---i.e., its closure (all its
 697 references, recursively)."
 698   (fold-path store cons '() path))
 699
 700 (define (topologically-sorted store paths)
 701   "Return a list containing PATHS and all their references sorted in
 702 topological order."
 703   (define (traverse)
 704     ;; Do a simple depth-first traversal of all of PATHS.
 705     (let loop ((paths   paths)
 706                (visited vlist-null)
 707                (result  '()))
 708       (define (visit n)
 709         (vhash-cons n #t visited))
 710
 711       (define (visited? n)
 712         (vhash-assoc n visited))
 713
 714       (match paths
 715         ((head tail ...)
 716          (if (visited? head)
 717              (loop tail visited result)
 718              (call-with-values
 719                  (lambda ()
 720                    (loop (references store head)
 721                          (visit head)
 722                          result))
 723                (lambda (visited result)
 724                  (loop tail
 725                        visited
 726                        (cons head result))))))
 727         (()
 728          (values visited result)))))
 729
 730   (call-with-values traverse
 731     (lambda (_ result)
 732       (reverse result))))
 733
 734 (define referrers
 735   (operation (query-referrers (store-path path))
 736              "Return the list of path that refer to PATH."
 737              store-path-list))
 738
 739 (define valid-derivers
 740   (operation (query-valid-derivers (store-path path))
 741              "Return the list of valid \"derivers\" of PATH---i.e., all the
 742 .drv present in the store that have PATH among their outputs."
 743              store-path-list))
 744
 745 (define query-derivation-outputs  ; avoid name clash with `derivation-outputs'
 746   (operation (query-derivation-outputs (store-path path))
 747              "Return the list of outputs of PATH, a .drv file."
 748              store-path-list))
 749
 750 (define-operation (has-substitutes? (store-path path))
 751   "Return #t if binary substitutes are available for PATH, and #f otherwise."
 752   boolean)
 753
 754 (define substitutable-paths
 755   (operation (query-substitutable-paths (store-path-list paths))
 756              "Return the subset of PATHS that is substitutable."
 757              store-path-list))
 758
 759 (define substitutable-path-info
 760   (operation (query-substitutable-path-infos (store-path-list paths))
 761              "Return information about the subset of PATHS that is
 762 substitutable.  For each substitutable path, a `substitutable?' object is
 763 returned."
 764              substitutable-path-list))
 765
 766 (define-operation (optimize-store)
 767   "Optimize the store by hard-linking identical files (\"deduplication\".)
 768 Return #t on success."
 769   ;; Note: the daemon in Guix <= 0.8.2 does not implement this RPC.
 770   boolean)
 771
 772 (define (run-gc server action to-delete min-freed)
 773   "Perform the garbage-collector operation ACTION, one of the
 774 `gc-action' values.  When ACTION is `delete-specific', the TO-DELETE is
 775 the list of store paths to delete.  IGNORE-LIVENESS? should always be
 776 #f.  MIN-FREED is the minimum amount of disk space to be freed, in
 777 bytes, before the GC can stop.  Return the list of store paths delete,
 778 and the number of bytes freed."
 779   (let ((s (nix-server-socket server)))
 780     (write-int (operation-id collect-garbage) s)
 781     (write-int action s)
 782     (write-store-path-list to-delete s)
 783     (write-arg boolean #f s)                      ; ignore-liveness?
 784     (write-long-long min-freed s)
 785     (write-int 0 s)                               ; obsolete
 786     (when (>= (nix-server-minor-version server) 5)
 787       ;; Obsolete `use-atime' and `max-atime' parameters.
 788       (write-int 0 s)
 789       (write-int 0 s))
 790
 791     ;; Loop until the server is done sending error output.
 792     (let loop ((done? (process-stderr server)))
 793       (or done? (loop (process-stderr server))))
 794
 795     (let ((paths    (read-store-path-list s))
 796           (freed    (read-long-long s))
 797           (obsolete (read-long-long s)))
 798       (unless (null? paths)
 799         ;; To be on the safe side, completely invalidate both caches.
 800         ;; Otherwise we could end up returning store paths that are no longer
 801         ;; valid.
 802         (hash-clear! (nix-server-add-to-store-cache server))
 803         (hash-clear! (nix-server-add-text-to-store-cache server)))
 804
 805      (values paths freed))))
 806
 807 (define-syntax-rule (%long-long-max)
 808   ;; Maximum unsigned 64-bit integer.
 809   (- (expt 2 64) 1))
 810
 811 (define (live-paths server)
 812   "Return the list of live store paths---i.e., store paths still
 813 referenced, and thus not subject to being garbage-collected."
 814   (run-gc server (gc-action return-live) '() (%long-long-max)))
 815
 816 (define (dead-paths server)
 817   "Return the list of dead store paths---i.e., store paths no longer
 818 referenced, and thus subject to being garbage-collected."
 819   (run-gc server (gc-action return-dead) '() (%long-long-max)))
 820
 821 (define* (collect-garbage server #:optional (min-freed (%long-long-max)))
 822   "Collect garbage from the store at SERVER.  If MIN-FREED is non-zero,
 823 then collect at least MIN-FREED bytes.  Return the paths that were
 824 collected, and the number of bytes freed."
 825   (run-gc server (gc-action delete-dead) '() min-freed))
 826
 827 (define* (delete-paths server paths #:optional (min-freed (%long-long-max)))
 828   "Delete PATHS from the store at SERVER, if they are no longer
 829 referenced.  If MIN-FREED is non-zero, then stop after at least
 830 MIN-FREED bytes have been collected.  Return the paths that were
 831 collected, and the number of bytes freed."
 832   (run-gc server (gc-action delete-specific) paths min-freed))
 833
 834 (define (import-paths server port)
 835   "Import the set of store paths read from PORT into SERVER's store.  An error
 836 is raised if the set of paths read from PORT is not signed (as per
 837 'export-path #:sign? #t'.)  Return the list of store paths imported."
 838   (let ((s (nix-server-socket server)))
 839     (write-int (operation-id import-paths) s)
 840     (let loop ((done? (process-stderr server port)))
 841       (or done? (loop (process-stderr server port))))
 842     (read-store-path-list s)))
 843
 844 (define* (export-path server path port #:key (sign? #t))
 845   "Export PATH to PORT.  When SIGN? is true, sign it."
 846   (let ((s (nix-server-socket server)))
 847     (write-int (operation-id export-path) s)
 848     (write-store-path path s)
 849     (write-arg boolean sign? s)
 850     (let loop ((done? (process-stderr server port)))
 851       (or done? (loop (process-stderr server port))))
 852     (= 1 (read-int s))))
 853
 854 (define* (export-paths server paths port #:key (sign? #t) recursive?)
 855   "Export the store paths listed in PATHS to PORT, in topological order,
 856 signing them if SIGN? is true.  When RECURSIVE? is true, export the closure of
 857 PATHS---i.e., PATHS and all their dependencies."
 858   (define ordered
 859     (let ((sorted (topologically-sorted server paths)))
 860       ;; When RECURSIVE? is #f, filter out the references of PATHS.
 861       (if recursive?
 862           sorted
 863           (filter (cut member <> paths) sorted))))
 864
 865   (let loop ((paths ordered))
 866     (match paths
 867       (()
 868        (write-int 0 port))
 869       ((head tail ...)
 870        (write-int 1 port)
 871        (and (export-path server head port #:sign? sign?)
 872             (loop tail))))))
 873
 874 (define* (register-path path
 875                         #:key (references '()) deriver prefix
 876                         state-directory)
 877   "Register PATH as a valid store file, with REFERENCES as its list of
 878 references, and DERIVER as its deriver (.drv that led to it.)  If PREFIX is
 879 not #f, it must be the name of the directory containing the new store to
 880 initialize; if STATE-DIRECTORY is not #f, it must be a string containing the
 881 absolute file name to the state directory of the store being initialized.
 882 Return #t on success.
 883
 884 Use with care as it directly modifies the store!  This is primarily meant to
 885 be used internally by the daemon's build hook."
 886   ;; Currently this is implemented by calling out to the fine C++ blob.
 887   (catch 'system-error
 888     (lambda ()
 889       (let ((pipe (apply open-pipe* OPEN_WRITE %guix-register-program
 890                          `(,@(if prefix
 891                                  `("--prefix" ,prefix)
 892                                  '())
 893                            ,@(if state-directory
 894                                  `("--state-directory" ,state-directory)
 895                                  '())))))
 896         (and pipe
 897              (begin
 898                (format pipe "~a~%~a~%~a~%"
 899                        path (or deriver "") (length references))
 900                (for-each (cut format pipe "~a~%" <>) references)
 901                (zero? (close-pipe pipe))))))
 902     (lambda args
 903       ;; Failed to run %GUIX-REGISTER-PROGRAM.
 904       #f)))
 905
 906 \f
 907 ;;;
 908 ;;; Store monad.
 909 ;;;
 910
 911 (define-syntax-rule (define-alias new old)
 912   (define-syntax new (identifier-syntax old)))
 913
 914 ;; The store monad allows us to (1) build sequences of operations in the
 915 ;; store, and (2) make the store an implicit part of the execution context,
 916 ;; rather than a parameter of every single function.
 917 (define-alias %store-monad %state-monad)
 918 (define-alias store-return state-return)
 919 (define-alias store-bind state-bind)
 920
 921 (define (preserve-documentation original proc)
 922   "Return PROC with documentation taken from ORIGINAL."
 923   (set-object-property! proc 'documentation
 924                         (procedure-property original 'documentation))
 925   proc)
 926
 927 (define (store-lift proc)
 928   "Lift PROC, a procedure whose first argument is a connection to the store,
 929 in the store monad."
 930   (preserve-documentation proc
 931                           (lambda args
 932                             (lambda (store)
 933                               (values (apply proc store args) store)))))
 934
 935 (define (store-lower proc)
 936   "Lower PROC, a monadic procedure in %STORE-MONAD, to a \"normal\" procedure
 937 taking the store as its first argument."
 938   (preserve-documentation proc
 939                           (lambda (store . args)
 940                             (run-with-store store (apply proc args)))))
 941
 942 ;;
 943 ;; Store monad operators.
 944 ;;
 945
 946 (define* (text-file name text
 947                     #:optional (references '()))
 948   "Return as a monadic value the absolute file name in the store of the file
 949 containing TEXT, a string.  REFERENCES is a list of store items that the
 950 resulting text file refers to; it defaults to the empty list."
 951   (lambda (store)
 952     (values (add-text-to-store store name text references)
 953             store)))
 954
 955 (define* (interned-file file #:optional name
 956                         #:key (recursive? #t))
 957   "Return the name of FILE once interned in the store.  Use NAME as its store
 958 name, or the basename of FILE if NAME is omitted.
 959
 960 When RECURSIVE? is true, the contents of FILE are added recursively; if FILE
 961 designates a flat file and RECURSIVE? is true, its contents are added, and its
 962 permission bits are kept."
 963   (lambda (store)
 964     (values (add-to-store store (or name (basename file))
 965                           recursive? "sha256" file)
 966             store)))
 967
 968 (define build
 969   ;; Monadic variant of 'build-things'.
 970   (store-lift build-things))
 971
 972 (define %guile-for-build
 973   ;; The derivation of the Guile to be used within the build environment,
 974   ;; when using 'gexp->derivation' and co.
 975   (make-parameter #f))
 976
 977 (define* (run-with-store store mval
 978                          #:key
 979                          (guile-for-build (%guile-for-build))
 980                          (system (%current-system)))
 981   "Run MVAL, a monadic value in the store monad, in STORE, an open store
 982 connection, and return the result."
 983   (parameterize ((%guile-for-build guile-for-build)
 984                  (%current-system system))
 985     (call-with-values (lambda ()
 986                         (run-with-state mval store))
 987       (lambda (result store)
 988         ;; Discard the state.
 989         result))))
 990
 991 \f
 992 ;;;
 993 ;;; Store paths.
 994 ;;;
 995
 996 (define %store-prefix
 997   ;; Absolute path to the Nix store.
 998   (make-parameter %store-directory))
 999
1000 (define (store-path? path)
1001   "Return #t if PATH is a store path."
1002   ;; This is a lightweight check, compared to using a regexp, but this has to
1003   ;; be fast as it's called often in `derivation', for instance.
1004   ;; `isStorePath' in Nix does something similar.
1005   (string-prefix? (%store-prefix) path))
1006
1007 (define (direct-store-path? path)
1008   "Return #t if PATH is a store path, and not a sub-directory of a store path.
1009 This predicate is sometimes needed because files *under* a store path are not
1010 valid inputs."
1011   (and (store-path? path)
1012        (not (string=? path (%store-prefix)))
1013        (let ((len (+ 1 (string-length (%store-prefix)))))
1014          (not (string-index (substring path len) #\/)))))
1015
1016 (define (direct-store-path path)
1017   "Return the direct store path part of PATH, stripping components after
1018 '/gnu/store/xxxx-foo'."
1019   (let ((prefix-length (+ (string-length (%store-prefix)) 35)))
1020     (if (> (string-length path) prefix-length)
1021         (let ((slash (string-index path #\/ prefix-length)))
1022           (if slash (string-take path slash) path))
1023         path)))
1024
1025 (define (derivation-path? path)
1026   "Return #t if PATH is a derivation path."
1027   (and (store-path? path) (string-suffix? ".drv" path)))
1028
1029 (define store-regexp*
1030   ;; The substituter makes repeated calls to 'store-path-hash-part', hence
1031   ;; this optimization.
1032   (memoize
1033    (lambda (store)
1034      "Return a regexp matching a file in STORE."
1035      (make-regexp (string-append "^" (regexp-quote store)
1036                                  "/([0-9a-df-np-sv-z]{32})-([^/]+)$")))))
1037
1038 (define (store-path-package-name path)
1039   "Return the package name part of PATH, a file name in the store."
1040   (let ((path-rx (store-regexp* (%store-prefix))))
1041     (and=> (regexp-exec path-rx path)
1042            (cut match:substring <> 2))))
1043
1044 (define (store-path-hash-part path)
1045   "Return the hash part of PATH as a base32 string, or #f if PATH is not a
1046 syntactically valid store path."
1047   (let ((path-rx (store-regexp* (%store-prefix))))
1048     (and=> (regexp-exec path-rx path)
1049            (cut match:substring <> 1))))
1050
1051 (define (log-file store file)
1052   "Return the build log file for FILE, or #f if none could be found.  FILE
1053 must be an absolute store file name, or a derivation file name."
1054   (cond ((derivation-path? file)
1055          (let* ((base    (basename file))
1056                 (log     (string-append (dirname %state-directory) ; XXX
1057                                         "/log/guix/drvs/"
1058                                         (string-take base 2) "/"
1059                                         (string-drop base 2)))
1060                 (log.bz2 (string-append log ".bz2")))
1061            (cond ((file-exists? log.bz2) log.bz2)
1062                  ((file-exists? log) log)
1063                  (else #f))))
1064         (else
1065          (match (valid-derivers store file)
1066            ((derivers ...)
1067             ;; Return the first that works.
1068             (any (cut log-file store <>) derivers))
1069            (_ #f)))))