guix/store.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2012, 2013, 2014, 2015, 2016 Ludovic Courtès <ludo@gnu.org>
   3 ;;;
   4 ;;; This file is part of GNU Guix.
   5 ;;;
   6 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   7 ;;; under the terms of the GNU General Public License as published by
   8 ;;; the Free Software Foundation; either version 3 of the License, or (at
   9 ;;; your option) any later version.
  10 ;;;
  11 ;;; GNU Guix is distributed in the hope that it will be useful, but
  12 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 ;;; GNU General Public License for more details.
  15 ;;;
  16 ;;; You should have received a copy of the GNU General Public License
  17 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 (define-module (guix store)
  20   #:use-module (guix utils)
  21   #:use-module (guix config)
  22   #:use-module (guix serialization)
  23   #:use-module (guix monads)
  24   #:autoload   (guix base32) (bytevector->base32-string)
  25   #:autoload   (guix build syscalls) (terminal-columns)
  26   #:use-module (rnrs bytevectors)
  27   #:use-module (rnrs io ports)
  28   #:use-module (srfi srfi-1)
  29   #:use-module (srfi srfi-9)
  30   #:use-module (srfi srfi-9 gnu)
  31   #:use-module (srfi srfi-11)
  32   #:use-module (srfi srfi-26)
  33   #:use-module (srfi srfi-34)
  34   #:use-module (srfi srfi-35)
  35   #:use-module (srfi srfi-39)
  36   #:use-module (ice-9 match)
  37   #:use-module (ice-9 regex)
  38   #:use-module (ice-9 vlist)
  39   #:use-module (ice-9 popen)
  40   #:export (%daemon-socket-file
  41             %gc-roots-directory
  42             %default-substitute-urls
  43
  44             nix-server?
  45             nix-server-major-version
  46             nix-server-minor-version
  47             nix-server-socket
  48
  49             &nix-error nix-error?
  50             &nix-connection-error nix-connection-error?
  51             nix-connection-error-file
  52             nix-connection-error-code
  53             &nix-protocol-error nix-protocol-error?
  54             nix-protocol-error-message
  55             nix-protocol-error-status
  56
  57             hash-algo
  58             build-mode
  59
  60             open-connection
  61             close-connection
  62             with-store
  63             set-build-options
  64             set-build-options*
  65             valid-path?
  66             query-path-hash
  67             hash-part->path
  68             query-path-info
  69             add-text-to-store
  70             add-to-store
  71             build-things
  72             build
  73             query-failed-paths
  74             clear-failed-paths
  75             add-temp-root
  76             add-indirect-root
  77             add-permanent-root
  78             remove-permanent-root
  79
  80             substitutable?
  81             substitutable-path
  82             substitutable-deriver
  83             substitutable-references
  84             substitutable-download-size
  85             substitutable-nar-size
  86             has-substitutes?
  87             substitutable-paths
  88             substitutable-path-info
  89
  90             path-info?
  91             path-info-deriver
  92             path-info-hash
  93             path-info-references
  94             path-info-registration-time
  95             path-info-nar-size
  96
  97             references
  98             references/substitutes
  99             requisites
 100             referrers
 101             optimize-store
 102             verify-store
 103             topologically-sorted
 104             valid-derivers
 105             query-derivation-outputs
 106             live-paths
 107             dead-paths
 108             collect-garbage
 109             delete-paths
 110             import-paths
 111             export-paths
 112
 113             current-build-output-port
 114
 115             register-path
 116
 117             %store-monad
 118             store-bind
 119             store-return
 120             store-lift
 121             store-lower
 122             run-with-store
 123             %guile-for-build
 124             current-system
 125             set-current-system
 126             text-file
 127             interned-file
 128
 129             %store-prefix
 130             store-path?
 131             direct-store-path?
 132             derivation-path?
 133             store-path-package-name
 134             store-path-hash-part
 135             direct-store-path
 136             log-file))
 137
 138 (define %protocol-version #x10f)
 139
 140 (define %worker-magic-1 #x6e697863)               ; "nixc"
 141 (define %worker-magic-2 #x6478696f)               ; "dxio"
 142
 143 (define (protocol-major magic)
 144   (logand magic #xff00))
 145 (define (protocol-minor magic)
 146   (logand magic #x00ff))
 147
 148 (define-syntax define-enumerate-type
 149   (syntax-rules ()
 150     ((_ name->int (name id) ...)
 151      (define-syntax name->int
 152        (syntax-rules (name ...)
 153          ((_ name) id) ...)))))
 154
 155 (define-enumerate-type operation-id
 156   ;; operation numbers from worker-protocol.hh
 157   (quit 0)
 158   (valid-path? 1)
 159   (has-substitutes? 3)
 160   (query-path-hash 4)
 161   (query-references 5)
 162   (query-referrers 6)
 163   (add-to-store 7)
 164   (add-text-to-store 8)
 165   (build-things 9)
 166   (ensure-path 10)
 167   (add-temp-root 11)
 168   (add-indirect-root 12)
 169   (sync-with-gc 13)
 170   (find-roots 14)
 171   (export-path 16)
 172   (query-deriver 18)
 173   (set-options 19)
 174   (collect-garbage 20)
 175   ;;(query-substitutable-path-info 21)  ; obsolete as of #x10c
 176   (query-derivation-outputs 22)
 177   (query-all-valid-paths 23)
 178   (query-failed-paths 24)
 179   (clear-failed-paths 25)
 180   (query-path-info 26)
 181   (import-paths 27)
 182   (query-derivation-output-names 28)
 183   (query-path-from-hash-part 29)
 184   (query-substitutable-path-infos 30)
 185   (query-valid-paths 31)
 186   (query-substitutable-paths 32)
 187   (query-valid-derivers 33)
 188   (optimize-store 34)
 189   (verify-store 35))
 190
 191 (define-enumerate-type hash-algo
 192   ;; hash.hh
 193   (md5 1)
 194   (sha1 2)
 195   (sha256 3))
 196
 197 (define-enumerate-type build-mode
 198   ;; store-api.hh
 199   (normal 0)
 200   (repair 1)
 201   (check 2))
 202
 203 (define-enumerate-type gc-action
 204   ;; store-api.hh
 205   (return-live 0)
 206   (return-dead 1)
 207   (delete-dead 2)
 208   (delete-specific 3))
 209
 210 (define %default-socket-path
 211   (string-append %state-directory "/daemon-socket/socket"))
 212
 213 (define %daemon-socket-file
 214   ;; File name of the socket the daemon listens too.
 215   (make-parameter (or (getenv "GUIX_DAEMON_SOCKET")
 216                       %default-socket-path)))
 217
 218
 219 \f
 220 ;; Information about a substitutable store path.
 221 (define-record-type <substitutable>
 222   (substitutable path deriver refs dl-size nar-size)
 223   substitutable?
 224   (path      substitutable-path)
 225   (deriver   substitutable-deriver)
 226   (refs      substitutable-references)
 227   (dl-size   substitutable-download-size)
 228   (nar-size  substitutable-nar-size))
 229
 230 (define (read-substitutable-path-list p)
 231   (let loop ((len    (read-int p))
 232              (result '()))
 233     (if (zero? len)
 234         (reverse result)
 235         (let ((path     (read-store-path p))
 236               (deriver  (read-store-path p))
 237               (refs     (read-store-path-list p))
 238               (dl-size  (read-long-long p))
 239               (nar-size (read-long-long p)))
 240           (loop (- len 1)
 241                 (cons (substitutable path deriver refs dl-size nar-size)
 242                       result))))))
 243
 244 ;; Information about a store path.
 245 (define-record-type <path-info>
 246   (path-info deriver hash references registration-time nar-size)
 247   path-info?
 248   (deriver path-info-deriver)                     ;string | #f
 249   (hash path-info-hash)
 250   (references path-info-references)
 251   (registration-time path-info-registration-time)
 252   (nar-size path-info-nar-size))
 253
 254 (define (read-path-info p)
 255   (let ((deriver  (match (read-store-path p)
 256                     ("" #f)
 257                     (x  x)))
 258         (hash     (base16-string->bytevector (read-string p)))
 259         (refs     (read-store-path-list p))
 260         (registration-time (read-int p))
 261         (nar-size (read-long-long p)))
 262     (path-info deriver hash refs registration-time nar-size)))
 263
 264 (define-syntax write-arg
 265   (syntax-rules (integer boolean file string string-list string-pairs
 266                  store-path store-path-list base16)
 267     ((_ integer arg p)
 268      (write-int arg p))
 269     ((_ boolean arg p)
 270      (write-int (if arg 1 0) p))
 271     ((_ file arg p)
 272      (write-file arg p))
 273     ((_ string arg p)
 274      (write-string arg p))
 275     ((_ string-list arg p)
 276      (write-string-list arg p))
 277     ((_ string-pairs arg p)
 278      (write-string-pairs arg p))
 279     ((_ store-path arg p)
 280      (write-store-path arg p))
 281     ((_ store-path-list arg p)
 282      (write-store-path-list arg p))
 283     ((_ base16 arg p)
 284      (write-string (bytevector->base16-string arg) p))))
 285
 286 (define-syntax read-arg
 287   (syntax-rules (integer boolean string store-path store-path-list
 288                  substitutable-path-list path-info base16)
 289     ((_ integer p)
 290      (read-int p))
 291     ((_ boolean p)
 292      (not (zero? (read-int p))))
 293     ((_ string p)
 294      (read-string p))
 295     ((_ store-path p)
 296      (read-store-path p))
 297     ((_ store-path-list p)
 298      (read-store-path-list p))
 299     ((_ substitutable-path-list p)
 300      (read-substitutable-path-list p))
 301     ((_ path-info p)
 302      (read-path-info p))
 303     ((_ base16 p)
 304      (base16-string->bytevector (read-string p)))))
 305
 306 \f
 307 ;; remote-store.cc
 308
 309 (define-record-type <nix-server>
 310   (%make-nix-server socket major minor
 311                     ats-cache atts-cache)
 312   nix-server?
 313   (socket nix-server-socket)
 314   (major  nix-server-major-version)
 315   (minor  nix-server-minor-version)
 316
 317   ;; Caches.  We keep them per-connection, because store paths build
 318   ;; during the session are temporary GC roots kept for the duration of
 319   ;; the session.
 320   (ats-cache  nix-server-add-to-store-cache)
 321   (atts-cache nix-server-add-text-to-store-cache))
 322
 323 (set-record-type-printer! <nix-server>
 324                           (lambda (obj port)
 325                             (format port "#<build-daemon ~a.~a ~a>"
 326                                     (nix-server-major-version obj)
 327                                     (nix-server-minor-version obj)
 328                                     (number->string (object-address obj)
 329                                                     16))))
 330
 331 (define-condition-type &nix-error &error
 332   nix-error?)
 333
 334 (define-condition-type &nix-connection-error &nix-error
 335   nix-connection-error?
 336   (file   nix-connection-error-file)
 337   (errno  nix-connection-error-code))
 338
 339 (define-condition-type &nix-protocol-error &nix-error
 340   nix-protocol-error?
 341   (message nix-protocol-error-message)
 342   (status  nix-protocol-error-status))
 343
 344 (define* (open-connection #:optional (file (%daemon-socket-file))
 345                           #:key (reserve-space? #t) cpu-affinity)
 346   "Connect to the daemon over the Unix-domain socket at FILE.  When
 347 RESERVE-SPACE? is true, instruct it to reserve a little bit of extra space on
 348 the file system so that the garbage collector can still operate, should the
 349 disk become full.  When CPU-AFFINITY is true, it must be an integer
 350 corresponding to an OS-level CPU number to which the daemon's worker process
 351 for this connection will be pinned.  Return a server object."
 352   (let ((s (with-fluids ((%default-port-encoding #f))
 353              ;; This trick allows use of the `scm_c_read' optimization.
 354              (socket PF_UNIX SOCK_STREAM 0)))
 355         (a (make-socket-address PF_UNIX file)))
 356
 357     (catch 'system-error
 358       (cut connect s a)
 359       (lambda args
 360         ;; Translate the error to something user-friendly.
 361         (let ((errno (system-error-errno args)))
 362           (raise (condition (&nix-connection-error
 363                              (file file)
 364                              (errno errno)))))))
 365
 366     (write-int %worker-magic-1 s)
 367     (let ((r (read-int s)))
 368       (and (eqv? r %worker-magic-2)
 369            (let ((v (read-int s)))
 370              (and (eqv? (protocol-major %protocol-version)
 371                         (protocol-major v))
 372                   (begin
 373                     (write-int %protocol-version s)
 374                     (when (>= (protocol-minor v) 14)
 375                       (write-int (if cpu-affinity 1 0) s)
 376                       (when cpu-affinity
 377                         (write-int cpu-affinity s)))
 378                     (when (>= (protocol-minor v) 11)
 379                       (write-int (if reserve-space? 1 0) s))
 380                     (let ((s (%make-nix-server s
 381                                                (protocol-major v)
 382                                                (protocol-minor v)
 383                                                (make-hash-table 100)
 384                                                (make-hash-table 100))))
 385                       (let loop ((done? (process-stderr s)))
 386                         (or done? (process-stderr s)))
 387                       s))))))))
 388
 389 (define (close-connection server)
 390   "Close the connection to SERVER."
 391   (close (nix-server-socket server)))
 392
 393 (define-syntax-rule (with-store store exp ...)
 394   "Bind STORE to an open connection to the store and evaluate EXPs;
 395 automatically close the store when the dynamic extent of EXP is left."
 396   (let ((store (open-connection)))
 397     (dynamic-wind
 398       (const #f)
 399       (lambda ()
 400         exp ...)
 401       (lambda ()
 402         (false-if-exception (close-connection store))))))
 403
 404 (define current-build-output-port
 405   ;; The port where build output is sent.
 406   (make-parameter (current-error-port)))
 407
 408 (define* (dump-port in out
 409                     #:optional len
 410                     #:key (buffer-size 16384))
 411   "Read LEN bytes from IN (or as much as possible if LEN is #f) and write it
 412 to OUT, using chunks of BUFFER-SIZE bytes."
 413   (define buffer
 414     (make-bytevector buffer-size))
 415
 416   (let loop ((total 0)
 417              (bytes (get-bytevector-n! in buffer 0
 418                                        (if len
 419                                            (min len buffer-size)
 420                                            buffer-size))))
 421     (or (eof-object? bytes)
 422         (and len (= total len))
 423         (let ((total (+ total bytes)))
 424           (put-bytevector out buffer 0 bytes)
 425           (loop total
 426                 (get-bytevector-n! in buffer 0
 427                                    (if len
 428                                        (min (- len total) buffer-size)
 429                                        buffer-size)))))))
 430
 431 (define %newlines
 432   ;; Newline characters triggering a flush of 'current-build-output-port'.
 433   ;; Unlike Guile's _IOLBF, we flush upon #\return so that progress reports
 434   ;; that use that trick are correctly displayed.
 435   (char-set #\newline #\return))
 436
 437 (define* (process-stderr server #:optional user-port)
 438   "Read standard output and standard error from SERVER, writing it to
 439 CURRENT-BUILD-OUTPUT-PORT.  Return #t when SERVER is done sending data, and
 440 #f otherwise; in the latter case, the caller should call `process-stderr'
 441 again until #t is returned or an error is raised.
 442
 443 Since the build process's output cannot be assumed to be UTF-8, we
 444 conservatively consider it to be Latin-1, thereby avoiding possible
 445 encoding conversion errors."
 446   (define p
 447     (nix-server-socket server))
 448
 449   ;; magic cookies from worker-protocol.hh
 450   (define %stderr-next  #x6f6c6d67)          ; "olmg", build log
 451   (define %stderr-read  #x64617461)          ; "data", data needed from source
 452   (define %stderr-write #x64617416)          ; "dat\x16", data for sink
 453   (define %stderr-last  #x616c7473)          ; "alts", we're done
 454   (define %stderr-error #x63787470)          ; "cxtp", error reporting
 455
 456   (let ((k (read-int p)))
 457     (cond ((= k %stderr-write)
 458            ;; Write a byte stream to USER-PORT.
 459            (let* ((len (read-int p))
 460                   (m   (modulo len 8)))
 461              (dump-port p user-port len)
 462              (unless (zero? m)
 463                ;; Consume padding, as for strings.
 464                (get-bytevector-n p (- 8 m))))
 465            #f)
 466           ((= k %stderr-read)
 467            ;; Read a byte stream from USER-PORT.
 468            ;; Note: Avoid 'get-bytevector-n' to work around
 469            ;; <http://bugs.gnu.org/17591> in Guile up to 2.0.11.
 470            (let* ((max-len (read-int p))
 471                   (data    (make-bytevector max-len))
 472                   (len     (get-bytevector-n! user-port data 0 max-len)))
 473              (write-int len p)
 474              (put-bytevector p data 0 len)
 475              (write-padding len p)
 476              #f))
 477           ((= k %stderr-next)
 478            ;; Log a string.  Build logs are usually UTF-8-encoded, but they
 479            ;; may also contain arbitrary byte sequences that should not cause
 480            ;; this to fail.  Thus, use the permissive
 481            ;; 'read-maybe-utf8-string'.
 482            (let ((s (read-maybe-utf8-string p)))
 483              (display s (current-build-output-port))
 484              (when (string-any %newlines s)
 485                (flush-output-port (current-build-output-port)))
 486              #f))
 487           ((= k %stderr-error)
 488            ;; Report an error.
 489            (let ((error  (read-maybe-utf8-string p))
 490                  ;; Currently the daemon fails to send a status code for early
 491                  ;; errors like DB schema version mismatches, so check for EOF.
 492                  (status (if (and (>= (nix-server-minor-version server) 8)
 493                                   (not (eof-object? (lookahead-u8 p))))
 494                              (read-int p)
 495                              1)))
 496              (raise (condition (&nix-protocol-error
 497                                 (message error)
 498                                 (status  status))))))
 499           ((= k %stderr-last)
 500            ;; The daemon is done (see `stopWork' in `nix-worker.cc'.)
 501            #t)
 502           (else
 503            (raise (condition (&nix-protocol-error
 504                               (message "invalid error code")
 505                               (status   k))))))))
 506
 507 (define %default-substitute-urls
 508   ;; Default list of substituters.  This is *not* the list baked in
 509   ;; 'guix-daemon', but it is used by 'guix-service-type' and and a couple of
 510   ;; clients ('guix build --log-file' uses it.)
 511   (map (if (false-if-exception (resolve-interface '(gnutls)))
 512            (cut string-append "https://" <>)
 513            (cut string-append "http://" <>))
 514        '("mirror.hydra.gnu.org" "hydra.gnu.org")))
 515
 516 (define* (set-build-options server
 517                             #:key keep-failed? keep-going? fallback?
 518                             (verbosity 0)
 519                             rounds                ;number of build rounds
 520                             (max-build-jobs 1)
 521                             timeout
 522                             (max-silent-time 3600)
 523                             (use-build-hook? #t)
 524                             (build-verbosity 0)
 525                             (log-type 0)
 526                             (print-build-trace #t)
 527                             (build-cores (current-processor-count))
 528                             (use-substitutes? #t)
 529
 530                             ;; Client-provided substitute URLs.  If it is #f,
 531                             ;; the daemon's settings are used.  Otherwise, it
 532                             ;; overrides the daemons settings; see 'guix
 533                             ;; substitute'.
 534                             (substitute-urls #f)
 535
 536                             ;; Number of columns in the client's terminal.
 537                             (terminal-columns (terminal-columns))
 538
 539                             ;; Locale of the client.
 540                             (locale (false-if-exception (setlocale LC_ALL))))
 541   ;; Must be called after `open-connection'.
 542
 543   (define socket
 544     (nix-server-socket server))
 545
 546   (let-syntax ((send (syntax-rules ()
 547                        ((_ (type option) ...)
 548                         (begin
 549                           (write-arg type option socket)
 550                           ...)))))
 551     (write-int (operation-id set-options) socket)
 552     (send (boolean keep-failed?) (boolean keep-going?)
 553           (boolean fallback?) (integer verbosity)
 554           (integer max-build-jobs) (integer max-silent-time))
 555     (when (>= (nix-server-minor-version server) 2)
 556       (send (boolean use-build-hook?)))
 557     (when (>= (nix-server-minor-version server) 4)
 558       (send (integer build-verbosity) (integer log-type)
 559             (boolean print-build-trace)))
 560     (when (>= (nix-server-minor-version server) 6)
 561       (send (integer build-cores)))
 562     (when (>= (nix-server-minor-version server) 10)
 563       (send (boolean use-substitutes?)))
 564     (when (>= (nix-server-minor-version server) 12)
 565       (let ((pairs `(,@(if timeout
 566                            `(("build-timeout" . ,(number->string timeout)))
 567                            '())
 568                      ,@(if substitute-urls
 569                            `(("substitute-urls"
 570                               . ,(string-join substitute-urls)))
 571                            '())
 572                      ,@(if rounds
 573                            `(("build-repeat"
 574                               . ,(number->string (max 0 (1- rounds)))))
 575                            '())
 576                      ,@(if terminal-columns
 577                            `(("terminal-columns"
 578                               . ,(number->string terminal-columns)))
 579                            '())
 580                      ,@(if locale
 581                            `(("locale" . ,locale))
 582                            '()))))
 583         (send (string-pairs pairs))))
 584     (let loop ((done? (process-stderr server)))
 585       (or done? (process-stderr server)))))
 586
 587 (define-syntax operation
 588   (syntax-rules ()
 589     "Define a client-side RPC stub for the given operation."
 590     ((_ (name (type arg) ...) docstring return ...)
 591      (lambda (server arg ...)
 592        docstring
 593        (let ((s (nix-server-socket server)))
 594          (write-int (operation-id name) s)
 595          (write-arg type arg s)
 596          ...
 597          ;; Loop until the server is done sending error output.
 598          (let loop ((done? (process-stderr server)))
 599            (or done? (loop (process-stderr server))))
 600          (values (read-arg return s) ...))))))
 601
 602 (define-syntax-rule (define-operation (name args ...)
 603                       docstring return ...)
 604   (define name
 605     (operation (name args ...) docstring return ...)))
 606
 607 (define-operation (valid-path? (string path))
 608   "Return #t when PATH designates a valid store item and #f otherwise (an
 609 invalid item may exist on disk but still be invalid, for instance because it
 610 is the result of an aborted or failed build.)
 611
 612 A '&nix-protocol-error' condition is raised if PATH is not prefixed by the
 613 store directory (/gnu/store)."
 614   boolean)
 615
 616 (define-operation (query-path-hash (store-path path))
 617   "Return the SHA256 hash of PATH as a bytevector."
 618   base16)
 619
 620 (define hash-part->path
 621   (let ((query-path-from-hash-part
 622          (operation (query-path-from-hash-part (string hash))
 623                     #f
 624                     store-path)))
 625    (lambda (server hash-part)
 626      "Return the store path whose hash part is HASH-PART (a nix-base32
 627 string).  Raise an error if no such path exists."
 628      ;; This RPC is primarily used by Hydra to reply to HTTP GETs of
 629      ;; /HASH.narinfo.
 630      (query-path-from-hash-part server hash-part))))
 631
 632 (define-operation (query-path-info (store-path path))
 633   "Return the info (hash, references, etc.) for PATH."
 634   path-info)
 635
 636 (define add-text-to-store
 637   ;; A memoizing version of `add-to-store', to avoid repeated RPCs with
 638   ;; the very same arguments during a given session.
 639   (let ((add-text-to-store
 640          (operation (add-text-to-store (string name) (string text)
 641                                        (string-list references))
 642                     #f
 643                     store-path)))
 644     (lambda* (server name text #:optional (references '()))
 645       "Add TEXT under file NAME in the store, and return its store path.
 646 REFERENCES is the list of store paths referred to by the resulting store
 647 path."
 648       (let ((args  `(,text ,name ,references))
 649             (cache (nix-server-add-text-to-store-cache server)))
 650         (or (hash-ref cache args)
 651             (let ((path (add-text-to-store server name text references)))
 652               (hash-set! cache args path)
 653               path))))))
 654
 655 (define add-to-store
 656   ;; A memoizing version of `add-to-store'.  This is important because
 657   ;; `add-to-store' leads to huge data transfers to the server, and
 658   ;; because it's often called many times with the very same argument.
 659   (let ((add-to-store (operation (add-to-store (string basename)
 660                                                (boolean fixed?) ; obsolete, must be #t
 661                                                (boolean recursive?)
 662                                                (string hash-algo)
 663                                                (file file-name))
 664                                  #f
 665                                  store-path)))
 666     (lambda (server basename recursive? hash-algo file-name)
 667       "Add the contents of FILE-NAME under BASENAME to the store.  When
 668 RECURSIVE? is false, FILE-NAME must designate a regular file--not a directory
 669 nor a symlink.  When RECURSIVE? is true and FILE-NAME designates a directory,
 670 the contents of FILE-NAME are added recursively; if FILE-NAME designates a
 671 flat file and RECURSIVE? is true, its contents are added, and its permission
 672 bits are kept.  HASH-ALGO must be a string such as \"sha256\"."
 673       (let* ((st    (false-if-exception (lstat file-name)))
 674              (args  `(,st ,basename ,recursive? ,hash-algo))
 675              (cache (nix-server-add-to-store-cache server)))
 676         (or (and st (hash-ref cache args))
 677             (let ((path (add-to-store server basename #t recursive?
 678                                       hash-algo file-name)))
 679               (hash-set! cache args path)
 680               path))))))
 681
 682 (define build-things
 683   (let ((build (operation (build-things (string-list things)
 684                                         (integer mode))
 685                           "Do it!"
 686                           boolean))
 687         (build/old (operation (build-things (string-list things))
 688                               "Do it!"
 689                               boolean)))
 690     (lambda* (store things #:optional (mode (build-mode normal)))
 691       "Build THINGS, a list of store items which may be either '.drv' files or
 692 outputs, and return when the worker is done building them.  Elements of THINGS
 693 that are not derivations can only be substituted and not built locally.
 694 Return #t on success."
 695       (if (>= (nix-server-minor-version store) 15)
 696           (build store things mode)
 697           (if (= mode (build-mode normal))
 698               (build/old store things)
 699               (raise (condition (&nix-protocol-error
 700                                  (message "unsupported build mode")
 701                                  (status  1)))))))))
 702
 703 (define-operation (add-temp-root (store-path path))
 704   "Make PATH a temporary root for the duration of the current session.
 705 Return #t."
 706   boolean)
 707
 708 (define-operation (add-indirect-root (string file-name))
 709   "Make the symlink FILE-NAME an indirect root for the garbage collector:
 710 whatever store item FILE-NAME points to will not be collected.  Return #t on
 711 success.
 712
 713 FILE-NAME can be anywhere on the file system, but it must be an absolute file
 714 name--it is the caller's responsibility to ensure that it is an absolute file
 715 name."
 716   boolean)
 717
 718 (define %gc-roots-directory
 719   ;; The place where garbage collector roots (symlinks) are kept.
 720   (string-append %state-directory "/gcroots"))
 721
 722 (define (add-permanent-root target)
 723   "Add a garbage collector root pointing to TARGET, an element of the store,
 724 preventing TARGET from even being collected.  This can also be used if TARGET
 725 does not exist yet.
 726
 727 Raise an error if the caller does not have write access to the GC root
 728 directory."
 729   (let* ((root (string-append %gc-roots-directory "/" (basename target))))
 730     (catch 'system-error
 731       (lambda ()
 732         (symlink target root))
 733       (lambda args
 734         ;; If ROOT already exists, this is fine; otherwise, re-throw.
 735         (unless (= EEXIST (system-error-errno args))
 736           (apply throw args))))))
 737
 738 (define (remove-permanent-root target)
 739   "Remove the permanent garbage collector root pointing to TARGET.  Raise an
 740 error if there is no such root."
 741   (delete-file (string-append %gc-roots-directory "/" (basename target))))
 742
 743 (define references
 744   (operation (query-references (store-path path))
 745              "Return the list of references of PATH."
 746              store-path-list))
 747
 748 (define %reference-cache
 749   ;; Brute-force cache mapping store items to their list of references.
 750   ;; Caching matters because when building a profile in the presence of
 751   ;; grafts, we keep calling 'graft-derivation', which in turn calls
 752   ;; 'references/substitutes' many times with the same arguments.  Ideally we
 753   ;; would use a cache associated with the daemon connection instead (XXX).
 754   (make-hash-table 100))
 755
 756 (define (references/substitutes store items)
 757   "Return the list of list of references of ITEMS; the result has the same
 758 length as ITEMS.  Query substitute information for any item missing from the
 759 store at once.  Raise a '&nix-protocol-error' exception if reference
 760 information for one of ITEMS is missing."
 761   (let* ((local-refs (map (lambda (item)
 762                             (or (hash-ref %reference-cache item)
 763                                 (guard (c ((nix-protocol-error? c) #f))
 764                                   (references store item))))
 765                           items))
 766          (missing    (fold-right (lambda (item local-ref result)
 767                                    (if local-ref
 768                                        result
 769                                        (cons item result)))
 770                                  '()
 771                                  items local-refs))
 772
 773          ;; Query all the substitutes at once to minimize the cost of
 774          ;; launching 'guix substitute' and making HTTP requests.
 775          (substs     (substitutable-path-info store missing)))
 776     (when (< (length substs) (length missing))
 777       (raise (condition (&nix-protocol-error
 778                          (message "cannot determine \
 779 the list of references")
 780                          (status 1)))))
 781
 782     ;; Intersperse SUBSTS and LOCAL-REFS.
 783     (let loop ((items       items)
 784                (local-refs  local-refs)
 785                (result      '()))
 786       (match items
 787         (()
 788          (let ((result (reverse result)))
 789            (for-each (cut hash-set! %reference-cache <> <>)
 790                      items result)
 791            result))
 792         ((item items ...)
 793          (match local-refs
 794            ((#f tail ...)
 795             (loop items tail
 796                   (cons (any (lambda (subst)
 797                                (and (string=? (substitutable-path subst) item)
 798                                     (substitutable-references subst)))
 799                              substs)
 800                         result)))
 801            ((head tail ...)
 802             (loop items tail
 803                   (cons head result)))))))))
 804
 805 (define* (fold-path store proc seed path
 806                     #:optional (relatives (cut references store <>)))
 807   "Call PROC for each of the RELATIVES of PATH, exactly once, and return the
 808 result formed from the successive calls to PROC, the first of which is passed
 809 SEED."
 810   (let loop ((paths  (list path))
 811              (result seed)
 812              (seen   vlist-null))
 813     (match paths
 814       ((path rest ...)
 815        (if (vhash-assoc path seen)
 816            (loop rest result seen)
 817            (let ((seen   (vhash-cons path #t seen))
 818                  (rest   (append rest (relatives path)))
 819                  (result (proc path result)))
 820              (loop rest result seen))))
 821       (()
 822        result))))
 823
 824 (define (requisites store path)
 825   "Return the requisites of PATH, including PATH---i.e., its closure (all its
 826 references, recursively)."
 827   (fold-path store cons '() path))
 828
 829 (define (topologically-sorted store paths)
 830   "Return a list containing PATHS and all their references sorted in
 831 topological order."
 832   (define (traverse)
 833     ;; Do a simple depth-first traversal of all of PATHS.
 834     (let loop ((paths   paths)
 835                (visited vlist-null)
 836                (result  '()))
 837       (define (visit n)
 838         (vhash-cons n #t visited))
 839
 840       (define (visited? n)
 841         (vhash-assoc n visited))
 842
 843       (match paths
 844         ((head tail ...)
 845          (if (visited? head)
 846              (loop tail visited result)
 847              (call-with-values
 848                  (lambda ()
 849                    (loop (references store head)
 850                          (visit head)
 851                          result))
 852                (lambda (visited result)
 853                  (loop tail
 854                        visited
 855                        (cons head result))))))
 856         (()
 857          (values visited result)))))
 858
 859   (call-with-values traverse
 860     (lambda (_ result)
 861       (reverse result))))
 862
 863 (define referrers
 864   (operation (query-referrers (store-path path))
 865              "Return the list of path that refer to PATH."
 866              store-path-list))
 867
 868 (define valid-derivers
 869   (operation (query-valid-derivers (store-path path))
 870              "Return the list of valid \"derivers\" of PATH---i.e., all the
 871 .drv present in the store that have PATH among their outputs."
 872              store-path-list))
 873
 874 (define query-derivation-outputs  ; avoid name clash with `derivation-outputs'
 875   (operation (query-derivation-outputs (store-path path))
 876              "Return the list of outputs of PATH, a .drv file."
 877              store-path-list))
 878
 879 (define-operation (has-substitutes? (store-path path))
 880   "Return #t if binary substitutes are available for PATH, and #f otherwise."
 881   boolean)
 882
 883 (define substitutable-paths
 884   (operation (query-substitutable-paths (store-path-list paths))
 885              "Return the subset of PATHS that is substitutable."
 886              store-path-list))
 887
 888 (define substitutable-path-info
 889   (operation (query-substitutable-path-infos (store-path-list paths))
 890              "Return information about the subset of PATHS that is
 891 substitutable.  For each substitutable path, a `substitutable?' object is
 892 returned; thus, the resulting list can be shorter than PATHS.  Furthermore,
 893 that there is no guarantee that the order of the resulting list matches the
 894 order of PATHS."
 895              substitutable-path-list))
 896
 897 (define-operation (optimize-store)
 898   "Optimize the store by hard-linking identical files (\"deduplication\".)
 899 Return #t on success."
 900   ;; Note: the daemon in Guix <= 0.8.2 does not implement this RPC.
 901   boolean)
 902
 903 (define verify-store
 904   (let ((verify (operation (verify-store (boolean check-contents?)
 905                                          (boolean repair?))
 906                            "Verify the store."
 907                            boolean)))
 908     (lambda* (store #:key check-contents? repair?)
 909       "Verify the integrity of the store and return false if errors remain,
 910 and true otherwise.  When REPAIR? is true, repair any missing or altered store
 911 items by substituting them (this typically requires root privileges because it
 912 is not an atomic operation.)  When CHECK-CONTENTS? is true, check the contents
 913 of store items; this can take a lot of time."
 914       (not (verify store check-contents? repair?)))))
 915
 916 (define (run-gc server action to-delete min-freed)
 917   "Perform the garbage-collector operation ACTION, one of the
 918 `gc-action' values.  When ACTION is `delete-specific', the TO-DELETE is
 919 the list of store paths to delete.  IGNORE-LIVENESS? should always be
 920 #f.  MIN-FREED is the minimum amount of disk space to be freed, in
 921 bytes, before the GC can stop.  Return the list of store paths delete,
 922 and the number of bytes freed."
 923   (let ((s (nix-server-socket server)))
 924     (write-int (operation-id collect-garbage) s)
 925     (write-int action s)
 926     (write-store-path-list to-delete s)
 927     (write-arg boolean #f s)                      ; ignore-liveness?
 928     (write-long-long min-freed s)
 929     (write-int 0 s)                               ; obsolete
 930     (when (>= (nix-server-minor-version server) 5)
 931       ;; Obsolete `use-atime' and `max-atime' parameters.
 932       (write-int 0 s)
 933       (write-int 0 s))
 934
 935     ;; Loop until the server is done sending error output.
 936     (let loop ((done? (process-stderr server)))
 937       (or done? (loop (process-stderr server))))
 938
 939     (let ((paths    (read-store-path-list s))
 940           (freed    (read-long-long s))
 941           (obsolete (read-long-long s)))
 942       (unless (null? paths)
 943         ;; To be on the safe side, completely invalidate both caches.
 944         ;; Otherwise we could end up returning store paths that are no longer
 945         ;; valid.
 946         (hash-clear! (nix-server-add-to-store-cache server))
 947         (hash-clear! (nix-server-add-text-to-store-cache server)))
 948
 949      (values paths freed))))
 950
 951 (define-syntax-rule (%long-long-max)
 952   ;; Maximum unsigned 64-bit integer.
 953   (- (expt 2 64) 1))
 954
 955 (define (live-paths server)
 956   "Return the list of live store paths---i.e., store paths still
 957 referenced, and thus not subject to being garbage-collected."
 958   (run-gc server (gc-action return-live) '() (%long-long-max)))
 959
 960 (define (dead-paths server)
 961   "Return the list of dead store paths---i.e., store paths no longer
 962 referenced, and thus subject to being garbage-collected."
 963   (run-gc server (gc-action return-dead) '() (%long-long-max)))
 964
 965 (define* (collect-garbage server #:optional (min-freed (%long-long-max)))
 966   "Collect garbage from the store at SERVER.  If MIN-FREED is non-zero,
 967 then collect at least MIN-FREED bytes.  Return the paths that were
 968 collected, and the number of bytes freed."
 969   (run-gc server (gc-action delete-dead) '() min-freed))
 970
 971 (define* (delete-paths server paths #:optional (min-freed (%long-long-max)))
 972   "Delete PATHS from the store at SERVER, if they are no longer
 973 referenced.  If MIN-FREED is non-zero, then stop after at least
 974 MIN-FREED bytes have been collected.  Return the paths that were
 975 collected, and the number of bytes freed."
 976   (run-gc server (gc-action delete-specific) paths min-freed))
 977
 978 (define (import-paths server port)
 979   "Import the set of store paths read from PORT into SERVER's store.  An error
 980 is raised if the set of paths read from PORT is not signed (as per
 981 'export-path #:sign? #t'.)  Return the list of store paths imported."
 982   (let ((s (nix-server-socket server)))
 983     (write-int (operation-id import-paths) s)
 984     (let loop ((done? (process-stderr server port)))
 985       (or done? (loop (process-stderr server port))))
 986     (read-store-path-list s)))
 987
 988 (define* (export-path server path port #:key (sign? #t))
 989   "Export PATH to PORT.  When SIGN? is true, sign it."
 990   (let ((s (nix-server-socket server)))
 991     (write-int (operation-id export-path) s)
 992     (write-store-path path s)
 993     (write-arg boolean sign? s)
 994     (let loop ((done? (process-stderr server port)))
 995       (or done? (loop (process-stderr server port))))
 996     (= 1 (read-int s))))
 997
 998 (define* (export-paths server paths port #:key (sign? #t) recursive?)
 999   "Export the store paths listed in PATHS to PORT, in topological order,
1000 signing them if SIGN? is true.  When RECURSIVE? is true, export the closure of
1001 PATHS---i.e., PATHS and all their dependencies."
1002   (define ordered
1003     (let ((sorted (topologically-sorted server paths)))
1004       ;; When RECURSIVE? is #f, filter out the references of PATHS.
1005       (if recursive?
1006           sorted
1007           (filter (cut member <> paths) sorted))))
1008
1009   (let loop ((paths ordered))
1010     (match paths
1011       (()
1012        (write-int 0 port))
1013       ((head tail ...)
1014        (write-int 1 port)
1015        (and (export-path server head port #:sign? sign?)
1016             (loop tail))))))
1017
1018 (define-operation (query-failed-paths)
1019   "Return the list of store items for which a build failure is cached.
1020
1021 The result is always the empty list unless the daemon was started with
1022 '--cache-failures'."
1023   store-path-list)
1024
1025 (define-operation (clear-failed-paths (store-path-list items))
1026   "Remove ITEMS from the list of cached build failures.
1027
1028 This makes sense only when the daemon was started with '--cache-failures'."
1029   boolean)
1030
1031 (define* (register-path path
1032                         #:key (references '()) deriver prefix
1033                         state-directory)
1034   "Register PATH as a valid store file, with REFERENCES as its list of
1035 references, and DERIVER as its deriver (.drv that led to it.)  If PREFIX is
1036 not #f, it must be the name of the directory containing the new store to
1037 initialize; if STATE-DIRECTORY is not #f, it must be a string containing the
1038 absolute file name to the state directory of the store being initialized.
1039 Return #t on success.
1040
1041 Use with care as it directly modifies the store!  This is primarily meant to
1042 be used internally by the daemon's build hook."
1043   ;; Currently this is implemented by calling out to the fine C++ blob.
1044   (catch 'system-error
1045     (lambda ()
1046       (let ((pipe (apply open-pipe* OPEN_WRITE %guix-register-program
1047                          `(,@(if prefix
1048                                  `("--prefix" ,prefix)
1049                                  '())
1050                            ,@(if state-directory
1051                                  `("--state-directory" ,state-directory)
1052                                  '())))))
1053         (and pipe
1054              (begin
1055                (format pipe "~a~%~a~%~a~%"
1056                        path (or deriver "") (length references))
1057                (for-each (cut format pipe "~a~%" <>) references)
1058                (zero? (close-pipe pipe))))))
1059     (lambda args
1060       ;; Failed to run %GUIX-REGISTER-PROGRAM.
1061       #f)))
1062
1063 \f
1064 ;;;
1065 ;;; Store monad.
1066 ;;;
1067
1068 (define-syntax-rule (define-alias new old)
1069   (define-syntax new (identifier-syntax old)))
1070
1071 ;; The store monad allows us to (1) build sequences of operations in the
1072 ;; store, and (2) make the store an implicit part of the execution context,
1073 ;; rather than a parameter of every single function.
1074 (define-alias %store-monad %state-monad)
1075 (define-alias store-return state-return)
1076 (define-alias store-bind state-bind)
1077
1078 (define (preserve-documentation original proc)
1079   "Return PROC with documentation taken from ORIGINAL."
1080   (set-object-property! proc 'documentation
1081                         (procedure-property original 'documentation))
1082   proc)
1083
1084 (define (store-lift proc)
1085   "Lift PROC, a procedure whose first argument is a connection to the store,
1086 in the store monad."
1087   (preserve-documentation proc
1088                           (lambda args
1089                             (lambda (store)
1090                               (values (apply proc store args) store)))))
1091
1092 (define (store-lower proc)
1093   "Lower PROC, a monadic procedure in %STORE-MONAD, to a \"normal\" procedure
1094 taking the store as its first argument."
1095   (preserve-documentation proc
1096                           (lambda (store . args)
1097                             (run-with-store store (apply proc args)))))
1098
1099 ;;
1100 ;; Store monad operators.
1101 ;;
1102
1103 (define* (text-file name text
1104                     #:optional (references '()))
1105   "Return as a monadic value the absolute file name in the store of the file
1106 containing TEXT, a string.  REFERENCES is a list of store items that the
1107 resulting text file refers to; it defaults to the empty list."
1108   (lambda (store)
1109     (values (add-text-to-store store name text references)
1110             store)))
1111
1112 (define* (interned-file file #:optional name
1113                         #:key (recursive? #t))
1114   "Return the name of FILE once interned in the store.  Use NAME as its store
1115 name, or the basename of FILE if NAME is omitted.
1116
1117 When RECURSIVE? is true, the contents of FILE are added recursively; if FILE
1118 designates a flat file and RECURSIVE? is true, its contents are added, and its
1119 permission bits are kept."
1120   (lambda (store)
1121     (values (add-to-store store (or name (basename file))
1122                           recursive? "sha256" file)
1123             store)))
1124
1125 (define build
1126   ;; Monadic variant of 'build-things'.
1127   (store-lift build-things))
1128
1129 (define set-build-options*
1130   (store-lift set-build-options))
1131
1132 (define-inlinable (current-system)
1133   ;; Consult the %CURRENT-SYSTEM fluid at bind time.  This is equivalent to
1134   ;; (lift0 %current-system %store-monad), but inlinable, thus avoiding
1135   ;; closure allocation in some cases.
1136   (lambda (state)
1137     (values (%current-system) state)))
1138
1139 (define-inlinable (set-current-system system)
1140   ;; Set the %CURRENT-SYSTEM fluid at bind time.
1141   (lambda (state)
1142     (values (%current-system system) state)))
1143
1144 (define %guile-for-build
1145   ;; The derivation of the Guile to be used within the build environment,
1146   ;; when using 'gexp->derivation' and co.
1147   (make-parameter #f))
1148
1149 (define* (run-with-store store mval
1150                          #:key
1151                          (guile-for-build (%guile-for-build))
1152                          (system (%current-system)))
1153   "Run MVAL, a monadic value in the store monad, in STORE, an open store
1154 connection, and return the result."
1155   ;; Initialize the dynamic bindings here to avoid bad surprises.  The
1156   ;; difficulty lies in the fact that dynamic bindings are resolved at
1157   ;; bind-time and not at call time, which can be disconcerting.
1158   (parameterize ((%guile-for-build guile-for-build)
1159                  (%current-system system)
1160                  (%current-target-system #f))
1161     (call-with-values (lambda ()
1162                         (run-with-state mval store))
1163       (lambda (result store)
1164         ;; Discard the state.
1165         result))))
1166
1167 \f
1168 ;;;
1169 ;;; Store paths.
1170 ;;;
1171
1172 (define %store-prefix
1173   ;; Absolute path to the Nix store.
1174   (make-parameter %store-directory))
1175
1176 (define (store-path? path)
1177   "Return #t if PATH is a store path."
1178   ;; This is a lightweight check, compared to using a regexp, but this has to
1179   ;; be fast as it's called often in `derivation', for instance.
1180   ;; `isStorePath' in Nix does something similar.
1181   (string-prefix? (%store-prefix) path))
1182
1183 (define (direct-store-path? path)
1184   "Return #t if PATH is a store path, and not a sub-directory of a store path.
1185 This predicate is sometimes needed because files *under* a store path are not
1186 valid inputs."
1187   (and (store-path? path)
1188        (not (string=? path (%store-prefix)))
1189        (let ((len (+ 1 (string-length (%store-prefix)))))
1190          (not (string-index (substring path len) #\/)))))
1191
1192 (define (direct-store-path path)
1193   "Return the direct store path part of PATH, stripping components after
1194 '/gnu/store/xxxx-foo'."
1195   (let ((prefix-length (+ (string-length (%store-prefix)) 35)))
1196     (if (> (string-length path) prefix-length)
1197         (let ((slash (string-index path #\/ prefix-length)))
1198           (if slash (string-take path slash) path))
1199         path)))
1200
1201 (define (derivation-path? path)
1202   "Return #t if PATH is a derivation path."
1203   (and (store-path? path) (string-suffix? ".drv" path)))
1204
1205 (define store-regexp*
1206   ;; The substituter makes repeated calls to 'store-path-hash-part', hence
1207   ;; this optimization.
1208   (memoize
1209    (lambda (store)
1210      "Return a regexp matching a file in STORE."
1211      (make-regexp (string-append "^" (regexp-quote store)
1212                                  "/([0-9a-df-np-sv-z]{32})-([^/]+)$")))))
1213
1214 (define (store-path-package-name path)
1215   "Return the package name part of PATH, a file name in the store."
1216   (let ((path-rx (store-regexp* (%store-prefix))))
1217     (and=> (regexp-exec path-rx path)
1218            (cut match:substring <> 2))))
1219
1220 (define (store-path-hash-part path)
1221   "Return the hash part of PATH as a base32 string, or #f if PATH is not a
1222 syntactically valid store path."
1223   (let ((path-rx (store-regexp* (%store-prefix))))
1224     (and=> (regexp-exec path-rx path)
1225            (cut match:substring <> 1))))
1226
1227 (define (log-file store file)
1228   "Return the build log file for FILE, or #f if none could be found.  FILE
1229 must be an absolute store file name, or a derivation file name."
1230   (cond ((derivation-path? file)
1231          (let* ((base    (basename file))
1232                 (log     (string-append (dirname %state-directory) ; XXX
1233                                         "/log/guix/drvs/"
1234                                         (string-take base 2) "/"
1235                                         (string-drop base 2)))
1236                 (log.bz2 (string-append log ".bz2")))
1237            (cond ((file-exists? log.bz2) log.bz2)
1238                  ((file-exists? log) log)
1239                  (else #f))))
1240         (else
1241          (match (valid-derivers store file)
1242            ((derivers ...)
1243             ;; Return the first that works.
1244             (any (cut log-file store <>) derivers))
1245            (_ #f)))))