guix/store.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2012, 2013, 2014, 2015, 2016 Ludovic Courtès <ludo@gnu.org>
   3 ;;;
   4 ;;; This file is part of GNU Guix.
   5 ;;;
   6 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   7 ;;; under the terms of the GNU General Public License as published by
   8 ;;; the Free Software Foundation; either version 3 of the License, or (at
   9 ;;; your option) any later version.
  10 ;;;
  11 ;;; GNU Guix is distributed in the hope that it will be useful, but
  12 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 ;;; GNU General Public License for more details.
  15 ;;;
  16 ;;; You should have received a copy of the GNU General Public License
  17 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 (define-module (guix store)
  20   #:use-module (guix utils)
  21   #:use-module (guix config)
  22   #:use-module (guix combinators)
  23   #:use-module (guix serialization)
  24   #:use-module (guix monads)
  25   #:autoload   (guix base32) (bytevector->base32-string)
  26   #:autoload   (guix build syscalls) (terminal-columns)
  27   #:use-module (rnrs bytevectors)
  28   #:use-module (rnrs io ports)
  29   #:use-module (srfi srfi-1)
  30   #:use-module (srfi srfi-9)
  31   #:use-module (srfi srfi-9 gnu)
  32   #:use-module (srfi srfi-11)
  33   #:use-module (srfi srfi-26)
  34   #:use-module (srfi srfi-34)
  35   #:use-module (srfi srfi-35)
  36   #:use-module (srfi srfi-39)
  37   #:use-module (ice-9 match)
  38   #:use-module (ice-9 regex)
  39   #:use-module (ice-9 vlist)
  40   #:use-module (ice-9 popen)
  41   #:export (%daemon-socket-file
  42             %gc-roots-directory
  43             %default-substitute-urls
  44
  45             nix-server?
  46             nix-server-major-version
  47             nix-server-minor-version
  48             nix-server-socket
  49
  50             &nix-error nix-error?
  51             &nix-connection-error nix-connection-error?
  52             nix-connection-error-file
  53             nix-connection-error-code
  54             &nix-protocol-error nix-protocol-error?
  55             nix-protocol-error-message
  56             nix-protocol-error-status
  57
  58             hash-algo
  59             build-mode
  60
  61             open-connection
  62             close-connection
  63             with-store
  64             set-build-options
  65             set-build-options*
  66             valid-path?
  67             query-path-hash
  68             hash-part->path
  69             query-path-info
  70             add-text-to-store
  71             add-to-store
  72             build-things
  73             build
  74             query-failed-paths
  75             clear-failed-paths
  76             add-temp-root
  77             add-indirect-root
  78             add-permanent-root
  79             remove-permanent-root
  80
  81             substitutable?
  82             substitutable-path
  83             substitutable-deriver
  84             substitutable-references
  85             substitutable-download-size
  86             substitutable-nar-size
  87             has-substitutes?
  88             substitutable-paths
  89             substitutable-path-info
  90
  91             path-info?
  92             path-info-deriver
  93             path-info-hash
  94             path-info-references
  95             path-info-registration-time
  96             path-info-nar-size
  97
  98             references
  99             references/substitutes
 100             requisites
 101             referrers
 102             optimize-store
 103             verify-store
 104             topologically-sorted
 105             valid-derivers
 106             query-derivation-outputs
 107             live-paths
 108             dead-paths
 109             collect-garbage
 110             delete-paths
 111             import-paths
 112             export-paths
 113
 114             current-build-output-port
 115
 116             register-path
 117
 118             %store-monad
 119             store-bind
 120             store-return
 121             store-lift
 122             store-lower
 123             run-with-store
 124             %guile-for-build
 125             current-system
 126             set-current-system
 127             text-file
 128             interned-file
 129
 130             %store-prefix
 131             store-path?
 132             direct-store-path?
 133             derivation-path?
 134             store-path-package-name
 135             store-path-hash-part
 136             direct-store-path
 137             log-file))
 138
 139 (define %protocol-version #x10f)
 140
 141 (define %worker-magic-1 #x6e697863)               ; "nixc"
 142 (define %worker-magic-2 #x6478696f)               ; "dxio"
 143
 144 (define (protocol-major magic)
 145   (logand magic #xff00))
 146 (define (protocol-minor magic)
 147   (logand magic #x00ff))
 148
 149 (define-syntax define-enumerate-type
 150   (syntax-rules ()
 151     ((_ name->int (name id) ...)
 152      (define-syntax name->int
 153        (syntax-rules (name ...)
 154          ((_ name) id) ...)))))
 155
 156 (define-enumerate-type operation-id
 157   ;; operation numbers from worker-protocol.hh
 158   (quit 0)
 159   (valid-path? 1)
 160   (has-substitutes? 3)
 161   (query-path-hash 4)
 162   (query-references 5)
 163   (query-referrers 6)
 164   (add-to-store 7)
 165   (add-text-to-store 8)
 166   (build-things 9)
 167   (ensure-path 10)
 168   (add-temp-root 11)
 169   (add-indirect-root 12)
 170   (sync-with-gc 13)
 171   (find-roots 14)
 172   (export-path 16)
 173   (query-deriver 18)
 174   (set-options 19)
 175   (collect-garbage 20)
 176   ;;(query-substitutable-path-info 21)  ; obsolete as of #x10c
 177   (query-derivation-outputs 22)
 178   (query-all-valid-paths 23)
 179   (query-failed-paths 24)
 180   (clear-failed-paths 25)
 181   (query-path-info 26)
 182   (import-paths 27)
 183   (query-derivation-output-names 28)
 184   (query-path-from-hash-part 29)
 185   (query-substitutable-path-infos 30)
 186   (query-valid-paths 31)
 187   (query-substitutable-paths 32)
 188   (query-valid-derivers 33)
 189   (optimize-store 34)
 190   (verify-store 35))
 191
 192 (define-enumerate-type hash-algo
 193   ;; hash.hh
 194   (md5 1)
 195   (sha1 2)
 196   (sha256 3))
 197
 198 (define-enumerate-type build-mode
 199   ;; store-api.hh
 200   (normal 0)
 201   (repair 1)
 202   (check 2))
 203
 204 (define-enumerate-type gc-action
 205   ;; store-api.hh
 206   (return-live 0)
 207   (return-dead 1)
 208   (delete-dead 2)
 209   (delete-specific 3))
 210
 211 (define %default-socket-path
 212   (string-append %state-directory "/daemon-socket/socket"))
 213
 214 (define %daemon-socket-file
 215   ;; File name of the socket the daemon listens too.
 216   (make-parameter (or (getenv "GUIX_DAEMON_SOCKET")
 217                       %default-socket-path)))
 218
 219
 220 \f
 221 ;; Information about a substitutable store path.
 222 (define-record-type <substitutable>
 223   (substitutable path deriver refs dl-size nar-size)
 224   substitutable?
 225   (path      substitutable-path)
 226   (deriver   substitutable-deriver)
 227   (refs      substitutable-references)
 228   (dl-size   substitutable-download-size)
 229   (nar-size  substitutable-nar-size))
 230
 231 (define (read-substitutable-path-list p)
 232   (let loop ((len    (read-int p))
 233              (result '()))
 234     (if (zero? len)
 235         (reverse result)
 236         (let ((path     (read-store-path p))
 237               (deriver  (read-store-path p))
 238               (refs     (read-store-path-list p))
 239               (dl-size  (read-long-long p))
 240               (nar-size (read-long-long p)))
 241           (loop (- len 1)
 242                 (cons (substitutable path deriver refs dl-size nar-size)
 243                       result))))))
 244
 245 ;; Information about a store path.
 246 (define-record-type <path-info>
 247   (path-info deriver hash references registration-time nar-size)
 248   path-info?
 249   (deriver path-info-deriver)                     ;string | #f
 250   (hash path-info-hash)
 251   (references path-info-references)
 252   (registration-time path-info-registration-time)
 253   (nar-size path-info-nar-size))
 254
 255 (define (read-path-info p)
 256   (let ((deriver  (match (read-store-path p)
 257                     ("" #f)
 258                     (x  x)))
 259         (hash     (base16-string->bytevector (read-string p)))
 260         (refs     (read-store-path-list p))
 261         (registration-time (read-int p))
 262         (nar-size (read-long-long p)))
 263     (path-info deriver hash refs registration-time nar-size)))
 264
 265 (define-syntax write-arg
 266   (syntax-rules (integer boolean string string-list string-pairs
 267                  store-path store-path-list base16)
 268     ((_ integer arg p)
 269      (write-int arg p))
 270     ((_ boolean arg p)
 271      (write-int (if arg 1 0) p))
 272     ((_ string arg p)
 273      (write-string arg p))
 274     ((_ string-list arg p)
 275      (write-string-list arg p))
 276     ((_ string-pairs arg p)
 277      (write-string-pairs arg p))
 278     ((_ store-path arg p)
 279      (write-store-path arg p))
 280     ((_ store-path-list arg p)
 281      (write-store-path-list arg p))
 282     ((_ base16 arg p)
 283      (write-string (bytevector->base16-string arg) p))))
 284
 285 (define-syntax read-arg
 286   (syntax-rules (integer boolean string store-path store-path-list
 287                  substitutable-path-list path-info base16)
 288     ((_ integer p)
 289      (read-int p))
 290     ((_ boolean p)
 291      (not (zero? (read-int p))))
 292     ((_ string p)
 293      (read-string p))
 294     ((_ store-path p)
 295      (read-store-path p))
 296     ((_ store-path-list p)
 297      (read-store-path-list p))
 298     ((_ substitutable-path-list p)
 299      (read-substitutable-path-list p))
 300     ((_ path-info p)
 301      (read-path-info p))
 302     ((_ base16 p)
 303      (base16-string->bytevector (read-string p)))))
 304
 305 \f
 306 ;; remote-store.cc
 307
 308 (define-record-type <nix-server>
 309   (%make-nix-server socket major minor
 310                     ats-cache atts-cache)
 311   nix-server?
 312   (socket nix-server-socket)
 313   (major  nix-server-major-version)
 314   (minor  nix-server-minor-version)
 315
 316   ;; Caches.  We keep them per-connection, because store paths build
 317   ;; during the session are temporary GC roots kept for the duration of
 318   ;; the session.
 319   (ats-cache  nix-server-add-to-store-cache)
 320   (atts-cache nix-server-add-text-to-store-cache))
 321
 322 (set-record-type-printer! <nix-server>
 323                           (lambda (obj port)
 324                             (format port "#<build-daemon ~a.~a ~a>"
 325                                     (nix-server-major-version obj)
 326                                     (nix-server-minor-version obj)
 327                                     (number->string (object-address obj)
 328                                                     16))))
 329
 330 (define-condition-type &nix-error &error
 331   nix-error?)
 332
 333 (define-condition-type &nix-connection-error &nix-error
 334   nix-connection-error?
 335   (file   nix-connection-error-file)
 336   (errno  nix-connection-error-code))
 337
 338 (define-condition-type &nix-protocol-error &nix-error
 339   nix-protocol-error?
 340   (message nix-protocol-error-message)
 341   (status  nix-protocol-error-status))
 342
 343 (define* (open-connection #:optional (file (%daemon-socket-file))
 344                           #:key (reserve-space? #t) cpu-affinity)
 345   "Connect to the daemon over the Unix-domain socket at FILE.  When
 346 RESERVE-SPACE? is true, instruct it to reserve a little bit of extra space on
 347 the file system so that the garbage collector can still operate, should the
 348 disk become full.  When CPU-AFFINITY is true, it must be an integer
 349 corresponding to an OS-level CPU number to which the daemon's worker process
 350 for this connection will be pinned.  Return a server object."
 351   (let ((s (with-fluids ((%default-port-encoding #f))
 352              ;; This trick allows use of the `scm_c_read' optimization.
 353              (socket PF_UNIX SOCK_STREAM 0)))
 354         (a (make-socket-address PF_UNIX file)))
 355
 356     (catch 'system-error
 357       (cut connect s a)
 358       (lambda args
 359         ;; Translate the error to something user-friendly.
 360         (let ((errno (system-error-errno args)))
 361           (raise (condition (&nix-connection-error
 362                              (file file)
 363                              (errno errno)))))))
 364
 365     (write-int %worker-magic-1 s)
 366     (let ((r (read-int s)))
 367       (and (eqv? r %worker-magic-2)
 368            (let ((v (read-int s)))
 369              (and (eqv? (protocol-major %protocol-version)
 370                         (protocol-major v))
 371                   (begin
 372                     (write-int %protocol-version s)
 373                     (when (>= (protocol-minor v) 14)
 374                       (write-int (if cpu-affinity 1 0) s)
 375                       (when cpu-affinity
 376                         (write-int cpu-affinity s)))
 377                     (when (>= (protocol-minor v) 11)
 378                       (write-int (if reserve-space? 1 0) s))
 379                     (let ((s (%make-nix-server s
 380                                                (protocol-major v)
 381                                                (protocol-minor v)
 382                                                (make-hash-table 100)
 383                                                (make-hash-table 100))))
 384                       (let loop ((done? (process-stderr s)))
 385                         (or done? (process-stderr s)))
 386                       s))))))))
 387
 388 (define (close-connection server)
 389   "Close the connection to SERVER."
 390   (close (nix-server-socket server)))
 391
 392 (define-syntax-rule (with-store store exp ...)
 393   "Bind STORE to an open connection to the store and evaluate EXPs;
 394 automatically close the store when the dynamic extent of EXP is left."
 395   (let ((store (open-connection)))
 396     (dynamic-wind
 397       (const #f)
 398       (lambda ()
 399         exp ...)
 400       (lambda ()
 401         (false-if-exception (close-connection store))))))
 402
 403 (define current-build-output-port
 404   ;; The port where build output is sent.
 405   (make-parameter (current-error-port)))
 406
 407 (define* (dump-port in out
 408                     #:optional len
 409                     #:key (buffer-size 16384))
 410   "Read LEN bytes from IN (or as much as possible if LEN is #f) and write it
 411 to OUT, using chunks of BUFFER-SIZE bytes."
 412   (define buffer
 413     (make-bytevector buffer-size))
 414
 415   (let loop ((total 0)
 416              (bytes (get-bytevector-n! in buffer 0
 417                                        (if len
 418                                            (min len buffer-size)
 419                                            buffer-size))))
 420     (or (eof-object? bytes)
 421         (and len (= total len))
 422         (let ((total (+ total bytes)))
 423           (put-bytevector out buffer 0 bytes)
 424           (loop total
 425                 (get-bytevector-n! in buffer 0
 426                                    (if len
 427                                        (min (- len total) buffer-size)
 428                                        buffer-size)))))))
 429
 430 (define %newlines
 431   ;; Newline characters triggering a flush of 'current-build-output-port'.
 432   ;; Unlike Guile's _IOLBF, we flush upon #\return so that progress reports
 433   ;; that use that trick are correctly displayed.
 434   (char-set #\newline #\return))
 435
 436 (define* (process-stderr server #:optional user-port)
 437   "Read standard output and standard error from SERVER, writing it to
 438 CURRENT-BUILD-OUTPUT-PORT.  Return #t when SERVER is done sending data, and
 439 #f otherwise; in the latter case, the caller should call `process-stderr'
 440 again until #t is returned or an error is raised.
 441
 442 Since the build process's output cannot be assumed to be UTF-8, we
 443 conservatively consider it to be Latin-1, thereby avoiding possible
 444 encoding conversion errors."
 445   (define p
 446     (nix-server-socket server))
 447
 448   ;; magic cookies from worker-protocol.hh
 449   (define %stderr-next  #x6f6c6d67)          ; "olmg", build log
 450   (define %stderr-read  #x64617461)          ; "data", data needed from source
 451   (define %stderr-write #x64617416)          ; "dat\x16", data for sink
 452   (define %stderr-last  #x616c7473)          ; "alts", we're done
 453   (define %stderr-error #x63787470)          ; "cxtp", error reporting
 454
 455   (let ((k (read-int p)))
 456     (cond ((= k %stderr-write)
 457            ;; Write a byte stream to USER-PORT.
 458            (let* ((len (read-int p))
 459                   (m   (modulo len 8)))
 460              (dump-port p user-port len)
 461              (unless (zero? m)
 462                ;; Consume padding, as for strings.
 463                (get-bytevector-n p (- 8 m))))
 464            #f)
 465           ((= k %stderr-read)
 466            ;; Read a byte stream from USER-PORT.
 467            ;; Note: Avoid 'get-bytevector-n' to work around
 468            ;; <http://bugs.gnu.org/17591> in Guile up to 2.0.11.
 469            (let* ((max-len (read-int p))
 470                   (data    (make-bytevector max-len))
 471                   (len     (get-bytevector-n! user-port data 0 max-len)))
 472              (write-int len p)
 473              (put-bytevector p data 0 len)
 474              (write-padding len p)
 475              #f))
 476           ((= k %stderr-next)
 477            ;; Log a string.  Build logs are usually UTF-8-encoded, but they
 478            ;; may also contain arbitrary byte sequences that should not cause
 479            ;; this to fail.  Thus, use the permissive
 480            ;; 'read-maybe-utf8-string'.
 481            (let ((s (read-maybe-utf8-string p)))
 482              (display s (current-build-output-port))
 483              (when (string-any %newlines s)
 484                (flush-output-port (current-build-output-port)))
 485              #f))
 486           ((= k %stderr-error)
 487            ;; Report an error.
 488            (let ((error  (read-maybe-utf8-string p))
 489                  ;; Currently the daemon fails to send a status code for early
 490                  ;; errors like DB schema version mismatches, so check for EOF.
 491                  (status (if (and (>= (nix-server-minor-version server) 8)
 492                                   (not (eof-object? (lookahead-u8 p))))
 493                              (read-int p)
 494                              1)))
 495              (raise (condition (&nix-protocol-error
 496                                 (message error)
 497                                 (status  status))))))
 498           ((= k %stderr-last)
 499            ;; The daemon is done (see `stopWork' in `nix-worker.cc'.)
 500            #t)
 501           (else
 502            (raise (condition (&nix-protocol-error
 503                               (message "invalid error code")
 504                               (status   k))))))))
 505
 506 (define %default-substitute-urls
 507   ;; Default list of substituters.  This is *not* the list baked in
 508   ;; 'guix-daemon', but it is used by 'guix-service-type' and and a couple of
 509   ;; clients ('guix build --log-file' uses it.)
 510   (map (if (false-if-exception (resolve-interface '(gnutls)))
 511            (cut string-append "https://" <>)
 512            (cut string-append "http://" <>))
 513        '("mirror.hydra.gnu.org" "hydra.gnu.org")))
 514
 515 (define* (set-build-options server
 516                             #:key keep-failed? keep-going? fallback?
 517                             (verbosity 0)
 518                             rounds                ;number of build rounds
 519                             (max-build-jobs 1)
 520                             timeout
 521                             (max-silent-time 3600)
 522                             (use-build-hook? #t)
 523                             (build-verbosity 0)
 524                             (log-type 0)
 525                             (print-build-trace #t)
 526                             (build-cores (current-processor-count))
 527                             (use-substitutes? #t)
 528
 529                             ;; Client-provided substitute URLs.  If it is #f,
 530                             ;; the daemon's settings are used.  Otherwise, it
 531                             ;; overrides the daemons settings; see 'guix
 532                             ;; substitute'.
 533                             (substitute-urls #f)
 534
 535                             ;; Number of columns in the client's terminal.
 536                             (terminal-columns (terminal-columns))
 537
 538                             ;; Locale of the client.
 539                             (locale (false-if-exception (setlocale LC_ALL))))
 540   ;; Must be called after `open-connection'.
 541
 542   (define socket
 543     (nix-server-socket server))
 544
 545   (let-syntax ((send (syntax-rules ()
 546                        ((_ (type option) ...)
 547                         (begin
 548                           (write-arg type option socket)
 549                           ...)))))
 550     (write-int (operation-id set-options) socket)
 551     (send (boolean keep-failed?) (boolean keep-going?)
 552           (boolean fallback?) (integer verbosity)
 553           (integer max-build-jobs) (integer max-silent-time))
 554     (when (>= (nix-server-minor-version server) 2)
 555       (send (boolean use-build-hook?)))
 556     (when (>= (nix-server-minor-version server) 4)
 557       (send (integer build-verbosity) (integer log-type)
 558             (boolean print-build-trace)))
 559     (when (>= (nix-server-minor-version server) 6)
 560       (send (integer build-cores)))
 561     (when (>= (nix-server-minor-version server) 10)
 562       (send (boolean use-substitutes?)))
 563     (when (>= (nix-server-minor-version server) 12)
 564       (let ((pairs `(,@(if timeout
 565                            `(("build-timeout" . ,(number->string timeout)))
 566                            '())
 567                      ,@(if substitute-urls
 568                            `(("substitute-urls"
 569                               . ,(string-join substitute-urls)))
 570                            '())
 571                      ,@(if rounds
 572                            `(("build-repeat"
 573                               . ,(number->string (max 0 (1- rounds)))))
 574                            '())
 575                      ,@(if terminal-columns
 576                            `(("terminal-columns"
 577                               . ,(number->string terminal-columns)))
 578                            '())
 579                      ,@(if locale
 580                            `(("locale" . ,locale))
 581                            '()))))
 582         (send (string-pairs pairs))))
 583     (let loop ((done? (process-stderr server)))
 584       (or done? (process-stderr server)))))
 585
 586 (define-syntax operation
 587   (syntax-rules ()
 588     "Define a client-side RPC stub for the given operation."
 589     ((_ (name (type arg) ...) docstring return ...)
 590      (lambda (server arg ...)
 591        docstring
 592        (let ((s (nix-server-socket server)))
 593          (write-int (operation-id name) s)
 594          (write-arg type arg s)
 595          ...
 596          ;; Loop until the server is done sending error output.
 597          (let loop ((done? (process-stderr server)))
 598            (or done? (loop (process-stderr server))))
 599          (values (read-arg return s) ...))))))
 600
 601 (define-syntax-rule (define-operation (name args ...)
 602                       docstring return ...)
 603   (define name
 604     (operation (name args ...) docstring return ...)))
 605
 606 (define-operation (valid-path? (string path))
 607   "Return #t when PATH designates a valid store item and #f otherwise (an
 608 invalid item may exist on disk but still be invalid, for instance because it
 609 is the result of an aborted or failed build.)
 610
 611 A '&nix-protocol-error' condition is raised if PATH is not prefixed by the
 612 store directory (/gnu/store)."
 613   boolean)
 614
 615 (define-operation (query-path-hash (store-path path))
 616   "Return the SHA256 hash of the nar serialization of PATH as a bytevector."
 617   base16)
 618
 619 (define hash-part->path
 620   (let ((query-path-from-hash-part
 621          (operation (query-path-from-hash-part (string hash))
 622                     #f
 623                     store-path)))
 624    (lambda (server hash-part)
 625      "Return the store path whose hash part is HASH-PART (a nix-base32
 626 string).  Raise an error if no such path exists."
 627      ;; This RPC is primarily used by Hydra to reply to HTTP GETs of
 628      ;; /HASH.narinfo.
 629      (query-path-from-hash-part server hash-part))))
 630
 631 (define-operation (query-path-info (store-path path))
 632   "Return the info (hash, references, etc.) for PATH."
 633   path-info)
 634
 635 (define add-text-to-store
 636   ;; A memoizing version of `add-to-store', to avoid repeated RPCs with
 637   ;; the very same arguments during a given session.
 638   (let ((add-text-to-store
 639          (operation (add-text-to-store (string name) (string text)
 640                                        (string-list references))
 641                     #f
 642                     store-path)))
 643     (lambda* (server name text #:optional (references '()))
 644       "Add TEXT under file NAME in the store, and return its store path.
 645 REFERENCES is the list of store paths referred to by the resulting store
 646 path."
 647       (let ((args  `(,text ,name ,references))
 648             (cache (nix-server-add-text-to-store-cache server)))
 649         (or (hash-ref cache args)
 650             (let ((path (add-text-to-store server name text references)))
 651               (hash-set! cache args path)
 652               path))))))
 653
 654 (define true
 655   ;; Define it once and for all since we use it as a default value for
 656   ;; 'add-to-store' and want to make sure two default values are 'eq?' for the
 657   ;; purposes or memoization.
 658   (lambda (file stat)
 659     #t))
 660
 661 (define add-to-store
 662   ;; A memoizing version of `add-to-store'.  This is important because
 663   ;; `add-to-store' leads to huge data transfers to the server, and
 664   ;; because it's often called many times with the very same argument.
 665   (let ((add-to-store
 666          (lambda* (server basename recursive? hash-algo file-name
 667                           #:key (select? true))
 668            ;; We don't use the 'operation' macro so we can pass SELECT? to
 669            ;; 'write-file'.
 670            (let ((port (nix-server-socket server)))
 671              (write-int (operation-id add-to-store) port)
 672              (write-string basename port)
 673              (write-int 1 port)                   ;obsolete, must be #t
 674              (write-int (if recursive? 1 0) port)
 675              (write-string hash-algo port)
 676              (write-file file-name port #:select? select?)
 677              (let loop ((done? (process-stderr server)))
 678                (or done? (loop (process-stderr server))))
 679              (read-store-path port)))))
 680     (lambda* (server basename recursive? hash-algo file-name
 681                      #:key (select? true))
 682       "Add the contents of FILE-NAME under BASENAME to the store.  When
 683 RECURSIVE? is false, FILE-NAME must designate a regular file--not a directory
 684 nor a symlink.  When RECURSIVE? is true and FILE-NAME designates a directory,
 685 the contents of FILE-NAME are added recursively; if FILE-NAME designates a
 686 flat file and RECURSIVE? is true, its contents are added, and its permission
 687 bits are kept.  HASH-ALGO must be a string such as \"sha256\".
 688
 689 When RECURSIVE? is true, call (SELECT?  FILE STAT) for each directory entry,
 690 where FILE is the entry's absolute file name and STAT is the result of
 691 'lstat'; exclude entries for which SELECT? does not return true."
 692       (let* ((st    (false-if-exception (lstat file-name)))
 693              (args  `(,st ,basename ,recursive? ,hash-algo ,select?))
 694              (cache (nix-server-add-to-store-cache server)))
 695         (or (and st (hash-ref cache args))
 696             (let ((path (add-to-store server basename recursive?
 697                                       hash-algo file-name
 698                                       #:select? select?)))
 699               (hash-set! cache args path)
 700               path))))))
 701
 702 (define build-things
 703   (let ((build (operation (build-things (string-list things)
 704                                         (integer mode))
 705                           "Do it!"
 706                           boolean))
 707         (build/old (operation (build-things (string-list things))
 708                               "Do it!"
 709                               boolean)))
 710     (lambda* (store things #:optional (mode (build-mode normal)))
 711       "Build THINGS, a list of store items which may be either '.drv' files or
 712 outputs, and return when the worker is done building them.  Elements of THINGS
 713 that are not derivations can only be substituted and not built locally.
 714 Return #t on success."
 715       (if (>= (nix-server-minor-version store) 15)
 716           (build store things mode)
 717           (if (= mode (build-mode normal))
 718               (build/old store things)
 719               (raise (condition (&nix-protocol-error
 720                                  (message "unsupported build mode")
 721                                  (status  1)))))))))
 722
 723 (define-operation (add-temp-root (store-path path))
 724   "Make PATH a temporary root for the duration of the current session.
 725 Return #t."
 726   boolean)
 727
 728 (define-operation (add-indirect-root (string file-name))
 729   "Make the symlink FILE-NAME an indirect root for the garbage collector:
 730 whatever store item FILE-NAME points to will not be collected.  Return #t on
 731 success.
 732
 733 FILE-NAME can be anywhere on the file system, but it must be an absolute file
 734 name--it is the caller's responsibility to ensure that it is an absolute file
 735 name."
 736   boolean)
 737
 738 (define %gc-roots-directory
 739   ;; The place where garbage collector roots (symlinks) are kept.
 740   (string-append %state-directory "/gcroots"))
 741
 742 (define (add-permanent-root target)
 743   "Add a garbage collector root pointing to TARGET, an element of the store,
 744 preventing TARGET from even being collected.  This can also be used if TARGET
 745 does not exist yet.
 746
 747 Raise an error if the caller does not have write access to the GC root
 748 directory."
 749   (let* ((root (string-append %gc-roots-directory "/" (basename target))))
 750     (catch 'system-error
 751       (lambda ()
 752         (symlink target root))
 753       (lambda args
 754         ;; If ROOT already exists, this is fine; otherwise, re-throw.
 755         (unless (= EEXIST (system-error-errno args))
 756           (apply throw args))))))
 757
 758 (define (remove-permanent-root target)
 759   "Remove the permanent garbage collector root pointing to TARGET.  Raise an
 760 error if there is no such root."
 761   (delete-file (string-append %gc-roots-directory "/" (basename target))))
 762
 763 (define references
 764   (operation (query-references (store-path path))
 765              "Return the list of references of PATH."
 766              store-path-list))
 767
 768 (define %reference-cache
 769   ;; Brute-force cache mapping store items to their list of references.
 770   ;; Caching matters because when building a profile in the presence of
 771   ;; grafts, we keep calling 'graft-derivation', which in turn calls
 772   ;; 'references/substitutes' many times with the same arguments.  Ideally we
 773   ;; would use a cache associated with the daemon connection instead (XXX).
 774   (make-hash-table 100))
 775
 776 (define (references/substitutes store items)
 777   "Return the list of list of references of ITEMS; the result has the same
 778 length as ITEMS.  Query substitute information for any item missing from the
 779 store at once.  Raise a '&nix-protocol-error' exception if reference
 780 information for one of ITEMS is missing."
 781   (let* ((local-refs (map (lambda (item)
 782                             (or (hash-ref %reference-cache item)
 783                                 (guard (c ((nix-protocol-error? c) #f))
 784                                   (references store item))))
 785                           items))
 786          (missing    (fold-right (lambda (item local-ref result)
 787                                    (if local-ref
 788                                        result
 789                                        (cons item result)))
 790                                  '()
 791                                  items local-refs))
 792
 793          ;; Query all the substitutes at once to minimize the cost of
 794          ;; launching 'guix substitute' and making HTTP requests.
 795          (substs     (substitutable-path-info store missing)))
 796     (when (< (length substs) (length missing))
 797       (raise (condition (&nix-protocol-error
 798                          (message "cannot determine \
 799 the list of references")
 800                          (status 1)))))
 801
 802     ;; Intersperse SUBSTS and LOCAL-REFS.
 803     (let loop ((items       items)
 804                (local-refs  local-refs)
 805                (result      '()))
 806       (match items
 807         (()
 808          (let ((result (reverse result)))
 809            (for-each (cut hash-set! %reference-cache <> <>)
 810                      items result)
 811            result))
 812         ((item items ...)
 813          (match local-refs
 814            ((#f tail ...)
 815             (loop items tail
 816                   (cons (any (lambda (subst)
 817                                (and (string=? (substitutable-path subst) item)
 818                                     (substitutable-references subst)))
 819                              substs)
 820                         result)))
 821            ((head tail ...)
 822             (loop items tail
 823                   (cons head result)))))))))
 824
 825 (define* (fold-path store proc seed paths
 826                     #:optional (relatives (cut references store <>)))
 827   "Call PROC for each of the RELATIVES of PATHS, exactly once, and return the
 828 result formed from the successive calls to PROC, the first of which is passed
 829 SEED."
 830   (let loop ((paths  paths)
 831              (result seed)
 832              (seen   vlist-null))
 833     (match paths
 834       ((path rest ...)
 835        (if (vhash-assoc path seen)
 836            (loop rest result seen)
 837            (let ((seen   (vhash-cons path #t seen))
 838                  (rest   (append rest (relatives path)))
 839                  (result (proc path result)))
 840              (loop rest result seen))))
 841       (()
 842        result))))
 843
 844 (define (requisites store paths)
 845   "Return the requisites of PATHS, including PATHS---i.e., their closures (all
 846 its references, recursively)."
 847   (fold-path store cons '() paths))
 848
 849 (define (topologically-sorted store paths)
 850   "Return a list containing PATHS and all their references sorted in
 851 topological order."
 852   (define (traverse)
 853     ;; Do a simple depth-first traversal of all of PATHS.
 854     (let loop ((paths   paths)
 855                (visited vlist-null)
 856                (result  '()))
 857       (define (visit n)
 858         (vhash-cons n #t visited))
 859
 860       (define (visited? n)
 861         (vhash-assoc n visited))
 862
 863       (match paths
 864         ((head tail ...)
 865          (if (visited? head)
 866              (loop tail visited result)
 867              (call-with-values
 868                  (lambda ()
 869                    (loop (references store head)
 870                          (visit head)
 871                          result))
 872                (lambda (visited result)
 873                  (loop tail
 874                        visited
 875                        (cons head result))))))
 876         (()
 877          (values visited result)))))
 878
 879   (call-with-values traverse
 880     (lambda (_ result)
 881       (reverse result))))
 882
 883 (define referrers
 884   (operation (query-referrers (store-path path))
 885              "Return the list of path that refer to PATH."
 886              store-path-list))
 887
 888 (define valid-derivers
 889   (operation (query-valid-derivers (store-path path))
 890              "Return the list of valid \"derivers\" of PATH---i.e., all the
 891 .drv present in the store that have PATH among their outputs."
 892              store-path-list))
 893
 894 (define query-derivation-outputs  ; avoid name clash with `derivation-outputs'
 895   (operation (query-derivation-outputs (store-path path))
 896              "Return the list of outputs of PATH, a .drv file."
 897              store-path-list))
 898
 899 (define-operation (has-substitutes? (store-path path))
 900   "Return #t if binary substitutes are available for PATH, and #f otherwise."
 901   boolean)
 902
 903 (define substitutable-paths
 904   (operation (query-substitutable-paths (store-path-list paths))
 905              "Return the subset of PATHS that is substitutable."
 906              store-path-list))
 907
 908 (define substitutable-path-info
 909   (operation (query-substitutable-path-infos (store-path-list paths))
 910              "Return information about the subset of PATHS that is
 911 substitutable.  For each substitutable path, a `substitutable?' object is
 912 returned; thus, the resulting list can be shorter than PATHS.  Furthermore,
 913 that there is no guarantee that the order of the resulting list matches the
 914 order of PATHS."
 915              substitutable-path-list))
 916
 917 (define-operation (optimize-store)
 918   "Optimize the store by hard-linking identical files (\"deduplication\".)
 919 Return #t on success."
 920   ;; Note: the daemon in Guix <= 0.8.2 does not implement this RPC.
 921   boolean)
 922
 923 (define verify-store
 924   (let ((verify (operation (verify-store (boolean check-contents?)
 925                                          (boolean repair?))
 926                            "Verify the store."
 927                            boolean)))
 928     (lambda* (store #:key check-contents? repair?)
 929       "Verify the integrity of the store and return false if errors remain,
 930 and true otherwise.  When REPAIR? is true, repair any missing or altered store
 931 items by substituting them (this typically requires root privileges because it
 932 is not an atomic operation.)  When CHECK-CONTENTS? is true, check the contents
 933 of store items; this can take a lot of time."
 934       (not (verify store check-contents? repair?)))))
 935
 936 (define (run-gc server action to-delete min-freed)
 937   "Perform the garbage-collector operation ACTION, one of the
 938 `gc-action' values.  When ACTION is `delete-specific', the TO-DELETE is
 939 the list of store paths to delete.  IGNORE-LIVENESS? should always be
 940 #f.  MIN-FREED is the minimum amount of disk space to be freed, in
 941 bytes, before the GC can stop.  Return the list of store paths delete,
 942 and the number of bytes freed."
 943   (let ((s (nix-server-socket server)))
 944     (write-int (operation-id collect-garbage) s)
 945     (write-int action s)
 946     (write-store-path-list to-delete s)
 947     (write-arg boolean #f s)                      ; ignore-liveness?
 948     (write-long-long min-freed s)
 949     (write-int 0 s)                               ; obsolete
 950     (when (>= (nix-server-minor-version server) 5)
 951       ;; Obsolete `use-atime' and `max-atime' parameters.
 952       (write-int 0 s)
 953       (write-int 0 s))
 954
 955     ;; Loop until the server is done sending error output.
 956     (let loop ((done? (process-stderr server)))
 957       (or done? (loop (process-stderr server))))
 958
 959     (let ((paths    (read-store-path-list s))
 960           (freed    (read-long-long s))
 961           (obsolete (read-long-long s)))
 962       (unless (null? paths)
 963         ;; To be on the safe side, completely invalidate both caches.
 964         ;; Otherwise we could end up returning store paths that are no longer
 965         ;; valid.
 966         (hash-clear! (nix-server-add-to-store-cache server))
 967         (hash-clear! (nix-server-add-text-to-store-cache server)))
 968
 969      (values paths freed))))
 970
 971 (define-syntax-rule (%long-long-max)
 972   ;; Maximum unsigned 64-bit integer.
 973   (- (expt 2 64) 1))
 974
 975 (define (live-paths server)
 976   "Return the list of live store paths---i.e., store paths still
 977 referenced, and thus not subject to being garbage-collected."
 978   (run-gc server (gc-action return-live) '() (%long-long-max)))
 979
 980 (define (dead-paths server)
 981   "Return the list of dead store paths---i.e., store paths no longer
 982 referenced, and thus subject to being garbage-collected."
 983   (run-gc server (gc-action return-dead) '() (%long-long-max)))
 984
 985 (define* (collect-garbage server #:optional (min-freed (%long-long-max)))
 986   "Collect garbage from the store at SERVER.  If MIN-FREED is non-zero,
 987 then collect at least MIN-FREED bytes.  Return the paths that were
 988 collected, and the number of bytes freed."
 989   (run-gc server (gc-action delete-dead) '() min-freed))
 990
 991 (define* (delete-paths server paths #:optional (min-freed (%long-long-max)))
 992   "Delete PATHS from the store at SERVER, if they are no longer
 993 referenced.  If MIN-FREED is non-zero, then stop after at least
 994 MIN-FREED bytes have been collected.  Return the paths that were
 995 collected, and the number of bytes freed."
 996   (run-gc server (gc-action delete-specific) paths min-freed))
 997
 998 (define (import-paths server port)
 999   "Import the set of store paths read from PORT into SERVER's store.  An error
1000 is raised if the set of paths read from PORT is not signed (as per
1001 'export-path #:sign? #t'.)  Return the list of store paths imported."
1002   (let ((s (nix-server-socket server)))
1003     (write-int (operation-id import-paths) s)
1004     (let loop ((done? (process-stderr server port)))
1005       (or done? (loop (process-stderr server port))))
1006     (read-store-path-list s)))
1007
1008 (define* (export-path server path port #:key (sign? #t))
1009   "Export PATH to PORT.  When SIGN? is true, sign it."
1010   (let ((s (nix-server-socket server)))
1011     (write-int (operation-id export-path) s)
1012     (write-store-path path s)
1013     (write-arg boolean sign? s)
1014     (let loop ((done? (process-stderr server port)))
1015       (or done? (loop (process-stderr server port))))
1016     (= 1 (read-int s))))
1017
1018 (define* (export-paths server paths port #:key (sign? #t) recursive?)
1019   "Export the store paths listed in PATHS to PORT, in topological order,
1020 signing them if SIGN? is true.  When RECURSIVE? is true, export the closure of
1021 PATHS---i.e., PATHS and all their dependencies."
1022   (define ordered
1023     (let ((sorted (topologically-sorted server paths)))
1024       ;; When RECURSIVE? is #f, filter out the references of PATHS.
1025       (if recursive?
1026           sorted
1027           (filter (cut member <> paths) sorted))))
1028
1029   (let loop ((paths ordered))
1030     (match paths
1031       (()
1032        (write-int 0 port))
1033       ((head tail ...)
1034        (write-int 1 port)
1035        (and (export-path server head port #:sign? sign?)
1036             (loop tail))))))
1037
1038 (define-operation (query-failed-paths)
1039   "Return the list of store items for which a build failure is cached.
1040
1041 The result is always the empty list unless the daemon was started with
1042 '--cache-failures'."
1043   store-path-list)
1044
1045 (define-operation (clear-failed-paths (store-path-list items))
1046   "Remove ITEMS from the list of cached build failures.
1047
1048 This makes sense only when the daemon was started with '--cache-failures'."
1049   boolean)
1050
1051 (define* (register-path path
1052                         #:key (references '()) deriver prefix
1053                         state-directory)
1054   "Register PATH as a valid store file, with REFERENCES as its list of
1055 references, and DERIVER as its deriver (.drv that led to it.)  If PREFIX is
1056 not #f, it must be the name of the directory containing the new store to
1057 initialize; if STATE-DIRECTORY is not #f, it must be a string containing the
1058 absolute file name to the state directory of the store being initialized.
1059 Return #t on success.
1060
1061 Use with care as it directly modifies the store!  This is primarily meant to
1062 be used internally by the daemon's build hook."
1063   ;; Currently this is implemented by calling out to the fine C++ blob.
1064   (catch 'system-error
1065     (lambda ()
1066       (let ((pipe (apply open-pipe* OPEN_WRITE %guix-register-program
1067                          `(,@(if prefix
1068                                  `("--prefix" ,prefix)
1069                                  '())
1070                            ,@(if state-directory
1071                                  `("--state-directory" ,state-directory)
1072                                  '())))))
1073         (and pipe
1074              (begin
1075                (format pipe "~a~%~a~%~a~%"
1076                        path (or deriver "") (length references))
1077                (for-each (cut format pipe "~a~%" <>) references)
1078                (zero? (close-pipe pipe))))))
1079     (lambda args
1080       ;; Failed to run %GUIX-REGISTER-PROGRAM.
1081       #f)))
1082
1083 \f
1084 ;;;
1085 ;;; Store monad.
1086 ;;;
1087
1088 (define-syntax-rule (define-alias new old)
1089   (define-syntax new (identifier-syntax old)))
1090
1091 ;; The store monad allows us to (1) build sequences of operations in the
1092 ;; store, and (2) make the store an implicit part of the execution context,
1093 ;; rather than a parameter of every single function.
1094 (define-alias %store-monad %state-monad)
1095 (define-alias store-return state-return)
1096 (define-alias store-bind state-bind)
1097
1098 (define (preserve-documentation original proc)
1099   "Return PROC with documentation taken from ORIGINAL."
1100   (set-object-property! proc 'documentation
1101                         (procedure-property original 'documentation))
1102   proc)
1103
1104 (define (store-lift proc)
1105   "Lift PROC, a procedure whose first argument is a connection to the store,
1106 in the store monad."
1107   (preserve-documentation proc
1108                           (lambda args
1109                             (lambda (store)
1110                               (values (apply proc store args) store)))))
1111
1112 (define (store-lower proc)
1113   "Lower PROC, a monadic procedure in %STORE-MONAD, to a \"normal\" procedure
1114 taking the store as its first argument."
1115   (preserve-documentation proc
1116                           (lambda (store . args)
1117                             (run-with-store store (apply proc args)))))
1118
1119 ;;
1120 ;; Store monad operators.
1121 ;;
1122
1123 (define* (text-file name text
1124                     #:optional (references '()))
1125   "Return as a monadic value the absolute file name in the store of the file
1126 containing TEXT, a string.  REFERENCES is a list of store items that the
1127 resulting text file refers to; it defaults to the empty list."
1128   (lambda (store)
1129     (values (add-text-to-store store name text references)
1130             store)))
1131
1132 (define* (interned-file file #:optional name
1133                         #:key (recursive? #t) (select? true))
1134   "Return the name of FILE once interned in the store.  Use NAME as its store
1135 name, or the basename of FILE if NAME is omitted.
1136
1137 When RECURSIVE? is true, the contents of FILE are added recursively; if FILE
1138 designates a flat file and RECURSIVE? is true, its contents are added, and its
1139 permission bits are kept.
1140
1141 When RECURSIVE? is true, call (SELECT?  FILE STAT) for each directory entry,
1142 where FILE is the entry's absolute file name and STAT is the result of
1143 'lstat'; exclude entries for which SELECT? does not return true."
1144   (lambda (store)
1145     (values (add-to-store store (or name (basename file))
1146                           recursive? "sha256" file
1147                           #:select? select?)
1148             store)))
1149
1150 (define build
1151   ;; Monadic variant of 'build-things'.
1152   (store-lift build-things))
1153
1154 (define set-build-options*
1155   (store-lift set-build-options))
1156
1157 (define-inlinable (current-system)
1158   ;; Consult the %CURRENT-SYSTEM fluid at bind time.  This is equivalent to
1159   ;; (lift0 %current-system %store-monad), but inlinable, thus avoiding
1160   ;; closure allocation in some cases.
1161   (lambda (state)
1162     (values (%current-system) state)))
1163
1164 (define-inlinable (set-current-system system)
1165   ;; Set the %CURRENT-SYSTEM fluid at bind time.
1166   (lambda (state)
1167     (values (%current-system system) state)))
1168
1169 (define %guile-for-build
1170   ;; The derivation of the Guile to be used within the build environment,
1171   ;; when using 'gexp->derivation' and co.
1172   (make-parameter #f))
1173
1174 (define* (run-with-store store mval
1175                          #:key
1176                          (guile-for-build (%guile-for-build))
1177                          (system (%current-system)))
1178   "Run MVAL, a monadic value in the store monad, in STORE, an open store
1179 connection, and return the result."
1180   ;; Initialize the dynamic bindings here to avoid bad surprises.  The
1181   ;; difficulty lies in the fact that dynamic bindings are resolved at
1182   ;; bind-time and not at call time, which can be disconcerting.
1183   (parameterize ((%guile-for-build guile-for-build)
1184                  (%current-system system)
1185                  (%current-target-system #f))
1186     (call-with-values (lambda ()
1187                         (run-with-state mval store))
1188       (lambda (result store)
1189         ;; Discard the state.
1190         result))))
1191
1192 \f
1193 ;;;
1194 ;;; Store paths.
1195 ;;;
1196
1197 (define %store-prefix
1198   ;; Absolute path to the Nix store.
1199   (make-parameter %store-directory))
1200
1201 (define (store-path? path)
1202   "Return #t if PATH is a store path."
1203   ;; This is a lightweight check, compared to using a regexp, but this has to
1204   ;; be fast as it's called often in `derivation', for instance.
1205   ;; `isStorePath' in Nix does something similar.
1206   (string-prefix? (%store-prefix) path))
1207
1208 (define (direct-store-path? path)
1209   "Return #t if PATH is a store path, and not a sub-directory of a store path.
1210 This predicate is sometimes needed because files *under* a store path are not
1211 valid inputs."
1212   (and (store-path? path)
1213        (not (string=? path (%store-prefix)))
1214        (let ((len (+ 1 (string-length (%store-prefix)))))
1215          (not (string-index (substring path len) #\/)))))
1216
1217 (define (direct-store-path path)
1218   "Return the direct store path part of PATH, stripping components after
1219 '/gnu/store/xxxx-foo'."
1220   (let ((prefix-length (+ (string-length (%store-prefix)) 35)))
1221     (if (> (string-length path) prefix-length)
1222         (let ((slash (string-index path #\/ prefix-length)))
1223           (if slash (string-take path slash) path))
1224         path)))
1225
1226 (define (derivation-path? path)
1227   "Return #t if PATH is a derivation path."
1228   (and (store-path? path) (string-suffix? ".drv" path)))
1229
1230 (define store-regexp*
1231   ;; The substituter makes repeated calls to 'store-path-hash-part', hence
1232   ;; this optimization.
1233   (memoize
1234    (lambda (store)
1235      "Return a regexp matching a file in STORE."
1236      (make-regexp (string-append "^" (regexp-quote store)
1237                                  "/([0-9a-df-np-sv-z]{32})-([^/]+)$")))))
1238
1239 (define (store-path-package-name path)
1240   "Return the package name part of PATH, a file name in the store."
1241   (let ((path-rx (store-regexp* (%store-prefix))))
1242     (and=> (regexp-exec path-rx path)
1243            (cut match:substring <> 2))))
1244
1245 (define (store-path-hash-part path)
1246   "Return the hash part of PATH as a base32 string, or #f if PATH is not a
1247 syntactically valid store path."
1248   (let ((path-rx (store-regexp* (%store-prefix))))
1249     (and=> (regexp-exec path-rx path)
1250            (cut match:substring <> 1))))
1251
1252 (define (log-file store file)
1253   "Return the build log file for FILE, or #f if none could be found.  FILE
1254 must be an absolute store file name, or a derivation file name."
1255   (cond ((derivation-path? file)
1256          (let* ((base    (basename file))
1257                 (log     (string-append (dirname %state-directory) ; XXX
1258                                         "/log/guix/drvs/"
1259                                         (string-take base 2) "/"
1260                                         (string-drop base 2)))
1261                 (log.bz2 (string-append log ".bz2")))
1262            (cond ((file-exists? log.bz2) log.bz2)
1263                  ((file-exists? log) log)
1264                  (else #f))))
1265         (else
1266          (match (valid-derivers store file)
1267            ((derivers ...)
1268             ;; Return the first that works.
1269             (any (cut log-file store <>) derivers))
1270            (_ #f)))))