guix/store.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
   3 ;;;
   4 ;;; This file is part of GNU Guix.
   5 ;;;
   6 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   7 ;;; under the terms of the GNU General Public License as published by
   8 ;;; the Free Software Foundation; either version 3 of the License, or (at
   9 ;;; your option) any later version.
  10 ;;;
  11 ;;; GNU Guix is distributed in the hope that it will be useful, but
  12 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 ;;; GNU General Public License for more details.
  15 ;;;
  16 ;;; You should have received a copy of the GNU General Public License
  17 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 (define-module (guix store)
  20   #:use-module (guix utils)
  21   #:use-module (guix config)
  22   #:use-module (guix memoization)
  23   #:use-module (guix serialization)
  24   #:use-module (guix monads)
  25   #:use-module (guix base16)
  26   #:use-module (guix base32)
  27   #:use-module (guix hash)
  28   #:use-module (guix profiling)
  29   #:autoload   (guix build syscalls) (terminal-columns)
  30   #:use-module (rnrs bytevectors)
  31   #:use-module (ice-9 binary-ports)
  32   #:use-module (srfi srfi-1)
  33   #:use-module (srfi srfi-9)
  34   #:use-module (srfi srfi-9 gnu)
  35   #:use-module (srfi srfi-11)
  36   #:use-module (srfi srfi-26)
  37   #:use-module (srfi srfi-34)
  38   #:use-module (srfi srfi-35)
  39   #:use-module (srfi srfi-39)
  40   #:use-module (ice-9 match)
  41   #:use-module (ice-9 regex)
  42   #:use-module (ice-9 vlist)
  43   #:use-module (ice-9 popen)
  44   #:use-module (ice-9 threads)
  45   #:use-module (ice-9 format)
  46   #:use-module (web uri)
  47   #:export (%daemon-socket-uri
  48             %gc-roots-directory
  49             %default-substitute-urls
  50
  51             nix-server?
  52             nix-server-major-version
  53             nix-server-minor-version
  54             nix-server-socket
  55
  56             &nix-error nix-error?
  57             &nix-connection-error nix-connection-error?
  58             nix-connection-error-file
  59             nix-connection-error-code
  60             &nix-protocol-error nix-protocol-error?
  61             nix-protocol-error-message
  62             nix-protocol-error-status
  63
  64             hash-algo
  65             build-mode
  66
  67             open-connection
  68             close-connection
  69             with-store
  70             set-build-options
  71             set-build-options*
  72             valid-path?
  73             query-path-hash
  74             hash-part->path
  75             query-path-info
  76             add-data-to-store
  77             add-text-to-store
  78             add-to-store
  79             build-things
  80             build
  81             query-failed-paths
  82             clear-failed-paths
  83             add-temp-root
  84             add-indirect-root
  85             add-permanent-root
  86             remove-permanent-root
  87
  88             substitutable?
  89             substitutable-path
  90             substitutable-deriver
  91             substitutable-references
  92             substitutable-download-size
  93             substitutable-nar-size
  94             has-substitutes?
  95             substitutable-paths
  96             substitutable-path-info
  97
  98             path-info?
  99             path-info-deriver
 100             path-info-hash
 101             path-info-references
 102             path-info-registration-time
 103             path-info-nar-size
 104
 105             built-in-builders
 106             references
 107             references/substitutes
 108             references*
 109             requisites
 110             referrers
 111             optimize-store
 112             verify-store
 113             topologically-sorted
 114             valid-derivers
 115             query-derivation-outputs
 116             live-paths
 117             dead-paths
 118             collect-garbage
 119             delete-paths
 120             import-paths
 121             export-paths
 122
 123             current-build-output-port
 124
 125             register-path
 126
 127             %store-monad
 128             store-bind
 129             store-return
 130             store-lift
 131             store-lower
 132             run-with-store
 133             %guile-for-build
 134             current-system
 135             set-current-system
 136             text-file
 137             interned-file
 138
 139             %store-prefix
 140             store-path
 141             output-path
 142             fixed-output-path
 143             store-path?
 144             direct-store-path?
 145             derivation-path?
 146             store-path-package-name
 147             store-path-hash-part
 148             direct-store-path
 149             log-file))
 150
 151 (define %protocol-version #x161)
 152
 153 (define %worker-magic-1 #x6e697863)               ; "nixc"
 154 (define %worker-magic-2 #x6478696f)               ; "dxio"
 155
 156 (define (protocol-major magic)
 157   (logand magic #xff00))
 158 (define (protocol-minor magic)
 159   (logand magic #x00ff))
 160
 161 (define-syntax define-enumerate-type
 162   (syntax-rules ()
 163     ((_ name->int (name id) ...)
 164      (define-syntax name->int
 165        (syntax-rules (name ...)
 166          ((_ name) id) ...)))))
 167
 168 (define-enumerate-type operation-id
 169   ;; operation numbers from worker-protocol.hh
 170   (quit 0)
 171   (valid-path? 1)
 172   (has-substitutes? 3)
 173   (query-path-hash 4)
 174   (query-references 5)
 175   (query-referrers 6)
 176   (add-to-store 7)
 177   (add-text-to-store 8)
 178   (build-things 9)
 179   (ensure-path 10)
 180   (add-temp-root 11)
 181   (add-indirect-root 12)
 182   (sync-with-gc 13)
 183   (find-roots 14)
 184   (export-path 16)
 185   (query-deriver 18)
 186   (set-options 19)
 187   (collect-garbage 20)
 188   ;;(query-substitutable-path-info 21)  ; obsolete as of #x10c
 189   (query-derivation-outputs 22)
 190   (query-all-valid-paths 23)
 191   (query-failed-paths 24)
 192   (clear-failed-paths 25)
 193   (query-path-info 26)
 194   (import-paths 27)
 195   (query-derivation-output-names 28)
 196   (query-path-from-hash-part 29)
 197   (query-substitutable-path-infos 30)
 198   (query-valid-paths 31)
 199   (query-substitutable-paths 32)
 200   (query-valid-derivers 33)
 201   (optimize-store 34)
 202   (verify-store 35)
 203   (built-in-builders 80))
 204
 205 (define-enumerate-type hash-algo
 206   ;; hash.hh
 207   (md5 1)
 208   (sha1 2)
 209   (sha256 3))
 210
 211 (define-enumerate-type build-mode
 212   ;; store-api.hh
 213   (normal 0)
 214   (repair 1)
 215   (check 2))
 216
 217 (define-enumerate-type gc-action
 218   ;; store-api.hh
 219   (return-live 0)
 220   (return-dead 1)
 221   (delete-dead 2)
 222   (delete-specific 3))
 223
 224 (define %default-socket-path
 225   (string-append %state-directory "/daemon-socket/socket"))
 226
 227 (define %daemon-socket-uri
 228   ;; URI or file name of the socket the daemon listens too.
 229   (make-parameter (or (getenv "GUIX_DAEMON_SOCKET")
 230                       %default-socket-path)))
 231
 232
 233 \f
 234 ;; Information about a substitutable store path.
 235 (define-record-type <substitutable>
 236   (substitutable path deriver refs dl-size nar-size)
 237   substitutable?
 238   (path      substitutable-path)
 239   (deriver   substitutable-deriver)
 240   (refs      substitutable-references)
 241   (dl-size   substitutable-download-size)
 242   (nar-size  substitutable-nar-size))
 243
 244 (define (read-substitutable-path-list p)
 245   (let loop ((len    (read-int p))
 246              (result '()))
 247     (if (zero? len)
 248         (reverse result)
 249         (let ((path     (read-store-path p))
 250               (deriver  (read-store-path p))
 251               (refs     (read-store-path-list p))
 252               (dl-size  (read-long-long p))
 253               (nar-size (read-long-long p)))
 254           (loop (- len 1)
 255                 (cons (substitutable path deriver refs dl-size nar-size)
 256                       result))))))
 257
 258 ;; Information about a store path.
 259 (define-record-type <path-info>
 260   (path-info deriver hash references registration-time nar-size)
 261   path-info?
 262   (deriver path-info-deriver)                     ;string | #f
 263   (hash path-info-hash)
 264   (references path-info-references)
 265   (registration-time path-info-registration-time)
 266   (nar-size path-info-nar-size))
 267
 268 (define (read-path-info p)
 269   (let ((deriver  (match (read-store-path p)
 270                     ("" #f)
 271                     (x  x)))
 272         (hash     (base16-string->bytevector (read-string p)))
 273         (refs     (read-store-path-list p))
 274         (registration-time (read-int p))
 275         (nar-size (read-long-long p)))
 276     (path-info deriver hash refs registration-time nar-size)))
 277
 278 (define-syntax write-arg
 279   (syntax-rules (integer boolean bytevector
 280                  string string-list string-pairs
 281                  store-path store-path-list base16)
 282     ((_ integer arg p)
 283      (write-int arg p))
 284     ((_ boolean arg p)
 285      (write-int (if arg 1 0) p))
 286     ((_ bytevector arg p)
 287      (write-bytevector arg p))
 288     ((_ string arg p)
 289      (write-string arg p))
 290     ((_ string-list arg p)
 291      (write-string-list arg p))
 292     ((_ string-pairs arg p)
 293      (write-string-pairs arg p))
 294     ((_ store-path arg p)
 295      (write-store-path arg p))
 296     ((_ store-path-list arg p)
 297      (write-store-path-list arg p))
 298     ((_ base16 arg p)
 299      (write-string (bytevector->base16-string arg) p))))
 300
 301 (define-syntax read-arg
 302   (syntax-rules (integer boolean string store-path store-path-list string-list
 303                  substitutable-path-list path-info base16)
 304     ((_ integer p)
 305      (read-int p))
 306     ((_ boolean p)
 307      (not (zero? (read-int p))))
 308     ((_ string p)
 309      (read-string p))
 310     ((_ store-path p)
 311      (read-store-path p))
 312     ((_ store-path-list p)
 313      (read-store-path-list p))
 314     ((_ string-list p)
 315      (read-string-list p))
 316     ((_ substitutable-path-list p)
 317      (read-substitutable-path-list p))
 318     ((_ path-info p)
 319      (read-path-info p))
 320     ((_ base16 p)
 321      (base16-string->bytevector (read-string p)))))
 322
 323 \f
 324 ;; remote-store.cc
 325
 326 (define-record-type <nix-server>
 327   (%make-nix-server socket major minor
 328                     buffer flush
 329                     ats-cache atts-cache)
 330   nix-server?
 331   (socket nix-server-socket)
 332   (major  nix-server-major-version)
 333   (minor  nix-server-minor-version)
 334
 335   (buffer nix-server-output-port)                 ;output port
 336   (flush  nix-server-flush-output)                ;thunk
 337
 338   ;; Caches.  We keep them per-connection, because store paths build
 339   ;; during the session are temporary GC roots kept for the duration of
 340   ;; the session.
 341   (ats-cache  nix-server-add-to-store-cache)
 342   (atts-cache nix-server-add-text-to-store-cache))
 343
 344 (set-record-type-printer! <nix-server>
 345                           (lambda (obj port)
 346                             (format port "#<build-daemon ~a.~a ~a>"
 347                                     (nix-server-major-version obj)
 348                                     (nix-server-minor-version obj)
 349                                     (number->string (object-address obj)
 350                                                     16))))
 351
 352 (define-condition-type &nix-error &error
 353   nix-error?)
 354
 355 (define-condition-type &nix-connection-error &nix-error
 356   nix-connection-error?
 357   (file   nix-connection-error-file)
 358   (errno  nix-connection-error-code))
 359
 360 (define-condition-type &nix-protocol-error &nix-error
 361   nix-protocol-error?
 362   (message nix-protocol-error-message)
 363   (status  nix-protocol-error-status))
 364
 365 (define-syntax-rule (system-error-to-connection-error file exp ...)
 366   "Catch 'system-error' exceptions and translate them to
 367 '&nix-connection-error'."
 368   (catch 'system-error
 369     (lambda ()
 370       exp ...)
 371     (lambda args
 372       (let ((errno (system-error-errno args)))
 373         (raise (condition (&nix-connection-error
 374                            (file file)
 375                            (errno errno))))))))
 376
 377 (define (open-unix-domain-socket file)
 378   "Connect to the Unix-domain socket at FILE and return it.  Raise a
 379 '&nix-connection-error' upon error."
 380   (let ((s (with-fluids ((%default-port-encoding #f))
 381              ;; This trick allows use of the `scm_c_read' optimization.
 382              (socket PF_UNIX SOCK_STREAM 0)))
 383         (a (make-socket-address PF_UNIX file)))
 384
 385     (system-error-to-connection-error file
 386       (connect s a)
 387       s)))
 388
 389 (define %default-guix-port
 390   ;; Default port when connecting to a daemon over TCP/IP.
 391   44146)
 392
 393 (define (open-inet-socket host port)
 394   "Connect to the Unix-domain socket at HOST:PORT and return it.  Raise a
 395 '&nix-connection-error' upon error."
 396   ;; Define 'TCP_NODELAY' on Guile 2.0.  The value is the same on all GNU
 397   ;; systems.
 398   (cond-expand (guile-2.2 #t)
 399                (else      (define TCP_NODELAY 1)))
 400
 401   (let ((sock (with-fluids ((%default-port-encoding #f))
 402                 ;; This trick allows use of the `scm_c_read' optimization.
 403                 (socket PF_UNIX SOCK_STREAM 0))))
 404     (define addresses
 405       (getaddrinfo host
 406                    (if (number? port) (number->string port) port)
 407                    (if (number? port)
 408                        (logior AI_ADDRCONFIG AI_NUMERICSERV)
 409                        AI_ADDRCONFIG)
 410                    0                              ;any address family
 411                    SOCK_STREAM))                  ;TCP only
 412
 413     (let loop ((addresses addresses))
 414       (match addresses
 415         ((ai rest ...)
 416          (let ((s (socket (addrinfo:fam ai)
 417                           ;; TCP/IP only
 418                           SOCK_STREAM IPPROTO_IP)))
 419
 420            (catch 'system-error
 421              (lambda ()
 422                (connect s (addrinfo:addr ai))
 423
 424                ;; Setting this option makes a dramatic difference because it
 425                ;; avoids the "ACK delay" on our RPC messages.
 426                (setsockopt s IPPROTO_TCP TCP_NODELAY 1)
 427                s)
 428              (lambda args
 429                ;; Connection failed, so try one of the other addresses.
 430                (close s)
 431                (if (null? rest)
 432                    (raise (condition (&nix-connection-error
 433                                       (file host)
 434                                       (errno (system-error-errno args)))))
 435                    (loop rest))))))))))
 436
 437 (define (connect-to-daemon uri)
 438   "Connect to the daemon at URI, a string that may be an actual URI or a file
 439 name."
 440   (define (not-supported)
 441     (raise (condition (&nix-connection-error
 442                        (file uri)
 443                        (errno ENOTSUP)))))
 444
 445   (define connect
 446     (match (string->uri uri)
 447       (#f                                         ;URI is a file name
 448        open-unix-domain-socket)
 449       ((? uri? uri)
 450        (match (uri-scheme uri)
 451          ((or #f 'file 'unix)
 452           (lambda (_)
 453             (open-unix-domain-socket (uri-path uri))))
 454          ('guix
 455           (lambda (_)
 456             (open-inet-socket (uri-host uri)
 457                               (or (uri-port uri) %default-guix-port))))
 458          ((? symbol? scheme)
 459           ;; Try to dynamically load a module for SCHEME.
 460           ;; XXX: Errors are swallowed.
 461           (match (false-if-exception
 462                   (resolve-interface `(guix store ,scheme)))
 463             ((? module? module)
 464              (match (false-if-exception
 465                      (module-ref module 'connect-to-daemon))
 466                ((? procedure? connect)
 467                 (lambda (_)
 468                   (connect uri)))
 469                (x (not-supported))))
 470             (#f (not-supported))))
 471          (x
 472           (not-supported))))))
 473
 474   (connect uri))
 475
 476 (define* (open-connection #:optional (uri (%daemon-socket-uri))
 477                           #:key port (reserve-space? #t) cpu-affinity)
 478   "Connect to the daemon at URI (a string), or, if PORT is not #f, use it as
 479 the I/O port over which to communicate to a build daemon.
 480
 481 When RESERVE-SPACE? is true, instruct it to reserve a little bit of extra
 482 space on the file system so that the garbage collector can still operate,
 483 should the disk become full.  When CPU-AFFINITY is true, it must be an integer
 484 corresponding to an OS-level CPU number to which the daemon's worker process
 485 for this connection will be pinned.  Return a server object."
 486   (guard (c ((nar-error? c)
 487              ;; One of the 'write-' or 'read-' calls below failed, but this is
 488              ;; really a connection error.
 489              (raise (condition
 490                      (&nix-connection-error (file (or port uri))
 491                                             (errno EPROTO))
 492                      (&message (message "build daemon handshake failed"))))))
 493     (let*-values (((port)
 494                    (or port (connect-to-daemon uri)))
 495                   ((output flush)
 496                    (buffering-output-port port
 497                                           (make-bytevector 8192))))
 498       (write-int %worker-magic-1 port)
 499       (let ((r (read-int port)))
 500         (and (eqv? r %worker-magic-2)
 501              (let ((v (read-int port)))
 502                (and (eqv? (protocol-major %protocol-version)
 503                           (protocol-major v))
 504                     (begin
 505                       (write-int %protocol-version port)
 506                       (when (>= (protocol-minor v) 14)
 507                         (write-int (if cpu-affinity 1 0) port)
 508                         (when cpu-affinity
 509                           (write-int cpu-affinity port)))
 510                       (when (>= (protocol-minor v) 11)
 511                         (write-int (if reserve-space? 1 0) port))
 512                       (let ((conn (%make-nix-server port
 513                                                     (protocol-major v)
 514                                                     (protocol-minor v)
 515                                                     output flush
 516                                                     (make-hash-table 100)
 517                                                     (make-hash-table 100))))
 518                         (let loop ((done? (process-stderr conn)))
 519                           (or done? (process-stderr conn)))
 520                         conn)))))))))
 521
 522 (define (write-buffered-output server)
 523   "Flush SERVER's output port."
 524   (force-output (nix-server-output-port server))
 525   ((nix-server-flush-output server)))
 526
 527 (define (close-connection server)
 528   "Close the connection to SERVER."
 529   (close (nix-server-socket server)))
 530
 531 (define-syntax-rule (with-store store exp ...)
 532   "Bind STORE to an open connection to the store and evaluate EXPs;
 533 automatically close the store when the dynamic extent of EXP is left."
 534   (let ((store (open-connection)))
 535     (dynamic-wind
 536       (const #f)
 537       (lambda ()
 538         exp ...)
 539       (lambda ()
 540         (false-if-exception (close-connection store))))))
 541
 542 (define current-build-output-port
 543   ;; The port where build output is sent.
 544   (make-parameter (current-error-port)))
 545
 546 (define* (dump-port in out
 547                     #:optional len
 548                     #:key (buffer-size 16384))
 549   "Read LEN bytes from IN (or as much as possible if LEN is #f) and write it
 550 to OUT, using chunks of BUFFER-SIZE bytes."
 551   (define buffer
 552     (make-bytevector buffer-size))
 553
 554   (let loop ((total 0)
 555              (bytes (get-bytevector-n! in buffer 0
 556                                        (if len
 557                                            (min len buffer-size)
 558                                            buffer-size))))
 559     (or (eof-object? bytes)
 560         (and len (= total len))
 561         (let ((total (+ total bytes)))
 562           (put-bytevector out buffer 0 bytes)
 563           (loop total
 564                 (get-bytevector-n! in buffer 0
 565                                    (if len
 566                                        (min (- len total) buffer-size)
 567                                        buffer-size)))))))
 568
 569 (define %newlines
 570   ;; Newline characters triggering a flush of 'current-build-output-port'.
 571   ;; Unlike Guile's _IOLBF, we flush upon #\return so that progress reports
 572   ;; that use that trick are correctly displayed.
 573   (char-set #\newline #\return))
 574
 575 (define* (process-stderr server #:optional user-port)
 576   "Read standard output and standard error from SERVER, writing it to
 577 CURRENT-BUILD-OUTPUT-PORT.  Return #t when SERVER is done sending data, and
 578 #f otherwise; in the latter case, the caller should call `process-stderr'
 579 again until #t is returned or an error is raised.
 580
 581 Since the build process's output cannot be assumed to be UTF-8, we
 582 conservatively consider it to be Latin-1, thereby avoiding possible
 583 encoding conversion errors."
 584   (define p
 585     (nix-server-socket server))
 586
 587   ;; magic cookies from worker-protocol.hh
 588   (define %stderr-next  #x6f6c6d67)          ; "olmg", build log
 589   (define %stderr-read  #x64617461)          ; "data", data needed from source
 590   (define %stderr-write #x64617416)          ; "dat\x16", data for sink
 591   (define %stderr-last  #x616c7473)          ; "alts", we're done
 592   (define %stderr-error #x63787470)          ; "cxtp", error reporting
 593
 594   (let ((k (read-int p)))
 595     (cond ((= k %stderr-write)
 596            ;; Write a byte stream to USER-PORT.
 597            (let* ((len (read-int p))
 598                   (m   (modulo len 8)))
 599              (dump-port p user-port len
 600                         #:buffer-size (if (<= len 16384) 16384 65536))
 601              (unless (zero? m)
 602                ;; Consume padding, as for strings.
 603                (get-bytevector-n p (- 8 m))))
 604            #f)
 605           ((= k %stderr-read)
 606            ;; Read a byte stream from USER-PORT.
 607            ;; Note: Avoid 'get-bytevector-n' to work around
 608            ;; <http://bugs.gnu.org/17591> in Guile up to 2.0.11.
 609            (let* ((max-len (read-int p))
 610                   (data    (make-bytevector max-len))
 611                   (len     (get-bytevector-n! user-port data 0 max-len)))
 612              (write-bytevector data p len)
 613              #f))
 614           ((= k %stderr-next)
 615            ;; Log a string.  Build logs are usually UTF-8-encoded, but they
 616            ;; may also contain arbitrary byte sequences that should not cause
 617            ;; this to fail.  Thus, use the permissive
 618            ;; 'read-maybe-utf8-string'.
 619            (let ((s (read-maybe-utf8-string p)))
 620              (display s (current-build-output-port))
 621              (when (string-any %newlines s)
 622                (force-output (current-build-output-port)))
 623              #f))
 624           ((= k %stderr-error)
 625            ;; Report an error.
 626            (let ((error  (read-maybe-utf8-string p))
 627                  ;; Currently the daemon fails to send a status code for early
 628                  ;; errors like DB schema version mismatches, so check for EOF.
 629                  (status (if (and (>= (nix-server-minor-version server) 8)
 630                                   (not (eof-object? (lookahead-u8 p))))
 631                              (read-int p)
 632                              1)))
 633              (raise (condition (&nix-protocol-error
 634                                 (message error)
 635                                 (status  status))))))
 636           ((= k %stderr-last)
 637            ;; The daemon is done (see `stopWork' in `nix-worker.cc'.)
 638            #t)
 639           (else
 640            (raise (condition (&nix-protocol-error
 641                               (message "invalid error code")
 642                               (status   k))))))))
 643
 644 (define %default-substitute-urls
 645   ;; Default list of substituters.  This is *not* the list baked in
 646   ;; 'guix-daemon', but it is used by 'guix-service-type' and and a couple of
 647   ;; clients ('guix build --log-file' uses it.)
 648   (map (if (false-if-exception (resolve-interface '(gnutls)))
 649            (cut string-append "https://" <>)
 650            (cut string-append "http://" <>))
 651        '("mirror.hydra.gnu.org")))
 652
 653 (define* (set-build-options server
 654                             #:key keep-failed? keep-going? fallback?
 655                             (verbosity 0)
 656                             rounds                ;number of build rounds
 657                             max-build-jobs
 658                             timeout
 659                             max-silent-time
 660                             (use-build-hook? #t)
 661                             (build-verbosity 0)
 662                             (log-type 0)
 663                             (print-build-trace #t)
 664                             build-cores
 665                             (use-substitutes? #t)
 666
 667                             ;; Client-provided substitute URLs.  If it is #f,
 668                             ;; the daemon's settings are used.  Otherwise, it
 669                             ;; overrides the daemons settings; see 'guix
 670                             ;; substitute'.
 671                             (substitute-urls #f)
 672
 673                             ;; Number of columns in the client's terminal.
 674                             (terminal-columns (terminal-columns))
 675
 676                             ;; Locale of the client.
 677                             (locale (false-if-exception (setlocale LC_ALL))))
 678   ;; Must be called after `open-connection'.
 679
 680   (define socket
 681     (nix-server-socket server))
 682
 683   (let-syntax ((send (syntax-rules ()
 684                        ((_ (type option) ...)
 685                         (begin
 686                           (write-arg type option socket)
 687                           ...)))))
 688     (write-int (operation-id set-options) socket)
 689     (send (boolean keep-failed?) (boolean keep-going?)
 690           (boolean fallback?) (integer verbosity))
 691     (when (< (nix-server-minor-version server) #x61)
 692       (let ((max-build-jobs (or max-build-jobs 1))
 693             (max-silent-time (or max-silent-time 3600)))
 694         (send (integer max-build-jobs) (integer max-silent-time))))
 695     (when (>= (nix-server-minor-version server) 2)
 696       (send (boolean use-build-hook?)))
 697     (when (>= (nix-server-minor-version server) 4)
 698       (send (integer build-verbosity) (integer log-type)
 699             (boolean print-build-trace)))
 700     (when (and (>= (nix-server-minor-version server) 6)
 701                (< (nix-server-minor-version server) #x61))
 702       (let ((build-cores (or build-cores (current-processor-count))))
 703         (send (integer build-cores))))
 704     (when (>= (nix-server-minor-version server) 10)
 705       (send (boolean use-substitutes?)))
 706     (when (>= (nix-server-minor-version server) 12)
 707       (let ((pairs `(,@(if timeout
 708                            `(("build-timeout" . ,(number->string timeout)))
 709                            '())
 710                      ,@(if max-silent-time
 711                            `(("build-max-silent-time"
 712                               . ,(number->string max-silent-time)))
 713                            '())
 714                      ,@(if max-build-jobs
 715                            `(("build-max-jobs"
 716                               . ,(number->string max-build-jobs)))
 717                            '())
 718                      ,@(if build-cores
 719                            `(("build-cores" . ,(number->string build-cores)))
 720                            '())
 721                      ,@(if substitute-urls
 722                            `(("substitute-urls"
 723                               . ,(string-join substitute-urls)))
 724                            '())
 725                      ,@(if rounds
 726                            `(("build-repeat"
 727                               . ,(number->string (max 0 (1- rounds)))))
 728                            '())
 729                      ,@(if terminal-columns
 730                            `(("terminal-columns"
 731                               . ,(number->string terminal-columns)))
 732                            '())
 733                      ,@(if locale
 734                            `(("locale" . ,locale))
 735                            '()))))
 736         (send (string-pairs pairs))))
 737     (let loop ((done? (process-stderr server)))
 738       (or done? (process-stderr server)))))
 739
 740 (define (buffering-output-port port buffer)
 741   "Return two value: an output port wrapped around PORT that uses BUFFER (a
 742 bytevector) as its internal buffer, and a thunk to flush this output port."
 743   ;; Note: In Guile 2.2.2, custom binary output ports already have their own
 744   ;; 4K internal buffer.
 745   (define size
 746     (bytevector-length buffer))
 747
 748   (define total 0)
 749
 750   (define (flush)
 751     (put-bytevector port buffer 0 total)
 752     (set! total 0))
 753
 754   (define (write bv offset count)
 755     (if (zero? count)                             ;end of file
 756         (flush)
 757         (let loop ((offset offset)
 758                    (count count)
 759                    (written 0))
 760           (cond ((= total size)
 761                  (flush)
 762                  (loop offset count written))
 763                 ((zero? count)
 764                  written)
 765                 (else
 766                  (let ((to-copy (min count (- size total))))
 767                    (bytevector-copy! bv offset buffer total to-copy)
 768                    (set! total (+ total to-copy))
 769                    (loop (+ offset to-copy) (- count to-copy)
 770                          (+ written to-copy))))))))
 771
 772   ;; Note: We need to return FLUSH because the custom binary port has no way
 773   ;; to be notified of a 'force-output' call on itself.
 774   (values (make-custom-binary-output-port "buffering-output-port"
 775                                           write #f #f flush)
 776           flush))
 777
 778 (define %rpc-calls
 779   ;; Mapping from RPC names (symbols) to invocation counts.
 780   (make-hash-table))
 781
 782 (define* (show-rpc-profile #:optional (port (current-error-port)))
 783   "Write to PORT a summary of the RPCs that have been made."
 784   (let ((profile (sort (hash-fold alist-cons '() %rpc-calls)
 785                        (lambda (rpc1 rpc2)
 786                          (< (cdr rpc1) (cdr rpc2))))))
 787     (format port "Remote procedure call summary: ~a RPCs~%"
 788             (match profile
 789               (((names . counts) ...)
 790                (reduce + 0 counts))))
 791     (for-each (match-lambda
 792                 ((rpc . count)
 793                  (format port "  ~30a ... ~5@a~%" rpc count)))
 794               profile)))
 795
 796 (define record-operation
 797   ;; Optionally, increment the number of calls of the given RPC.
 798   (if (profiled? "rpc")
 799       (begin
 800         (register-profiling-hook! "rpc" show-rpc-profile)
 801         (lambda (name)
 802           (let ((count (or (hashq-ref %rpc-calls name) 0)))
 803             (hashq-set! %rpc-calls name (+ count 1)))))
 804       (lambda (_)
 805         #t)))
 806
 807 (define-syntax operation
 808   (syntax-rules ()
 809     "Define a client-side RPC stub for the given operation."
 810     ((_ (name (type arg) ...) docstring return ...)
 811      (lambda (server arg ...)
 812        docstring
 813        (let* ((s (nix-server-socket server))
 814               (buffered (nix-server-output-port server)))
 815          (record-operation 'name)
 816          (write-int (operation-id name) buffered)
 817          (write-arg type arg buffered)
 818          ...
 819          (write-buffered-output server)
 820
 821          ;; Loop until the server is done sending error output.
 822          (let loop ((done? (process-stderr server)))
 823            (or done? (loop (process-stderr server))))
 824          (values (read-arg return s) ...))))))
 825
 826 (define-syntax-rule (define-operation (name args ...)
 827                       docstring return ...)
 828   (define name
 829     (operation (name args ...) docstring return ...)))
 830
 831 (define-operation (valid-path? (string path))
 832   "Return #t when PATH designates a valid store item and #f otherwise (an
 833 invalid item may exist on disk but still be invalid, for instance because it
 834 is the result of an aborted or failed build.)
 835
 836 A '&nix-protocol-error' condition is raised if PATH is not prefixed by the
 837 store directory (/gnu/store)."
 838   boolean)
 839
 840 (define-operation (query-path-hash (store-path path))
 841   "Return the SHA256 hash of the nar serialization of PATH as a bytevector."
 842   base16)
 843
 844 (define hash-part->path
 845   (let ((query-path-from-hash-part
 846          (operation (query-path-from-hash-part (string hash))
 847                     #f
 848                     store-path)))
 849    (lambda (server hash-part)
 850      "Return the store path whose hash part is HASH-PART (a nix-base32
 851 string).  Raise an error if no such path exists."
 852      ;; This RPC is primarily used by Hydra to reply to HTTP GETs of
 853      ;; /HASH.narinfo.
 854      (query-path-from-hash-part server hash-part))))
 855
 856 (define-operation (query-path-info (store-path path))
 857   "Return the info (hash, references, etc.) for PATH."
 858   path-info)
 859
 860 (define add-data-to-store
 861   ;; A memoizing version of `add-to-store', to avoid repeated RPCs with
 862   ;; the very same arguments during a given session.
 863   (let ((add-text-to-store
 864          (operation (add-text-to-store (string name) (bytevector text)
 865                                        (string-list references))
 866                     #f
 867                     store-path)))
 868     (lambda* (server name bytes #:optional (references '()))
 869       "Add BYTES under file NAME in the store, and return its store path.
 870 REFERENCES is the list of store paths referred to by the resulting store
 871 path."
 872       (let* ((args  `(,bytes ,name ,references))
 873              (cache (nix-server-add-text-to-store-cache server)))
 874         (or (hash-ref cache args)
 875             (let ((path (add-text-to-store server name bytes references)))
 876               (hash-set! cache args path)
 877               path))))))
 878
 879 (define* (add-text-to-store store name text #:optional (references '()))
 880   "Add TEXT under file NAME in the store, and return its store path.
 881 REFERENCES is the list of store paths referred to by the resulting store
 882 path."
 883   (add-data-to-store store name (string->utf8 text) references))
 884
 885 (define true
 886   ;; Define it once and for all since we use it as a default value for
 887   ;; 'add-to-store' and want to make sure two default values are 'eq?' for the
 888   ;; purposes or memoization.
 889   (lambda (file stat)
 890     #t))
 891
 892 (define add-to-store
 893   ;; A memoizing version of `add-to-store'.  This is important because
 894   ;; `add-to-store' leads to huge data transfers to the server, and
 895   ;; because it's often called many times with the very same argument.
 896   (let ((add-to-store
 897          (lambda* (server basename recursive? hash-algo file-name
 898                           #:key (select? true))
 899            ;; We don't use the 'operation' macro so we can pass SELECT? to
 900            ;; 'write-file'.
 901            (record-operation 'add-to-store)
 902            (let ((port (nix-server-socket server)))
 903              (write-int (operation-id add-to-store) port)
 904              (write-string basename port)
 905              (write-int 1 port)                   ;obsolete, must be #t
 906              (write-int (if recursive? 1 0) port)
 907              (write-string hash-algo port)
 908              (write-file file-name port #:select? select?)
 909              (let loop ((done? (process-stderr server)))
 910                (or done? (loop (process-stderr server))))
 911              (read-store-path port)))))
 912     (lambda* (server basename recursive? hash-algo file-name
 913                      #:key (select? true))
 914       "Add the contents of FILE-NAME under BASENAME to the store.  When
 915 RECURSIVE? is false, FILE-NAME must designate a regular file--not a directory
 916 nor a symlink.  When RECURSIVE? is true and FILE-NAME designates a directory,
 917 the contents of FILE-NAME are added recursively; if FILE-NAME designates a
 918 flat file and RECURSIVE? is true, its contents are added, and its permission
 919 bits are kept.  HASH-ALGO must be a string such as \"sha256\".
 920
 921 When RECURSIVE? is true, call (SELECT?  FILE STAT) for each directory entry,
 922 where FILE is the entry's absolute file name and STAT is the result of
 923 'lstat'; exclude entries for which SELECT? does not return true."
 924       ;; Note: We don't stat FILE-NAME at each call, and thus we assume that
 925       ;; the file remains unchanged for the lifetime of SERVER.
 926       (let* ((args  `(,file-name ,basename ,recursive? ,hash-algo ,select?))
 927              (cache (nix-server-add-to-store-cache server)))
 928         (or (hash-ref cache args)
 929             (let ((path (add-to-store server basename recursive?
 930                                       hash-algo file-name
 931                                       #:select? select?)))
 932               (hash-set! cache args path)
 933               path))))))
 934
 935 (define build-things
 936   (let ((build (operation (build-things (string-list things)
 937                                         (integer mode))
 938                           "Do it!"
 939                           boolean))
 940         (build/old (operation (build-things (string-list things))
 941                               "Do it!"
 942                               boolean)))
 943     (lambda* (store things #:optional (mode (build-mode normal)))
 944       "Build THINGS, a list of store items which may be either '.drv' files or
 945 outputs, and return when the worker is done building them.  Elements of THINGS
 946 that are not derivations can only be substituted and not built locally.
 947 Return #t on success."
 948       (if (>= (nix-server-minor-version store) 15)
 949           (build store things mode)
 950           (if (= mode (build-mode normal))
 951               (build/old store things)
 952               (raise (condition (&nix-protocol-error
 953                                  (message "unsupported build mode")
 954                                  (status  1)))))))))
 955
 956 (define-operation (add-temp-root (store-path path))
 957   "Make PATH a temporary root for the duration of the current session.
 958 Return #t."
 959   boolean)
 960
 961 (define-operation (add-indirect-root (string file-name))
 962   "Make the symlink FILE-NAME an indirect root for the garbage collector:
 963 whatever store item FILE-NAME points to will not be collected.  Return #t on
 964 success.
 965
 966 FILE-NAME can be anywhere on the file system, but it must be an absolute file
 967 name--it is the caller's responsibility to ensure that it is an absolute file
 968 name."
 969   boolean)
 970
 971 (define %gc-roots-directory
 972   ;; The place where garbage collector roots (symlinks) are kept.
 973   (string-append %state-directory "/gcroots"))
 974
 975 (define (add-permanent-root target)
 976   "Add a garbage collector root pointing to TARGET, an element of the store,
 977 preventing TARGET from even being collected.  This can also be used if TARGET
 978 does not exist yet.
 979
 980 Raise an error if the caller does not have write access to the GC root
 981 directory."
 982   (let* ((root (string-append %gc-roots-directory "/" (basename target))))
 983     (catch 'system-error
 984       (lambda ()
 985         (symlink target root))
 986       (lambda args
 987         ;; If ROOT already exists, this is fine; otherwise, re-throw.
 988         (unless (= EEXIST (system-error-errno args))
 989           (apply throw args))))))
 990
 991 (define (remove-permanent-root target)
 992   "Remove the permanent garbage collector root pointing to TARGET.  Raise an
 993 error if there is no such root."
 994   (delete-file (string-append %gc-roots-directory "/" (basename target))))
 995
 996 (define references
 997   (operation (query-references (store-path path))
 998              "Return the list of references of PATH."
 999              store-path-list))
1000
1001 (define %reference-cache
1002   ;; Brute-force cache mapping store items to their list of references.
1003   ;; Caching matters because when building a profile in the presence of
1004   ;; grafts, we keep calling 'graft-derivation', which in turn calls
1005   ;; 'references/substitutes' many times with the same arguments.  Ideally we
1006   ;; would use a cache associated with the daemon connection instead (XXX).
1007   (make-hash-table 100))
1008
1009 (define (references/substitutes store items)
1010   "Return the list of list of references of ITEMS; the result has the same
1011 length as ITEMS.  Query substitute information for any item missing from the
1012 store at once.  Raise a '&nix-protocol-error' exception if reference
1013 information for one of ITEMS is missing."
1014   (let* ((requested  items)
1015          (local-refs (map (lambda (item)
1016                             (or (hash-ref %reference-cache item)
1017                                 (guard (c ((nix-protocol-error? c) #f))
1018                                   (references store item))))
1019                           items))
1020          (missing    (fold-right (lambda (item local-ref result)
1021                                    (if local-ref
1022                                        result
1023                                        (cons item result)))
1024                                  '()
1025                                  items local-refs))
1026
1027          ;; Query all the substitutes at once to minimize the cost of
1028          ;; launching 'guix substitute' and making HTTP requests.
1029          (substs     (if (null? missing)
1030                          '()
1031                          (substitutable-path-info store missing))))
1032     (when (< (length substs) (length missing))
1033       (raise (condition (&nix-protocol-error
1034                          (message "cannot determine \
1035 the list of references")
1036                          (status 1)))))
1037
1038     ;; Intersperse SUBSTS and LOCAL-REFS.
1039     (let loop ((items       items)
1040                (local-refs  local-refs)
1041                (result      '()))
1042       (match items
1043         (()
1044          (let ((result (reverse result)))
1045            (for-each (cut hash-set! %reference-cache <> <>)
1046                      requested result)
1047            result))
1048         ((item items ...)
1049          (match local-refs
1050            ((#f tail ...)
1051             (loop items tail
1052                   (cons (any (lambda (subst)
1053                                (and (string=? (substitutable-path subst) item)
1054                                     (substitutable-references subst)))
1055                              substs)
1056                         result)))
1057            ((head tail ...)
1058             (loop items tail
1059                   (cons head result)))))))))
1060
1061 (define* (fold-path store proc seed paths
1062                     #:optional (relatives (cut references store <>)))
1063   "Call PROC for each of the RELATIVES of PATHS, exactly once, and return the
1064 result formed from the successive calls to PROC, the first of which is passed
1065 SEED."
1066   (let loop ((paths  paths)
1067              (result seed)
1068              (seen   vlist-null))
1069     (match paths
1070       ((path rest ...)
1071        (if (vhash-assoc path seen)
1072            (loop rest result seen)
1073            (let ((seen   (vhash-cons path #t seen))
1074                  (rest   (append rest (relatives path)))
1075                  (result (proc path result)))
1076              (loop rest result seen))))
1077       (()
1078        result))))
1079
1080 (define (requisites store paths)
1081   "Return the requisites of PATHS, including PATHS---i.e., their closures (all
1082 its references, recursively)."
1083   (fold-path store cons '() paths))
1084
1085 (define (topologically-sorted store paths)
1086   "Return a list containing PATHS and all their references sorted in
1087 topological order."
1088   (define (traverse)
1089     ;; Do a simple depth-first traversal of all of PATHS.
1090     (let loop ((paths   paths)
1091                (visited vlist-null)
1092                (result  '()))
1093       (define (visit n)
1094         (vhash-cons n #t visited))
1095
1096       (define (visited? n)
1097         (vhash-assoc n visited))
1098
1099       (match paths
1100         ((head tail ...)
1101          (if (visited? head)
1102              (loop tail visited result)
1103              (call-with-values
1104                  (lambda ()
1105                    (loop (references store head)
1106                          (visit head)
1107                          result))
1108                (lambda (visited result)
1109                  (loop tail
1110                        visited
1111                        (cons head result))))))
1112         (()
1113          (values visited result)))))
1114
1115   (call-with-values traverse
1116     (lambda (_ result)
1117       (reverse result))))
1118
1119 (define referrers
1120   (operation (query-referrers (store-path path))
1121              "Return the list of path that refer to PATH."
1122              store-path-list))
1123
1124 (define valid-derivers
1125   (operation (query-valid-derivers (store-path path))
1126              "Return the list of valid \"derivers\" of PATH---i.e., all the
1127 .drv present in the store that have PATH among their outputs."
1128              store-path-list))
1129
1130 (define query-derivation-outputs  ; avoid name clash with `derivation-outputs'
1131   (operation (query-derivation-outputs (store-path path))
1132              "Return the list of outputs of PATH, a .drv file."
1133              store-path-list))
1134
1135 (define-operation (has-substitutes? (store-path path))
1136   "Return #t if binary substitutes are available for PATH, and #f otherwise."
1137   boolean)
1138
1139 (define substitutable-paths
1140   (operation (query-substitutable-paths (store-path-list paths))
1141              "Return the subset of PATHS that is substitutable."
1142              store-path-list))
1143
1144 (define substitutable-path-info
1145   (operation (query-substitutable-path-infos (store-path-list paths))
1146              "Return information about the subset of PATHS that is
1147 substitutable.  For each substitutable path, a `substitutable?' object is
1148 returned; thus, the resulting list can be shorter than PATHS.  Furthermore,
1149 that there is no guarantee that the order of the resulting list matches the
1150 order of PATHS."
1151              substitutable-path-list))
1152
1153 (define built-in-builders
1154   (let ((builders (operation (built-in-builders)
1155                              "Return the built-in builders."
1156                              string-list)))
1157     (lambda (store)
1158       "Return the names of the supported built-in derivation builders
1159 supported by STORE."
1160       ;; Check whether STORE's version supports this RPC and built-in
1161       ;; derivation builders in general, which appeared in Guix > 0.11.0.
1162       ;; Return the empty list if it doesn't.  Note that this RPC does not
1163       ;; exist in 'nix-daemon'.
1164       (if (or (> (nix-server-major-version store) #x100)
1165               (and (= (nix-server-major-version store) #x100)
1166                    (>= (nix-server-minor-version store) #x60)))
1167           (builders store)
1168           '()))))
1169
1170 (define-operation (optimize-store)
1171   "Optimize the store by hard-linking identical files (\"deduplication\".)
1172 Return #t on success."
1173   ;; Note: the daemon in Guix <= 0.8.2 does not implement this RPC.
1174   boolean)
1175
1176 (define verify-store
1177   (let ((verify (operation (verify-store (boolean check-contents?)
1178                                          (boolean repair?))
1179                            "Verify the store."
1180                            boolean)))
1181     (lambda* (store #:key check-contents? repair?)
1182       "Verify the integrity of the store and return false if errors remain,
1183 and true otherwise.  When REPAIR? is true, repair any missing or altered store
1184 items by substituting them (this typically requires root privileges because it
1185 is not an atomic operation.)  When CHECK-CONTENTS? is true, check the contents
1186 of store items; this can take a lot of time."
1187       (not (verify store check-contents? repair?)))))
1188
1189 (define (run-gc server action to-delete min-freed)
1190   "Perform the garbage-collector operation ACTION, one of the
1191 `gc-action' values.  When ACTION is `delete-specific', the TO-DELETE is
1192 the list of store paths to delete.  IGNORE-LIVENESS? should always be
1193 #f.  MIN-FREED is the minimum amount of disk space to be freed, in
1194 bytes, before the GC can stop.  Return the list of store paths delete,
1195 and the number of bytes freed."
1196   (let ((s (nix-server-socket server)))
1197     (write-int (operation-id collect-garbage) s)
1198     (write-int action s)
1199     (write-store-path-list to-delete s)
1200     (write-arg boolean #f s)                      ; ignore-liveness?
1201     (write-long-long min-freed s)
1202     (write-int 0 s)                               ; obsolete
1203     (when (>= (nix-server-minor-version server) 5)
1204       ;; Obsolete `use-atime' and `max-atime' parameters.
1205       (write-int 0 s)
1206       (write-int 0 s))
1207
1208     ;; Loop until the server is done sending error output.
1209     (let loop ((done? (process-stderr server)))
1210       (or done? (loop (process-stderr server))))
1211
1212     (let ((paths    (read-store-path-list s))
1213           (freed    (read-long-long s))
1214           (obsolete (read-long-long s)))
1215       (unless (null? paths)
1216         ;; To be on the safe side, completely invalidate both caches.
1217         ;; Otherwise we could end up returning store paths that are no longer
1218         ;; valid.
1219         (hash-clear! (nix-server-add-to-store-cache server))
1220         (hash-clear! (nix-server-add-text-to-store-cache server)))
1221
1222      (values paths freed))))
1223
1224 (define-syntax-rule (%long-long-max)
1225   ;; Maximum unsigned 64-bit integer.
1226   (- (expt 2 64) 1))
1227
1228 (define (live-paths server)
1229   "Return the list of live store paths---i.e., store paths still
1230 referenced, and thus not subject to being garbage-collected."
1231   (run-gc server (gc-action return-live) '() (%long-long-max)))
1232
1233 (define (dead-paths server)
1234   "Return the list of dead store paths---i.e., store paths no longer
1235 referenced, and thus subject to being garbage-collected."
1236   (run-gc server (gc-action return-dead) '() (%long-long-max)))
1237
1238 (define* (collect-garbage server #:optional (min-freed (%long-long-max)))
1239   "Collect garbage from the store at SERVER.  If MIN-FREED is non-zero,
1240 then collect at least MIN-FREED bytes.  Return the paths that were
1241 collected, and the number of bytes freed."
1242   (run-gc server (gc-action delete-dead) '() min-freed))
1243
1244 (define* (delete-paths server paths #:optional (min-freed (%long-long-max)))
1245   "Delete PATHS from the store at SERVER, if they are no longer
1246 referenced.  If MIN-FREED is non-zero, then stop after at least
1247 MIN-FREED bytes have been collected.  Return the paths that were
1248 collected, and the number of bytes freed."
1249   (run-gc server (gc-action delete-specific) paths min-freed))
1250
1251 (define (import-paths server port)
1252   "Import the set of store paths read from PORT into SERVER's store.  An error
1253 is raised if the set of paths read from PORT is not signed (as per
1254 'export-path #:sign? #t'.)  Return the list of store paths imported."
1255   (let ((s (nix-server-socket server)))
1256     (write-int (operation-id import-paths) s)
1257     (let loop ((done? (process-stderr server port)))
1258       (or done? (loop (process-stderr server port))))
1259     (read-store-path-list s)))
1260
1261 (define* (export-path server path port #:key (sign? #t))
1262   "Export PATH to PORT.  When SIGN? is true, sign it."
1263   (let ((s (nix-server-socket server)))
1264     (write-int (operation-id export-path) s)
1265     (write-store-path path s)
1266     (write-arg boolean sign? s)
1267     (let loop ((done? (process-stderr server port)))
1268       (or done? (loop (process-stderr server port))))
1269     (= 1 (read-int s))))
1270
1271 (define* (export-paths server paths port #:key (sign? #t) recursive?)
1272   "Export the store paths listed in PATHS to PORT, in topological order,
1273 signing them if SIGN? is true.  When RECURSIVE? is true, export the closure of
1274 PATHS---i.e., PATHS and all their dependencies."
1275   (define ordered
1276     (let ((sorted (topologically-sorted server paths)))
1277       ;; When RECURSIVE? is #f, filter out the references of PATHS.
1278       (if recursive?
1279           sorted
1280           (filter (cut member <> paths) sorted))))
1281
1282   (let loop ((paths ordered))
1283     (match paths
1284       (()
1285        (write-int 0 port))
1286       ((head tail ...)
1287        (write-int 1 port)
1288        (and (export-path server head port #:sign? sign?)
1289             (loop tail))))))
1290
1291 (define-operation (query-failed-paths)
1292   "Return the list of store items for which a build failure is cached.
1293
1294 The result is always the empty list unless the daemon was started with
1295 '--cache-failures'."
1296   store-path-list)
1297
1298 (define-operation (clear-failed-paths (store-path-list items))
1299   "Remove ITEMS from the list of cached build failures.
1300
1301 This makes sense only when the daemon was started with '--cache-failures'."
1302   boolean)
1303
1304 (define* (register-path path
1305                         #:key (references '()) deriver prefix
1306                         state-directory)
1307   "Register PATH as a valid store file, with REFERENCES as its list of
1308 references, and DERIVER as its deriver (.drv that led to it.)  If PREFIX is
1309 not #f, it must be the name of the directory containing the new store to
1310 initialize; if STATE-DIRECTORY is not #f, it must be a string containing the
1311 absolute file name to the state directory of the store being initialized.
1312 Return #t on success.
1313
1314 Use with care as it directly modifies the store!  This is primarily meant to
1315 be used internally by the daemon's build hook."
1316   ;; Currently this is implemented by calling out to the fine C++ blob.
1317   (let ((pipe (apply open-pipe* OPEN_WRITE %guix-register-program
1318                      `(,@(if prefix
1319                              `("--prefix" ,prefix)
1320                              '())
1321                        ,@(if state-directory
1322                              `("--state-directory" ,state-directory)
1323                              '())))))
1324     (and pipe
1325          (begin
1326            (format pipe "~a~%~a~%~a~%"
1327                    path (or deriver "") (length references))
1328            (for-each (cut format pipe "~a~%" <>) references)
1329            (zero? (close-pipe pipe))))))
1330
1331 \f
1332 ;;;
1333 ;;; Store monad.
1334 ;;;
1335
1336 (define-syntax-rule (define-alias new old)
1337   (define-syntax new (identifier-syntax old)))
1338
1339 ;; The store monad allows us to (1) build sequences of operations in the
1340 ;; store, and (2) make the store an implicit part of the execution context,
1341 ;; rather than a parameter of every single function.
1342 (define-alias %store-monad %state-monad)
1343 (define-alias store-return state-return)
1344 (define-alias store-bind state-bind)
1345
1346 ;; Instantiate templates for %STORE-MONAD since it's syntactically different
1347 ;; from %STATE-MONAD.
1348 (template-directory instantiations %store-monad)
1349
1350 (define (preserve-documentation original proc)
1351   "Return PROC with documentation taken from ORIGINAL."
1352   (set-object-property! proc 'documentation
1353                         (procedure-property original 'documentation))
1354   proc)
1355
1356 (define (store-lift proc)
1357   "Lift PROC, a procedure whose first argument is a connection to the store,
1358 in the store monad."
1359   (preserve-documentation proc
1360                           (lambda args
1361                             (lambda (store)
1362                               (values (apply proc store args) store)))))
1363
1364 (define (store-lower proc)
1365   "Lower PROC, a monadic procedure in %STORE-MONAD, to a \"normal\" procedure
1366 taking the store as its first argument."
1367   (preserve-documentation proc
1368                           (lambda (store . args)
1369                             (run-with-store store (apply proc args)))))
1370
1371 ;;
1372 ;; Store monad operators.
1373 ;;
1374
1375 (define* (text-file name text
1376                     #:optional (references '()))
1377   "Return as a monadic value the absolute file name in the store of the file
1378 containing TEXT, a string.  REFERENCES is a list of store items that the
1379 resulting text file refers to; it defaults to the empty list."
1380   (lambda (store)
1381     (values (add-text-to-store store name text references)
1382             store)))
1383
1384 (define* (interned-file file #:optional name
1385                         #:key (recursive? #t) (select? true))
1386   "Return the name of FILE once interned in the store.  Use NAME as its store
1387 name, or the basename of FILE if NAME is omitted.
1388
1389 When RECURSIVE? is true, the contents of FILE are added recursively; if FILE
1390 designates a flat file and RECURSIVE? is true, its contents are added, and its
1391 permission bits are kept.
1392
1393 When RECURSIVE? is true, call (SELECT?  FILE STAT) for each directory entry,
1394 where FILE is the entry's absolute file name and STAT is the result of
1395 'lstat'; exclude entries for which SELECT? does not return true."
1396   (lambda (store)
1397     (values (add-to-store store (or name (basename file))
1398                           recursive? "sha256" file
1399                           #:select? select?)
1400             store)))
1401
1402 (define build
1403   ;; Monadic variant of 'build-things'.
1404   (store-lift build-things))
1405
1406 (define set-build-options*
1407   (store-lift set-build-options))
1408
1409 (define references*
1410   (store-lift references))
1411
1412 (define-inlinable (current-system)
1413   ;; Consult the %CURRENT-SYSTEM fluid at bind time.  This is equivalent to
1414   ;; (lift0 %current-system %store-monad), but inlinable, thus avoiding
1415   ;; closure allocation in some cases.
1416   (lambda (state)
1417     (values (%current-system) state)))
1418
1419 (define-inlinable (set-current-system system)
1420   ;; Set the %CURRENT-SYSTEM fluid at bind time.
1421   (lambda (state)
1422     (values (%current-system system) state)))
1423
1424 (define %guile-for-build
1425   ;; The derivation of the Guile to be used within the build environment,
1426   ;; when using 'gexp->derivation' and co.
1427   (make-parameter #f))
1428
1429 (define* (run-with-store store mval
1430                          #:key
1431                          (guile-for-build (%guile-for-build))
1432                          (system (%current-system))
1433                          (target #f))
1434   "Run MVAL, a monadic value in the store monad, in STORE, an open store
1435 connection, and return the result."
1436   ;; Initialize the dynamic bindings here to avoid bad surprises.  The
1437   ;; difficulty lies in the fact that dynamic bindings are resolved at
1438   ;; bind-time and not at call time, which can be disconcerting.
1439   (parameterize ((%guile-for-build guile-for-build)
1440                  (%current-system system)
1441                  (%current-target-system target))
1442     (call-with-values (lambda ()
1443                         (run-with-state mval store))
1444       (lambda (result store)
1445         ;; Discard the state.
1446         result))))
1447
1448 \f
1449 ;;;
1450 ;;; Store paths.
1451 ;;;
1452
1453 (define %store-prefix
1454   ;; Absolute path to the Nix store.
1455   (make-parameter %store-directory))
1456
1457 (define (compressed-hash bv size)                 ; `compressHash'
1458   "Given the hash stored in BV, return a compressed version thereof that fits
1459 in SIZE bytes."
1460   (define new (make-bytevector size 0))
1461   (define old-size (bytevector-length bv))
1462   (let loop ((i 0))
1463     (if (= i old-size)
1464         new
1465         (let* ((j (modulo i size))
1466                (o (bytevector-u8-ref new j)))
1467           (bytevector-u8-set! new j
1468                               (logxor o (bytevector-u8-ref bv i)))
1469           (loop (+ 1 i))))))
1470
1471 (define (store-path type hash name)               ; makeStorePath
1472   "Return the store path for NAME/HASH/TYPE."
1473   (let* ((s (string-append type ":sha256:"
1474                            (bytevector->base16-string hash) ":"
1475                            (%store-prefix) ":" name))
1476          (h (sha256 (string->utf8 s)))
1477          (c (compressed-hash h 20)))
1478     (string-append (%store-prefix) "/"
1479                    (bytevector->nix-base32-string c) "-"
1480                    name)))
1481
1482 (define (output-path output hash name)            ; makeOutputPath
1483   "Return an output path for OUTPUT (the name of the output as a string) of
1484 the derivation called NAME with hash HASH."
1485   (store-path (string-append "output:" output) hash
1486               (if (string=? output "out")
1487                   name
1488                   (string-append name "-" output))))
1489
1490 (define* (fixed-output-path name hash
1491                             #:key
1492                             (output "out")
1493                             (hash-algo 'sha256)
1494                             (recursive? #t))
1495   "Return an output path for the fixed output OUTPUT defined by HASH of type
1496 HASH-ALGO, of the derivation NAME.  RECURSIVE? has the same meaning as for
1497 'add-to-store'."
1498   (if (and recursive? (eq? hash-algo 'sha256))
1499       (store-path "source" hash name)
1500       (let ((tag (string-append "fixed:" output ":"
1501                                 (if recursive? "r:" "")
1502                                 (symbol->string hash-algo) ":"
1503                                 (bytevector->base16-string hash) ":")))
1504         (store-path (string-append "output:" output)
1505                     (sha256 (string->utf8 tag))
1506                     name))))
1507
1508 (define (store-path? path)
1509   "Return #t if PATH is a store path."
1510   ;; This is a lightweight check, compared to using a regexp, but this has to
1511   ;; be fast as it's called often in `derivation', for instance.
1512   ;; `isStorePath' in Nix does something similar.
1513   (string-prefix? (%store-prefix) path))
1514
1515 (define (direct-store-path? path)
1516   "Return #t if PATH is a store path, and not a sub-directory of a store path.
1517 This predicate is sometimes needed because files *under* a store path are not
1518 valid inputs."
1519   (and (store-path? path)
1520        (not (string=? path (%store-prefix)))
1521        (let ((len (+ 1 (string-length (%store-prefix)))))
1522          (not (string-index (substring path len) #\/)))))
1523
1524 (define (direct-store-path path)
1525   "Return the direct store path part of PATH, stripping components after
1526 '/gnu/store/xxxx-foo'."
1527   (let ((prefix-length (+ (string-length (%store-prefix)) 35)))
1528     (if (> (string-length path) prefix-length)
1529         (let ((slash (string-index path #\/ prefix-length)))
1530           (if slash (string-take path slash) path))
1531         path)))
1532
1533 (define (derivation-path? path)
1534   "Return #t if PATH is a derivation path."
1535   (and (store-path? path) (string-suffix? ".drv" path)))
1536
1537 (define store-regexp*
1538   ;; The substituter makes repeated calls to 'store-path-hash-part', hence
1539   ;; this optimization.
1540   (mlambda (store)
1541     "Return a regexp matching a file in STORE."
1542     (make-regexp (string-append "^" (regexp-quote store)
1543                                 "/([0-9a-df-np-sv-z]{32})-([^/]+)$"))))
1544
1545 (define (store-path-package-name path)
1546   "Return the package name part of PATH, a file name in the store."
1547   (let ((path-rx (store-regexp* (%store-prefix))))
1548     (and=> (regexp-exec path-rx path)
1549            (cut match:substring <> 2))))
1550
1551 (define (store-path-hash-part path)
1552   "Return the hash part of PATH as a base32 string, or #f if PATH is not a
1553 syntactically valid store path."
1554   (and (string-prefix? (%store-prefix) path)
1555        (let ((base (string-drop path (+ 1 (string-length (%store-prefix))))))
1556          (and (> (string-length base) 33)
1557               (let ((hash (string-take base 32)))
1558                 (and (string-every %nix-base32-charset hash)
1559                      hash))))))
1560
1561 (define (log-file store file)
1562   "Return the build log file for FILE, or #f if none could be found.  FILE
1563 must be an absolute store file name, or a derivation file name."
1564   (cond ((derivation-path? file)
1565          (let* ((base    (basename file))
1566                 (log     (string-append (dirname %state-directory) ; XXX
1567                                         "/log/guix/drvs/"
1568                                         (string-take base 2) "/"
1569                                         (string-drop base 2)))
1570                 (log.gz  (string-append log ".gz"))
1571                 (log.bz2 (string-append log ".bz2")))
1572            (cond ((file-exists? log.gz) log.gz)
1573                  ((file-exists? log.bz2) log.bz2)
1574                  ((file-exists? log) log)
1575                  (else #f))))
1576         (else
1577          (match (valid-derivers store file)
1578            ((derivers ...)
1579             ;; Return the first that works.
1580             (any (cut log-file store <>) derivers))
1581            (_ #f)))))
1582
1583 ;;; Local Variables:
1584 ;;; eval: (put 'system-error-to-connection-error 'scheme-indent-function 1)
1585 ;;; End: