utils: Move base16 procedures to (guix base16).
[jackhill/guix/guix.git] / guix / derivations.scm
1 ;;; GNU Guix --- Functional package management for GNU
2 ;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org>
3 ;;; Copyright © 2016, 2017 Mathieu Lirzin <mthl@gnu.org>
4 ;;;
5 ;;; This file is part of GNU Guix.
6 ;;;
7 ;;; GNU Guix is free software; you can redistribute it and/or modify it
8 ;;; under the terms of the GNU General Public License as published by
9 ;;; the Free Software Foundation; either version 3 of the License, or (at
10 ;;; your option) any later version.
11 ;;;
12 ;;; GNU Guix is distributed in the hope that it will be useful, but
13 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 ;;; GNU General Public License for more details.
16 ;;;
17 ;;; You should have received a copy of the GNU General Public License
18 ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
19
20 (define-module (guix derivations)
21 #:use-module (srfi srfi-1)
22 #:use-module (srfi srfi-9)
23 #:use-module (srfi srfi-9 gnu)
24 #:use-module (srfi srfi-26)
25 #:use-module (srfi srfi-34)
26 #:use-module (srfi srfi-35)
27 #:use-module (ice-9 binary-ports)
28 #:use-module (rnrs bytevectors)
29 #:use-module (ice-9 match)
30 #:use-module (ice-9 rdelim)
31 #:use-module (ice-9 vlist)
32 #:use-module (guix store)
33 #:use-module (guix utils)
34 #:use-module (guix base16)
35 #:use-module (guix memoization)
36 #:use-module (guix combinators)
37 #:use-module (guix monads)
38 #:use-module (guix hash)
39 #:use-module (guix base32)
40 #:use-module (guix records)
41 #:use-module (guix sets)
42 #:export (<derivation>
43 derivation?
44 derivation-outputs
45 derivation-inputs
46 derivation-sources
47 derivation-system
48 derivation-builder
49 derivation-builder-arguments
50 derivation-builder-environment-vars
51 derivation-file-name
52 derivation-prerequisites
53 derivation-prerequisites-to-build
54
55 <derivation-output>
56 derivation-output?
57 derivation-output-path
58 derivation-output-hash-algo
59 derivation-output-hash
60 derivation-output-recursive?
61
62 <derivation-input>
63 derivation-input?
64 derivation-input-path
65 derivation-input-sub-derivations
66 derivation-input-output-paths
67 valid-derivation-input?
68
69 &derivation-error
70 derivation-error?
71 derivation-error-derivation
72 &derivation-missing-output-error
73 derivation-missing-output-error?
74 derivation-missing-output
75
76 derivation-name
77 derivation-output-names
78 fixed-output-derivation?
79 fixed-output-path
80 offloadable-derivation?
81 substitutable-derivation?
82 substitution-oracle
83 derivation-hash
84
85 read-derivation
86 write-derivation
87 derivation->output-path
88 derivation->output-paths
89 derivation-path->output-path
90 derivation-path->output-paths
91 derivation
92 raw-derivation
93
94 map-derivation
95
96 build-derivations
97 built-derivations
98
99 file-search-error?
100 file-search-error-file-name
101 file-search-error-search-path
102
103 search-path*
104 module->source-file-name
105 build-expression->derivation)
106
107 ;; Re-export it from here for backward compatibility.
108 #:re-export (%guile-for-build))
109
110 ;;;
111 ;;; Error conditions.
112 ;;;
113
114 (define-condition-type &derivation-error &nix-error
115 derivation-error?
116 (derivation derivation-error-derivation))
117
118 (define-condition-type &derivation-missing-output-error &derivation-error
119 derivation-missing-output-error?
120 (output derivation-missing-output))
121
122 ;;;
123 ;;; Nix derivations, as implemented in Nix's `derivations.cc'.
124 ;;;
125
126 (define-immutable-record-type <derivation>
127 (make-derivation outputs inputs sources system builder args env-vars
128 file-name)
129 derivation?
130 (outputs derivation-outputs) ; list of name/<derivation-output> pairs
131 (inputs derivation-inputs) ; list of <derivation-input>
132 (sources derivation-sources) ; list of store paths
133 (system derivation-system) ; string
134 (builder derivation-builder) ; store path
135 (args derivation-builder-arguments) ; list of strings
136 (env-vars derivation-builder-environment-vars) ; list of name/value pairs
137 (file-name derivation-file-name)) ; the .drv file name
138
139 (define-record-type <derivation-output>
140 (make-derivation-output path hash-algo hash recursive?)
141 derivation-output?
142 (path derivation-output-path) ; store path
143 (hash-algo derivation-output-hash-algo) ; symbol | #f
144 (hash derivation-output-hash) ; bytevector | #f
145 (recursive? derivation-output-recursive?)) ; Boolean
146
147 (define-record-type <derivation-input>
148 (make-derivation-input path sub-derivations)
149 derivation-input?
150 (path derivation-input-path) ; store path
151 (sub-derivations derivation-input-sub-derivations)) ; list of strings
152
153 (set-record-type-printer! <derivation>
154 (lambda (drv port)
155 (format port "#<derivation ~a => ~a ~a>"
156 (derivation-file-name drv)
157 (string-join
158 (map (match-lambda
159 ((_ . output)
160 (derivation-output-path output)))
161 (derivation-outputs drv)))
162 (number->string (object-address drv) 16))))
163
164 (define (derivation-name drv)
165 "Return the base name of DRV."
166 (let ((base (store-path-package-name (derivation-file-name drv))))
167 (string-drop-right base 4)))
168
169 (define (derivation-output-names drv)
170 "Return the names of the outputs of DRV."
171 (match (derivation-outputs drv)
172 (((names . _) ...)
173 names)))
174
175 (define (fixed-output-derivation? drv)
176 "Return #t if DRV is a fixed-output derivation, such as the result of a
177 download with a fixed hash (aka. `fetchurl')."
178 (match drv
179 (($ <derivation>
180 (("out" . ($ <derivation-output> _ (? symbol?) (? bytevector?)))))
181 #t)
182 (_ #f)))
183
184 (define (derivation-input<? input1 input2)
185 "Compare INPUT1 and INPUT2, two <derivation-input>."
186 (string<? (derivation-input-path input1)
187 (derivation-input-path input2)))
188
189 (define (derivation-input-output-paths input)
190 "Return the list of output paths corresponding to INPUT, a
191 <derivation-input>."
192 (match input
193 (($ <derivation-input> path sub-drvs)
194 (map (cut derivation-path->output-path path <>)
195 sub-drvs))))
196
197 (define (valid-derivation-input? store input)
198 "Return true if INPUT is valid--i.e., if all the outputs it requests are in
199 the store."
200 (every (cut valid-path? store <>)
201 (derivation-input-output-paths input)))
202
203 (define (coalesce-duplicate-inputs inputs)
204 "Return a list of inputs, such that when INPUTS contains the same DRV twice,
205 they are coalesced, with their sub-derivations merged. This is needed because
206 Nix itself keeps only one of them."
207 (fold (lambda (input result)
208 (match input
209 (($ <derivation-input> path sub-drvs)
210 ;; XXX: quadratic
211 (match (find (match-lambda
212 (($ <derivation-input> p s)
213 (string=? p path)))
214 result)
215 (#f
216 (cons input result))
217 ((and dup ($ <derivation-input> _ sub-drvs2))
218 ;; Merge DUP with INPUT.
219 (let ((sub-drvs (delete-duplicates
220 (append sub-drvs sub-drvs2))))
221 (cons (make-derivation-input path
222 (sort sub-drvs string<?))
223 (delq dup result))))))))
224 '()
225 inputs))
226
227 (define* (derivation-prerequisites drv #:optional (cut? (const #f)))
228 "Return the list of derivation-inputs required to build DRV, recursively.
229
230 CUT? is a predicate that is passed a derivation-input and returns true to
231 eliminate the given input and its dependencies from the search. An example of
232 search a predicate is 'valid-derivation-input?'; when it is used as CUT?, the
233 result is the set of prerequisites of DRV not already in valid."
234 (let loop ((drv drv)
235 (result '())
236 (input-set (set)))
237 (let ((inputs (remove (lambda (input)
238 (or (set-contains? input-set input)
239 (cut? input)))
240 (derivation-inputs drv))))
241 (fold2 loop
242 (append inputs result)
243 (fold set-insert input-set inputs)
244 (map (lambda (i)
245 (call-with-input-file (derivation-input-path i)
246 read-derivation))
247 inputs)))))
248
249 (define (offloadable-derivation? drv)
250 "Return true if DRV can be offloaded, false otherwise."
251 (match (assoc "preferLocalBuild"
252 (derivation-builder-environment-vars drv))
253 (("preferLocalBuild" . "1") #f)
254 (_ #t)))
255
256 (define (substitutable-derivation? drv)
257 "Return #t if DRV can be substituted."
258 (match (assoc "allowSubstitutes"
259 (derivation-builder-environment-vars drv))
260 (("allowSubstitutes" . value)
261 (string=? value "1"))
262 (_ #t)))
263
264 (define (derivation-output-paths drv sub-drvs)
265 "Return the output paths of outputs SUB-DRVS of DRV."
266 (match drv
267 (($ <derivation> outputs)
268 (map (lambda (sub-drv)
269 (derivation-output-path (assoc-ref outputs sub-drv)))
270 sub-drvs))))
271
272 (define* (substitution-oracle store drv
273 #:key (mode (build-mode normal)))
274 "Return a one-argument procedure that, when passed a store file name,
275 returns #t if it's substitutable and #f otherwise. The returned procedure
276 knows about all substitutes for all the derivations listed in DRV, *except*
277 those that are already valid (that is, it won't bother checking whether an
278 item is substitutable if it's already on disk); it also knows about their
279 prerequisites, unless they are themselves substitutable.
280
281 Creating a single oracle (thus making a single 'substitutable-paths' call) and
282 reusing it is much more efficient than calling 'has-substitutes?' or similar
283 repeatedly, because it avoids the costs associated with launching the
284 substituter many times."
285 (define valid?
286 (cut valid-path? store <>))
287
288 (define valid-input?
289 (cut valid-derivation-input? store <>))
290
291 (define (dependencies drv)
292 ;; Skip prerequisite sub-trees of DRV whose root is valid. This allows us
293 ;; to ask the substituter for just as much as needed, instead of asking it
294 ;; for the whole world, which can be significantly faster when substitute
295 ;; info is not already in cache.
296 (append-map derivation-input-output-paths
297 (derivation-prerequisites drv valid-input?)))
298
299 (let* ((paths (delete-duplicates
300 (concatenate
301 (fold (lambda (drv result)
302 (let ((self (match (derivation->output-paths drv)
303 (((names . paths) ...)
304 paths))))
305 (cond ((eqv? mode (build-mode check))
306 (cons (dependencies drv) result))
307 ((every valid? self)
308 result)
309 (else
310 (cons* self (dependencies drv) result)))))
311 '()
312 drv))))
313 (subst (list->set (substitutable-paths store paths))))
314 (cut set-contains? subst <>)))
315
316 (define* (derivation-prerequisites-to-build store drv
317 #:key
318 (mode (build-mode normal))
319 (outputs
320 (derivation-output-names drv))
321 (substitutable?
322 (substitution-oracle store
323 (list drv)
324 #:mode mode)))
325 "Return two values: the list of derivation-inputs required to build the
326 OUTPUTS of DRV and not already available in STORE, recursively, and the list
327 of required store paths that can be substituted. SUBSTITUTABLE? must be a
328 one-argument procedure similar to that returned by 'substitution-oracle'."
329 (define built?
330 (cut valid-path? store <>))
331
332 (define input-built?
333 (compose (cut any built? <>) derivation-input-output-paths))
334
335 (define input-substitutable?
336 ;; Return true if and only if all of SUB-DRVS are subsitutable. If at
337 ;; least one is missing, then everything must be rebuilt.
338 (compose (cut every substitutable? <>) derivation-input-output-paths))
339
340 (define (derivation-built? drv* sub-drvs)
341 ;; In 'check' mode, assume that DRV is not built.
342 (and (not (and (eqv? mode (build-mode check))
343 (eq? drv* drv)))
344 (every built? (derivation-output-paths drv* sub-drvs))))
345
346 (define (derivation-substitutable? drv sub-drvs)
347 (and (substitutable-derivation? drv)
348 (every substitutable? (derivation-output-paths drv sub-drvs))))
349
350 (let loop ((drv drv)
351 (sub-drvs outputs)
352 (build '())
353 (substitute '()))
354 (cond ((derivation-built? drv sub-drvs)
355 (values build substitute))
356 ((derivation-substitutable? drv sub-drvs)
357 (values build
358 (append (derivation-output-paths drv sub-drvs)
359 substitute)))
360 (else
361 (let ((build (if (substitutable-derivation? drv)
362 build
363 (cons (make-derivation-input
364 (derivation-file-name drv) sub-drvs)
365 build)))
366 (inputs (remove (lambda (i)
367 (or (member i build) ; XXX: quadratic
368 (input-built? i)
369 (input-substitutable? i)))
370 (derivation-inputs drv))))
371 (fold2 loop
372 (append inputs build)
373 (append (append-map (lambda (input)
374 (if (and (not (input-built? input))
375 (input-substitutable? input))
376 (derivation-input-output-paths
377 input)
378 '()))
379 (derivation-inputs drv))
380 substitute)
381 (map (lambda (i)
382 (call-with-input-file (derivation-input-path i)
383 read-derivation))
384 inputs)
385 (map derivation-input-sub-derivations inputs)))))))
386
387 (define (%read-derivation drv-port)
388 ;; Actually read derivation from DRV-PORT.
389
390 (define comma (string->symbol ","))
391
392 (define (ununquote x)
393 (match x
394 (('unquote x) (ununquote x))
395 ((x ...) (map ununquote x))
396 (_ x)))
397
398 (define (outputs->alist x)
399 (fold-right (lambda (output result)
400 (match output
401 ((name path "" "")
402 (alist-cons name
403 (make-derivation-output path #f #f #f)
404 result))
405 ((name path hash-algo hash)
406 ;; fixed-output
407 (let* ((rec? (string-prefix? "r:" hash-algo))
408 (algo (string->symbol
409 (if rec?
410 (string-drop hash-algo 2)
411 hash-algo)))
412 (hash (base16-string->bytevector hash)))
413 (alist-cons name
414 (make-derivation-output path algo
415 hash rec?)
416 result)))))
417 '()
418 x))
419
420 (define (make-input-drvs x)
421 (fold-right (lambda (input result)
422 (match input
423 ((path (sub-drvs ...))
424 (cons (make-derivation-input path sub-drvs)
425 result))))
426 '()
427 x))
428
429 ;; The contents of a derivation are typically ASCII, but choosing
430 ;; UTF-8 allows us to take the fast path for Guile's `scm_getc'.
431 (set-port-encoding! drv-port "UTF-8")
432
433 (let loop ((exp (read drv-port))
434 (result '()))
435 (match exp
436 ((? eof-object?)
437 (let ((result (reverse result)))
438 (match result
439 (('Derive ((outputs ...) (input-drvs ...)
440 (input-srcs ...)
441 (? string? system)
442 (? string? builder)
443 ((? string? args) ...)
444 ((var value) ...)))
445 (make-derivation (outputs->alist outputs)
446 (make-input-drvs input-drvs)
447 input-srcs
448 system builder args
449 (fold-right alist-cons '() var value)
450 (port-filename drv-port)))
451 (_
452 (error "failed to parse derivation" drv-port result)))))
453 ((? (cut eq? <> comma))
454 (loop (read drv-port) result))
455 (_
456 (loop (read drv-port)
457 (cons (ununquote exp) result))))))
458
459 (define %derivation-cache
460 ;; Maps derivation file names to <derivation> objects.
461 ;; XXX: This is redundant with 'atts-cache' in the store.
462 (make-weak-value-hash-table 200))
463
464 (define (read-derivation drv-port)
465 "Read the derivation from DRV-PORT and return the corresponding
466 <derivation> object."
467 ;; Memoize that operation because `%read-derivation' is quite expensive,
468 ;; and because the same argument is read more than 15 times on average
469 ;; during something like (package-derivation s gdb).
470 (let ((file (port-filename drv-port)))
471 (or (and file (hash-ref %derivation-cache file))
472 (let ((drv (%read-derivation drv-port)))
473 (hash-set! %derivation-cache file drv)
474 drv))))
475
476 (define-inlinable (write-sequence lst write-item port)
477 ;; Write each element of LST with WRITE-ITEM to PORT, separating them with a
478 ;; comma.
479 (match lst
480 (()
481 #t)
482 ((prefix (... ...) last)
483 (for-each (lambda (item)
484 (write-item item port)
485 (display "," port))
486 prefix)
487 (write-item last port))))
488
489 (define-inlinable (write-list lst write-item port)
490 ;; Write LST as a derivation list to PORT, using WRITE-ITEM to write each
491 ;; element.
492 (display "[" port)
493 (write-sequence lst write-item port)
494 (display "]" port))
495
496 (define-inlinable (write-tuple lst write-item port)
497 ;; Same, but write LST as a tuple.
498 (display "(" port)
499 (write-sequence lst write-item port)
500 (display ")" port))
501
502 (define (write-derivation drv port)
503 "Write the ATerm-like serialization of DRV to PORT. See Section 2.4 of
504 Eelco Dolstra's PhD dissertation for an overview of a previous version of
505 that form."
506
507 ;; Make sure we're using the faster implementation.
508 (define format simple-format)
509
510 (define (write-string-list lst)
511 (write-list lst write port))
512
513 (define (write-output output port)
514 (match output
515 ((name . ($ <derivation-output> path hash-algo hash recursive?))
516 (write-tuple (list name path
517 (if hash-algo
518 (string-append (if recursive? "r:" "")
519 (symbol->string hash-algo))
520 "")
521 (or (and=> hash bytevector->base16-string)
522 ""))
523 write
524 port))))
525
526 (define (write-input input port)
527 (match input
528 (($ <derivation-input> path sub-drvs)
529 (display "(\"" port)
530 (display path port)
531 (display "\"," port)
532 (write-string-list sub-drvs)
533 (display ")" port))))
534
535 (define (write-env-var env-var port)
536 (match env-var
537 ((name . value)
538 (display "(" port)
539 (write name port)
540 (display "," port)
541 (write value port)
542 (display ")" port))))
543
544 ;; Assume all the lists we are writing are already sorted.
545 (match drv
546 (($ <derivation> outputs inputs sources
547 system builder args env-vars)
548 (display "Derive(" port)
549 (write-list outputs write-output port)
550 (display "," port)
551 (write-list inputs write-input port)
552 (display "," port)
553 (write-string-list sources)
554 (simple-format port ",\"~a\",\"~a\"," system builder)
555 (write-string-list args)
556 (display "," port)
557 (write-list env-vars write-env-var port)
558 (display ")" port))))
559
560 (define derivation->string
561 (mlambda (drv)
562 "Return the external representation of DRV as a string."
563 (with-fluids ((%default-port-encoding "UTF-8"))
564 (call-with-output-string
565 (cut write-derivation drv <>)))))
566
567 (define* (derivation->output-path drv #:optional (output "out"))
568 "Return the store path of its output OUTPUT. Raise a
569 '&derivation-missing-output-error' condition if OUTPUT is not an output of
570 DRV."
571 (let ((output* (assoc-ref (derivation-outputs drv) output)))
572 (if output*
573 (derivation-output-path output*)
574 (raise (condition (&derivation-missing-output-error
575 (derivation drv)
576 (output output)))))))
577
578 (define (derivation->output-paths drv)
579 "Return the list of name/path pairs of the outputs of DRV."
580 (map (match-lambda
581 ((name . output)
582 (cons name (derivation-output-path output))))
583 (derivation-outputs drv)))
584
585 (define derivation-path->output-path
586 ;; This procedure is called frequently, so memoize it.
587 (let ((memoized (mlambda (path output)
588 (derivation->output-path (call-with-input-file path
589 read-derivation)
590 output))))
591 (lambda* (path #:optional (output "out"))
592 "Read the derivation from PATH (`/gnu/store/xxx.drv'), and return the store
593 path of its output OUTPUT."
594 (memoized path output))))
595
596 (define (derivation-path->output-paths path)
597 "Read the derivation from PATH (`/gnu/store/xxx.drv'), and return the
598 list of name/path pairs of its outputs."
599 (derivation->output-paths (call-with-input-file path read-derivation)))
600
601 \f
602 ;;;
603 ;;; Derivation primitive.
604 ;;;
605
606 (define (compressed-hash bv size) ; `compressHash'
607 "Given the hash stored in BV, return a compressed version thereof that fits
608 in SIZE bytes."
609 (define new (make-bytevector size 0))
610 (define old-size (bytevector-length bv))
611 (let loop ((i 0))
612 (if (= i old-size)
613 new
614 (let* ((j (modulo i size))
615 (o (bytevector-u8-ref new j)))
616 (bytevector-u8-set! new j
617 (logxor o (bytevector-u8-ref bv i)))
618 (loop (+ 1 i))))))
619
620 (define derivation-path->base16-hash
621 (mlambda (file)
622 "Return a string containing the base16 representation of the hash of the
623 derivation at FILE."
624 (call-with-input-file file
625 (compose bytevector->base16-string
626 derivation-hash
627 read-derivation))))
628
629 (define derivation-hash ; `hashDerivationModulo' in derivations.cc
630 (mlambda (drv)
631 "Return the hash of DRV, modulo its fixed-output inputs, as a bytevector."
632 (match drv
633 (($ <derivation> ((_ . ($ <derivation-output> path
634 (? symbol? hash-algo) (? bytevector? hash)
635 (? boolean? recursive?)))))
636 ;; A fixed-output derivation.
637 (sha256
638 (string->utf8
639 (string-append "fixed:out:"
640 (if recursive? "r:" "")
641 (symbol->string hash-algo)
642 ":" (bytevector->base16-string hash)
643 ":" path))))
644 (($ <derivation> outputs inputs sources
645 system builder args env-vars)
646 ;; A regular derivation: replace the path of each input with that
647 ;; input's hash; return the hash of serialization of the resulting
648 ;; derivation.
649 (let* ((inputs (map (match-lambda
650 (($ <derivation-input> path sub-drvs)
651 (let ((hash (derivation-path->base16-hash path)))
652 (make-derivation-input hash sub-drvs))))
653 inputs))
654 (drv (make-derivation outputs
655 (sort (coalesce-duplicate-inputs inputs)
656 derivation-input<?)
657 sources
658 system builder args env-vars
659 #f)))
660
661 ;; XXX: At this point this remains faster than `port-sha256', because
662 ;; the SHA256 port's `write' method gets called for every single
663 ;; character.
664 (sha256
665 (string->utf8 (derivation->string drv))))))))
666
667 (define (store-path type hash name) ; makeStorePath
668 "Return the store path for NAME/HASH/TYPE."
669 (let* ((s (string-append type ":sha256:"
670 (bytevector->base16-string hash) ":"
671 (%store-prefix) ":" name))
672 (h (sha256 (string->utf8 s)))
673 (c (compressed-hash h 20)))
674 (string-append (%store-prefix) "/"
675 (bytevector->nix-base32-string c) "-"
676 name)))
677
678 (define (output-path output hash name) ; makeOutputPath
679 "Return an output path for OUTPUT (the name of the output as a string) of
680 the derivation called NAME with hash HASH."
681 (store-path (string-append "output:" output) hash
682 (if (string=? output "out")
683 name
684 (string-append name "-" output))))
685
686 (define* (fixed-output-path name hash
687 #:key
688 (output "out")
689 (hash-algo 'sha256)
690 (recursive? #t))
691 "Return an output path for the fixed output OUTPUT defined by HASH of type
692 HASH-ALGO, of the derivation NAME. RECURSIVE? has the same meaning as for
693 'add-to-store'."
694 (if (and recursive? (eq? hash-algo 'sha256))
695 (store-path "source" hash name)
696 (let ((tag (string-append "fixed:" output ":"
697 (if recursive? "r:" "")
698 (symbol->string hash-algo) ":"
699 (bytevector->base16-string hash) ":")))
700 (store-path (string-append "output:" output)
701 (sha256 (string->utf8 tag))
702 name))))
703
704 (define* (derivation store name builder args
705 #:key
706 (system (%current-system)) (env-vars '())
707 (inputs '()) (outputs '("out"))
708 hash hash-algo recursive?
709 references-graphs
710 allowed-references disallowed-references
711 leaked-env-vars local-build?
712 (substitutable? #t))
713 "Build a derivation with the given arguments, and return the resulting
714 <derivation> object. When HASH and HASH-ALGO are given, a
715 fixed-output derivation is created---i.e., one whose result is known in
716 advance, such as a file download. If, in addition, RECURSIVE? is true, then
717 that fixed output may be an executable file or a directory and HASH must be
718 the hash of an archive containing this output.
719
720 When REFERENCES-GRAPHS is true, it must be a list of file name/store path
721 pairs. In that case, the reference graph of each store path is exported in
722 the build environment in the corresponding file, in a simple text format.
723
724 When ALLOWED-REFERENCES is true, it must be a list of store items or outputs
725 that the derivation's outputs may refer to. Likewise, DISALLOWED-REFERENCES,
726 if true, must be a list of things the outputs may not refer to.
727
728 When LEAKED-ENV-VARS is true, it must be a list of strings denoting
729 environment variables that are allowed to \"leak\" from the daemon's
730 environment to the build environment. This is only applicable to fixed-output
731 derivations--i.e., when HASH is true. The main use is to allow variables such
732 as \"http_proxy\" to be passed to derivations that download files.
733
734 When LOCAL-BUILD? is true, declare that the derivation is not a good candidate
735 for offloading and should rather be built locally. This is the case for small
736 derivations where the costs of data transfers would outweigh the benefits.
737
738 When SUBSTITUTABLE? is false, declare that substitutes of the derivation's
739 output should not be used."
740 (define (add-output-paths drv)
741 ;; Return DRV with an actual store path for each of its output and the
742 ;; corresponding environment variable.
743 (match drv
744 (($ <derivation> outputs inputs sources
745 system builder args env-vars)
746 (let* ((drv-hash (derivation-hash drv))
747 (outputs (map (match-lambda
748 ((output-name . ($ <derivation-output>
749 _ algo hash rec?))
750 (let ((path
751 (if hash
752 (fixed-output-path name hash
753 #:hash-algo algo
754 #:output output-name
755 #:recursive? rec?)
756 (output-path output-name
757 drv-hash name))))
758 (cons output-name
759 (make-derivation-output path algo
760 hash rec?)))))
761 outputs)))
762 (make-derivation outputs inputs sources system builder args
763 (map (match-lambda
764 ((name . value)
765 (cons name
766 (or (and=> (assoc-ref outputs name)
767 derivation-output-path)
768 value))))
769 env-vars)
770 #f)))))
771
772 (define (user+system-env-vars)
773 ;; Some options are passed to the build daemon via the env. vars of
774 ;; derivations (urgh!). We hide that from our API, but here is the place
775 ;; where we kludgify those options.
776 (let ((env-vars `(,@(if local-build?
777 `(("preferLocalBuild" . "1"))
778 '())
779 ,@(if (not substitutable?)
780 `(("allowSubstitutes" . "0"))
781 '())
782 ,@(if allowed-references
783 `(("allowedReferences"
784 . ,(string-join allowed-references)))
785 '())
786 ,@(if disallowed-references
787 `(("disallowedReferences"
788 . ,(string-join disallowed-references)))
789 '())
790 ,@(if leaked-env-vars
791 `(("impureEnvVars"
792 . ,(string-join leaked-env-vars)))
793 '())
794 ,@env-vars)))
795 (match references-graphs
796 (((file . path) ...)
797 (let ((value (map (cut string-append <> " " <>)
798 file path)))
799 ;; XXX: This all breaks down if an element of FILE or PATH contains
800 ;; white space.
801 `(("exportReferencesGraph" . ,(string-join value " "))
802 ,@env-vars)))
803 (#f
804 env-vars))))
805
806 (define (env-vars-with-empty-outputs env-vars)
807 ;; Return a variant of ENV-VARS where each OUTPUTS is associated with an
808 ;; empty string, even outputs that do not appear in ENV-VARS.
809 (let ((e (map (match-lambda
810 ((name . val)
811 (if (member name outputs)
812 (cons name "")
813 (cons name val))))
814 env-vars)))
815 (fold (lambda (output-name env-vars)
816 (if (assoc output-name env-vars)
817 env-vars
818 (append env-vars `((,output-name . "")))))
819 e
820 outputs)))
821
822 (define input->derivation-input
823 (match-lambda
824 (((? derivation? drv))
825 (make-derivation-input (derivation-file-name drv) '("out")))
826 (((? derivation? drv) sub-drvs ...)
827 (make-derivation-input (derivation-file-name drv) sub-drvs))
828 (((? direct-store-path? input))
829 (make-derivation-input input '("out")))
830 (((? direct-store-path? input) sub-drvs ...)
831 (make-derivation-input input sub-drvs))
832 ((input . _)
833 (let ((path (add-to-store store (basename input)
834 #t "sha256" input)))
835 (make-derivation-input path '())))))
836
837 ;; Note: lists are sorted alphabetically, to conform with the behavior of
838 ;; C++ `std::map' in Nix itself.
839
840 (let* ((outputs (map (lambda (name)
841 ;; Return outputs with an empty path.
842 (cons name
843 (make-derivation-output "" hash-algo
844 hash recursive?)))
845 (sort outputs string<?)))
846 (inputs (sort (coalesce-duplicate-inputs
847 (map input->derivation-input
848 (delete-duplicates inputs)))
849 derivation-input<?))
850 (env-vars (sort (env-vars-with-empty-outputs
851 (user+system-env-vars))
852 (lambda (e1 e2)
853 (string<? (car e1) (car e2)))))
854 (drv-masked (make-derivation outputs
855 (filter (compose derivation-path?
856 derivation-input-path)
857 inputs)
858 (filter-map (lambda (i)
859 (let ((p (derivation-input-path i)))
860 (and (not (derivation-path? p))
861 p)))
862 inputs)
863 system builder args env-vars #f))
864 (drv (add-output-paths drv-masked)))
865
866 (let* ((file (add-text-to-store store (string-append name ".drv")
867 (derivation->string drv)
868 (map derivation-input-path inputs)))
869 (drv* (set-field drv (derivation-file-name) file)))
870 (hash-set! %derivation-cache file drv*)
871 drv*)))
872
873 (define* (map-derivation store drv mapping
874 #:key (system (%current-system)))
875 "Given MAPPING, a list of pairs of derivations, return a derivation based on
876 DRV where all the 'car's of MAPPING have been replaced by its 'cdr's,
877 recursively."
878 (define (substitute str initial replacements)
879 (fold (lambda (path replacement result)
880 (string-replace-substring result path
881 replacement))
882 str
883 initial replacements))
884
885 (define (substitute-file file initial replacements)
886 (define contents
887 (with-fluids ((%default-port-encoding #f))
888 (call-with-input-file file read-string)))
889
890 (let ((updated (substitute contents initial replacements)))
891 (if (string=? updated contents)
892 file
893 ;; XXX: permissions aren't preserved.
894 (add-text-to-store store (store-path-package-name file)
895 updated))))
896
897 (define input->output-paths
898 (match-lambda
899 (((? derivation? drv))
900 (list (derivation->output-path drv)))
901 (((? derivation? drv) sub-drvs ...)
902 (map (cut derivation->output-path drv <>)
903 sub-drvs))
904 ((file)
905 (list file))))
906
907 (let ((mapping (fold (lambda (pair result)
908 (match pair
909 (((? derivation? orig) . replacement)
910 (vhash-cons (derivation-file-name orig)
911 replacement result))
912 ((file . replacement)
913 (vhash-cons file replacement result))))
914 vlist-null
915 mapping)))
916 (define rewritten-input
917 ;; Rewrite the given input according to MAPPING, and return an input
918 ;; in the format used in 'derivation' calls.
919 (mlambda (input loop)
920 (match input
921 (($ <derivation-input> path (sub-drvs ...))
922 (match (vhash-assoc path mapping)
923 ((_ . (? derivation? replacement))
924 (cons replacement sub-drvs))
925 ((_ . replacement)
926 (list replacement))
927 (#f
928 (let* ((drv (loop (call-with-input-file path read-derivation))))
929 (cons drv sub-drvs))))))))
930
931 (let loop ((drv drv))
932 (let* ((inputs (map (cut rewritten-input <> loop)
933 (derivation-inputs drv)))
934 (initial (append-map derivation-input-output-paths
935 (derivation-inputs drv)))
936 (replacements (append-map input->output-paths inputs))
937
938 ;; Sources typically refer to the output directories of the
939 ;; original inputs, INITIAL. Rewrite them by substituting
940 ;; REPLACEMENTS.
941 (sources (map (lambda (source)
942 (match (vhash-assoc source mapping)
943 ((_ . replacement)
944 replacement)
945 (#f
946 (substitute-file source
947 initial replacements))))
948 (derivation-sources drv)))
949
950 ;; Now augment the lists of initials and replacements.
951 (initial (append (derivation-sources drv) initial))
952 (replacements (append sources replacements))
953 (name (store-path-package-name
954 (string-drop-right (derivation-file-name drv)
955 4))))
956 (derivation store name
957 (substitute (derivation-builder drv)
958 initial replacements)
959 (map (cut substitute <> initial replacements)
960 (derivation-builder-arguments drv))
961 #:system system
962 #:env-vars (map (match-lambda
963 ((var . value)
964 `(,var
965 . ,(substitute value initial
966 replacements))))
967 (derivation-builder-environment-vars drv))
968 #:inputs (append (map list sources) inputs)
969 #:outputs (derivation-output-names drv)
970 #:hash (match (derivation-outputs drv)
971 ((($ <derivation-output> _ algo hash))
972 hash)
973 (_ #f))
974 #:hash-algo (match (derivation-outputs drv)
975 ((($ <derivation-output> _ algo hash))
976 algo)
977 (_ #f)))))))
978
979 \f
980 ;;;
981 ;;; Store compatibility layer.
982 ;;;
983
984 (define* (build-derivations store derivations
985 #:optional (mode (build-mode normal)))
986 "Build DERIVATIONS, a list of <derivation> objects or .drv file names, using
987 the specified MODE."
988 (build-things store (map (match-lambda
989 ((? string? file) file)
990 ((and drv ($ <derivation>))
991 (derivation-file-name drv)))
992 derivations)
993 mode))
994
995 \f
996 ;;;
997 ;;; Guile-based builders.
998 ;;;
999
1000 (define (parent-directories file-name)
1001 "Return the list of parent dirs of FILE-NAME, in the order in which an
1002 `mkdir -p' implementation would make them."
1003 (let ((not-slash (char-set-complement (char-set #\/))))
1004 (reverse
1005 (fold (lambda (dir result)
1006 (match result
1007 (()
1008 (list dir))
1009 ((prev _ ...)
1010 (cons (string-append prev "/" dir)
1011 result))))
1012 '()
1013 (remove (cut string=? <> ".")
1014 (string-tokenize (dirname file-name) not-slash))))))
1015
1016 (define* (imported-files store files ;deprecated
1017 #:key (name "file-import")
1018 (system (%current-system))
1019 (guile (%guile-for-build)))
1020 "Return a derivation that imports FILES into STORE. FILES must be a list
1021 of (FINAL-PATH . FILE-NAME) pairs; each FILE-NAME is read from the file
1022 system, imported, and appears under FINAL-PATH in the resulting store path."
1023 (let* ((files (map (match-lambda
1024 ((final-path . file-name)
1025 (list final-path
1026 (add-to-store store (basename final-path) #f
1027 "sha256" file-name))))
1028 files))
1029 (builder
1030 `(begin
1031 (mkdir %output) (chdir %output)
1032 ,@(append-map (match-lambda
1033 ((final-path store-path)
1034 (append (match (parent-directories final-path)
1035 (() '())
1036 ((head ... tail)
1037 (append (map (lambda (d)
1038 `(false-if-exception
1039 (mkdir ,d)))
1040 head)
1041 `((or (file-exists? ,tail)
1042 (mkdir ,tail))))))
1043 `((symlink ,store-path ,final-path)))))
1044 files))))
1045 (build-expression->derivation store name builder
1046 #:system system
1047 #:inputs files
1048 #:guile-for-build guile
1049 #:local-build? #t)))
1050
1051 ;; The "file not found" error condition.
1052 (define-condition-type &file-search-error &error
1053 file-search-error?
1054 (file file-search-error-file-name)
1055 (path file-search-error-search-path))
1056
1057 (define search-path*
1058 ;; A memoizing version of 'search-path' so 'imported-modules' does not end
1059 ;; up looking for the same files over and over again.
1060 (mlambda (path file)
1061 "Search for FILE in PATH and memoize the result. Raise a
1062 '&file-search-error' condition if it could not be found."
1063 (or (search-path path file)
1064 (raise (condition
1065 (&file-search-error (file file)
1066 (path path)))))))
1067
1068 (define (module->source-file-name module)
1069 "Return the file name corresponding to MODULE, a Guile module name (a list
1070 of symbols.)"
1071 (string-append (string-join (map symbol->string module) "/")
1072 ".scm"))
1073
1074 (define* (%imported-modules store modules ;deprecated
1075 #:key (name "module-import")
1076 (system (%current-system))
1077 (guile (%guile-for-build))
1078 (module-path %load-path))
1079 "Return a derivation that contains the source files of MODULES, a list of
1080 module names such as `(ice-9 q)'. All of MODULES must be in the MODULE-PATH
1081 search path."
1082 ;; TODO: Determine the closure of MODULES, build the `.go' files,
1083 ;; canonicalize the source files through read/write, etc.
1084 (let ((files (map (lambda (m)
1085 (let ((f (module->source-file-name m)))
1086 (cons f (search-path* module-path f))))
1087 modules)))
1088 (imported-files store files #:name name #:system system
1089 #:guile guile)))
1090
1091 (define* (%compiled-modules store modules ;deprecated
1092 #:key (name "module-import-compiled")
1093 (system (%current-system))
1094 (guile (%guile-for-build))
1095 (module-path %load-path))
1096 "Return a derivation that builds a tree containing the `.go' files
1097 corresponding to MODULES. All the MODULES are built in a context where
1098 they can refer to each other."
1099 (let* ((module-drv (%imported-modules store modules
1100 #:system system
1101 #:guile guile
1102 #:module-path module-path))
1103 (module-dir (derivation->output-path module-drv))
1104 (files (map (lambda (m)
1105 (let ((f (string-join (map symbol->string m)
1106 "/")))
1107 (cons (string-append f ".go")
1108 (string-append module-dir "/" f ".scm"))))
1109 modules)))
1110 (define builder
1111 `(begin
1112 (use-modules (system base compile))
1113 (let ((out (assoc-ref %outputs "out")))
1114 (mkdir out)
1115 (chdir out))
1116
1117 (set! %load-path
1118 (cons ,module-dir %load-path))
1119
1120 ,@(map (match-lambda
1121 ((output . input)
1122 (let ((make-parent-dirs (map (lambda (dir)
1123 `(unless (file-exists? ,dir)
1124 (mkdir ,dir)))
1125 (parent-directories output))))
1126 `(begin
1127 ,@make-parent-dirs
1128 (compile-file ,input
1129 #:output-file ,output
1130 #:opts %auto-compilation-options)))))
1131 files)))
1132
1133 (build-expression->derivation store name builder
1134 #:inputs `(("modules" ,module-drv))
1135 #:system system
1136 #:guile-for-build guile
1137 #:local-build? #t)))
1138
1139 (define* (build-expression->derivation store name exp ;deprecated
1140 #:key
1141 (system (%current-system))
1142 (inputs '())
1143 (outputs '("out"))
1144 hash hash-algo recursive?
1145 (env-vars '())
1146 (modules '())
1147 guile-for-build
1148 references-graphs
1149 allowed-references
1150 disallowed-references
1151 local-build? (substitutable? #t))
1152 "Return a derivation that executes Scheme expression EXP as a builder
1153 for derivation NAME. INPUTS must be a list of (NAME DRV-PATH SUB-DRV)
1154 tuples; when SUB-DRV is omitted, \"out\" is assumed. MODULES is a list
1155 of names of Guile modules from the current search path to be copied in
1156 the store, compiled, and made available in the load path during the
1157 execution of EXP.
1158
1159 EXP is evaluated in an environment where %OUTPUT is bound to the main
1160 output path, %OUTPUTS is bound to a list of output/path pairs, and where
1161 %BUILD-INPUTS is bound to an alist of string/output-path pairs made from
1162 INPUTS. Optionally, ENV-VARS is a list of string pairs specifying the
1163 name and value of environment variables visible to the builder. The
1164 builder terminates by passing the result of EXP to `exit'; thus, when
1165 EXP returns #f, the build is considered to have failed.
1166
1167 EXP is built using GUILE-FOR-BUILD (a derivation). When GUILE-FOR-BUILD is
1168 omitted or is #f, the value of the `%guile-for-build' fluid is used instead.
1169
1170 See the `derivation' procedure for the meaning of REFERENCES-GRAPHS,
1171 ALLOWED-REFERENCES, DISALLOWED-REFERENCES, LOCAL-BUILD?, and SUBSTITUTABLE?."
1172 (define guile-drv
1173 (or guile-for-build (%guile-for-build)))
1174
1175 (define guile
1176 (string-append (derivation->output-path guile-drv)
1177 "/bin/guile"))
1178
1179 (define module-form?
1180 (match-lambda
1181 (((or 'define-module 'use-modules) _ ...) #t)
1182 (_ #f)))
1183
1184 (define source-path
1185 ;; When passed an input that is a source, return its path; otherwise
1186 ;; return #f.
1187 (match-lambda
1188 ((_ (? derivation?) _ ...)
1189 #f)
1190 ((_ path _ ...)
1191 (and (not (derivation-path? path))
1192 path))))
1193
1194 (let* ((prologue `(begin
1195 ,@(match exp
1196 ((_ ...)
1197 ;; Module forms must appear at the top-level so
1198 ;; that any macros they export can be expanded.
1199 (filter module-form? exp))
1200 (_ `(,exp)))
1201
1202 (define %output (getenv "out"))
1203 (define %outputs
1204 (map (lambda (o)
1205 (cons o (getenv o)))
1206 ',outputs))
1207 (define %build-inputs
1208 ',(map (match-lambda
1209 ((name drv . rest)
1210 (let ((sub (match rest
1211 (() "out")
1212 ((x) x))))
1213 (cons name
1214 (cond
1215 ((derivation? drv)
1216 (derivation->output-path drv sub))
1217 ((derivation-path? drv)
1218 (derivation-path->output-path drv
1219 sub))
1220 (else drv))))))
1221 inputs))
1222
1223 ,@(if (null? modules)
1224 '()
1225 ;; Remove our own settings.
1226 '((unsetenv "GUILE_LOAD_COMPILED_PATH")))
1227
1228 ;; Guile sets it, but remove it to avoid conflicts when
1229 ;; building Guile-using packages.
1230 (unsetenv "LD_LIBRARY_PATH")))
1231 (builder (add-text-to-store store
1232 (string-append name "-guile-builder")
1233
1234 ;; Explicitly use UTF-8 for determinism,
1235 ;; and also because UTF-8 output is faster.
1236 (with-fluids ((%default-port-encoding
1237 "UTF-8"))
1238 (call-with-output-string
1239 (lambda (port)
1240 (write prologue port)
1241 (write
1242 `(exit
1243 ,(match exp
1244 ((_ ...)
1245 (remove module-form? exp))
1246 (_ `(,exp))))
1247 port))))
1248
1249 ;; The references don't really matter
1250 ;; since the builder is always used in
1251 ;; conjunction with the drv that needs
1252 ;; it. For clarity, we add references
1253 ;; to the subset of INPUTS that are
1254 ;; sources, avoiding references to other
1255 ;; .drv; otherwise, BUILDER's hash would
1256 ;; depend on those, even if they are
1257 ;; fixed-output.
1258 (filter-map source-path inputs)))
1259
1260 (mod-drv (and (pair? modules)
1261 (%imported-modules store modules
1262 #:guile guile-drv
1263 #:system system)))
1264 (mod-dir (and mod-drv
1265 (derivation->output-path mod-drv)))
1266 (go-drv (and (pair? modules)
1267 (%compiled-modules store modules
1268 #:guile guile-drv
1269 #:system system)))
1270 (go-dir (and go-drv
1271 (derivation->output-path go-drv))))
1272 (derivation store name guile
1273 `("--no-auto-compile"
1274 ,@(if mod-dir `("-L" ,mod-dir) '())
1275 ,builder)
1276
1277 #:system system
1278
1279 #:inputs `((,(or guile-for-build (%guile-for-build)))
1280 (,builder)
1281 ,@(map cdr inputs)
1282 ,@(if mod-drv `((,mod-drv) (,go-drv)) '()))
1283
1284 ;; When MODULES is non-empty, shamelessly clobber
1285 ;; $GUILE_LOAD_COMPILED_PATH.
1286 #:env-vars (if go-dir
1287 `(("GUILE_LOAD_COMPILED_PATH" . ,go-dir)
1288 ,@(alist-delete "GUILE_LOAD_COMPILED_PATH"
1289 env-vars))
1290 env-vars)
1291
1292 #:hash hash #:hash-algo hash-algo
1293 #:recursive? recursive?
1294 #:outputs outputs
1295 #:references-graphs references-graphs
1296 #:allowed-references allowed-references
1297 #:disallowed-references disallowed-references
1298 #:local-build? local-build?
1299 #:substitutable? substitutable?)))
1300
1301 \f
1302 ;;;
1303 ;;; Monadic interface.
1304 ;;;
1305
1306 (define built-derivations
1307 (store-lift build-derivations))
1308
1309 (define raw-derivation
1310 (store-lift derivation))