guix system: Do not import the user's (guix config).
[jackhill/guix/guix.git] / guix / gexp.scm
1 ;;; GNU Guix --- Functional package management for GNU
2 ;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org>
3 ;;; Copyright © 2018 Clément Lassieur <clement@lassieur.org>
4 ;;; Copyright © 2018 Jan Nieuwenhuizen <janneke@gnu.org>
5 ;;; Copyright © 2019, 2020 Mathieu Othacehe <m.othacehe@gmail.com>
6 ;;;
7 ;;; This file is part of GNU Guix.
8 ;;;
9 ;;; GNU Guix is free software; you can redistribute it and/or modify it
10 ;;; under the terms of the GNU General Public License as published by
11 ;;; the Free Software Foundation; either version 3 of the License, or (at
12 ;;; your option) any later version.
13 ;;;
14 ;;; GNU Guix is distributed in the hope that it will be useful, but
15 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
16 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 ;;; GNU General Public License for more details.
18 ;;;
19 ;;; You should have received a copy of the GNU General Public License
20 ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
21
22 (define-module (guix gexp)
23 #:use-module (guix store)
24 #:use-module (guix monads)
25 #:use-module (guix derivations)
26 #:use-module (guix grafts)
27 #:use-module (guix utils)
28 #:use-module (rnrs bytevectors)
29 #:use-module (srfi srfi-1)
30 #:use-module (srfi srfi-9)
31 #:use-module (srfi srfi-9 gnu)
32 #:use-module (srfi srfi-26)
33 #:use-module (srfi srfi-34)
34 #:use-module (srfi srfi-35)
35 #:use-module (ice-9 match)
36 #:export (gexp
37 gexp?
38 with-imported-modules
39 with-extensions
40
41 gexp-input
42 gexp-input?
43 gexp-input-thing
44 gexp-input-output
45 gexp-input-native?
46
47 local-file
48 local-file?
49 local-file-file
50 local-file-absolute-file-name
51 local-file-name
52 local-file-recursive?
53
54 plain-file
55 plain-file?
56 plain-file-name
57 plain-file-content
58
59 computed-file
60 computed-file?
61 computed-file-name
62 computed-file-gexp
63 computed-file-options
64
65 program-file
66 program-file?
67 program-file-name
68 program-file-gexp
69 program-file-guile
70 program-file-module-path
71
72 scheme-file
73 scheme-file?
74 scheme-file-name
75 scheme-file-gexp
76
77 file-append
78 file-append?
79 file-append-base
80 file-append-suffix
81
82 raw-derivation-file
83 raw-derivation-file?
84
85 with-parameters
86 parameterized?
87
88 load-path-expression
89 gexp-modules
90
91 lower-gexp
92 lowered-gexp?
93 lowered-gexp-sexp
94 lowered-gexp-inputs
95 lowered-gexp-sources
96 lowered-gexp-guile
97 lowered-gexp-load-path
98 lowered-gexp-load-compiled-path
99
100 gexp->derivation
101 gexp->file
102 gexp->script
103 text-file*
104 mixed-text-file
105 file-union
106 directory-union
107 imported-files
108 imported-modules
109 compiled-modules
110
111 define-gexp-compiler
112 gexp-compiler?
113 file-like?
114 lower-object
115
116 lower-inputs
117
118 &gexp-error
119 gexp-error?
120 &gexp-input-error
121 gexp-input-error?
122 gexp-error-invalid-input))
123
124 ;;; Commentary:
125 ;;;
126 ;;; This module implements "G-expressions", or "gexps". Gexps are like
127 ;;; S-expressions (sexps), with two differences:
128 ;;;
129 ;;; 1. References (un-quotations) to derivations or packages in a gexp are
130 ;;; replaced by the corresponding output file name; in addition, the
131 ;;; 'ungexp-native' unquote-like form allows code to explicitly refer to
132 ;;; the native code of a given package, in case of cross-compilation;
133 ;;;
134 ;;; 2. Gexps embed information about the derivations they refer to.
135 ;;;
136 ;;; Gexps make it easy to write to files Scheme code that refers to store
137 ;;; items, or to write Scheme code to build derivations.
138 ;;;
139 ;;; Code:
140
141 ;; "G expressions".
142 (define-record-type <gexp>
143 (make-gexp references modules extensions proc)
144 gexp?
145 (references gexp-references) ;list of <gexp-input>
146 (modules gexp-self-modules) ;list of module names
147 (extensions gexp-self-extensions) ;list of lowerable things
148 (proc gexp-proc)) ;procedure
149
150 (define (write-gexp gexp port)
151 "Write GEXP on PORT."
152 (display "#<gexp " port)
153
154 ;; Try to write the underlying sexp. Now, this trick doesn't work when
155 ;; doing things like (ungexp-splicing (gexp ())) because GEXP's procedure
156 ;; tries to use 'append' on that, which fails with wrong-type-arg.
157 (false-if-exception
158 (write (apply (gexp-proc gexp)
159 (gexp-references gexp))
160 port))
161 (format port " ~a>"
162 (number->string (object-address gexp) 16)))
163
164 (set-record-type-printer! <gexp> write-gexp)
165
166 \f
167 ;;;
168 ;;; Methods.
169 ;;;
170
171 ;; Compiler for a type of objects that may be introduced in a gexp.
172 (define-record-type <gexp-compiler>
173 (gexp-compiler type lower expand)
174 gexp-compiler?
175 (type gexp-compiler-type) ;record type descriptor
176 (lower gexp-compiler-lower)
177 (expand gexp-compiler-expand)) ;#f | DRV -> sexp
178
179 (define-condition-type &gexp-error &error
180 gexp-error?)
181
182 (define-condition-type &gexp-input-error &gexp-error
183 gexp-input-error?
184 (input gexp-error-invalid-input))
185
186
187 (define %gexp-compilers
188 ;; 'eq?' mapping of record type descriptor to <gexp-compiler>.
189 (make-hash-table 20))
190
191 (define (default-expander thing obj output)
192 "This is the default expander for \"things\" that appear in gexps. It
193 returns its output file name of OBJ's OUTPUT."
194 (match obj
195 ((? derivation? drv)
196 (derivation->output-path drv output))
197 ((? string? file)
198 file)))
199
200 (define (register-compiler! compiler)
201 "Register COMPILER as a gexp compiler."
202 (hashq-set! %gexp-compilers
203 (gexp-compiler-type compiler) compiler))
204
205 (define (lookup-compiler object)
206 "Search for a compiler for OBJECT. Upon success, return the three argument
207 procedure to lower it; otherwise return #f."
208 (and=> (hashq-ref %gexp-compilers (struct-vtable object))
209 gexp-compiler-lower))
210
211 (define (file-like? object)
212 "Return #t if OBJECT leads to a file in the store once unquoted in a
213 G-expression; otherwise return #f."
214 (and (struct? object) (->bool (lookup-compiler object))))
215
216 (define (lookup-expander object)
217 "Search for an expander for OBJECT. Upon success, return the three argument
218 procedure to expand it; otherwise return #f."
219 (and=> (hashq-ref %gexp-compilers (struct-vtable object))
220 gexp-compiler-expand))
221
222 (define* (lower-object obj
223 #:optional (system (%current-system))
224 #:key (target 'current))
225 "Return as a value in %STORE-MONAD the derivation or store item
226 corresponding to OBJ for SYSTEM, cross-compiling for TARGET if TARGET is true.
227 OBJ must be an object that has an associated gexp compiler, such as a
228 <package>."
229 (match (lookup-compiler obj)
230 (#f
231 (raise (condition (&gexp-input-error (input obj)))))
232 (lower
233 ;; Cache in STORE the result of lowering OBJ.
234 (mlet %store-monad ((target (if (eq? target 'current)
235 (current-target-system)
236 (return target)))
237 (graft? (grafting?)))
238 (mcached (let ((lower (lookup-compiler obj)))
239 (lower obj system target))
240 obj
241 system target graft?)))))
242
243 (define-syntax define-gexp-compiler
244 (syntax-rules (=> compiler expander)
245 "Define NAME as a compiler for objects matching PREDICATE encountered in
246 gexps.
247
248 In the simplest form of the macro, BODY must return a derivation for PARAM, an
249 object that matches PREDICATE, for SYSTEM and TARGET (the latter of which is
250 #f except when cross-compiling.)
251
252 The more elaborate form allows you to specify an expander:
253
254 (define-gexp-compiler something something?
255 compiler => (lambda (param system target) ...)
256 expander => (lambda (param drv output) ...))
257
258 The expander specifies how an object is converted to its sexp representation."
259 ((_ (name (param record-type) system target) body ...)
260 (define-gexp-compiler name record-type
261 compiler => (lambda (param system target) body ...)
262 expander => default-expander))
263 ((_ name record-type
264 compiler => compile
265 expander => expand)
266 (begin
267 (define name
268 (gexp-compiler record-type compile expand))
269 (register-compiler! name)))))
270
271 (define-gexp-compiler (derivation-compiler (drv <derivation>) system target)
272 ;; Derivations are the lowest-level representation, so this is the identity
273 ;; compiler.
274 (with-monad %store-monad
275 (return drv)))
276
277 ;; Expand to a raw ".drv" file for the lowerable object it wraps. In other
278 ;; words, this gives the raw ".drv" file instead of its build result.
279 (define-record-type <raw-derivation-file>
280 (raw-derivation-file obj)
281 raw-derivation-file?
282 (obj raw-derivation-file-object)) ;lowerable object
283
284 (define-gexp-compiler raw-derivation-file-compiler <raw-derivation-file>
285 compiler => (lambda (obj system target)
286 (mlet %store-monad ((obj (lower-object
287 (raw-derivation-file-object obj)
288 system #:target target)))
289 ;; Returning the .drv file name instead of the <derivation>
290 ;; record ensures that 'lower-gexp' will classify it as a
291 ;; "source" and not as an "input".
292 (return (if (derivation? obj)
293 (derivation-file-name obj)
294 obj))))
295 expander => (lambda (obj lowered output)
296 (if (derivation? lowered)
297 (derivation-file-name lowered)
298 lowered)))
299
300 \f
301 ;;;
302 ;;; File declarations.
303 ;;;
304
305 ;; A local file name. FILE is the file name the user entered, which can be a
306 ;; relative file name, and ABSOLUTE is a promise that computes its canonical
307 ;; absolute file name. We keep it in a promise to compute it lazily and avoid
308 ;; repeated 'stat' calls.
309 (define-record-type <local-file>
310 (%%local-file file absolute name recursive? select?)
311 local-file?
312 (file local-file-file) ;string
313 (absolute %local-file-absolute-file-name) ;promise string
314 (name local-file-name) ;string
315 (recursive? local-file-recursive?) ;Boolean
316 (select? local-file-select?)) ;string stat -> Boolean
317
318 (define (true file stat) #t)
319
320 (define* (%local-file file promise #:optional (name (basename file))
321 #:key recursive? (select? true))
322 ;; This intermediate procedure is part of our ABI, but the underlying
323 ;; %%LOCAL-FILE is not.
324 (%%local-file file promise name recursive? select?))
325
326 (define (absolute-file-name file directory)
327 "Return the canonical absolute file name for FILE, which lives in the
328 vicinity of DIRECTORY."
329 (canonicalize-path
330 (cond ((string-prefix? "/" file) file)
331 ((not directory) file)
332 ((string-prefix? "/" directory)
333 (string-append directory "/" file))
334 (else file))))
335
336 (define-syntax local-file
337 (lambda (s)
338 "Return an object representing local file FILE to add to the store; this
339 object can be used in a gexp. If FILE is a relative file name, it is looked
340 up relative to the source file where this form appears. FILE will be added to
341 the store under NAME--by default the base name of FILE.
342
343 When RECURSIVE? is true, the contents of FILE are added recursively; if FILE
344 designates a flat file and RECURSIVE? is true, its contents are added, and its
345 permission bits are kept.
346
347 When RECURSIVE? is true, call (SELECT? FILE STAT) for each directory entry,
348 where FILE is the entry's absolute file name and STAT is the result of
349 'lstat'; exclude entries for which SELECT? does not return true.
350
351 This is the declarative counterpart of the 'interned-file' monadic procedure.
352 It is implemented as a macro to capture the current source directory where it
353 appears."
354 (syntax-case s ()
355 ((_ file rest ...)
356 (string? (syntax->datum #'file))
357 ;; FILE is a literal, so resolve it relative to the source directory.
358 #'(%local-file file
359 (delay (absolute-file-name file (current-source-directory)))
360 rest ...))
361 ((_ file rest ...)
362 ;; Resolve FILE relative to the current directory.
363 #'(%local-file file
364 (delay (absolute-file-name file (getcwd)))
365 rest ...))
366 ((_)
367 #'(syntax-error "missing file name"))
368 (id
369 (identifier? #'id)
370 ;; XXX: We could return #'(lambda (file . rest) ...). However,
371 ;; (syntax-source #'id) is #f so (current-source-directory) would not
372 ;; work. Thus, simply forbid this form.
373 #'(syntax-error
374 "'local-file' is a macro and cannot be used like this")))))
375
376 (define (local-file-absolute-file-name file)
377 "Return the absolute file name for FILE, a <local-file> instance. A
378 'system-error' exception is raised if FILE could not be found."
379 (force (%local-file-absolute-file-name file)))
380
381 (define-gexp-compiler (local-file-compiler (file <local-file>) system target)
382 ;; "Compile" FILE by adding it to the store.
383 (match file
384 (($ <local-file> file (= force absolute) name recursive? select?)
385 ;; Canonicalize FILE so that if it's a symlink, it is resolved. Failing
386 ;; to do that, when RECURSIVE? is #t, we could end up creating a dangling
387 ;; symlink in the store, and when RECURSIVE? is #f 'add-to-store' would
388 ;; just throw an error, both of which are inconvenient.
389 (interned-file absolute name
390 #:recursive? recursive? #:select? select?))))
391
392 (define-record-type <plain-file>
393 (%plain-file name content references)
394 plain-file?
395 (name plain-file-name) ;string
396 (content plain-file-content) ;string or bytevector
397 (references plain-file-references)) ;list (currently unused)
398
399 (define (plain-file name content)
400 "Return an object representing a text file called NAME with the given
401 CONTENT (a string) to be added to the store.
402
403 This is the declarative counterpart of 'text-file'."
404 ;; XXX: For now just ignore 'references' because it's not clear how to use
405 ;; them in a declarative context.
406 (%plain-file name content '()))
407
408 (define-gexp-compiler (plain-file-compiler (file <plain-file>) system target)
409 ;; "Compile" FILE by adding it to the store.
410 (match file
411 (($ <plain-file> name (and (? string?) content) references)
412 (text-file name content references))
413 (($ <plain-file> name (and (? bytevector?) content) references)
414 (binary-file name content references))))
415
416 (define-record-type <computed-file>
417 (%computed-file name gexp guile options)
418 computed-file?
419 (name computed-file-name) ;string
420 (gexp computed-file-gexp) ;gexp
421 (guile computed-file-guile) ;<package>
422 (options computed-file-options)) ;list of arguments
423
424 (define* (computed-file name gexp
425 #:key guile (options '(#:local-build? #t)))
426 "Return an object representing the store item NAME, a file or directory
427 computed by GEXP. OPTIONS is a list of additional arguments to pass
428 to 'gexp->derivation'.
429
430 This is the declarative counterpart of 'gexp->derivation'."
431 (%computed-file name gexp guile options))
432
433 (define-gexp-compiler (computed-file-compiler (file <computed-file>)
434 system target)
435 ;; Compile FILE by returning a derivation whose build expression is its
436 ;; gexp.
437 (match file
438 (($ <computed-file> name gexp guile options)
439 (if guile
440 (mlet %store-monad ((guile (lower-object guile system
441 #:target target)))
442 (apply gexp->derivation name gexp #:guile-for-build guile
443 #:system system #:target target options))
444 (apply gexp->derivation name gexp
445 #:system system #:target target options)))))
446
447 (define-record-type <program-file>
448 (%program-file name gexp guile path)
449 program-file?
450 (name program-file-name) ;string
451 (gexp program-file-gexp) ;gexp
452 (guile program-file-guile) ;package
453 (path program-file-module-path)) ;list of strings
454
455 (define* (program-file name gexp #:key (guile #f) (module-path %load-path))
456 "Return an object representing the executable store item NAME that runs
457 GEXP. GUILE is the Guile package used to execute that script. Imported
458 modules of GEXP are looked up in MODULE-PATH.
459
460 This is the declarative counterpart of 'gexp->script'."
461 (%program-file name gexp guile module-path))
462
463 (define-gexp-compiler (program-file-compiler (file <program-file>)
464 system target)
465 ;; Compile FILE by returning a derivation that builds the script.
466 (match file
467 (($ <program-file> name gexp guile module-path)
468 (gexp->script name gexp
469 #:module-path module-path
470 #:guile (or guile (default-guile))
471 #:system system
472 #:target target))))
473
474 (define-record-type <scheme-file>
475 (%scheme-file name gexp splice?)
476 scheme-file?
477 (name scheme-file-name) ;string
478 (gexp scheme-file-gexp) ;gexp
479 (splice? scheme-file-splice?)) ;Boolean
480
481 (define* (scheme-file name gexp #:key splice?)
482 "Return an object representing the Scheme file NAME that contains GEXP.
483
484 This is the declarative counterpart of 'gexp->file'."
485 (%scheme-file name gexp splice?))
486
487 (define-gexp-compiler (scheme-file-compiler (file <scheme-file>)
488 system target)
489 ;; Compile FILE by returning a derivation that builds the file.
490 (match file
491 (($ <scheme-file> name gexp splice?)
492 (gexp->file name gexp
493 #:splice? splice?
494 #:system system
495 #:target target))))
496
497 ;; Appending SUFFIX to BASE's output file name.
498 (define-record-type <file-append>
499 (%file-append base suffix)
500 file-append?
501 (base file-append-base) ;<package> | <derivation> | ...
502 (suffix file-append-suffix)) ;list of strings
503
504 (define (write-file-append file port)
505 (match file
506 (($ <file-append> base suffix)
507 (format port "#<file-append ~s ~s>" base
508 (string-join suffix)))))
509
510 (set-record-type-printer! <file-append> write-file-append)
511
512 (define (file-append base . suffix)
513 "Return a <file-append> object that expands to the concatenation of BASE and
514 SUFFIX."
515 (%file-append base suffix))
516
517 (define-gexp-compiler file-append-compiler <file-append>
518 compiler => (lambda (obj system target)
519 (match obj
520 (($ <file-append> base _)
521 (lower-object base system #:target target))))
522 expander => (lambda (obj lowered output)
523 (match obj
524 (($ <file-append> base suffix)
525 (let* ((expand (lookup-expander base))
526 (base (expand base lowered output)))
527 (string-append base (string-concatenate suffix)))))))
528
529 ;; Representation of SRFI-39 parameter settings in the dynamic scope of an
530 ;; object lowering.
531 (define-record-type <parameterized>
532 (parameterized bindings thunk)
533 parameterized?
534 (bindings parameterized-bindings) ;list of parameter/value pairs
535 (thunk parameterized-thunk)) ;thunk
536
537 (define-syntax-rule (with-parameters ((param value) ...) body ...)
538 "Bind each PARAM to the corresponding VALUE for the extent during which BODY
539 is lowered. Consider this example:
540
541 (with-parameters ((%current-system \"x86_64-linux\"))
542 coreutils)
543
544 It returns a <parameterized> object that ensures %CURRENT-SYSTEM is set to
545 x86_64-linux when COREUTILS is lowered."
546 (parameterized (list (list param (lambda () value)) ...)
547 (lambda ()
548 body ...)))
549
550 (define-gexp-compiler compile-parameterized <parameterized>
551 compiler =>
552 (lambda (parameterized system target)
553 (match (parameterized-bindings parameterized)
554 (((parameters values) ...)
555 (let ((fluids (map parameter-fluid parameters))
556 (thunk (parameterized-thunk parameterized)))
557 ;; Install the PARAMETERS for the dynamic extent of THUNK.
558 (with-fluids* fluids
559 (map (lambda (thunk) (thunk)) values)
560 (lambda ()
561 ;; Special-case '%current-system' and '%current-target-system' to
562 ;; make sure we get the desired effect.
563 (let ((system (if (memq %current-system parameters)
564 (%current-system)
565 system))
566 (target (if (memq %current-target-system parameters)
567 (%current-target-system)
568 target)))
569 (lower-object (thunk) system #:target target))))))))
570
571 expander => (lambda (parameterized lowered output)
572 (match (parameterized-bindings parameterized)
573 (((parameters values) ...)
574 (let ((fluids (map parameter-fluid parameters))
575 (thunk (parameterized-thunk parameterized)))
576 ;; Install the PARAMETERS for the dynamic extent of THUNK.
577 (with-fluids* fluids
578 (map (lambda (thunk) (thunk)) values)
579 (lambda ()
580 ;; Delegate to the expander of the wrapped object.
581 (let* ((base (thunk))
582 (expand (lookup-expander base)))
583 (expand base lowered output)))))))))
584
585 \f
586 ;;;
587 ;;; Inputs & outputs.
588 ;;;
589
590 ;; The input of a gexp.
591 (define-record-type <gexp-input>
592 (%gexp-input thing output native?)
593 gexp-input?
594 (thing gexp-input-thing) ;<package> | <origin> | <derivation> | ...
595 (output gexp-input-output) ;string
596 (native? gexp-input-native?)) ;Boolean
597
598 (define (write-gexp-input input port)
599 (match input
600 (($ <gexp-input> thing output #f)
601 (format port "#<gexp-input ~s:~a>" thing output))
602 (($ <gexp-input> thing output #t)
603 (format port "#<gexp-input native ~s:~a>" thing output))))
604
605 (set-record-type-printer! <gexp-input> write-gexp-input)
606
607 (define* (gexp-input thing ;convenience procedure
608 #:optional (output "out")
609 #:key native?)
610 "Return a new <gexp-input> for the OUTPUT of THING; NATIVE? determines
611 whether this should be considered a \"native\" input or not."
612 (%gexp-input thing output native?))
613
614 ;; Reference to one of the derivation's outputs, for gexps used in
615 ;; derivations.
616 (define-record-type <gexp-output>
617 (gexp-output name)
618 gexp-output?
619 (name gexp-output-name))
620
621 (define (write-gexp-output output port)
622 (match output
623 (($ <gexp-output> name)
624 (format port "#<gexp-output ~a>" name))))
625
626 (set-record-type-printer! <gexp-output> write-gexp-output)
627
628 (define* (gexp-attribute gexp self-attribute #:optional (equal? equal?))
629 "Recurse on GEXP and the expressions it refers to, summing the items
630 returned by SELF-ATTRIBUTE, a procedure that takes a gexp. Use EQUAL? as the
631 second argument to 'delete-duplicates'."
632 (if (gexp? gexp)
633 (delete-duplicates
634 (append (self-attribute gexp)
635 (append-map (match-lambda
636 (($ <gexp-input> (? gexp? exp))
637 (gexp-attribute exp self-attribute))
638 (($ <gexp-input> (lst ...))
639 (append-map (lambda (item)
640 (if (gexp? item)
641 (gexp-attribute item
642 self-attribute)
643 '()))
644 lst))
645 (_
646 '()))
647 (gexp-references gexp)))
648 equal?)
649 '())) ;plain Scheme data type
650
651 (define (gexp-modules gexp)
652 "Return the list of Guile module names GEXP relies on. If (gexp? GEXP) is
653 false, meaning that GEXP is a plain Scheme object, return the empty list."
654 (define (module=? m1 m2)
655 ;; Return #t when M1 equals M2. Special-case '=>' specs because their
656 ;; right-hand side may not be comparable with 'equal?': it's typically a
657 ;; file-like object that embeds a gexp, which in turn embeds closure;
658 ;; those closures may be 'eq?' when running compiled code but are unlikely
659 ;; to be 'eq?' when running on 'eval'. Ignore the right-hand side to
660 ;; avoid this discrepancy.
661 (match m1
662 (((name1 ...) '=> _)
663 (match m2
664 (((name2 ...) '=> _) (equal? name1 name2))
665 (_ #f)))
666 (_
667 (equal? m1 m2))))
668
669 (gexp-attribute gexp gexp-self-modules module=?))
670
671 (define (gexp-extensions gexp)
672 "Return the list of Guile extensions (packages) GEXP relies on. If (gexp?
673 GEXP) is false, meaning that GEXP is a plain Scheme object, return the empty
674 list."
675 (gexp-attribute gexp gexp-self-extensions))
676
677 (define* (lower-inputs inputs
678 #:key system target)
679 "Turn any object from INPUTS into a derivation input for SYSTEM or a store
680 item (a \"source\"); return the corresponding input list as a monadic value.
681 When TARGET is true, use it as the cross-compilation target triplet."
682 (define (store-item? obj)
683 (and (string? obj) (store-path? obj)))
684
685 (with-monad %store-monad
686 (mapm/accumulate-builds
687 (match-lambda
688 (((? struct? thing) sub-drv ...)
689 (mlet %store-monad ((obj (lower-object
690 thing system #:target target)))
691 (return (match obj
692 ((? derivation? drv)
693 (let ((outputs (if (null? sub-drv)
694 '("out")
695 sub-drv)))
696 (derivation-input drv outputs)))
697 ((? store-item? item)
698 item)))))
699 (((? store-item? item))
700 (return item)))
701 inputs)))
702
703 (define* (lower-reference-graphs graphs #:key system target)
704 "Given GRAPHS, a list of (FILE-NAME INPUT ...) lists for use as a
705 #:reference-graphs argument, lower it such that each INPUT is replaced by the
706 corresponding <derivation-input> or store item."
707 (match graphs
708 (((file-names . inputs) ...)
709 (mlet %store-monad ((inputs (lower-inputs inputs
710 #:system system
711 #:target target)))
712 (return (map cons file-names inputs))))))
713
714 (define* (lower-references lst #:key system target)
715 "Based on LST, a list of output names and packages, return a list of output
716 names and file names suitable for the #:allowed-references argument to
717 'derivation'."
718 (with-monad %store-monad
719 (define lower
720 (match-lambda
721 ((? string? output)
722 (return output))
723 (($ <gexp-input> thing output native?)
724 (mlet %store-monad ((drv (lower-object thing system
725 #:target (if native?
726 #f target))))
727 (return (derivation->output-path drv output))))
728 (thing
729 (mlet %store-monad ((drv (lower-object thing system
730 #:target target)))
731 (return (derivation->output-path drv))))))
732
733 (mapm/accumulate-builds lower lst)))
734
735 (define default-guile-derivation
736 ;; Here we break the abstraction by talking to the higher-level layer.
737 ;; Thus, do the resolution lazily to hide the circular dependency.
738 (let ((proc (delay
739 (let ((iface (resolve-interface '(guix packages))))
740 (module-ref iface 'default-guile-derivation)))))
741 (lambda (system)
742 ((force proc) system))))
743
744 ;; Representation of a gexp instantiated for a given target and system.
745 ;; It's an intermediate representation between <gexp> and <derivation>.
746 (define-record-type <lowered-gexp>
747 (lowered-gexp sexp inputs sources guile load-path load-compiled-path)
748 lowered-gexp?
749 (sexp lowered-gexp-sexp) ;sexp
750 (inputs lowered-gexp-inputs) ;list of <derivation-input>
751 (sources lowered-gexp-sources) ;list of store items
752 (guile lowered-gexp-guile) ;<derivation-input> | #f
753 (load-path lowered-gexp-load-path) ;list of store items
754 (load-compiled-path lowered-gexp-load-compiled-path)) ;list of store items
755
756 (define* (imported+compiled-modules modules system
757 #:key (extensions '())
758 deprecation-warnings guile
759 (module-path %load-path))
760 "Return a pair where the first element is the imported MODULES and the
761 second element is the derivation to compile them."
762 (mcached equal?
763 (mlet %store-monad ((modules (if (pair? modules)
764 (imported-modules modules
765 #:system system
766 #:module-path module-path)
767 (return #f)))
768 (compiled (if (pair? modules)
769 (compiled-modules modules
770 #:system system
771 #:module-path module-path
772 #:extensions extensions
773 #:guile guile
774 #:deprecation-warnings
775 deprecation-warnings)
776 (return #f))))
777 (return (cons modules compiled)))
778 modules
779 system extensions guile deprecation-warnings module-path))
780
781 (define* (lower-gexp exp
782 #:key
783 (module-path %load-path)
784 (system (%current-system))
785 (target 'current)
786 (graft? (%graft?))
787 (guile-for-build (%guile-for-build))
788 (effective-version "2.2")
789
790 deprecation-warnings)
791 "*Note: This API is subject to change; use at your own risk!*
792
793 Lower EXP, a gexp, instantiating it for SYSTEM and TARGET. Return a
794 <lowered-gexp> ready to be used.
795
796 Lowered gexps are an intermediate representation that's useful for
797 applications that deal with gexps outside in a way that is disconnected from
798 derivations--e.g., code evaluated for its side effects."
799 (define %modules
800 (delete-duplicates (gexp-modules exp)))
801
802 (define (search-path modules extensions suffix)
803 (append (match modules
804 ((? derivation? drv)
805 (list (derivation->output-path drv)))
806 (#f
807 '())
808 ((? store-path? item)
809 (list item)))
810 (map (lambda (extension)
811 (string-append (match extension
812 ((? derivation? drv)
813 (derivation->output-path drv))
814 ((? store-path? item)
815 item))
816 suffix))
817 extensions)))
818
819 (mlet* %store-monad ( ;; The following binding forces '%current-system' and
820 ;; '%current-target-system' to be looked up at >>=
821 ;; time.
822 (graft? (set-grafting graft?))
823
824 (system -> (or system (%current-system)))
825 (target -> (if (eq? target 'current)
826 (%current-target-system)
827 target))
828 (guile (if guile-for-build
829 (return guile-for-build)
830 (default-guile-derivation system)))
831 (normals (lower-inputs (gexp-inputs exp)
832 #:system system
833 #:target target))
834 (natives (lower-inputs (gexp-native-inputs exp)
835 #:system system
836 #:target #f))
837 (inputs -> (append normals natives))
838 (sexp (gexp->sexp exp
839 #:system system
840 #:target target))
841 (extensions -> (gexp-extensions exp))
842 (exts (mapm %store-monad
843 (lambda (obj)
844 (lower-object obj system
845 #:target #f))
846 extensions))
847 (modules+compiled (imported+compiled-modules
848 %modules system
849 #:extensions extensions
850 #:deprecation-warnings
851 deprecation-warnings
852 #:guile guile
853 #:module-path module-path))
854 (modules -> (car modules+compiled))
855 (compiled -> (cdr modules+compiled)))
856 (define load-path
857 (search-path modules exts
858 (string-append "/share/guile/site/" effective-version)))
859
860 (define load-compiled-path
861 (search-path compiled exts
862 (string-append "/lib/guile/" effective-version
863 "/site-ccache")))
864
865 (mbegin %store-monad
866 (set-grafting graft?) ;restore the initial setting
867 (return (lowered-gexp sexp
868 `(,@(if (derivation? modules)
869 (list (derivation-input modules))
870 '())
871 ,@(if compiled
872 (list (derivation-input compiled))
873 '())
874 ,@(map derivation-input exts)
875 ,@(filter derivation-input? inputs))
876 (filter string? (cons modules inputs))
877 (derivation-input guile '("out"))
878 load-path
879 load-compiled-path)))))
880
881 (define* (gexp->derivation name exp
882 #:key
883 system (target 'current)
884 hash hash-algo recursive?
885 (env-vars '())
886 (modules '())
887 (module-path %load-path)
888 (guile-for-build (%guile-for-build))
889 (effective-version "2.2")
890 (graft? (%graft?))
891 references-graphs
892 allowed-references disallowed-references
893 leaked-env-vars
894 local-build? (substitutable? #t)
895 (properties '())
896 deprecation-warnings
897 (script-name (string-append name "-builder")))
898 "Return a derivation NAME that runs EXP (a gexp) with GUILE-FOR-BUILD (a
899 derivation) on SYSTEM; EXP is stored in a file called SCRIPT-NAME. When
900 TARGET is true, it is used as the cross-compilation target triplet for
901 packages referred to by EXP.
902
903 MODULES is deprecated in favor of 'with-imported-modules'. Its meaning is to
904 make MODULES available in the evaluation context of EXP; MODULES is a list of
905 names of Guile modules searched in MODULE-PATH to be copied in the store,
906 compiled, and made available in the load path during the execution of
907 EXP---e.g., '((guix build utils) (guix build gnu-build-system)).
908
909 EFFECTIVE-VERSION determines the string to use when adding extensions of
910 EXP (see 'with-extensions') to the search path---e.g., \"2.2\".
911
912 GRAFT? determines whether packages referred to by EXP should be grafted when
913 applicable.
914
915 When REFERENCES-GRAPHS is true, it must be a list of tuples of one of the
916 following forms:
917
918 (FILE-NAME PACKAGE)
919 (FILE-NAME PACKAGE OUTPUT)
920 (FILE-NAME DERIVATION)
921 (FILE-NAME DERIVATION OUTPUT)
922 (FILE-NAME STORE-ITEM)
923
924 The right-hand-side of each element of REFERENCES-GRAPHS is automatically made
925 an input of the build process of EXP. In the build environment, each
926 FILE-NAME contains the reference graph of the corresponding item, in a simple
927 text format.
928
929 ALLOWED-REFERENCES must be either #f or a list of output names and packages.
930 In the latter case, the list denotes store items that the result is allowed to
931 refer to. Any reference to another store item will lead to a build error.
932 Similarly for DISALLOWED-REFERENCES, which can list items that must not be
933 referenced by the outputs.
934
935 DEPRECATION-WARNINGS determines whether to show deprecation warnings while
936 compiling modules. It can be #f, #t, or 'detailed.
937
938 The other arguments are as for 'derivation'."
939 (define outputs (gexp-outputs exp))
940 (define requested-graft? graft?)
941
942 (define (graphs-file-names graphs)
943 ;; Return a list of (FILE-NAME . STORE-PATH) pairs made from GRAPHS.
944 (map (match-lambda
945 ((file-name . (? derivation-input? input))
946 (cons file-name (first (derivation-input-output-paths input))))
947 ((file-name . (? string? item))
948 (cons file-name item)))
949 graphs))
950
951 (define (add-modules exp modules)
952 (if (null? modules)
953 exp
954 (make-gexp (gexp-references exp)
955 (append modules (gexp-self-modules exp))
956 (gexp-self-extensions exp)
957 (gexp-proc exp))))
958
959 (mlet* %store-monad ( ;; The following binding forces '%current-system' and
960 ;; '%current-target-system' to be looked up at >>=
961 ;; time.
962 (graft? (set-grafting graft?))
963
964 (system -> (or system (%current-system)))
965 (target -> (if (eq? target 'current)
966 (%current-target-system)
967 target))
968 (exp -> (add-modules exp modules))
969 (lowered (lower-gexp exp
970 #:module-path module-path
971 #:system system
972 #:target target
973 #:graft? requested-graft?
974 #:guile-for-build
975 guile-for-build
976 #:effective-version
977 effective-version
978 #:deprecation-warnings
979 deprecation-warnings))
980
981 (graphs (if references-graphs
982 (lower-reference-graphs references-graphs
983 #:system system
984 #:target target)
985 (return #f)))
986 (allowed (if allowed-references
987 (lower-references allowed-references
988 #:system system
989 #:target target)
990 (return #f)))
991 (disallowed (if disallowed-references
992 (lower-references disallowed-references
993 #:system system
994 #:target target)
995 (return #f)))
996 (guile -> (lowered-gexp-guile lowered))
997 (builder (text-file script-name
998 (object->string
999 (lowered-gexp-sexp lowered)))))
1000 (mbegin %store-monad
1001 (set-grafting graft?) ;restore the initial setting
1002 (raw-derivation name
1003 (string-append (derivation-input-output-path guile)
1004 "/bin/guile")
1005 `("--no-auto-compile"
1006 ,@(append-map (lambda (directory)
1007 `("-L" ,directory))
1008 (lowered-gexp-load-path lowered))
1009 ,@(append-map (lambda (directory)
1010 `("-C" ,directory))
1011 (lowered-gexp-load-compiled-path lowered))
1012 ,builder)
1013 #:outputs outputs
1014 #:env-vars env-vars
1015 #:system system
1016 #:inputs `(,guile
1017 ,@(lowered-gexp-inputs lowered)
1018 ,@(match graphs
1019 (((_ . inputs) ...)
1020 (filter derivation-input? inputs))
1021 (#f '())))
1022 #:sources `(,builder
1023 ,@(if (and (string? modules)
1024 (store-path? modules))
1025 (list modules)
1026 '())
1027 ,@(lowered-gexp-sources lowered)
1028 ,@(match graphs
1029 (((_ . inputs) ...)
1030 (filter string? inputs))
1031 (#f '())))
1032
1033 #:hash hash #:hash-algo hash-algo #:recursive? recursive?
1034 #:references-graphs (and=> graphs graphs-file-names)
1035 #:allowed-references allowed
1036 #:disallowed-references disallowed
1037 #:leaked-env-vars leaked-env-vars
1038 #:local-build? local-build?
1039 #:substitutable? substitutable?
1040 #:properties properties))))
1041
1042 (define* (gexp-inputs exp #:key native?)
1043 "Return the input list for EXP. When NATIVE? is true, return only native
1044 references; otherwise, return only non-native references."
1045 ;; TODO: Return <gexp-input> records instead of tuples.
1046 (define (add-reference-inputs ref result)
1047 (match ref
1048 (($ <gexp-input> (? gexp? exp) _ #t)
1049 (if native?
1050 (append (gexp-inputs exp)
1051 (gexp-inputs exp #:native? #t)
1052 result)
1053 result))
1054 (($ <gexp-input> (? gexp? exp) _ #f)
1055 (append (gexp-inputs exp #:native? native?)
1056 result))
1057 (($ <gexp-input> (? string? str))
1058 (if (direct-store-path? str)
1059 (cons `(,str) result)
1060 result))
1061 (($ <gexp-input> (? struct? thing) output n?)
1062 (if (and (eqv? n? native?) (lookup-compiler thing))
1063 ;; THING is a derivation, or a package, or an origin, etc.
1064 (cons `(,thing ,output) result)
1065 result))
1066 (($ <gexp-input> (lst ...) output n?)
1067 (fold-right add-reference-inputs result
1068 ;; XXX: For now, automatically convert LST to a list of
1069 ;; gexp-inputs. Inherit N?.
1070 (map (match-lambda
1071 ((? gexp-input? x)
1072 (%gexp-input (gexp-input-thing x)
1073 (gexp-input-output x)
1074 n?))
1075 (x
1076 (%gexp-input x "out" n?)))
1077 lst)))
1078 (_
1079 ;; Ignore references to other kinds of objects.
1080 result)))
1081
1082 (fold-right add-reference-inputs
1083 '()
1084 (gexp-references exp)))
1085
1086 (define gexp-native-inputs
1087 (cut gexp-inputs <> #:native? #t))
1088
1089 (define (gexp-outputs exp)
1090 "Return the outputs referred to by EXP as a list of strings."
1091 (define (add-reference-output ref result)
1092 (match ref
1093 (($ <gexp-output> name)
1094 (cons name result))
1095 (($ <gexp-input> (? gexp? exp))
1096 (append (gexp-outputs exp) result))
1097 (($ <gexp-input> (lst ...) output native?)
1098 ;; XXX: Automatically convert LST.
1099 (add-reference-output (map (match-lambda
1100 ((? gexp-input? x) x)
1101 (x (%gexp-input x "out" native?)))
1102 lst)
1103 result))
1104 ((lst ...)
1105 (fold-right add-reference-output result lst))
1106 (_
1107 result)))
1108
1109 (delete-duplicates
1110 (add-reference-output (gexp-references exp) '())))
1111
1112 (define* (gexp->sexp exp #:key
1113 (system (%current-system))
1114 (target (%current-target-system)))
1115 "Return (monadically) the sexp corresponding to EXP for the given OUTPUT,
1116 and in the current monad setting (system type, etc.)"
1117 (define (self-quoting? x)
1118 (letrec-syntax ((one-of (syntax-rules ()
1119 ((_) #f)
1120 ((_ pred rest ...)
1121 (or (pred x)
1122 (one-of rest ...))))))
1123 (one-of symbol? string? keyword? pair? null? array?
1124 number? boolean? char?)))
1125
1126 (define* (reference->sexp ref #:optional native?)
1127 (with-monad %store-monad
1128 (match ref
1129 (($ <gexp-output> output)
1130 ;; Output file names are not known in advance but the daemon defines
1131 ;; an environment variable for each of them at build time, so use
1132 ;; that trick.
1133 (return `((@ (guile) getenv) ,output)))
1134 (($ <gexp-input> (? gexp? exp) output n?)
1135 (gexp->sexp exp
1136 #:system system
1137 #:target (if (or n? native?) #f target)))
1138 (($ <gexp-input> (refs ...) output n?)
1139 (mapm %store-monad
1140 (lambda (ref)
1141 ;; XXX: Automatically convert REF to an gexp-input.
1142 (reference->sexp
1143 (if (gexp-input? ref)
1144 ref
1145 (%gexp-input ref "out" n?))
1146 (or n? native?)))
1147 refs))
1148 (($ <gexp-input> (? struct? thing) output n?)
1149 (let ((target (if (or n? native?) #f target))
1150 (expand (lookup-expander thing)))
1151 (mlet %store-monad ((obj (lower-object thing system
1152 #:target target)))
1153 ;; OBJ must be either a derivation or a store file name.
1154 (return (expand thing obj output)))))
1155 (($ <gexp-input> (? self-quoting? x))
1156 (return x))
1157 (($ <gexp-input> x)
1158 (raise (condition (&gexp-input-error (input x)))))
1159 (x
1160 (return x)))))
1161
1162 (mlet %store-monad
1163 ((args (mapm %store-monad
1164 reference->sexp (gexp-references exp))))
1165 (return (apply (gexp-proc exp) args))))
1166
1167 (define-syntax-rule (define-syntax-parameter-once name proc)
1168 ;; Like 'define-syntax-parameter' but ensure the top-level binding for NAME
1169 ;; does not get redefined. This works around a race condition in a
1170 ;; multi-threaded context with Guile <= 2.2.4: <https://bugs.gnu.org/27476>.
1171 (eval-when (load eval expand compile)
1172 (define name
1173 (if (module-locally-bound? (current-module) 'name)
1174 (module-ref (current-module) 'name)
1175 (make-syntax-transformer 'name 'syntax-parameter
1176 (list proc))))))
1177
1178 (define-syntax-parameter-once current-imported-modules
1179 ;; Current list of imported modules.
1180 (identifier-syntax '()))
1181
1182 (define-syntax-rule (with-imported-modules modules body ...)
1183 "Mark the gexps defined in BODY... as requiring MODULES in their execution
1184 environment."
1185 (syntax-parameterize ((current-imported-modules
1186 (identifier-syntax modules)))
1187 body ...))
1188
1189 (define-syntax-parameter-once current-imported-extensions
1190 ;; Current list of extensions.
1191 (identifier-syntax '()))
1192
1193 (define-syntax-rule (with-extensions extensions body ...)
1194 "Mark the gexps defined in BODY... as requiring EXTENSIONS in their
1195 execution environment."
1196 (syntax-parameterize ((current-imported-extensions
1197 (identifier-syntax extensions)))
1198 body ...))
1199
1200 (define-syntax gexp
1201 (lambda (s)
1202 (define (collect-escapes exp)
1203 ;; Return all the 'ungexp' present in EXP.
1204 (let loop ((exp exp)
1205 (result '()))
1206 (syntax-case exp (ungexp
1207 ungexp-splicing
1208 ungexp-native
1209 ungexp-native-splicing)
1210 ((ungexp _)
1211 (cons exp result))
1212 ((ungexp _ _)
1213 (cons exp result))
1214 ((ungexp-splicing _ ...)
1215 (cons exp result))
1216 ((ungexp-native _ ...)
1217 (cons exp result))
1218 ((ungexp-native-splicing _ ...)
1219 (cons exp result))
1220 ((exp0 . exp)
1221 (let ((result (loop #'exp0 result)))
1222 (loop #'exp result)))
1223 (_
1224 result))))
1225
1226 (define (escape->ref exp)
1227 ;; Turn 'ungexp' form EXP into a "reference".
1228 (syntax-case exp (ungexp ungexp-splicing
1229 ungexp-native ungexp-native-splicing
1230 output)
1231 ((ungexp output)
1232 #'(gexp-output "out"))
1233 ((ungexp output name)
1234 #'(gexp-output name))
1235 ((ungexp thing)
1236 #'(%gexp-input thing "out" #f))
1237 ((ungexp drv-or-pkg out)
1238 #'(%gexp-input drv-or-pkg out #f))
1239 ((ungexp-splicing lst)
1240 #'(%gexp-input lst "out" #f))
1241 ((ungexp-native thing)
1242 #'(%gexp-input thing "out" #t))
1243 ((ungexp-native drv-or-pkg out)
1244 #'(%gexp-input drv-or-pkg out #t))
1245 ((ungexp-native-splicing lst)
1246 #'(%gexp-input lst "out" #t))))
1247
1248 (define (substitute-ungexp exp substs)
1249 ;; Given EXP, an 'ungexp' or 'ungexp-native' form, substitute it with
1250 ;; the corresponding form in SUBSTS.
1251 (match (assoc exp substs)
1252 ((_ id)
1253 id)
1254 (_ ;internal error
1255 (with-syntax ((exp exp))
1256 #'(syntax-error "error: no 'ungexp' substitution" exp)))))
1257
1258 (define (substitute-ungexp-splicing exp substs)
1259 (syntax-case exp ()
1260 ((exp rest ...)
1261 (match (assoc #'exp substs)
1262 ((_ id)
1263 (with-syntax ((id id))
1264 #`(append id
1265 #,(substitute-references #'(rest ...) substs))))
1266 (_
1267 #'(syntax-error "error: no 'ungexp-splicing' substitution"
1268 exp))))))
1269
1270 (define (substitute-references exp substs)
1271 ;; Return a variant of EXP where all the cars of SUBSTS have been
1272 ;; replaced by the corresponding cdr.
1273 (syntax-case exp (ungexp ungexp-native
1274 ungexp-splicing ungexp-native-splicing)
1275 ((ungexp _ ...)
1276 (substitute-ungexp exp substs))
1277 ((ungexp-native _ ...)
1278 (substitute-ungexp exp substs))
1279 (((ungexp-splicing _ ...) rest ...)
1280 (substitute-ungexp-splicing exp substs))
1281 (((ungexp-native-splicing _ ...) rest ...)
1282 (substitute-ungexp-splicing exp substs))
1283 ((exp0 . exp)
1284 #`(cons #,(substitute-references #'exp0 substs)
1285 #,(substitute-references #'exp substs)))
1286 (x #''x)))
1287
1288 (syntax-case s (ungexp output)
1289 ((_ exp)
1290 (let* ((escapes (delete-duplicates (collect-escapes #'exp)))
1291 (formals (generate-temporaries escapes))
1292 (sexp (substitute-references #'exp (zip escapes formals)))
1293 (refs (map escape->ref escapes)))
1294 #`(make-gexp (list #,@refs)
1295 current-imported-modules
1296 current-imported-extensions
1297 (lambda #,formals
1298 #,sexp)))))))
1299
1300 \f
1301 ;;;
1302 ;;; Module handling.
1303 ;;;
1304
1305 (define %not-slash
1306 (char-set-complement (char-set #\/)))
1307
1308 (define (file-mapping->tree mapping)
1309 "Convert MAPPING, an alist like:
1310
1311 ((\"guix/build/utils.scm\" . \"…/utils.scm\"))
1312
1313 to a tree suitable for 'interned-file-tree'."
1314 (let ((mapping (map (match-lambda
1315 ((destination . source)
1316 (cons (string-tokenize destination
1317 %not-slash)
1318 source)))
1319 mapping)))
1320 (fold (lambda (pair result)
1321 (match pair
1322 ((destination . source)
1323 (let loop ((destination destination)
1324 (result result))
1325 (match destination
1326 ((file)
1327 (let* ((mode (stat:mode (stat source)))
1328 (type (if (zero? (logand mode #o100))
1329 'regular
1330 'executable)))
1331 (alist-cons file
1332 `(,type (file ,source))
1333 result)))
1334 ((file rest ...)
1335 (let ((directory (assoc-ref result file)))
1336 (alist-cons file
1337 `(directory
1338 ,@(loop rest
1339 (match directory
1340 (('directory . entries) entries)
1341 (#f '()))))
1342 (if directory
1343 (alist-delete file result)
1344 result)))))))))
1345 '()
1346 mapping)))
1347
1348 (define %utils-module
1349 ;; This file provides 'mkdir-p', needed to implement 'imported-files' and
1350 ;; other primitives below. Note: We give the file name relative to this
1351 ;; file you are currently reading; 'search-path' could return a file name
1352 ;; relative to the current working directory.
1353 (local-file "build/utils.scm"
1354 "build-utils.scm"))
1355
1356 (define* (imported-files/derivation files
1357 #:key (name "file-import")
1358 (symlink? #f)
1359 (system (%current-system))
1360 (guile (%guile-for-build)))
1361 "Return a derivation that imports FILES into STORE. FILES must be a list
1362 of (FINAL-PATH . FILE) pairs. Each FILE is mapped to FINAL-PATH in the
1363 resulting store path. FILE can be either a file name, or a file-like object,
1364 as returned by 'local-file' for example. If SYMLINK? is true, create symlinks
1365 to the source files instead of copying them."
1366 (define file-pair
1367 (match-lambda
1368 ((final-path . (? string? file-name))
1369 (mlet %store-monad ((file (interned-file file-name
1370 (basename final-path))))
1371 (return (list final-path file))))
1372 ((final-path . file-like)
1373 (mlet %store-monad ((file (lower-object file-like system)))
1374 (return (list final-path file))))))
1375
1376 (mlet %store-monad ((files (mapm %store-monad file-pair files)))
1377 (define build
1378 (gexp
1379 (begin
1380 (primitive-load (ungexp %utils-module)) ;for 'mkdir-p'
1381 (use-modules (ice-9 match))
1382
1383 (mkdir (ungexp output)) (chdir (ungexp output))
1384 (for-each (match-lambda
1385 ((final-path store-path)
1386 (mkdir-p (dirname final-path))
1387 ((ungexp (if symlink? 'symlink 'copy-file))
1388 store-path final-path)))
1389 '(ungexp files)))))
1390
1391 ;; TODO: Pass FILES as an environment variable so that BUILD remains
1392 ;; exactly the same regardless of FILES: less disk space, and fewer
1393 ;; 'add-to-store' RPCs.
1394 (gexp->derivation name build
1395 #:system system
1396 #:guile-for-build guile
1397 #:local-build? #t
1398
1399 ;; Avoid deprecation warnings about the use of the _IO*
1400 ;; constants in (guix build utils).
1401 #:env-vars
1402 '(("GUILE_WARN_DEPRECATED" . "no")))))
1403
1404 (define* (imported-files files
1405 #:key (name "file-import")
1406 ;; The following parameters make sense when creating
1407 ;; an actual derivation.
1408 (system (%current-system))
1409 (guile (%guile-for-build)))
1410 "Import FILES into the store and return the resulting derivation or store
1411 file name (a derivation is created if and only if some elements of FILES are
1412 file-like objects and not local file names.) FILES must be a list
1413 of (FINAL-PATH . FILE) pairs. Each FILE is mapped to FINAL-PATH in the
1414 resulting store path. FILE can be either a file name, or a file-like object,
1415 as returned by 'local-file' for example."
1416 (if (any (match-lambda
1417 ((_ . (? struct? source)) #t)
1418 (_ #f))
1419 files)
1420 (imported-files/derivation files #:name name
1421 #:symlink? derivation?
1422 #:system system #:guile guile)
1423 (interned-file-tree `(,name directory
1424 ,@(file-mapping->tree files)))))
1425
1426 (define* (imported-modules modules
1427 #:key (name "module-import")
1428 (system (%current-system))
1429 (guile (%guile-for-build))
1430 (module-path %load-path))
1431 "Return a derivation that contains the source files of MODULES, a list of
1432 module names such as `(ice-9 q)'. All of MODULES must be either names of
1433 modules to be found in the MODULE-PATH search path, or a module name followed
1434 by an arrow followed by a file-like object. For example:
1435
1436 (imported-modules `((guix build utils)
1437 (guix gcrypt)
1438 ((guix config) => ,(scheme-file …))))
1439
1440 In this example, the first two modules are taken from MODULE-PATH, and the
1441 last one is created from the given <scheme-file> object."
1442 (let ((files (map (match-lambda
1443 (((module ...) '=> file)
1444 (cons (module->source-file-name module)
1445 file))
1446 ((module ...)
1447 (let ((f (module->source-file-name module)))
1448 (cons f (search-path* module-path f)))))
1449 modules)))
1450 (imported-files files #:name name
1451 #:system system
1452 #:guile guile)))
1453
1454 (define* (compiled-modules modules
1455 #:key (name "module-import-compiled")
1456 (system (%current-system))
1457 target
1458 (guile (%guile-for-build))
1459 (module-path %load-path)
1460 (extensions '())
1461 (deprecation-warnings #f))
1462 "Return a derivation that builds a tree containing the `.go' files
1463 corresponding to MODULES. All the MODULES are built in a context where
1464 they can refer to each other. When TARGET is true, cross-compile MODULES for
1465 TARGET, a GNU triplet."
1466 (define total (length modules))
1467
1468 (mlet %store-monad ((modules (imported-modules modules
1469 #:system system
1470 #:guile guile
1471 #:module-path
1472 module-path)))
1473 (define build
1474 (gexp
1475 (begin
1476 (primitive-load (ungexp %utils-module)) ;for 'mkdir-p'
1477
1478 (use-modules (ice-9 ftw)
1479 (ice-9 format)
1480 (srfi srfi-1)
1481 (srfi srfi-26)
1482 (system base compile))
1483
1484 ;; TODO: Inline this on the next rebuild cycle.
1485 (ungexp-splicing
1486 (if target
1487 (gexp ((use-modules (system base target))))
1488 (gexp ())))
1489
1490 (define (regular? file)
1491 (not (member file '("." ".."))))
1492
1493 (define (process-entry entry output processed)
1494 (if (file-is-directory? entry)
1495 (let ((output (string-append output "/" (basename entry))))
1496 (mkdir-p output)
1497 (process-directory entry output processed))
1498 (let* ((base (basename entry ".scm"))
1499 (output (string-append output "/" base ".go")))
1500 (format #t "[~2@a/~2@a] Compiling '~a'...~%"
1501 (+ 1 processed (ungexp total))
1502 (ungexp (* total 2))
1503 entry)
1504
1505 (ungexp-splicing
1506 (if target
1507 (gexp ((with-target (ungexp target)
1508 (lambda ()
1509 (compile-file entry
1510 #:output-file output
1511 #:opts
1512 %auto-compilation-options)))))
1513 (gexp ((compile-file entry
1514 #:output-file output
1515 #:opts %auto-compilation-options)))))
1516
1517 (+ 1 processed))))
1518
1519 (define (process-directory directory output processed)
1520 (let ((entries (map (cut string-append directory "/" <>)
1521 (scandir directory regular?))))
1522 (fold (cut process-entry <> output <>)
1523 processed
1524 entries)))
1525
1526 (define* (load-from-directory directory
1527 #:optional (loaded 0))
1528 "Load all the source files found in DIRECTORY."
1529 ;; XXX: This works around <https://bugs.gnu.org/15602>.
1530 (let ((entries (map (cut string-append directory "/" <>)
1531 (scandir directory regular?))))
1532 (fold (lambda (file loaded)
1533 (if (file-is-directory? file)
1534 (load-from-directory file loaded)
1535 (begin
1536 (format #t "[~2@a/~2@a] Loading '~a'...~%"
1537 (+ 1 loaded) (ungexp (* 2 total))
1538 file)
1539 (save-module-excursion
1540 (lambda ()
1541 (primitive-load file)))
1542 (+ 1 loaded))))
1543 loaded
1544 entries)))
1545
1546 (setvbuf (current-output-port)
1547 (cond-expand (guile-2.2 'line) (else _IOLBF)))
1548
1549 (define mkdir-p
1550 ;; Capture 'mkdir-p'.
1551 (@ (guix build utils) mkdir-p))
1552
1553 ;; Add EXTENSIONS to the search path.
1554 (set! %load-path
1555 (append (map (lambda (extension)
1556 (string-append extension
1557 "/share/guile/site/"
1558 (effective-version)))
1559 '((ungexp-native-splicing extensions)))
1560 %load-path))
1561 (set! %load-compiled-path
1562 (append (map (lambda (extension)
1563 (string-append extension "/lib/guile/"
1564 (effective-version)
1565 "/site-ccache"))
1566 '((ungexp-native-splicing extensions)))
1567 %load-compiled-path))
1568
1569 (set! %load-path (cons (ungexp modules) %load-path))
1570
1571 ;; Above we loaded our own (guix build utils) but now we may need to
1572 ;; load a compile a different one. Thus, force a reload.
1573 (let ((utils (string-append (ungexp modules)
1574 "/guix/build/utils.scm")))
1575 (when (file-exists? utils)
1576 (load utils)))
1577
1578 (mkdir (ungexp output))
1579 (chdir (ungexp modules))
1580
1581 (load-from-directory ".")
1582 (process-directory "." (ungexp output) 0))))
1583
1584 ;; TODO: Pass MODULES as an environment variable.
1585 (gexp->derivation name build
1586 #:system system
1587 #:guile-for-build guile
1588 #:local-build? #t
1589 #:env-vars
1590 (case deprecation-warnings
1591 ((#f)
1592 '(("GUILE_WARN_DEPRECATED" . "no")))
1593 ((detailed)
1594 '(("GUILE_WARN_DEPRECATED" . "detailed")))
1595 (else
1596 '())))))
1597
1598 \f
1599 ;;;
1600 ;;; Convenience procedures.
1601 ;;;
1602
1603 (define (default-guile)
1604 ;; Lazily resolve 'guile-2.2' (not 'guile-final' because this is for
1605 ;; programs returned by 'program-file' and we don't want to keep references
1606 ;; to several Guile packages). This module must not refer to (gnu …)
1607 ;; modules directly, to avoid circular dependencies, hence this hack.
1608 (module-ref (resolve-interface '(gnu packages guile))
1609 'guile-2.2))
1610
1611 (define* (load-path-expression modules #:optional (path %load-path)
1612 #:key (extensions '()) system target)
1613 "Return as a monadic value a gexp that sets '%load-path' and
1614 '%load-compiled-path' to point to MODULES, a list of module names. MODULES
1615 are searched for in PATH. Return #f when MODULES and EXTENSIONS are empty."
1616 (if (and (null? modules) (null? extensions))
1617 (with-monad %store-monad
1618 (return #f))
1619 (mlet %store-monad ((modules (imported-modules modules
1620 #:module-path path
1621 #:system system))
1622 (compiled (compiled-modules modules
1623 #:extensions extensions
1624 #:module-path path
1625 #:system system
1626 #:target target)))
1627 (return
1628 (gexp (eval-when (expand load eval)
1629 ;; Augment the load paths and delete duplicates. Do that
1630 ;; without loading (srfi srfi-1) or anything.
1631 (let ((extensions '((ungexp-splicing extensions)))
1632 (prepend (lambda (items lst)
1633 ;; This is O(N²) but N is typically small.
1634 (let loop ((items items)
1635 (lst lst))
1636 (if (null? items)
1637 lst
1638 (loop (cdr items)
1639 (cons (car items)
1640 (delete (car items) lst))))))))
1641 (set! %load-path
1642 (prepend (cons (ungexp modules)
1643 (map (lambda (extension)
1644 (string-append extension
1645 "/share/guile/site/"
1646 (effective-version)))
1647 extensions))
1648 %load-path))
1649 (set! %load-compiled-path
1650 (prepend (cons (ungexp compiled)
1651 (map (lambda (extension)
1652 (string-append extension
1653 "/lib/guile/"
1654 (effective-version)
1655 "/site-ccache"))
1656 extensions))
1657 %load-compiled-path)))))))))
1658
1659 (define* (gexp->script name exp
1660 #:key (guile (default-guile))
1661 (module-path %load-path)
1662 (system (%current-system))
1663 (target 'current))
1664 "Return an executable script NAME that runs EXP using GUILE, with EXP's
1665 imported modules in its search path. Look up EXP's modules in MODULE-PATH."
1666 (mlet* %store-monad ((target (if (eq? target 'current)
1667 (current-target-system)
1668 (return target)))
1669 (set-load-path
1670 (load-path-expression (gexp-modules exp)
1671 module-path
1672 #:extensions
1673 (gexp-extensions exp)
1674 #:system system
1675 #:target target)))
1676 (gexp->derivation name
1677 (gexp
1678 (call-with-output-file (ungexp output)
1679 (lambda (port)
1680 ;; Note: that makes a long shebang. When the store
1681 ;; is /gnu/store, that fits within the 128-byte
1682 ;; limit imposed by Linux, but that may go beyond
1683 ;; when running tests.
1684 (format port
1685 "#!~a/bin/guile --no-auto-compile~%!#~%"
1686 (ungexp guile))
1687
1688 (ungexp-splicing
1689 (if set-load-path
1690 (gexp ((write '(ungexp set-load-path) port)))
1691 (gexp ())))
1692
1693 (write '(ungexp exp) port)
1694 (chmod port #o555))))
1695 #:system system
1696 #:target target
1697 #:module-path module-path
1698
1699 ;; These derivations are not worth offloading or
1700 ;; substituting.
1701 #:local-build? #t
1702 #:substitutable? #f)))
1703
1704 (define* (gexp->file name exp #:key
1705 (set-load-path? #t)
1706 (module-path %load-path)
1707 (splice? #f)
1708 (system (%current-system))
1709 (target 'current))
1710 "Return a derivation that builds a file NAME containing EXP. When SPLICE?
1711 is true, EXP is considered to be a list of expressions that will be spliced in
1712 the resulting file.
1713
1714 When SET-LOAD-PATH? is true, emit code in the resulting file to set
1715 '%load-path' and '%load-compiled-path' to honor EXP's imported modules.
1716 Lookup EXP's modules in MODULE-PATH."
1717 (define modules (gexp-modules exp))
1718 (define extensions (gexp-extensions exp))
1719
1720 (mlet* %store-monad
1721 ((target (if (eq? target 'current)
1722 (current-target-system)
1723 (return target)))
1724 (no-load-path? -> (or (not set-load-path?)
1725 (and (null? modules)
1726 (null? extensions))))
1727 (set-load-path
1728 (load-path-expression modules module-path
1729 #:extensions extensions
1730 #:system system
1731 #:target target)))
1732 (if no-load-path?
1733 (gexp->derivation name
1734 (gexp
1735 (call-with-output-file (ungexp output)
1736 (lambda (port)
1737 (for-each
1738 (lambda (exp)
1739 (write exp port))
1740 '(ungexp (if splice?
1741 exp
1742 (gexp ((ungexp exp)))))))))
1743 #:local-build? #t
1744 #:substitutable? #f
1745 #:system system
1746 #:target target)
1747 (gexp->derivation name
1748 (gexp
1749 (call-with-output-file (ungexp output)
1750 (lambda (port)
1751 (write '(ungexp set-load-path) port)
1752 (for-each
1753 (lambda (exp)
1754 (write exp port))
1755 '(ungexp (if splice?
1756 exp
1757 (gexp ((ungexp exp)))))))))
1758 #:module-path module-path
1759 #:local-build? #t
1760 #:substitutable? #f
1761 #:system system
1762 #:target target))))
1763
1764 (define* (text-file* name #:rest text)
1765 "Return as a monadic value a derivation that builds a text file containing
1766 all of TEXT. TEXT may list, in addition to strings, objects of any type that
1767 can be used in a gexp: packages, derivations, local file objects, etc. The
1768 resulting store file holds references to all these."
1769 (define builder
1770 (gexp (call-with-output-file (ungexp output "out")
1771 (lambda (port)
1772 (display (string-append (ungexp-splicing text)) port)))))
1773
1774 (gexp->derivation name builder
1775 #:local-build? #t
1776 #:substitutable? #f))
1777
1778 (define* (mixed-text-file name #:rest text)
1779 "Return an object representing store file NAME containing TEXT. TEXT is a
1780 sequence of strings and file-like objects, as in:
1781
1782 (mixed-text-file \"profile\"
1783 \"export PATH=\" coreutils \"/bin:\" grep \"/bin\")
1784
1785 This is the declarative counterpart of 'text-file*'."
1786 (define build
1787 (gexp (call-with-output-file (ungexp output "out")
1788 (lambda (port)
1789 (display (string-append (ungexp-splicing text)) port)))))
1790
1791 (computed-file name build))
1792
1793 (define (file-union name files)
1794 "Return a <computed-file> that builds a directory containing all of FILES.
1795 Each item in FILES must be a two-element list where the first element is the
1796 file name to use in the new directory, and the second element is a gexp
1797 denoting the target file. Here's an example:
1798
1799 (file-union \"etc\"
1800 `((\"hosts\" ,(plain-file \"hosts\"
1801 \"127.0.0.1 localhost\"))
1802 (\"bashrc\" ,(plain-file \"bashrc\"
1803 \"alias ls='ls --color'\"))
1804 (\"libvirt/qemu.conf\" ,(plain-file \"qemu.conf\" \"\"))))
1805
1806 This yields an 'etc' directory containing these two files."
1807 (computed-file name
1808 (with-imported-modules '((guix build utils))
1809 (gexp
1810 (begin
1811 (use-modules (guix build utils))
1812
1813 (mkdir (ungexp output))
1814 (chdir (ungexp output))
1815 (ungexp-splicing
1816 (map (match-lambda
1817 ((target source)
1818 (gexp
1819 (begin
1820 ;; Stat the source to abort early if it does
1821 ;; not exist.
1822 (stat (ungexp source))
1823
1824 (mkdir-p (dirname (ungexp target)))
1825 (symlink (ungexp source)
1826 (ungexp target))))))
1827 files)))))))
1828
1829 (define* (directory-union name things
1830 #:key (copy? #f) (quiet? #f)
1831 (resolve-collision 'warn-about-collision))
1832 "Return a directory that is the union of THINGS, where THINGS is a list of
1833 file-like objects denoting directories. For example:
1834
1835 (directory-union \"guile+emacs\" (list guile emacs))
1836
1837 yields a directory that is the union of the 'guile' and 'emacs' packages.
1838
1839 Call RESOLVE-COLLISION when several files collide, passing it the list of
1840 colliding files. RESOLVE-COLLISION must return the chosen file or #f, in
1841 which case the colliding entry is skipped altogether.
1842
1843 When HARD-LINKS? is true, create hard links instead of symlinks. When QUIET?
1844 is true, the derivation will not print anything."
1845 (define symlink
1846 (if copy?
1847 (gexp (lambda (old new)
1848 (if (file-is-directory? old)
1849 (symlink old new)
1850 (copy-file old new))))
1851 (gexp symlink)))
1852
1853 (define log-port
1854 (if quiet?
1855 (gexp (%make-void-port "w"))
1856 (gexp (current-error-port))))
1857
1858 (match things
1859 ((one)
1860 ;; Only one thing; return it.
1861 one)
1862 (_
1863 (computed-file name
1864 (with-imported-modules '((guix build union))
1865 (gexp (begin
1866 (use-modules (guix build union)
1867 (srfi srfi-1)) ;for 'first' and 'last'
1868
1869 (union-build (ungexp output)
1870 '(ungexp things)
1871
1872 #:log-port (ungexp log-port)
1873 #:symlink (ungexp symlink)
1874 #:resolve-collision
1875 (ungexp resolve-collision)))))))))
1876
1877 \f
1878 ;;;
1879 ;;; Syntactic sugar.
1880 ;;;
1881
1882 (eval-when (expand load eval)
1883 (define* (read-ungexp chr port #:optional native?)
1884 "Read an 'ungexp' or 'ungexp-splicing' form from PORT. When NATIVE? is
1885 true, use 'ungexp-native' and 'ungexp-native-splicing' instead."
1886 (define unquote-symbol
1887 (match (peek-char port)
1888 (#\@
1889 (read-char port)
1890 (if native?
1891 'ungexp-native-splicing
1892 'ungexp-splicing))
1893 (_
1894 (if native?
1895 'ungexp-native
1896 'ungexp))))
1897
1898 (match (read port)
1899 ((? symbol? symbol)
1900 (let ((str (symbol->string symbol)))
1901 (match (string-index-right str #\:)
1902 (#f
1903 `(,unquote-symbol ,symbol))
1904 (colon
1905 (let ((name (string->symbol (substring str 0 colon)))
1906 (output (substring str (+ colon 1))))
1907 `(,unquote-symbol ,name ,output))))))
1908 (x
1909 `(,unquote-symbol ,x))))
1910
1911 (define (read-gexp chr port)
1912 "Read a 'gexp' form from PORT."
1913 `(gexp ,(read port)))
1914
1915 ;; Extend the reader
1916 (read-hash-extend #\~ read-gexp)
1917 (read-hash-extend #\$ read-ungexp)
1918 (read-hash-extend #\+ (cut read-ungexp <> <> #t)))
1919
1920 ;;; gexp.scm ends here