guix/gexp.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019 Ludovic Courtès <ludo@gnu.org>
   3 ;;; Copyright © 2018 Clément Lassieur <clement@lassieur.org>
   4 ;;; Copyright © 2018 Jan Nieuwenhuizen <janneke@gnu.org>
   5 ;;; Copyright © 2019 Mathieu Othacehe <m.othacehe@gmail.com>
   6 ;;;
   7 ;;; This file is part of GNU Guix.
   8 ;;;
   9 ;;; GNU Guix is free software; you can redistribute it and/or modify it
  10 ;;; under the terms of the GNU General Public License as published by
  11 ;;; the Free Software Foundation; either version 3 of the License, or (at
  12 ;;; your option) any later version.
  13 ;;;
  14 ;;; GNU Guix is distributed in the hope that it will be useful, but
  15 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  16 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 ;;; GNU General Public License for more details.
  18 ;;;
  19 ;;; You should have received a copy of the GNU General Public License
  20 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 (define-module (guix gexp)
  23   #:use-module (guix store)
  24   #:use-module (guix monads)
  25   #:use-module (guix derivations)
  26   #:use-module (guix grafts)
  27   #:use-module (guix utils)
  28   #:use-module (rnrs bytevectors)
  29   #:use-module (srfi srfi-1)
  30   #:use-module (srfi srfi-9)
  31   #:use-module (srfi srfi-9 gnu)
  32   #:use-module (srfi srfi-26)
  33   #:use-module (srfi srfi-34)
  34   #:use-module (srfi srfi-35)
  35   #:use-module (ice-9 match)
  36   #:export (gexp
  37             gexp?
  38             with-imported-modules
  39             with-extensions
  40
  41             gexp-input
  42             gexp-input?
  43             gexp-input-thing
  44             gexp-input-output
  45             gexp-input-native?
  46
  47             local-file
  48             local-file?
  49             local-file-file
  50             local-file-absolute-file-name
  51             local-file-name
  52             local-file-recursive?
  53
  54             plain-file
  55             plain-file?
  56             plain-file-name
  57             plain-file-content
  58
  59             computed-file
  60             computed-file?
  61             computed-file-name
  62             computed-file-gexp
  63             computed-file-options
  64
  65             program-file
  66             program-file?
  67             program-file-name
  68             program-file-gexp
  69             program-file-guile
  70             program-file-module-path
  71
  72             scheme-file
  73             scheme-file?
  74             scheme-file-name
  75             scheme-file-gexp
  76
  77             file-append
  78             file-append?
  79             file-append-base
  80             file-append-suffix
  81
  82             load-path-expression
  83             gexp-modules
  84
  85             lower-gexp
  86             lowered-gexp?
  87             lowered-gexp-sexp
  88             lowered-gexp-inputs
  89             lowered-gexp-sources
  90             lowered-gexp-guile
  91             lowered-gexp-load-path
  92             lowered-gexp-load-compiled-path
  93
  94             gexp->derivation
  95             gexp->file
  96             gexp->script
  97             text-file*
  98             mixed-text-file
  99             file-union
 100             directory-union
 101             imported-files
 102             imported-modules
 103             compiled-modules
 104
 105             define-gexp-compiler
 106             gexp-compiler?
 107             file-like?
 108             lower-object
 109
 110             lower-inputs
 111
 112             &gexp-error
 113             gexp-error?
 114             &gexp-input-error
 115             gexp-input-error?
 116             gexp-error-invalid-input))
 117
 118 ;;; Commentary:
 119 ;;;
 120 ;;; This module implements "G-expressions", or "gexps".  Gexps are like
 121 ;;; S-expressions (sexps), with two differences:
 122 ;;;
 123 ;;;   1. References (un-quotations) to derivations or packages in a gexp are
 124 ;;;      replaced by the corresponding output file name; in addition, the
 125 ;;;      'ungexp-native' unquote-like form allows code to explicitly refer to
 126 ;;;      the native code of a given package, in case of cross-compilation;
 127 ;;;
 128 ;;;   2. Gexps embed information about the derivations they refer to.
 129 ;;;
 130 ;;; Gexps make it easy to write to files Scheme code that refers to store
 131 ;;; items, or to write Scheme code to build derivations.
 132 ;;;
 133 ;;; Code:
 134
 135 ;; "G expressions".
 136 (define-record-type <gexp>
 137   (make-gexp references modules extensions proc)
 138   gexp?
 139   (references gexp-references)                    ;list of <gexp-input>
 140   (modules    gexp-self-modules)                  ;list of module names
 141   (extensions gexp-self-extensions)               ;list of lowerable things
 142   (proc       gexp-proc))                         ;procedure
 143
 144 (define (write-gexp gexp port)
 145   "Write GEXP on PORT."
 146   (display "#<gexp " port)
 147
 148   ;; Try to write the underlying sexp.  Now, this trick doesn't work when
 149   ;; doing things like (ungexp-splicing (gexp ())) because GEXP's procedure
 150   ;; tries to use 'append' on that, which fails with wrong-type-arg.
 151   (false-if-exception
 152    (write (apply (gexp-proc gexp)
 153                  (gexp-references gexp))
 154           port))
 155   (format port " ~a>"
 156           (number->string (object-address gexp) 16)))
 157
 158 (set-record-type-printer! <gexp> write-gexp)
 159
 160 \f
 161 ;;;
 162 ;;; Methods.
 163 ;;;
 164
 165 ;; Compiler for a type of objects that may be introduced in a gexp.
 166 (define-record-type <gexp-compiler>
 167   (gexp-compiler type lower expand)
 168   gexp-compiler?
 169   (type       gexp-compiler-type)                 ;record type descriptor
 170   (lower      gexp-compiler-lower)
 171   (expand     gexp-compiler-expand))              ;#f | DRV -> sexp
 172
 173 (define-condition-type &gexp-error &error
 174   gexp-error?)
 175
 176 (define-condition-type &gexp-input-error &gexp-error
 177   gexp-input-error?
 178   (input gexp-error-invalid-input))
 179
 180
 181 (define %gexp-compilers
 182   ;; 'eq?' mapping of record type descriptor to <gexp-compiler>.
 183   (make-hash-table 20))
 184
 185 (define (default-expander thing obj output)
 186   "This is the default expander for \"things\" that appear in gexps.  It
 187 returns its output file name of OBJ's OUTPUT."
 188   (match obj
 189     ((? derivation? drv)
 190      (derivation->output-path drv output))
 191     ((? string? file)
 192      file)))
 193
 194 (define (register-compiler! compiler)
 195   "Register COMPILER as a gexp compiler."
 196   (hashq-set! %gexp-compilers
 197               (gexp-compiler-type compiler) compiler))
 198
 199 (define (lookup-compiler object)
 200   "Search for a compiler for OBJECT.  Upon success, return the three argument
 201 procedure to lower it; otherwise return #f."
 202   (and=> (hashq-ref %gexp-compilers (struct-vtable object))
 203          gexp-compiler-lower))
 204
 205 (define (file-like? object)
 206   "Return #t if OBJECT leads to a file in the store once unquoted in a
 207 G-expression; otherwise return #f."
 208   (and (struct? object) (->bool (lookup-compiler object))))
 209
 210 (define (lookup-expander object)
 211   "Search for an expander for OBJECT.  Upon success, return the three argument
 212 procedure to expand it; otherwise return #f."
 213   (and=> (hashq-ref %gexp-compilers (struct-vtable object))
 214          gexp-compiler-expand))
 215
 216 (define* (lower-object obj
 217                        #:optional (system (%current-system))
 218                        #:key target)
 219   "Return as a value in %STORE-MONAD the derivation or store item
 220 corresponding to OBJ for SYSTEM, cross-compiling for TARGET if TARGET is true.
 221 OBJ must be an object that has an associated gexp compiler, such as a
 222 <package>."
 223   (match (lookup-compiler obj)
 224     (#f
 225      (raise (condition (&gexp-input-error (input obj)))))
 226     (lower
 227      ;; Cache in STORE the result of lowering OBJ.
 228      (mlet %store-monad ((graft? (grafting?)))
 229        (mcached (let ((lower (lookup-compiler obj)))
 230                   (lower obj system target))
 231                 obj
 232                 system target graft?)))))
 233
 234 (define-syntax define-gexp-compiler
 235   (syntax-rules (=> compiler expander)
 236     "Define NAME as a compiler for objects matching PREDICATE encountered in
 237 gexps.
 238
 239 In the simplest form of the macro, BODY must return a derivation for PARAM, an
 240 object that matches PREDICATE, for SYSTEM and TARGET (the latter of which is
 241 #f except when cross-compiling.)
 242
 243 The more elaborate form allows you to specify an expander:
 244
 245   (define-gexp-compiler something something?
 246     compiler => (lambda (param system target) ...)
 247     expander => (lambda (param drv output) ...))
 248
 249 The expander specifies how an object is converted to its sexp representation."
 250     ((_ (name (param record-type) system target) body ...)
 251      (define-gexp-compiler name record-type
 252        compiler => (lambda (param system target) body ...)
 253        expander => default-expander))
 254     ((_ name record-type
 255         compiler => compile
 256         expander => expand)
 257      (begin
 258        (define name
 259          (gexp-compiler record-type compile expand))
 260        (register-compiler! name)))))
 261
 262 (define-gexp-compiler (derivation-compiler (drv <derivation>) system target)
 263   ;; Derivations are the lowest-level representation, so this is the identity
 264   ;; compiler.
 265   (with-monad %store-monad
 266     (return drv)))
 267
 268 \f
 269 ;;;
 270 ;;; File declarations.
 271 ;;;
 272
 273 ;; A local file name.  FILE is the file name the user entered, which can be a
 274 ;; relative file name, and ABSOLUTE is a promise that computes its canonical
 275 ;; absolute file name.  We keep it in a promise to compute it lazily and avoid
 276 ;; repeated 'stat' calls.
 277 (define-record-type <local-file>
 278   (%%local-file file absolute name recursive? select?)
 279   local-file?
 280   (file       local-file-file)                    ;string
 281   (absolute   %local-file-absolute-file-name)     ;promise string
 282   (name       local-file-name)                    ;string
 283   (recursive? local-file-recursive?)              ;Boolean
 284   (select?    local-file-select?))                ;string stat -> Boolean
 285
 286 (define (true file stat) #t)
 287
 288 (define* (%local-file file promise #:optional (name (basename file))
 289                       #:key recursive? (select? true))
 290   ;; This intermediate procedure is part of our ABI, but the underlying
 291   ;; %%LOCAL-FILE is not.
 292   (%%local-file file promise name recursive? select?))
 293
 294 (define (absolute-file-name file directory)
 295   "Return the canonical absolute file name for FILE, which lives in the
 296 vicinity of DIRECTORY."
 297   (canonicalize-path
 298    (cond ((string-prefix? "/" file) file)
 299          ((not directory) file)
 300          ((string-prefix? "/" directory)
 301           (string-append directory "/" file))
 302          (else file))))
 303
 304 (define-syntax local-file
 305   (lambda (s)
 306     "Return an object representing local file FILE to add to the store; this
 307 object can be used in a gexp.  If FILE is a relative file name, it is looked
 308 up relative to the source file where this form appears.  FILE will be added to
 309 the store under NAME--by default the base name of FILE.
 310
 311 When RECURSIVE? is true, the contents of FILE are added recursively; if FILE
 312 designates a flat file and RECURSIVE? is true, its contents are added, and its
 313 permission bits are kept.
 314
 315 When RECURSIVE? is true, call (SELECT?  FILE STAT) for each directory entry,
 316 where FILE is the entry's absolute file name and STAT is the result of
 317 'lstat'; exclude entries for which SELECT? does not return true.
 318
 319 This is the declarative counterpart of the 'interned-file' monadic procedure.
 320 It is implemented as a macro to capture the current source directory where it
 321 appears."
 322     (syntax-case s ()
 323       ((_ file rest ...)
 324        (string? (syntax->datum #'file))
 325        ;; FILE is a literal, so resolve it relative to the source directory.
 326        #'(%local-file file
 327                       (delay (absolute-file-name file (current-source-directory)))
 328                       rest ...))
 329       ((_ file rest ...)
 330        ;; Resolve FILE relative to the current directory.
 331        #'(%local-file file
 332                       (delay (absolute-file-name file (getcwd)))
 333                       rest ...))
 334       ((_)
 335        #'(syntax-error "missing file name"))
 336       (id
 337        (identifier? #'id)
 338        ;; XXX: We could return #'(lambda (file . rest) ...).  However,
 339        ;; (syntax-source #'id) is #f so (current-source-directory) would not
 340        ;; work.  Thus, simply forbid this form.
 341        #'(syntax-error
 342           "'local-file' is a macro and cannot be used like this")))))
 343
 344 (define (local-file-absolute-file-name file)
 345   "Return the absolute file name for FILE, a <local-file> instance.  A
 346 'system-error' exception is raised if FILE could not be found."
 347   (force (%local-file-absolute-file-name file)))
 348
 349 (define-gexp-compiler (local-file-compiler (file <local-file>) system target)
 350   ;; "Compile" FILE by adding it to the store.
 351   (match file
 352     (($ <local-file> file (= force absolute) name recursive? select?)
 353      ;; Canonicalize FILE so that if it's a symlink, it is resolved.  Failing
 354      ;; to do that, when RECURSIVE? is #t, we could end up creating a dangling
 355      ;; symlink in the store, and when RECURSIVE? is #f 'add-to-store' would
 356      ;; just throw an error, both of which are inconvenient.
 357      (interned-file absolute name
 358                     #:recursive? recursive? #:select? select?))))
 359
 360 (define-record-type <plain-file>
 361   (%plain-file name content references)
 362   plain-file?
 363   (name        plain-file-name)                   ;string
 364   (content     plain-file-content)                ;string or bytevector
 365   (references  plain-file-references))            ;list (currently unused)
 366
 367 (define (plain-file name content)
 368   "Return an object representing a text file called NAME with the given
 369 CONTENT (a string) to be added to the store.
 370
 371 This is the declarative counterpart of 'text-file'."
 372   ;; XXX: For now just ignore 'references' because it's not clear how to use
 373   ;; them in a declarative context.
 374   (%plain-file name content '()))
 375
 376 (define-gexp-compiler (plain-file-compiler (file <plain-file>) system target)
 377   ;; "Compile" FILE by adding it to the store.
 378   (match file
 379     (($ <plain-file> name (and (? string?) content) references)
 380      (text-file name content references))
 381     (($ <plain-file> name (and (? bytevector?) content) references)
 382      (binary-file name content references))))
 383
 384 (define-record-type <computed-file>
 385   (%computed-file name gexp guile options)
 386   computed-file?
 387   (name       computed-file-name)                 ;string
 388   (gexp       computed-file-gexp)                 ;gexp
 389   (guile      computed-file-guile)                ;<package>
 390   (options    computed-file-options))             ;list of arguments
 391
 392 (define* (computed-file name gexp
 393                         #:key guile (options '(#:local-build? #t)))
 394   "Return an object representing the store item NAME, a file or directory
 395 computed by GEXP.  OPTIONS is a list of additional arguments to pass
 396 to 'gexp->derivation'.
 397
 398 This is the declarative counterpart of 'gexp->derivation'."
 399   (%computed-file name gexp guile options))
 400
 401 (define-gexp-compiler (computed-file-compiler (file <computed-file>)
 402                                               system target)
 403   ;; Compile FILE by returning a derivation whose build expression is its
 404   ;; gexp.
 405   (match file
 406     (($ <computed-file> name gexp guile options)
 407      (if guile
 408          (mlet %store-monad ((guile (lower-object guile system
 409                                                   #:target target)))
 410            (apply gexp->derivation name gexp #:guile-for-build guile
 411                   #:system system #:target target options))
 412          (apply gexp->derivation name gexp
 413                 #:system system #:target target options)))))
 414
 415 (define-record-type <program-file>
 416   (%program-file name gexp guile path)
 417   program-file?
 418   (name       program-file-name)                  ;string
 419   (gexp       program-file-gexp)                  ;gexp
 420   (guile      program-file-guile)                 ;package
 421   (path       program-file-module-path))          ;list of strings
 422
 423 (define* (program-file name gexp #:key (guile #f) (module-path %load-path))
 424   "Return an object representing the executable store item NAME that runs
 425 GEXP.  GUILE is the Guile package used to execute that script.  Imported
 426 modules of GEXP are looked up in MODULE-PATH.
 427
 428 This is the declarative counterpart of 'gexp->script'."
 429   (%program-file name gexp guile module-path))
 430
 431 (define-gexp-compiler (program-file-compiler (file <program-file>)
 432                                              system target)
 433   ;; Compile FILE by returning a derivation that builds the script.
 434   (match file
 435     (($ <program-file> name gexp guile module-path)
 436      (gexp->script name gexp
 437                    #:module-path module-path
 438                    #:guile (or guile (default-guile))
 439                    #:system system
 440                    #:target target))))
 441
 442 (define-record-type <scheme-file>
 443   (%scheme-file name gexp splice?)
 444   scheme-file?
 445   (name       scheme-file-name)                  ;string
 446   (gexp       scheme-file-gexp)                  ;gexp
 447   (splice?    scheme-file-splice?))              ;Boolean
 448
 449 (define* (scheme-file name gexp #:key splice?)
 450   "Return an object representing the Scheme file NAME that contains GEXP.
 451
 452 This is the declarative counterpart of 'gexp->file'."
 453   (%scheme-file name gexp splice?))
 454
 455 (define-gexp-compiler (scheme-file-compiler (file <scheme-file>)
 456                                             system target)
 457   ;; Compile FILE by returning a derivation that builds the file.
 458   (match file
 459     (($ <scheme-file> name gexp splice?)
 460      (gexp->file name gexp
 461                  #:splice? splice?
 462                  #:system system
 463                  #:target target))))
 464
 465 ;; Appending SUFFIX to BASE's output file name.
 466 (define-record-type <file-append>
 467   (%file-append base suffix)
 468   file-append?
 469   (base   file-append-base)                    ;<package> | <derivation> | ...
 470   (suffix file-append-suffix))                 ;list of strings
 471
 472 (define (write-file-append file port)
 473   (match file
 474     (($ <file-append> base suffix)
 475      (format port "#<file-append ~s ~s>" base
 476              (string-join suffix)))))
 477
 478 (set-record-type-printer! <file-append> write-file-append)
 479
 480 (define (file-append base . suffix)
 481   "Return a <file-append> object that expands to the concatenation of BASE and
 482 SUFFIX."
 483   (%file-append base suffix))
 484
 485 (define-gexp-compiler file-append-compiler <file-append>
 486   compiler => (lambda (obj system target)
 487                 (match obj
 488                   (($ <file-append> base _)
 489                    (lower-object base system #:target target))))
 490   expander => (lambda (obj lowered output)
 491                 (match obj
 492                   (($ <file-append> base suffix)
 493                    (let* ((expand (lookup-expander base))
 494                           (base   (expand base lowered output)))
 495                      (string-append base (string-concatenate suffix)))))))
 496
 497 \f
 498 ;;;
 499 ;;; Inputs & outputs.
 500 ;;;
 501
 502 ;; The input of a gexp.
 503 (define-record-type <gexp-input>
 504   (%gexp-input thing output native?)
 505   gexp-input?
 506   (thing     gexp-input-thing)       ;<package> | <origin> | <derivation> | ...
 507   (output    gexp-input-output)      ;string
 508   (native?   gexp-input-native?))    ;Boolean
 509
 510 (define (write-gexp-input input port)
 511   (match input
 512     (($ <gexp-input> thing output #f)
 513      (format port "#<gexp-input ~s:~a>" thing output))
 514     (($ <gexp-input> thing output #t)
 515      (format port "#<gexp-input native ~s:~a>" thing output))))
 516
 517 (set-record-type-printer! <gexp-input> write-gexp-input)
 518
 519 (define* (gexp-input thing                        ;convenience procedure
 520                      #:optional (output "out")
 521                      #:key native?)
 522   "Return a new <gexp-input> for the OUTPUT of THING; NATIVE? determines
 523 whether this should be considered a \"native\" input or not."
 524   (%gexp-input thing output native?))
 525
 526 ;; Reference to one of the derivation's outputs, for gexps used in
 527 ;; derivations.
 528 (define-record-type <gexp-output>
 529   (gexp-output name)
 530   gexp-output?
 531   (name gexp-output-name))
 532
 533 (define (write-gexp-output output port)
 534   (match output
 535     (($ <gexp-output> name)
 536      (format port "#<gexp-output ~a>" name))))
 537
 538 (set-record-type-printer! <gexp-output> write-gexp-output)
 539
 540 (define* (gexp-attribute gexp self-attribute #:optional (equal? equal?))
 541   "Recurse on GEXP and the expressions it refers to, summing the items
 542 returned by SELF-ATTRIBUTE, a procedure that takes a gexp.  Use EQUAL? as the
 543 second argument to 'delete-duplicates'."
 544   (if (gexp? gexp)
 545       (delete-duplicates
 546        (append (self-attribute gexp)
 547                (append-map (match-lambda
 548                              (($ <gexp-input> (? gexp? exp))
 549                               (gexp-attribute exp self-attribute))
 550                              (($ <gexp-input> (lst ...))
 551                               (append-map (lambda (item)
 552                                             (if (gexp? item)
 553                                                 (gexp-attribute item
 554                                                                 self-attribute)
 555                                                 '()))
 556                                           lst))
 557                              (_
 558                               '()))
 559                            (gexp-references gexp)))
 560        equal?)
 561       '()))                                       ;plain Scheme data type
 562
 563 (define (gexp-modules gexp)
 564   "Return the list of Guile module names GEXP relies on.  If (gexp? GEXP) is
 565 false, meaning that GEXP is a plain Scheme object, return the empty list."
 566   (define (module=? m1 m2)
 567     ;; Return #t when M1 equals M2.  Special-case '=>' specs because their
 568     ;; right-hand side may not be comparable with 'equal?': it's typically a
 569     ;; file-like object that embeds a gexp, which in turn embeds closure;
 570     ;; those closures may be 'eq?' when running compiled code but are unlikely
 571     ;; to be 'eq?' when running on 'eval'.  Ignore the right-hand side to
 572     ;; avoid this discrepancy.
 573     (match m1
 574       (((name1 ...) '=> _)
 575        (match m2
 576          (((name2 ...) '=> _) (equal? name1 name2))
 577          (_ #f)))
 578       (_
 579        (equal? m1 m2))))
 580
 581   (gexp-attribute gexp gexp-self-modules module=?))
 582
 583 (define (gexp-extensions gexp)
 584   "Return the list of Guile extensions (packages) GEXP relies on.  If (gexp?
 585 GEXP) is false, meaning that GEXP is a plain Scheme object, return the empty
 586 list."
 587   (gexp-attribute gexp gexp-self-extensions))
 588
 589 (define* (lower-inputs inputs
 590                        #:key system target)
 591   "Turn any object from INPUTS into a derivation input for SYSTEM or a store
 592 item (a \"source\"); return the corresponding input list as a monadic value.
 593 When TARGET is true, use it as the cross-compilation target triplet."
 594   (define (store-item? obj)
 595     (and (string? obj) (store-path? obj)))
 596
 597   (with-monad %store-monad
 598     (mapm %store-monad
 599           (match-lambda
 600             (((? struct? thing) sub-drv ...)
 601              (mlet %store-monad ((obj (lower-object
 602                                        thing system #:target target)))
 603                (return (match obj
 604                          ((? derivation? drv)
 605                           (let ((outputs (if (null? sub-drv)
 606                                              '("out")
 607                                              sub-drv)))
 608                             (derivation-input drv outputs)))
 609                          ((? store-item? item)
 610                           item)))))
 611             (((? store-item? item))
 612              (return item)))
 613           inputs)))
 614
 615 (define* (lower-reference-graphs graphs #:key system target)
 616   "Given GRAPHS, a list of (FILE-NAME INPUT ...) lists for use as a
 617 #:reference-graphs argument, lower it such that each INPUT is replaced by the
 618 corresponding <derivation-input> or store item."
 619   (match graphs
 620     (((file-names . inputs) ...)
 621      (mlet %store-monad ((inputs (lower-inputs inputs
 622                                                #:system system
 623                                                #:target target)))
 624        (return (map cons file-names inputs))))))
 625
 626 (define* (lower-references lst #:key system target)
 627   "Based on LST, a list of output names and packages, return a list of output
 628 names and file names suitable for the #:allowed-references argument to
 629 'derivation'."
 630   (with-monad %store-monad
 631     (define lower
 632       (match-lambda
 633        ((? string? output)
 634         (return output))
 635        (($ <gexp-input> thing output native?)
 636         (mlet %store-monad ((drv (lower-object thing system
 637                                                #:target (if native?
 638                                                             #f target))))
 639           (return (derivation->output-path drv output))))
 640        (thing
 641         (mlet %store-monad ((drv (lower-object thing system
 642                                                #:target target)))
 643           (return (derivation->output-path drv))))))
 644
 645     (mapm %store-monad lower lst)))
 646
 647 (define default-guile-derivation
 648   ;; Here we break the abstraction by talking to the higher-level layer.
 649   ;; Thus, do the resolution lazily to hide the circular dependency.
 650   (let ((proc (delay
 651                 (let ((iface (resolve-interface '(guix packages))))
 652                   (module-ref iface 'default-guile-derivation)))))
 653     (lambda (system)
 654       ((force proc) system))))
 655
 656 ;; Representation of a gexp instantiated for a given target and system.
 657 ;; It's an intermediate representation between <gexp> and <derivation>.
 658 (define-record-type <lowered-gexp>
 659   (lowered-gexp sexp inputs sources guile load-path load-compiled-path)
 660   lowered-gexp?
 661   (sexp                lowered-gexp-sexp)         ;sexp
 662   (inputs              lowered-gexp-inputs)       ;list of <derivation-input>
 663   (sources             lowered-gexp-sources)      ;list of store items
 664   (guile               lowered-gexp-guile)        ;<derivation-input> | #f
 665   (load-path           lowered-gexp-load-path)    ;list of store items
 666   (load-compiled-path  lowered-gexp-load-compiled-path)) ;list of store items
 667
 668 (define* (imported+compiled-modules modules system
 669                                     #:key (extensions '())
 670                                     deprecation-warnings guile
 671                                     (module-path %load-path))
 672   "Return a pair where the first element is the imported MODULES and the
 673 second element is the derivation to compile them."
 674   (mcached equal?
 675            (mlet %store-monad ((modules  (if (pair? modules)
 676                                              (imported-modules modules
 677                                                                #:system system
 678                                                                #:module-path module-path)
 679                                              (return #f)))
 680                                (compiled (if (pair? modules)
 681                                              (compiled-modules modules
 682                                                                #:system system
 683                                                                #:module-path module-path
 684                                                                #:extensions extensions
 685                                                                #:guile guile
 686                                                                #:deprecation-warnings
 687                                                                deprecation-warnings)
 688                                              (return #f))))
 689              (return (cons modules compiled)))
 690            modules
 691            system extensions guile deprecation-warnings module-path))
 692
 693 (define* (lower-gexp exp
 694                      #:key
 695                      (module-path %load-path)
 696                      (system (%current-system))
 697                      (target 'current)
 698                      (graft? (%graft?))
 699                      (guile-for-build (%guile-for-build))
 700                      (effective-version "2.2")
 701
 702                      deprecation-warnings)
 703   "*Note: This API is subject to change; use at your own risk!*
 704
 705 Lower EXP, a gexp, instantiating it for SYSTEM and TARGET.  Return a
 706 <lowered-gexp> ready to be used.
 707
 708 Lowered gexps are an intermediate representation that's useful for
 709 applications that deal with gexps outside in a way that is disconnected from
 710 derivations--e.g., code evaluated for its side effects."
 711   (define %modules
 712     (delete-duplicates (gexp-modules exp)))
 713
 714   (define (search-path modules extensions suffix)
 715     (append (match modules
 716               ((? derivation? drv)
 717                (list (derivation->output-path drv)))
 718               (#f
 719                '())
 720               ((? store-path? item)
 721                (list item)))
 722             (map (lambda (extension)
 723                    (string-append (match extension
 724                                     ((? derivation? drv)
 725                                      (derivation->output-path drv))
 726                                     ((? store-path? item)
 727                                      item))
 728                                   suffix))
 729                  extensions)))
 730
 731   (mlet* %store-monad ( ;; The following binding forces '%current-system' and
 732                        ;; '%current-target-system' to be looked up at >>=
 733                        ;; time.
 734                        (graft?    (set-grafting graft?))
 735
 736                        (system -> (or system (%current-system)))
 737                        (target -> (if (eq? target 'current)
 738                                       (%current-target-system)
 739                                       target))
 740                        (guile     (if guile-for-build
 741                                       (return guile-for-build)
 742                                       (default-guile-derivation system)))
 743                        (normals  (lower-inputs (gexp-inputs exp)
 744                                                #:system system
 745                                                #:target target))
 746                        (natives  (lower-inputs (gexp-native-inputs exp)
 747                                                #:system system
 748                                                #:target #f))
 749                        (inputs -> (append normals natives))
 750                        (sexp     (gexp->sexp exp
 751                                              #:system system
 752                                              #:target target))
 753                        (extensions -> (gexp-extensions exp))
 754                        (exts     (mapm %store-monad
 755                                        (lambda (obj)
 756                                          (lower-object obj system))
 757                                        extensions))
 758                        (modules+compiled (imported+compiled-modules
 759                                           %modules system
 760                                           #:extensions extensions
 761                                           #:deprecation-warnings
 762                                           deprecation-warnings
 763                                           #:guile guile
 764                                           #:module-path module-path))
 765                        (modules ->  (car modules+compiled))
 766                        (compiled -> (cdr modules+compiled)))
 767     (define load-path
 768       (search-path modules exts
 769                    (string-append "/share/guile/site/" effective-version)))
 770
 771     (define load-compiled-path
 772       (search-path compiled exts
 773                    (string-append "/lib/guile/" effective-version
 774                                   "/site-ccache")))
 775
 776     (mbegin %store-monad
 777       (set-grafting graft?)                       ;restore the initial setting
 778       (return (lowered-gexp sexp
 779                             `(,@(if (derivation? modules)
 780                                     (list (derivation-input modules))
 781                                     '())
 782                               ,@(if compiled
 783                                     (list (derivation-input compiled))
 784                                     '())
 785                               ,@(map derivation-input exts)
 786                               ,@(filter derivation-input? inputs))
 787                             (filter string? (cons modules inputs))
 788                             (derivation-input guile '("out"))
 789                             load-path
 790                             load-compiled-path)))))
 791
 792 (define* (gexp->derivation name exp
 793                            #:key
 794                            system (target 'current)
 795                            hash hash-algo recursive?
 796                            (env-vars '())
 797                            (modules '())
 798                            (module-path %load-path)
 799                            (guile-for-build (%guile-for-build))
 800                            (effective-version "2.2")
 801                            (graft? (%graft?))
 802                            references-graphs
 803                            allowed-references disallowed-references
 804                            leaked-env-vars
 805                            local-build? (substitutable? #t)
 806                            (properties '())
 807                            deprecation-warnings
 808                            (script-name (string-append name "-builder")))
 809   "Return a derivation NAME that runs EXP (a gexp) with GUILE-FOR-BUILD (a
 810 derivation) on SYSTEM; EXP is stored in a file called SCRIPT-NAME.  When
 811 TARGET is true, it is used as the cross-compilation target triplet for
 812 packages referred to by EXP.
 813
 814 MODULES is deprecated in favor of 'with-imported-modules'.  Its meaning is to
 815 make MODULES available in the evaluation context of EXP; MODULES is a list of
 816 names of Guile modules searched in MODULE-PATH to be copied in the store,
 817 compiled, and made available in the load path during the execution of
 818 EXP---e.g., '((guix build utils) (guix build gnu-build-system)).
 819
 820 EFFECTIVE-VERSION determines the string to use when adding extensions of
 821 EXP (see 'with-extensions') to the search path---e.g., \"2.2\".
 822
 823 GRAFT? determines whether packages referred to by EXP should be grafted when
 824 applicable.
 825
 826 When REFERENCES-GRAPHS is true, it must be a list of tuples of one of the
 827 following forms:
 828
 829   (FILE-NAME PACKAGE)
 830   (FILE-NAME PACKAGE OUTPUT)
 831   (FILE-NAME DERIVATION)
 832   (FILE-NAME DERIVATION OUTPUT)
 833   (FILE-NAME STORE-ITEM)
 834
 835 The right-hand-side of each element of REFERENCES-GRAPHS is automatically made
 836 an input of the build process of EXP.  In the build environment, each
 837 FILE-NAME contains the reference graph of the corresponding item, in a simple
 838 text format.
 839
 840 ALLOWED-REFERENCES must be either #f or a list of output names and packages.
 841 In the latter case, the list denotes store items that the result is allowed to
 842 refer to.  Any reference to another store item will lead to a build error.
 843 Similarly for DISALLOWED-REFERENCES, which can list items that must not be
 844 referenced by the outputs.
 845
 846 DEPRECATION-WARNINGS determines whether to show deprecation warnings while
 847 compiling modules.  It can be #f, #t, or 'detailed.
 848
 849 The other arguments are as for 'derivation'."
 850   (define outputs (gexp-outputs exp))
 851   (define requested-graft? graft?)
 852
 853   (define (graphs-file-names graphs)
 854     ;; Return a list of (FILE-NAME . STORE-PATH) pairs made from GRAPHS.
 855     (map (match-lambda
 856            ((file-name . (? derivation-input? input))
 857             (cons file-name (first (derivation-input-output-paths input))))
 858            ((file-name . (? string? item))
 859             (cons file-name item)))
 860          graphs))
 861
 862   (define (add-modules exp modules)
 863     (if (null? modules)
 864         exp
 865         (make-gexp (gexp-references exp)
 866                    (append modules (gexp-self-modules exp))
 867                    (gexp-self-extensions exp)
 868                    (gexp-proc exp))))
 869
 870   (mlet* %store-monad ( ;; The following binding forces '%current-system' and
 871                        ;; '%current-target-system' to be looked up at >>=
 872                        ;; time.
 873                        (graft?    (set-grafting graft?))
 874
 875                        (system -> (or system (%current-system)))
 876                        (target -> (if (eq? target 'current)
 877                                       (%current-target-system)
 878                                       target))
 879                        (exp ->    (add-modules exp modules))
 880                        (lowered   (lower-gexp exp
 881                                               #:module-path module-path
 882                                               #:system system
 883                                               #:target target
 884                                               #:graft? requested-graft?
 885                                               #:guile-for-build
 886                                               guile-for-build
 887                                               #:effective-version
 888                                               effective-version
 889                                               #:deprecation-warnings
 890                                               deprecation-warnings))
 891
 892                        (graphs   (if references-graphs
 893                                      (lower-reference-graphs references-graphs
 894                                                              #:system system
 895                                                              #:target target)
 896                                      (return #f)))
 897                        (allowed  (if allowed-references
 898                                      (lower-references allowed-references
 899                                                        #:system system
 900                                                        #:target target)
 901                                      (return #f)))
 902                        (disallowed (if disallowed-references
 903                                        (lower-references disallowed-references
 904                                                          #:system system
 905                                                          #:target target)
 906                                        (return #f)))
 907                        (guile -> (lowered-gexp-guile lowered))
 908                        (builder  (text-file script-name
 909                                             (object->string
 910                                              (lowered-gexp-sexp lowered)))))
 911     (mbegin %store-monad
 912       (set-grafting graft?)                       ;restore the initial setting
 913       (raw-derivation name
 914                       (string-append (derivation-input-output-path guile)
 915                                      "/bin/guile")
 916                       `("--no-auto-compile"
 917                         ,@(append-map (lambda (directory)
 918                                         `("-L" ,directory))
 919                                       (lowered-gexp-load-path lowered))
 920                         ,@(append-map (lambda (directory)
 921                                         `("-C" ,directory))
 922                                       (lowered-gexp-load-compiled-path lowered))
 923                         ,builder)
 924                       #:outputs outputs
 925                       #:env-vars env-vars
 926                       #:system system
 927                       #:inputs `(,guile
 928                                  ,@(lowered-gexp-inputs lowered)
 929                                  ,@(match graphs
 930                                      (((_ . inputs) ...)
 931                                       (filter derivation-input? inputs))
 932                                      (#f '())))
 933                       #:sources `(,builder
 934                                   ,@(if (and (string? modules)
 935                                              (store-path? modules))
 936                                         (list modules)
 937                                         '())
 938                                   ,@(lowered-gexp-sources lowered)
 939                                   ,@(match graphs
 940                                       (((_ . inputs) ...)
 941                                        (filter string? inputs))
 942                                       (#f '())))
 943
 944                       #:hash hash #:hash-algo hash-algo #:recursive? recursive?
 945                       #:references-graphs (and=> graphs graphs-file-names)
 946                       #:allowed-references allowed
 947                       #:disallowed-references disallowed
 948                       #:leaked-env-vars leaked-env-vars
 949                       #:local-build? local-build?
 950                       #:substitutable? substitutable?
 951                       #:properties properties))))
 952
 953 (define* (gexp-inputs exp #:key native?)
 954   "Return the input list for EXP.  When NATIVE? is true, return only native
 955 references; otherwise, return only non-native references."
 956   ;; TODO: Return <gexp-input> records instead of tuples.
 957   (define (add-reference-inputs ref result)
 958     (match ref
 959       (($ <gexp-input> (? gexp? exp) _ #t)
 960        (if native?
 961            (append (gexp-inputs exp)
 962                    (gexp-inputs exp #:native? #t)
 963                    result)
 964            result))
 965       (($ <gexp-input> (? gexp? exp) _ #f)
 966        (append (gexp-inputs exp #:native? native?)
 967                result))
 968       (($ <gexp-input> (? string? str))
 969        (if (direct-store-path? str)
 970            (cons `(,str) result)
 971            result))
 972       (($ <gexp-input> (? struct? thing) output n?)
 973        (if (and (eqv? n? native?) (lookup-compiler thing))
 974            ;; THING is a derivation, or a package, or an origin, etc.
 975            (cons `(,thing ,output) result)
 976            result))
 977       (($ <gexp-input> (lst ...) output n?)
 978        (fold-right add-reference-inputs result
 979                    ;; XXX: For now, automatically convert LST to a list of
 980                    ;; gexp-inputs.  Inherit N?.
 981                    (map (match-lambda
 982                           ((? gexp-input? x)
 983                            (%gexp-input (gexp-input-thing x)
 984                                         (gexp-input-output x)
 985                                         n?))
 986                           (x
 987                            (%gexp-input x "out" n?)))
 988                         lst)))
 989       (_
 990        ;; Ignore references to other kinds of objects.
 991        result)))
 992
 993   (fold-right add-reference-inputs
 994               '()
 995               (gexp-references exp)))
 996
 997 (define gexp-native-inputs
 998   (cut gexp-inputs <> #:native? #t))
 999
1000 (define (gexp-outputs exp)
1001   "Return the outputs referred to by EXP as a list of strings."
1002   (define (add-reference-output ref result)
1003     (match ref
1004       (($ <gexp-output> name)
1005        (cons name result))
1006       (($ <gexp-input> (? gexp? exp))
1007        (append (gexp-outputs exp) result))
1008       (($ <gexp-input> (lst ...) output native?)
1009        ;; XXX: Automatically convert LST.
1010        (add-reference-output (map (match-lambda
1011                                    ((? gexp-input? x) x)
1012                                    (x (%gexp-input x "out" native?)))
1013                                   lst)
1014                              result))
1015       ((lst ...)
1016        (fold-right add-reference-output result lst))
1017       (_
1018        result)))
1019
1020   (delete-duplicates
1021    (add-reference-output (gexp-references exp) '())))
1022
1023 (define* (gexp->sexp exp #:key
1024                      (system (%current-system))
1025                      (target (%current-target-system)))
1026   "Return (monadically) the sexp corresponding to EXP for the given OUTPUT,
1027 and in the current monad setting (system type, etc.)"
1028   (define (self-quoting? x)
1029     (letrec-syntax ((one-of (syntax-rules ()
1030                               ((_) #f)
1031                               ((_ pred rest ...)
1032                                (or (pred x)
1033                                    (one-of rest ...))))))
1034       (one-of symbol? string? keyword? pair? null? array?
1035               number? boolean? char?)))
1036
1037   (define* (reference->sexp ref #:optional native?)
1038     (with-monad %store-monad
1039       (match ref
1040         (($ <gexp-output> output)
1041          ;; Output file names are not known in advance but the daemon defines
1042          ;; an environment variable for each of them at build time, so use
1043          ;; that trick.
1044          (return `((@ (guile) getenv) ,output)))
1045         (($ <gexp-input> (? gexp? exp) output n?)
1046          (gexp->sexp exp
1047                      #:system system
1048                      #:target (if (or n? native?) #f target)))
1049         (($ <gexp-input> (refs ...) output n?)
1050          (mapm %store-monad
1051                (lambda (ref)
1052                  ;; XXX: Automatically convert REF to an gexp-input.
1053                  (reference->sexp
1054                   (if (gexp-input? ref)
1055                       ref
1056                       (%gexp-input ref "out" n?))
1057                   (or n? native?)))
1058                refs))
1059         (($ <gexp-input> (? struct? thing) output n?)
1060          (let ((target (if (or n? native?) #f target))
1061                (expand (lookup-expander thing)))
1062            (mlet %store-monad ((obj (lower-object thing system
1063                                                   #:target target)))
1064              ;; OBJ must be either a derivation or a store file name.
1065              (return (expand thing obj output)))))
1066         (($ <gexp-input> (? self-quoting? x))
1067          (return x))
1068         (($ <gexp-input> x)
1069          (raise (condition (&gexp-input-error (input x)))))
1070         (x
1071          (return x)))))
1072
1073   (mlet %store-monad
1074       ((args (mapm %store-monad
1075                    reference->sexp (gexp-references exp))))
1076     (return (apply (gexp-proc exp) args))))
1077
1078 (define-syntax-rule (define-syntax-parameter-once name proc)
1079   ;; Like 'define-syntax-parameter' but ensure the top-level binding for NAME
1080   ;; does not get redefined.  This works around a race condition in a
1081   ;; multi-threaded context with Guile <= 2.2.4: <https://bugs.gnu.org/27476>.
1082   (eval-when (load eval expand compile)
1083     (define name
1084       (if (module-locally-bound? (current-module) 'name)
1085           (module-ref (current-module) 'name)
1086           (make-syntax-transformer 'name 'syntax-parameter
1087                                    (list proc))))))
1088
1089 (define-syntax-parameter-once current-imported-modules
1090   ;; Current list of imported modules.
1091   (identifier-syntax '()))
1092
1093 (define-syntax-rule (with-imported-modules modules body ...)
1094   "Mark the gexps defined in BODY... as requiring MODULES in their execution
1095 environment."
1096   (syntax-parameterize ((current-imported-modules
1097                          (identifier-syntax modules)))
1098     body ...))
1099
1100 (define-syntax-parameter-once current-imported-extensions
1101   ;; Current list of extensions.
1102   (identifier-syntax '()))
1103
1104 (define-syntax-rule (with-extensions extensions body ...)
1105   "Mark the gexps defined in BODY... as requiring EXTENSIONS in their
1106 execution environment."
1107   (syntax-parameterize ((current-imported-extensions
1108                          (identifier-syntax extensions)))
1109     body ...))
1110
1111 (define-syntax gexp
1112   (lambda (s)
1113     (define (collect-escapes exp)
1114       ;; Return all the 'ungexp' present in EXP.
1115       (let loop ((exp    exp)
1116                  (result '()))
1117         (syntax-case exp (ungexp
1118                           ungexp-splicing
1119                           ungexp-native
1120                           ungexp-native-splicing)
1121           ((ungexp _)
1122            (cons exp result))
1123           ((ungexp _ _)
1124            (cons exp result))
1125           ((ungexp-splicing _ ...)
1126            (cons exp result))
1127           ((ungexp-native _ ...)
1128            (cons exp result))
1129           ((ungexp-native-splicing _ ...)
1130            (cons exp result))
1131           ((exp0 . exp)
1132            (let ((result (loop #'exp0 result)))
1133              (loop  #'exp result)))
1134           (_
1135            result))))
1136
1137     (define (escape->ref exp)
1138       ;; Turn 'ungexp' form EXP into a "reference".
1139       (syntax-case exp (ungexp ungexp-splicing
1140                         ungexp-native ungexp-native-splicing
1141                         output)
1142         ((ungexp output)
1143          #'(gexp-output "out"))
1144         ((ungexp output name)
1145          #'(gexp-output name))
1146         ((ungexp thing)
1147          #'(%gexp-input thing "out" #f))
1148         ((ungexp drv-or-pkg out)
1149          #'(%gexp-input drv-or-pkg out #f))
1150         ((ungexp-splicing lst)
1151          #'(%gexp-input lst "out" #f))
1152         ((ungexp-native thing)
1153          #'(%gexp-input thing "out" #t))
1154         ((ungexp-native drv-or-pkg out)
1155          #'(%gexp-input drv-or-pkg out #t))
1156         ((ungexp-native-splicing lst)
1157          #'(%gexp-input lst "out" #t))))
1158
1159     (define (substitute-ungexp exp substs)
1160       ;; Given EXP, an 'ungexp' or 'ungexp-native' form, substitute it with
1161       ;; the corresponding form in SUBSTS.
1162       (match (assoc exp substs)
1163         ((_ id)
1164          id)
1165         (_                                        ;internal error
1166          (with-syntax ((exp exp))
1167            #'(syntax-error "error: no 'ungexp' substitution" exp)))))
1168
1169     (define (substitute-ungexp-splicing exp substs)
1170       (syntax-case exp ()
1171         ((exp rest ...)
1172          (match (assoc #'exp substs)
1173            ((_ id)
1174             (with-syntax ((id id))
1175               #`(append id
1176                         #,(substitute-references #'(rest ...) substs))))
1177            (_
1178             #'(syntax-error "error: no 'ungexp-splicing' substitution"
1179                             exp))))))
1180
1181     (define (substitute-references exp substs)
1182       ;; Return a variant of EXP where all the cars of SUBSTS have been
1183       ;; replaced by the corresponding cdr.
1184       (syntax-case exp (ungexp ungexp-native
1185                         ungexp-splicing ungexp-native-splicing)
1186         ((ungexp _ ...)
1187          (substitute-ungexp exp substs))
1188         ((ungexp-native _ ...)
1189          (substitute-ungexp exp substs))
1190         (((ungexp-splicing _ ...) rest ...)
1191          (substitute-ungexp-splicing exp substs))
1192         (((ungexp-native-splicing _ ...) rest ...)
1193          (substitute-ungexp-splicing exp substs))
1194         ((exp0 . exp)
1195          #`(cons #,(substitute-references #'exp0 substs)
1196                  #,(substitute-references #'exp substs)))
1197         (x #''x)))
1198
1199     (syntax-case s (ungexp output)
1200       ((_ exp)
1201        (let* ((escapes (delete-duplicates (collect-escapes #'exp)))
1202               (formals (generate-temporaries escapes))
1203               (sexp    (substitute-references #'exp (zip escapes formals)))
1204               (refs    (map escape->ref escapes)))
1205          #`(make-gexp (list #,@refs)
1206                       current-imported-modules
1207                       current-imported-extensions
1208                       (lambda #,formals
1209                         #,sexp)))))))
1210
1211 \f
1212 ;;;
1213 ;;; Module handling.
1214 ;;;
1215
1216 (define %not-slash
1217   (char-set-complement (char-set #\/)))
1218
1219 (define (file-mapping->tree mapping)
1220   "Convert MAPPING, an alist like:
1221
1222   ((\"guix/build/utils.scm\" . \"…/utils.scm\"))
1223
1224 to a tree suitable for 'interned-file-tree'."
1225   (let ((mapping (map (match-lambda
1226                         ((destination . source)
1227                          (cons (string-tokenize destination
1228                                                 %not-slash)
1229                                source)))
1230                       mapping)))
1231     (fold (lambda (pair result)
1232             (match pair
1233               ((destination . source)
1234                (let loop ((destination destination)
1235                           (result result))
1236                  (match destination
1237                    ((file)
1238                     (let* ((mode (stat:mode (stat source)))
1239                            (type (if (zero? (logand mode #o100))
1240                                      'regular
1241                                      'executable)))
1242                       (alist-cons file
1243                                   `(,type (file ,source))
1244                                   result)))
1245                    ((file rest ...)
1246                     (let ((directory (assoc-ref result file)))
1247                       (alist-cons file
1248                                   `(directory
1249                                     ,@(loop rest
1250                                             (match directory
1251                                               (('directory . entries) entries)
1252                                               (#f '()))))
1253                                   (if directory
1254                                       (alist-delete file result)
1255                                       result)))))))))
1256           '()
1257           mapping)))
1258
1259 (define %utils-module
1260   ;; This file provides 'mkdir-p', needed to implement 'imported-files' and
1261   ;; other primitives below.  Note: We give the file name relative to this
1262   ;; file you are currently reading; 'search-path' could return a file name
1263   ;; relative to the current working directory.
1264   (local-file "build/utils.scm"
1265               "build-utils.scm"))
1266
1267 (define* (imported-files/derivation files
1268                                     #:key (name "file-import")
1269                                     (symlink? #f)
1270                                     (system (%current-system))
1271                                     (guile (%guile-for-build)))
1272   "Return a derivation that imports FILES into STORE.  FILES must be a list
1273 of (FINAL-PATH . FILE) pairs.  Each FILE is mapped to FINAL-PATH in the
1274 resulting store path.  FILE can be either a file name, or a file-like object,
1275 as returned by 'local-file' for example.  If SYMLINK? is true, create symlinks
1276 to the source files instead of copying them."
1277   (define file-pair
1278     (match-lambda
1279      ((final-path . (? string? file-name))
1280       (mlet %store-monad ((file (interned-file file-name
1281                                                (basename final-path))))
1282         (return (list final-path file))))
1283      ((final-path . file-like)
1284       (mlet %store-monad ((file (lower-object file-like system)))
1285         (return (list final-path file))))))
1286
1287   (mlet %store-monad ((files (mapm %store-monad file-pair files)))
1288     (define build
1289       (gexp
1290        (begin
1291          (primitive-load (ungexp %utils-module))  ;for 'mkdir-p'
1292          (use-modules (ice-9 match))
1293
1294          (mkdir (ungexp output)) (chdir (ungexp output))
1295          (for-each (match-lambda
1296                     ((final-path store-path)
1297                      (mkdir-p (dirname final-path))
1298                      ((ungexp (if symlink? 'symlink 'copy-file))
1299                       store-path final-path)))
1300                    '(ungexp files)))))
1301
1302     ;; TODO: Pass FILES as an environment variable so that BUILD remains
1303     ;; exactly the same regardless of FILES: less disk space, and fewer
1304     ;; 'add-to-store' RPCs.
1305     (gexp->derivation name build
1306                       #:system system
1307                       #:guile-for-build guile
1308                       #:local-build? #t
1309
1310                       ;; Avoid deprecation warnings about the use of the _IO*
1311                       ;; constants in (guix build utils).
1312                       #:env-vars
1313                       '(("GUILE_WARN_DEPRECATED" . "no")))))
1314
1315 (define* (imported-files files
1316                          #:key (name "file-import")
1317                          ;; The following parameters make sense when creating
1318                          ;; an actual derivation.
1319                          (system (%current-system))
1320                          (guile (%guile-for-build)))
1321   "Import FILES into the store and return the resulting derivation or store
1322 file name (a derivation is created if and only if some elements of FILES are
1323 file-like objects and not local file names.)  FILES must be a list
1324 of (FINAL-PATH . FILE) pairs.  Each FILE is mapped to FINAL-PATH in the
1325 resulting store path.  FILE can be either a file name, or a file-like object,
1326 as returned by 'local-file' for example."
1327   (if (any (match-lambda
1328              ((_ . (? struct? source)) #t)
1329              (_ #f))
1330            files)
1331       (imported-files/derivation files #:name name
1332                                  #:symlink? derivation?
1333                                  #:system system #:guile guile)
1334       (interned-file-tree `(,name directory
1335                                   ,@(file-mapping->tree files)))))
1336
1337 (define* (imported-modules modules
1338                            #:key (name "module-import")
1339                            (system (%current-system))
1340                            (guile (%guile-for-build))
1341                            (module-path %load-path))
1342   "Return a derivation that contains the source files of MODULES, a list of
1343 module names such as `(ice-9 q)'.  All of MODULES must be either names of
1344 modules to be found in the MODULE-PATH search path, or a module name followed
1345 by an arrow followed by a file-like object.  For example:
1346
1347   (imported-modules `((guix build utils)
1348                       (guix gcrypt)
1349                       ((guix config) => ,(scheme-file …))))
1350
1351 In this example, the first two modules are taken from MODULE-PATH, and the
1352 last one is created from the given <scheme-file> object."
1353   (let ((files (map (match-lambda
1354                       (((module ...) '=> file)
1355                        (cons (module->source-file-name module)
1356                              file))
1357                       ((module ...)
1358                        (let ((f (module->source-file-name module)))
1359                          (cons f (search-path* module-path f)))))
1360                     modules)))
1361     (imported-files files #:name name
1362                     #:system system
1363                     #:guile guile)))
1364
1365 (define* (compiled-modules modules
1366                            #:key (name "module-import-compiled")
1367                            (system (%current-system))
1368                            target
1369                            (guile (%guile-for-build))
1370                            (module-path %load-path)
1371                            (extensions '())
1372                            (deprecation-warnings #f))
1373   "Return a derivation that builds a tree containing the `.go' files
1374 corresponding to MODULES.  All the MODULES are built in a context where
1375 they can refer to each other.  When TARGET is true, cross-compile MODULES for
1376 TARGET, a GNU triplet."
1377   (define total (length modules))
1378
1379   (mlet %store-monad ((modules (imported-modules modules
1380                                                  #:system system
1381                                                  #:guile guile
1382                                                  #:module-path
1383                                                  module-path)))
1384     (define build
1385       (gexp
1386        (begin
1387          (primitive-load (ungexp %utils-module))  ;for 'mkdir-p'
1388
1389          (use-modules (ice-9 ftw)
1390                       (ice-9 format)
1391                       (srfi srfi-1)
1392                       (srfi srfi-26)
1393                       (system base compile))
1394
1395          ;; TODO: Inline this on the next rebuild cycle.
1396          (ungexp-splicing
1397           (if target
1398               (gexp ((use-modules (system base target))))
1399               (gexp ())))
1400
1401          (define (regular? file)
1402            (not (member file '("." ".."))))
1403
1404          (define (process-entry entry output processed)
1405            (if (file-is-directory? entry)
1406                (let ((output (string-append output "/" (basename entry))))
1407                  (mkdir-p output)
1408                  (process-directory entry output processed))
1409                (let* ((base   (basename entry ".scm"))
1410                       (output (string-append output "/" base ".go")))
1411                  (format #t "[~2@a/~2@a] Compiling '~a'...~%"
1412                          (+ 1 processed (ungexp total))
1413                          (ungexp (* total 2))
1414                          entry)
1415
1416                  (ungexp-splicing
1417                   (if target
1418                       (gexp ((with-target (ungexp target)
1419                                (lambda ()
1420                                  (compile-file entry
1421                                                #:output-file output
1422                                                #:opts
1423                                                %auto-compilation-options)))))
1424                       (gexp ((compile-file entry
1425                                            #:output-file output
1426                                            #:opts %auto-compilation-options)))))
1427
1428                  (+ 1 processed))))
1429
1430          (define (process-directory directory output processed)
1431            (let ((entries (map (cut string-append directory "/" <>)
1432                                (scandir directory regular?))))
1433              (fold (cut process-entry <> output <>)
1434                    processed
1435                    entries)))
1436
1437          (define* (load-from-directory directory
1438                                        #:optional (loaded 0))
1439            "Load all the source files found in DIRECTORY."
1440            ;; XXX: This works around <https://bugs.gnu.org/15602>.
1441            (let ((entries (map (cut string-append directory "/" <>)
1442                                (scandir directory regular?))))
1443              (fold (lambda (file loaded)
1444                      (if (file-is-directory? file)
1445                          (load-from-directory file loaded)
1446                          (begin
1447                            (format #t "[~2@a/~2@a] Loading '~a'...~%"
1448                                    (+ 1 loaded) (ungexp (* 2 total))
1449                                    file)
1450                            (save-module-excursion
1451                             (lambda ()
1452                               (primitive-load file)))
1453                            (+ 1 loaded))))
1454                    loaded
1455                    entries)))
1456
1457          (setvbuf (current-output-port)
1458                   (cond-expand (guile-2.2 'line) (else _IOLBF)))
1459
1460          (define mkdir-p
1461            ;; Capture 'mkdir-p'.
1462            (@ (guix build utils) mkdir-p))
1463
1464          ;; Add EXTENSIONS to the search path.
1465          (set! %load-path
1466            (append (map (lambda (extension)
1467                           (string-append extension
1468                                          "/share/guile/site/"
1469                                          (effective-version)))
1470                         '((ungexp-native-splicing extensions)))
1471                    %load-path))
1472          (set! %load-compiled-path
1473            (append (map (lambda (extension)
1474                           (string-append extension "/lib/guile/"
1475                                          (effective-version)
1476                                          "/site-ccache"))
1477                         '((ungexp-native-splicing extensions)))
1478                    %load-compiled-path))
1479
1480          (set! %load-path (cons (ungexp modules) %load-path))
1481
1482          ;; Above we loaded our own (guix build utils) but now we may need to
1483          ;; load a compile a different one.  Thus, force a reload.
1484          (let ((utils (string-append (ungexp modules)
1485                                      "/guix/build/utils.scm")))
1486            (when (file-exists? utils)
1487              (load utils)))
1488
1489          (mkdir (ungexp output))
1490          (chdir (ungexp modules))
1491
1492          (load-from-directory ".")
1493          (process-directory "." (ungexp output) 0))))
1494
1495     ;; TODO: Pass MODULES as an environment variable.
1496     (gexp->derivation name build
1497                       #:system system
1498                       #:guile-for-build guile
1499                       #:local-build? #t
1500                       #:env-vars
1501                       (case deprecation-warnings
1502                         ((#f)
1503                          '(("GUILE_WARN_DEPRECATED" . "no")))
1504                         ((detailed)
1505                          '(("GUILE_WARN_DEPRECATED" . "detailed")))
1506                         (else
1507                          '())))))
1508
1509 \f
1510 ;;;
1511 ;;; Convenience procedures.
1512 ;;;
1513
1514 (define (default-guile)
1515   ;; Lazily resolve 'guile-2.2' (not 'guile-final' because this is for
1516   ;; programs returned by 'program-file' and we don't want to keep references
1517   ;; to several Guile packages).  This module must not refer to (gnu …)
1518   ;; modules directly, to avoid circular dependencies, hence this hack.
1519   (module-ref (resolve-interface '(gnu packages guile))
1520               'guile-2.2))
1521
1522 (define* (load-path-expression modules #:optional (path %load-path)
1523                                #:key (extensions '()) system target)
1524   "Return as a monadic value a gexp that sets '%load-path' and
1525 '%load-compiled-path' to point to MODULES, a list of module names.  MODULES
1526 are searched for in PATH.  Return #f when MODULES and EXTENSIONS are empty."
1527   (if (and (null? modules) (null? extensions))
1528       (with-monad %store-monad
1529         (return #f))
1530       (mlet %store-monad ((modules  (imported-modules modules
1531                                                       #:module-path path
1532                                                       #:system system))
1533                           (compiled (compiled-modules modules
1534                                                       #:extensions extensions
1535                                                       #:module-path path
1536                                                       #:system system
1537                                                       #:target target)))
1538         (return
1539          (gexp (eval-when (expand load eval)
1540                  ;; Augment the load paths and delete duplicates.  Do that
1541                  ;; without loading (srfi srfi-1) or anything.
1542                  (let ((extensions '((ungexp-splicing extensions)))
1543                        (prepend (lambda (items lst)
1544                                   ;; This is O(N²) but N is typically small.
1545                                   (let loop ((items items)
1546                                              (lst lst))
1547                                     (if (null? items)
1548                                         lst
1549                                         (loop (cdr items)
1550                                               (cons (car items)
1551                                                     (delete (car items) lst))))))))
1552                    (set! %load-path
1553                      (prepend (cons (ungexp modules)
1554                                     (map (lambda (extension)
1555                                            (string-append extension
1556                                                           "/share/guile/site/"
1557                                                           (effective-version)))
1558                                          extensions))
1559                               %load-path))
1560                    (set! %load-compiled-path
1561                      (prepend (cons (ungexp compiled)
1562                                     (map (lambda (extension)
1563                                            (string-append extension
1564                                                           "/lib/guile/"
1565                                                           (effective-version)
1566                                                           "/site-ccache"))
1567                                          extensions))
1568                               %load-compiled-path)))))))))
1569
1570 (define* (gexp->script name exp
1571                        #:key (guile (default-guile))
1572                        (module-path %load-path)
1573                        (system (%current-system))
1574                        target)
1575   "Return an executable script NAME that runs EXP using GUILE, with EXP's
1576 imported modules in its search path.  Look up EXP's modules in MODULE-PATH."
1577   (mlet %store-monad ((set-load-path
1578                        (load-path-expression (gexp-modules exp)
1579                                              module-path
1580                                              #:extensions
1581                                              (gexp-extensions exp)
1582                                              #:system system
1583                                              #:target target)))
1584     (gexp->derivation name
1585                       (gexp
1586                        (call-with-output-file (ungexp output)
1587                          (lambda (port)
1588                            ;; Note: that makes a long shebang.  When the store
1589                            ;; is /gnu/store, that fits within the 128-byte
1590                            ;; limit imposed by Linux, but that may go beyond
1591                            ;; when running tests.
1592                            (format port
1593                                    "#!~a/bin/guile --no-auto-compile~%!#~%"
1594                                    (ungexp guile))
1595
1596                            (ungexp-splicing
1597                             (if set-load-path
1598                                 (gexp ((write '(ungexp set-load-path) port)))
1599                                 (gexp ())))
1600
1601                            (write '(ungexp exp) port)
1602                            (chmod port #o555))))
1603                       #:system system
1604                       #:target target
1605                       #:module-path module-path)))
1606
1607 (define* (gexp->file name exp #:key
1608                      (set-load-path? #t)
1609                      (module-path %load-path)
1610                      (splice? #f)
1611                      (system (%current-system))
1612                      target)
1613   "Return a derivation that builds a file NAME containing EXP.  When SPLICE?
1614 is true, EXP is considered to be a list of expressions that will be spliced in
1615 the resulting file.
1616
1617 When SET-LOAD-PATH? is true, emit code in the resulting file to set
1618 '%load-path' and '%load-compiled-path' to honor EXP's imported modules.
1619 Lookup EXP's modules in MODULE-PATH."
1620   (define modules (gexp-modules exp))
1621   (define extensions (gexp-extensions exp))
1622
1623   (if (or (not set-load-path?)
1624           (and (null? modules) (null? extensions)))
1625       (gexp->derivation name
1626                         (gexp
1627                          (call-with-output-file (ungexp output)
1628                            (lambda (port)
1629                              (for-each (lambda (exp)
1630                                          (write exp port))
1631                                        '(ungexp (if splice?
1632                                                     exp
1633                                                     (gexp ((ungexp exp)))))))))
1634                         #:local-build? #t
1635                         #:substitutable? #f
1636                         #:system system
1637                         #:target target)
1638       (mlet %store-monad ((set-load-path
1639                            (load-path-expression modules module-path
1640                                                  #:extensions extensions
1641                                                  #:system system
1642                                                  #:target target)))
1643         (gexp->derivation name
1644                           (gexp
1645                            (call-with-output-file (ungexp output)
1646                              (lambda (port)
1647                                (write '(ungexp set-load-path) port)
1648                                (for-each (lambda (exp)
1649                                            (write exp port))
1650                                          '(ungexp (if splice?
1651                                                       exp
1652                                                       (gexp ((ungexp exp)))))))))
1653                           #:module-path module-path
1654                           #:local-build? #t
1655                           #:substitutable? #f
1656                           #:system system
1657                           #:target target))))
1658
1659 (define* (text-file* name #:rest text)
1660   "Return as a monadic value a derivation that builds a text file containing
1661 all of TEXT.  TEXT may list, in addition to strings, objects of any type that
1662 can be used in a gexp: packages, derivations, local file objects, etc.  The
1663 resulting store file holds references to all these."
1664   (define builder
1665     (gexp (call-with-output-file (ungexp output "out")
1666             (lambda (port)
1667               (display (string-append (ungexp-splicing text)) port)))))
1668
1669   (gexp->derivation name builder
1670                     #:local-build? #t
1671                     #:substitutable? #f))
1672
1673 (define* (mixed-text-file name #:rest text)
1674   "Return an object representing store file NAME containing TEXT.  TEXT is a
1675 sequence of strings and file-like objects, as in:
1676
1677   (mixed-text-file \"profile\"
1678                    \"export PATH=\" coreutils \"/bin:\" grep \"/bin\")
1679
1680 This is the declarative counterpart of 'text-file*'."
1681   (define build
1682     (gexp (call-with-output-file (ungexp output "out")
1683             (lambda (port)
1684               (display (string-append (ungexp-splicing text)) port)))))
1685
1686   (computed-file name build))
1687
1688 (define (file-union name files)
1689   "Return a <computed-file> that builds a directory containing all of FILES.
1690 Each item in FILES must be a two-element list where the first element is the
1691 file name to use in the new directory, and the second element is a gexp
1692 denoting the target file.  Here's an example:
1693
1694   (file-union \"etc\"
1695               `((\"hosts\" ,(plain-file \"hosts\"
1696                                         \"127.0.0.1 localhost\"))
1697                 (\"bashrc\" ,(plain-file \"bashrc\"
1698                                          \"alias ls='ls --color'\"))
1699                 (\"libvirt/qemu.conf\" ,(plain-file \"qemu.conf\" \"\"))))
1700
1701 This yields an 'etc' directory containing these two files."
1702   (computed-file name
1703                  (with-imported-modules '((guix build utils))
1704                    (gexp
1705                     (begin
1706                       (use-modules (guix build utils))
1707
1708                       (mkdir (ungexp output))
1709                       (chdir (ungexp output))
1710                       (ungexp-splicing
1711                        (map (match-lambda
1712                               ((target source)
1713                                (gexp
1714                                 (begin
1715                                   ;; Stat the source to abort early if it does
1716                                   ;; not exist.
1717                                   (stat (ungexp source))
1718
1719                                   (mkdir-p (dirname (ungexp target)))
1720                                   (symlink (ungexp source)
1721                                            (ungexp target))))))
1722                             files)))))))
1723
1724 (define* (directory-union name things
1725                           #:key (copy? #f) (quiet? #f)
1726                           (resolve-collision 'warn-about-collision))
1727   "Return a directory that is the union of THINGS, where THINGS is a list of
1728 file-like objects denoting directories.  For example:
1729
1730   (directory-union \"guile+emacs\" (list guile emacs))
1731
1732 yields a directory that is the union of the 'guile' and 'emacs' packages.
1733
1734 Call RESOLVE-COLLISION when several files collide, passing it the list of
1735 colliding files.  RESOLVE-COLLISION must return the chosen file or #f, in
1736 which case the colliding entry is skipped altogether.
1737
1738 When HARD-LINKS? is true, create hard links instead of symlinks.  When QUIET?
1739 is true, the derivation will not print anything."
1740   (define symlink
1741     (if copy?
1742         (gexp (lambda (old new)
1743                 (if (file-is-directory? old)
1744                     (symlink old new)
1745                     (copy-file old new))))
1746         (gexp symlink)))
1747
1748   (define log-port
1749     (if quiet?
1750         (gexp (%make-void-port "w"))
1751         (gexp (current-error-port))))
1752
1753   (match things
1754     ((one)
1755      ;; Only one thing; return it.
1756      one)
1757     (_
1758      (computed-file name
1759                     (with-imported-modules '((guix build union))
1760                       (gexp (begin
1761                               (use-modules (guix build union)
1762                                            (srfi srfi-1)) ;for 'first' and 'last'
1763
1764                               (union-build (ungexp output)
1765                                            '(ungexp things)
1766
1767                                            #:log-port (ungexp log-port)
1768                                            #:symlink (ungexp symlink)
1769                                            #:resolve-collision
1770                                            (ungexp resolve-collision)))))))))
1771
1772 \f
1773 ;;;
1774 ;;; Syntactic sugar.
1775 ;;;
1776
1777 (eval-when (expand load eval)
1778   (define* (read-ungexp chr port #:optional native?)
1779     "Read an 'ungexp' or 'ungexp-splicing' form from PORT.  When NATIVE? is
1780 true, use 'ungexp-native' and 'ungexp-native-splicing' instead."
1781     (define unquote-symbol
1782       (match (peek-char port)
1783         (#\@
1784          (read-char port)
1785          (if native?
1786              'ungexp-native-splicing
1787              'ungexp-splicing))
1788         (_
1789          (if native?
1790              'ungexp-native
1791              'ungexp))))
1792
1793     (match (read port)
1794       ((? symbol? symbol)
1795        (let ((str (symbol->string symbol)))
1796          (match (string-index-right str #\:)
1797            (#f
1798             `(,unquote-symbol ,symbol))
1799            (colon
1800             (let ((name   (string->symbol (substring str 0 colon)))
1801                   (output (substring str (+ colon 1))))
1802               `(,unquote-symbol ,name ,output))))))
1803       (x
1804        `(,unquote-symbol ,x))))
1805
1806   (define (read-gexp chr port)
1807     "Read a 'gexp' form from PORT."
1808     `(gexp ,(read port)))
1809
1810   ;; Extend the reader
1811   (read-hash-extend #\~ read-gexp)
1812   (read-hash-extend #\$ read-ungexp)
1813   (read-hash-extend #\+ (cut read-ungexp <> <> #t)))
1814
1815 ;;; gexp.scm ends here