guix/gexp.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org>
   3 ;;; Copyright © 2018 Clément Lassieur <clement@lassieur.org>
   4 ;;; Copyright © 2018 Jan Nieuwenhuizen <janneke@gnu.org>
   5 ;;; Copyright © 2019 Mathieu Othacehe <m.othacehe@gmail.com>
   6 ;;;
   7 ;;; This file is part of GNU Guix.
   8 ;;;
   9 ;;; GNU Guix is free software; you can redistribute it and/or modify it
  10 ;;; under the terms of the GNU General Public License as published by
  11 ;;; the Free Software Foundation; either version 3 of the License, or (at
  12 ;;; your option) any later version.
  13 ;;;
  14 ;;; GNU Guix is distributed in the hope that it will be useful, but
  15 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  16 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 ;;; GNU General Public License for more details.
  18 ;;;
  19 ;;; You should have received a copy of the GNU General Public License
  20 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  21
  22 (define-module (guix gexp)
  23   #:use-module (guix store)
  24   #:use-module (guix monads)
  25   #:use-module (guix derivations)
  26   #:use-module (guix grafts)
  27   #:use-module (guix utils)
  28   #:use-module (rnrs bytevectors)
  29   #:use-module (srfi srfi-1)
  30   #:use-module (srfi srfi-9)
  31   #:use-module (srfi srfi-9 gnu)
  32   #:use-module (srfi srfi-26)
  33   #:use-module (srfi srfi-34)
  34   #:use-module (srfi srfi-35)
  35   #:use-module (ice-9 match)
  36   #:export (gexp
  37             gexp?
  38             with-imported-modules
  39             with-extensions
  40
  41             gexp-input
  42             gexp-input?
  43             gexp-input-thing
  44             gexp-input-output
  45             gexp-input-native?
  46
  47             local-file
  48             local-file?
  49             local-file-file
  50             local-file-absolute-file-name
  51             local-file-name
  52             local-file-recursive?
  53
  54             plain-file
  55             plain-file?
  56             plain-file-name
  57             plain-file-content
  58
  59             computed-file
  60             computed-file?
  61             computed-file-name
  62             computed-file-gexp
  63             computed-file-options
  64
  65             program-file
  66             program-file?
  67             program-file-name
  68             program-file-gexp
  69             program-file-guile
  70             program-file-module-path
  71
  72             scheme-file
  73             scheme-file?
  74             scheme-file-name
  75             scheme-file-gexp
  76
  77             file-append
  78             file-append?
  79             file-append-base
  80             file-append-suffix
  81
  82             raw-derivation-file
  83             raw-derivation-file?
  84
  85             load-path-expression
  86             gexp-modules
  87
  88             lower-gexp
  89             lowered-gexp?
  90             lowered-gexp-sexp
  91             lowered-gexp-inputs
  92             lowered-gexp-sources
  93             lowered-gexp-guile
  94             lowered-gexp-load-path
  95             lowered-gexp-load-compiled-path
  96
  97             gexp->derivation
  98             gexp->file
  99             gexp->script
 100             text-file*
 101             mixed-text-file
 102             file-union
 103             directory-union
 104             imported-files
 105             imported-modules
 106             compiled-modules
 107
 108             define-gexp-compiler
 109             gexp-compiler?
 110             file-like?
 111             lower-object
 112
 113             lower-inputs
 114
 115             &gexp-error
 116             gexp-error?
 117             &gexp-input-error
 118             gexp-input-error?
 119             gexp-error-invalid-input))
 120
 121 ;;; Commentary:
 122 ;;;
 123 ;;; This module implements "G-expressions", or "gexps".  Gexps are like
 124 ;;; S-expressions (sexps), with two differences:
 125 ;;;
 126 ;;;   1. References (un-quotations) to derivations or packages in a gexp are
 127 ;;;      replaced by the corresponding output file name; in addition, the
 128 ;;;      'ungexp-native' unquote-like form allows code to explicitly refer to
 129 ;;;      the native code of a given package, in case of cross-compilation;
 130 ;;;
 131 ;;;   2. Gexps embed information about the derivations they refer to.
 132 ;;;
 133 ;;; Gexps make it easy to write to files Scheme code that refers to store
 134 ;;; items, or to write Scheme code to build derivations.
 135 ;;;
 136 ;;; Code:
 137
 138 ;; "G expressions".
 139 (define-record-type <gexp>
 140   (make-gexp references modules extensions proc)
 141   gexp?
 142   (references gexp-references)                    ;list of <gexp-input>
 143   (modules    gexp-self-modules)                  ;list of module names
 144   (extensions gexp-self-extensions)               ;list of lowerable things
 145   (proc       gexp-proc))                         ;procedure
 146
 147 (define (write-gexp gexp port)
 148   "Write GEXP on PORT."
 149   (display "#<gexp " port)
 150
 151   ;; Try to write the underlying sexp.  Now, this trick doesn't work when
 152   ;; doing things like (ungexp-splicing (gexp ())) because GEXP's procedure
 153   ;; tries to use 'append' on that, which fails with wrong-type-arg.
 154   (false-if-exception
 155    (write (apply (gexp-proc gexp)
 156                  (gexp-references gexp))
 157           port))
 158   (format port " ~a>"
 159           (number->string (object-address gexp) 16)))
 160
 161 (set-record-type-printer! <gexp> write-gexp)
 162
 163 \f
 164 ;;;
 165 ;;; Methods.
 166 ;;;
 167
 168 ;; Compiler for a type of objects that may be introduced in a gexp.
 169 (define-record-type <gexp-compiler>
 170   (gexp-compiler type lower expand)
 171   gexp-compiler?
 172   (type       gexp-compiler-type)                 ;record type descriptor
 173   (lower      gexp-compiler-lower)
 174   (expand     gexp-compiler-expand))              ;#f | DRV -> sexp
 175
 176 (define-condition-type &gexp-error &error
 177   gexp-error?)
 178
 179 (define-condition-type &gexp-input-error &gexp-error
 180   gexp-input-error?
 181   (input gexp-error-invalid-input))
 182
 183
 184 (define %gexp-compilers
 185   ;; 'eq?' mapping of record type descriptor to <gexp-compiler>.
 186   (make-hash-table 20))
 187
 188 (define (default-expander thing obj output)
 189   "This is the default expander for \"things\" that appear in gexps.  It
 190 returns its output file name of OBJ's OUTPUT."
 191   (match obj
 192     ((? derivation? drv)
 193      (derivation->output-path drv output))
 194     ((? string? file)
 195      file)))
 196
 197 (define (register-compiler! compiler)
 198   "Register COMPILER as a gexp compiler."
 199   (hashq-set! %gexp-compilers
 200               (gexp-compiler-type compiler) compiler))
 201
 202 (define (lookup-compiler object)
 203   "Search for a compiler for OBJECT.  Upon success, return the three argument
 204 procedure to lower it; otherwise return #f."
 205   (and=> (hashq-ref %gexp-compilers (struct-vtable object))
 206          gexp-compiler-lower))
 207
 208 (define (file-like? object)
 209   "Return #t if OBJECT leads to a file in the store once unquoted in a
 210 G-expression; otherwise return #f."
 211   (and (struct? object) (->bool (lookup-compiler object))))
 212
 213 (define (lookup-expander object)
 214   "Search for an expander for OBJECT.  Upon success, return the three argument
 215 procedure to expand it; otherwise return #f."
 216   (and=> (hashq-ref %gexp-compilers (struct-vtable object))
 217          gexp-compiler-expand))
 218
 219 (define* (lower-object obj
 220                        #:optional (system (%current-system))
 221                        #:key target)
 222   "Return as a value in %STORE-MONAD the derivation or store item
 223 corresponding to OBJ for SYSTEM, cross-compiling for TARGET if TARGET is true.
 224 OBJ must be an object that has an associated gexp compiler, such as a
 225 <package>."
 226   (match (lookup-compiler obj)
 227     (#f
 228      (raise (condition (&gexp-input-error (input obj)))))
 229     (lower
 230      ;; Cache in STORE the result of lowering OBJ.
 231      (mlet %store-monad ((graft? (grafting?)))
 232        (mcached (let ((lower (lookup-compiler obj)))
 233                   (lower obj system target))
 234                 obj
 235                 system target graft?)))))
 236
 237 (define-syntax define-gexp-compiler
 238   (syntax-rules (=> compiler expander)
 239     "Define NAME as a compiler for objects matching PREDICATE encountered in
 240 gexps.
 241
 242 In the simplest form of the macro, BODY must return a derivation for PARAM, an
 243 object that matches PREDICATE, for SYSTEM and TARGET (the latter of which is
 244 #f except when cross-compiling.)
 245
 246 The more elaborate form allows you to specify an expander:
 247
 248   (define-gexp-compiler something something?
 249     compiler => (lambda (param system target) ...)
 250     expander => (lambda (param drv output) ...))
 251
 252 The expander specifies how an object is converted to its sexp representation."
 253     ((_ (name (param record-type) system target) body ...)
 254      (define-gexp-compiler name record-type
 255        compiler => (lambda (param system target) body ...)
 256        expander => default-expander))
 257     ((_ name record-type
 258         compiler => compile
 259         expander => expand)
 260      (begin
 261        (define name
 262          (gexp-compiler record-type compile expand))
 263        (register-compiler! name)))))
 264
 265 (define-gexp-compiler (derivation-compiler (drv <derivation>) system target)
 266   ;; Derivations are the lowest-level representation, so this is the identity
 267   ;; compiler.
 268   (with-monad %store-monad
 269     (return drv)))
 270
 271 ;; Expand to a raw ".drv" file for the lowerable object it wraps.  In other
 272 ;; words, this gives the raw ".drv" file instead of its build result.
 273 (define-record-type <raw-derivation-file>
 274   (raw-derivation-file obj)
 275   raw-derivation-file?
 276   (obj  raw-derivation-file-object))              ;lowerable object
 277
 278 (define-gexp-compiler raw-derivation-file-compiler <raw-derivation-file>
 279   compiler => (lambda (obj system target)
 280                 (mlet %store-monad ((obj (lower-object
 281                                           (raw-derivation-file-object obj)
 282                                           system #:target target)))
 283                   ;; Returning the .drv file name instead of the <derivation>
 284                   ;; record ensures that 'lower-gexp' will classify it as a
 285                   ;; "source" and not as an "input".
 286                   (return (if (derivation? obj)
 287                               (derivation-file-name obj)
 288                               obj))))
 289   expander => (lambda (obj lowered output)
 290                 (if (derivation? lowered)
 291                     (derivation-file-name lowered)
 292                     lowered)))
 293
 294 \f
 295 ;;;
 296 ;;; File declarations.
 297 ;;;
 298
 299 ;; A local file name.  FILE is the file name the user entered, which can be a
 300 ;; relative file name, and ABSOLUTE is a promise that computes its canonical
 301 ;; absolute file name.  We keep it in a promise to compute it lazily and avoid
 302 ;; repeated 'stat' calls.
 303 (define-record-type <local-file>
 304   (%%local-file file absolute name recursive? select?)
 305   local-file?
 306   (file       local-file-file)                    ;string
 307   (absolute   %local-file-absolute-file-name)     ;promise string
 308   (name       local-file-name)                    ;string
 309   (recursive? local-file-recursive?)              ;Boolean
 310   (select?    local-file-select?))                ;string stat -> Boolean
 311
 312 (define (true file stat) #t)
 313
 314 (define* (%local-file file promise #:optional (name (basename file))
 315                       #:key recursive? (select? true))
 316   ;; This intermediate procedure is part of our ABI, but the underlying
 317   ;; %%LOCAL-FILE is not.
 318   (%%local-file file promise name recursive? select?))
 319
 320 (define (absolute-file-name file directory)
 321   "Return the canonical absolute file name for FILE, which lives in the
 322 vicinity of DIRECTORY."
 323   (canonicalize-path
 324    (cond ((string-prefix? "/" file) file)
 325          ((not directory) file)
 326          ((string-prefix? "/" directory)
 327           (string-append directory "/" file))
 328          (else file))))
 329
 330 (define-syntax local-file
 331   (lambda (s)
 332     "Return an object representing local file FILE to add to the store; this
 333 object can be used in a gexp.  If FILE is a relative file name, it is looked
 334 up relative to the source file where this form appears.  FILE will be added to
 335 the store under NAME--by default the base name of FILE.
 336
 337 When RECURSIVE? is true, the contents of FILE are added recursively; if FILE
 338 designates a flat file and RECURSIVE? is true, its contents are added, and its
 339 permission bits are kept.
 340
 341 When RECURSIVE? is true, call (SELECT?  FILE STAT) for each directory entry,
 342 where FILE is the entry's absolute file name and STAT is the result of
 343 'lstat'; exclude entries for which SELECT? does not return true.
 344
 345 This is the declarative counterpart of the 'interned-file' monadic procedure.
 346 It is implemented as a macro to capture the current source directory where it
 347 appears."
 348     (syntax-case s ()
 349       ((_ file rest ...)
 350        (string? (syntax->datum #'file))
 351        ;; FILE is a literal, so resolve it relative to the source directory.
 352        #'(%local-file file
 353                       (delay (absolute-file-name file (current-source-directory)))
 354                       rest ...))
 355       ((_ file rest ...)
 356        ;; Resolve FILE relative to the current directory.
 357        #'(%local-file file
 358                       (delay (absolute-file-name file (getcwd)))
 359                       rest ...))
 360       ((_)
 361        #'(syntax-error "missing file name"))
 362       (id
 363        (identifier? #'id)
 364        ;; XXX: We could return #'(lambda (file . rest) ...).  However,
 365        ;; (syntax-source #'id) is #f so (current-source-directory) would not
 366        ;; work.  Thus, simply forbid this form.
 367        #'(syntax-error
 368           "'local-file' is a macro and cannot be used like this")))))
 369
 370 (define (local-file-absolute-file-name file)
 371   "Return the absolute file name for FILE, a <local-file> instance.  A
 372 'system-error' exception is raised if FILE could not be found."
 373   (force (%local-file-absolute-file-name file)))
 374
 375 (define-gexp-compiler (local-file-compiler (file <local-file>) system target)
 376   ;; "Compile" FILE by adding it to the store.
 377   (match file
 378     (($ <local-file> file (= force absolute) name recursive? select?)
 379      ;; Canonicalize FILE so that if it's a symlink, it is resolved.  Failing
 380      ;; to do that, when RECURSIVE? is #t, we could end up creating a dangling
 381      ;; symlink in the store, and when RECURSIVE? is #f 'add-to-store' would
 382      ;; just throw an error, both of which are inconvenient.
 383      (interned-file absolute name
 384                     #:recursive? recursive? #:select? select?))))
 385
 386 (define-record-type <plain-file>
 387   (%plain-file name content references)
 388   plain-file?
 389   (name        plain-file-name)                   ;string
 390   (content     plain-file-content)                ;string or bytevector
 391   (references  plain-file-references))            ;list (currently unused)
 392
 393 (define (plain-file name content)
 394   "Return an object representing a text file called NAME with the given
 395 CONTENT (a string) to be added to the store.
 396
 397 This is the declarative counterpart of 'text-file'."
 398   ;; XXX: For now just ignore 'references' because it's not clear how to use
 399   ;; them in a declarative context.
 400   (%plain-file name content '()))
 401
 402 (define-gexp-compiler (plain-file-compiler (file <plain-file>) system target)
 403   ;; "Compile" FILE by adding it to the store.
 404   (match file
 405     (($ <plain-file> name (and (? string?) content) references)
 406      (text-file name content references))
 407     (($ <plain-file> name (and (? bytevector?) content) references)
 408      (binary-file name content references))))
 409
 410 (define-record-type <computed-file>
 411   (%computed-file name gexp guile options)
 412   computed-file?
 413   (name       computed-file-name)                 ;string
 414   (gexp       computed-file-gexp)                 ;gexp
 415   (guile      computed-file-guile)                ;<package>
 416   (options    computed-file-options))             ;list of arguments
 417
 418 (define* (computed-file name gexp
 419                         #:key guile (options '(#:local-build? #t)))
 420   "Return an object representing the store item NAME, a file or directory
 421 computed by GEXP.  OPTIONS is a list of additional arguments to pass
 422 to 'gexp->derivation'.
 423
 424 This is the declarative counterpart of 'gexp->derivation'."
 425   (%computed-file name gexp guile options))
 426
 427 (define-gexp-compiler (computed-file-compiler (file <computed-file>)
 428                                               system target)
 429   ;; Compile FILE by returning a derivation whose build expression is its
 430   ;; gexp.
 431   (match file
 432     (($ <computed-file> name gexp guile options)
 433      (if guile
 434          (mlet %store-monad ((guile (lower-object guile system
 435                                                   #:target target)))
 436            (apply gexp->derivation name gexp #:guile-for-build guile
 437                   #:system system #:target target options))
 438          (apply gexp->derivation name gexp
 439                 #:system system #:target target options)))))
 440
 441 (define-record-type <program-file>
 442   (%program-file name gexp guile path)
 443   program-file?
 444   (name       program-file-name)                  ;string
 445   (gexp       program-file-gexp)                  ;gexp
 446   (guile      program-file-guile)                 ;package
 447   (path       program-file-module-path))          ;list of strings
 448
 449 (define* (program-file name gexp #:key (guile #f) (module-path %load-path))
 450   "Return an object representing the executable store item NAME that runs
 451 GEXP.  GUILE is the Guile package used to execute that script.  Imported
 452 modules of GEXP are looked up in MODULE-PATH.
 453
 454 This is the declarative counterpart of 'gexp->script'."
 455   (%program-file name gexp guile module-path))
 456
 457 (define-gexp-compiler (program-file-compiler (file <program-file>)
 458                                              system target)
 459   ;; Compile FILE by returning a derivation that builds the script.
 460   (match file
 461     (($ <program-file> name gexp guile module-path)
 462      (gexp->script name gexp
 463                    #:module-path module-path
 464                    #:guile (or guile (default-guile))
 465                    #:system system
 466                    #:target target))))
 467
 468 (define-record-type <scheme-file>
 469   (%scheme-file name gexp splice?)
 470   scheme-file?
 471   (name       scheme-file-name)                  ;string
 472   (gexp       scheme-file-gexp)                  ;gexp
 473   (splice?    scheme-file-splice?))              ;Boolean
 474
 475 (define* (scheme-file name gexp #:key splice?)
 476   "Return an object representing the Scheme file NAME that contains GEXP.
 477
 478 This is the declarative counterpart of 'gexp->file'."
 479   (%scheme-file name gexp splice?))
 480
 481 (define-gexp-compiler (scheme-file-compiler (file <scheme-file>)
 482                                             system target)
 483   ;; Compile FILE by returning a derivation that builds the file.
 484   (match file
 485     (($ <scheme-file> name gexp splice?)
 486      (gexp->file name gexp
 487                  #:splice? splice?
 488                  #:system system
 489                  #:target target))))
 490
 491 ;; Appending SUFFIX to BASE's output file name.
 492 (define-record-type <file-append>
 493   (%file-append base suffix)
 494   file-append?
 495   (base   file-append-base)                    ;<package> | <derivation> | ...
 496   (suffix file-append-suffix))                 ;list of strings
 497
 498 (define (write-file-append file port)
 499   (match file
 500     (($ <file-append> base suffix)
 501      (format port "#<file-append ~s ~s>" base
 502              (string-join suffix)))))
 503
 504 (set-record-type-printer! <file-append> write-file-append)
 505
 506 (define (file-append base . suffix)
 507   "Return a <file-append> object that expands to the concatenation of BASE and
 508 SUFFIX."
 509   (%file-append base suffix))
 510
 511 (define-gexp-compiler file-append-compiler <file-append>
 512   compiler => (lambda (obj system target)
 513                 (match obj
 514                   (($ <file-append> base _)
 515                    (lower-object base system #:target target))))
 516   expander => (lambda (obj lowered output)
 517                 (match obj
 518                   (($ <file-append> base suffix)
 519                    (let* ((expand (lookup-expander base))
 520                           (base   (expand base lowered output)))
 521                      (string-append base (string-concatenate suffix)))))))
 522
 523 \f
 524 ;;;
 525 ;;; Inputs & outputs.
 526 ;;;
 527
 528 ;; The input of a gexp.
 529 (define-record-type <gexp-input>
 530   (%gexp-input thing output native?)
 531   gexp-input?
 532   (thing     gexp-input-thing)       ;<package> | <origin> | <derivation> | ...
 533   (output    gexp-input-output)      ;string
 534   (native?   gexp-input-native?))    ;Boolean
 535
 536 (define (write-gexp-input input port)
 537   (match input
 538     (($ <gexp-input> thing output #f)
 539      (format port "#<gexp-input ~s:~a>" thing output))
 540     (($ <gexp-input> thing output #t)
 541      (format port "#<gexp-input native ~s:~a>" thing output))))
 542
 543 (set-record-type-printer! <gexp-input> write-gexp-input)
 544
 545 (define* (gexp-input thing                        ;convenience procedure
 546                      #:optional (output "out")
 547                      #:key native?)
 548   "Return a new <gexp-input> for the OUTPUT of THING; NATIVE? determines
 549 whether this should be considered a \"native\" input or not."
 550   (%gexp-input thing output native?))
 551
 552 ;; Reference to one of the derivation's outputs, for gexps used in
 553 ;; derivations.
 554 (define-record-type <gexp-output>
 555   (gexp-output name)
 556   gexp-output?
 557   (name gexp-output-name))
 558
 559 (define (write-gexp-output output port)
 560   (match output
 561     (($ <gexp-output> name)
 562      (format port "#<gexp-output ~a>" name))))
 563
 564 (set-record-type-printer! <gexp-output> write-gexp-output)
 565
 566 (define* (gexp-attribute gexp self-attribute #:optional (equal? equal?))
 567   "Recurse on GEXP and the expressions it refers to, summing the items
 568 returned by SELF-ATTRIBUTE, a procedure that takes a gexp.  Use EQUAL? as the
 569 second argument to 'delete-duplicates'."
 570   (if (gexp? gexp)
 571       (delete-duplicates
 572        (append (self-attribute gexp)
 573                (append-map (match-lambda
 574                              (($ <gexp-input> (? gexp? exp))
 575                               (gexp-attribute exp self-attribute))
 576                              (($ <gexp-input> (lst ...))
 577                               (append-map (lambda (item)
 578                                             (if (gexp? item)
 579                                                 (gexp-attribute item
 580                                                                 self-attribute)
 581                                                 '()))
 582                                           lst))
 583                              (_
 584                               '()))
 585                            (gexp-references gexp)))
 586        equal?)
 587       '()))                                       ;plain Scheme data type
 588
 589 (define (gexp-modules gexp)
 590   "Return the list of Guile module names GEXP relies on.  If (gexp? GEXP) is
 591 false, meaning that GEXP is a plain Scheme object, return the empty list."
 592   (define (module=? m1 m2)
 593     ;; Return #t when M1 equals M2.  Special-case '=>' specs because their
 594     ;; right-hand side may not be comparable with 'equal?': it's typically a
 595     ;; file-like object that embeds a gexp, which in turn embeds closure;
 596     ;; those closures may be 'eq?' when running compiled code but are unlikely
 597     ;; to be 'eq?' when running on 'eval'.  Ignore the right-hand side to
 598     ;; avoid this discrepancy.
 599     (match m1
 600       (((name1 ...) '=> _)
 601        (match m2
 602          (((name2 ...) '=> _) (equal? name1 name2))
 603          (_ #f)))
 604       (_
 605        (equal? m1 m2))))
 606
 607   (gexp-attribute gexp gexp-self-modules module=?))
 608
 609 (define (gexp-extensions gexp)
 610   "Return the list of Guile extensions (packages) GEXP relies on.  If (gexp?
 611 GEXP) is false, meaning that GEXP is a plain Scheme object, return the empty
 612 list."
 613   (gexp-attribute gexp gexp-self-extensions))
 614
 615 (define* (lower-inputs inputs
 616                        #:key system target)
 617   "Turn any object from INPUTS into a derivation input for SYSTEM or a store
 618 item (a \"source\"); return the corresponding input list as a monadic value.
 619 When TARGET is true, use it as the cross-compilation target triplet."
 620   (define (store-item? obj)
 621     (and (string? obj) (store-path? obj)))
 622
 623   (with-monad %store-monad
 624     (mapm %store-monad
 625           (match-lambda
 626             (((? struct? thing) sub-drv ...)
 627              (mlet %store-monad ((obj (lower-object
 628                                        thing system #:target target)))
 629                (return (match obj
 630                          ((? derivation? drv)
 631                           (let ((outputs (if (null? sub-drv)
 632                                              '("out")
 633                                              sub-drv)))
 634                             (derivation-input drv outputs)))
 635                          ((? store-item? item)
 636                           item)))))
 637             (((? store-item? item))
 638              (return item)))
 639           inputs)))
 640
 641 (define* (lower-reference-graphs graphs #:key system target)
 642   "Given GRAPHS, a list of (FILE-NAME INPUT ...) lists for use as a
 643 #:reference-graphs argument, lower it such that each INPUT is replaced by the
 644 corresponding <derivation-input> or store item."
 645   (match graphs
 646     (((file-names . inputs) ...)
 647      (mlet %store-monad ((inputs (lower-inputs inputs
 648                                                #:system system
 649                                                #:target target)))
 650        (return (map cons file-names inputs))))))
 651
 652 (define* (lower-references lst #:key system target)
 653   "Based on LST, a list of output names and packages, return a list of output
 654 names and file names suitable for the #:allowed-references argument to
 655 'derivation'."
 656   (with-monad %store-monad
 657     (define lower
 658       (match-lambda
 659        ((? string? output)
 660         (return output))
 661        (($ <gexp-input> thing output native?)
 662         (mlet %store-monad ((drv (lower-object thing system
 663                                                #:target (if native?
 664                                                             #f target))))
 665           (return (derivation->output-path drv output))))
 666        (thing
 667         (mlet %store-monad ((drv (lower-object thing system
 668                                                #:target target)))
 669           (return (derivation->output-path drv))))))
 670
 671     (mapm %store-monad lower lst)))
 672
 673 (define default-guile-derivation
 674   ;; Here we break the abstraction by talking to the higher-level layer.
 675   ;; Thus, do the resolution lazily to hide the circular dependency.
 676   (let ((proc (delay
 677                 (let ((iface (resolve-interface '(guix packages))))
 678                   (module-ref iface 'default-guile-derivation)))))
 679     (lambda (system)
 680       ((force proc) system))))
 681
 682 ;; Representation of a gexp instantiated for a given target and system.
 683 ;; It's an intermediate representation between <gexp> and <derivation>.
 684 (define-record-type <lowered-gexp>
 685   (lowered-gexp sexp inputs sources guile load-path load-compiled-path)
 686   lowered-gexp?
 687   (sexp                lowered-gexp-sexp)         ;sexp
 688   (inputs              lowered-gexp-inputs)       ;list of <derivation-input>
 689   (sources             lowered-gexp-sources)      ;list of store items
 690   (guile               lowered-gexp-guile)        ;<derivation-input> | #f
 691   (load-path           lowered-gexp-load-path)    ;list of store items
 692   (load-compiled-path  lowered-gexp-load-compiled-path)) ;list of store items
 693
 694 (define* (imported+compiled-modules modules system
 695                                     #:key (extensions '())
 696                                     deprecation-warnings guile
 697                                     (module-path %load-path))
 698   "Return a pair where the first element is the imported MODULES and the
 699 second element is the derivation to compile them."
 700   (mcached equal?
 701            (mlet %store-monad ((modules  (if (pair? modules)
 702                                              (imported-modules modules
 703                                                                #:system system
 704                                                                #:module-path module-path)
 705                                              (return #f)))
 706                                (compiled (if (pair? modules)
 707                                              (compiled-modules modules
 708                                                                #:system system
 709                                                                #:module-path module-path
 710                                                                #:extensions extensions
 711                                                                #:guile guile
 712                                                                #:deprecation-warnings
 713                                                                deprecation-warnings)
 714                                              (return #f))))
 715              (return (cons modules compiled)))
 716            modules
 717            system extensions guile deprecation-warnings module-path))
 718
 719 (define* (lower-gexp exp
 720                      #:key
 721                      (module-path %load-path)
 722                      (system (%current-system))
 723                      (target 'current)
 724                      (graft? (%graft?))
 725                      (guile-for-build (%guile-for-build))
 726                      (effective-version "2.2")
 727
 728                      deprecation-warnings)
 729   "*Note: This API is subject to change; use at your own risk!*
 730
 731 Lower EXP, a gexp, instantiating it for SYSTEM and TARGET.  Return a
 732 <lowered-gexp> ready to be used.
 733
 734 Lowered gexps are an intermediate representation that's useful for
 735 applications that deal with gexps outside in a way that is disconnected from
 736 derivations--e.g., code evaluated for its side effects."
 737   (define %modules
 738     (delete-duplicates (gexp-modules exp)))
 739
 740   (define (search-path modules extensions suffix)
 741     (append (match modules
 742               ((? derivation? drv)
 743                (list (derivation->output-path drv)))
 744               (#f
 745                '())
 746               ((? store-path? item)
 747                (list item)))
 748             (map (lambda (extension)
 749                    (string-append (match extension
 750                                     ((? derivation? drv)
 751                                      (derivation->output-path drv))
 752                                     ((? store-path? item)
 753                                      item))
 754                                   suffix))
 755                  extensions)))
 756
 757   (mlet* %store-monad ( ;; The following binding forces '%current-system' and
 758                        ;; '%current-target-system' to be looked up at >>=
 759                        ;; time.
 760                        (graft?    (set-grafting graft?))
 761
 762                        (system -> (or system (%current-system)))
 763                        (target -> (if (eq? target 'current)
 764                                       (%current-target-system)
 765                                       target))
 766                        (guile     (if guile-for-build
 767                                       (return guile-for-build)
 768                                       (default-guile-derivation system)))
 769                        (normals  (lower-inputs (gexp-inputs exp)
 770                                                #:system system
 771                                                #:target target))
 772                        (natives  (lower-inputs (gexp-native-inputs exp)
 773                                                #:system system
 774                                                #:target #f))
 775                        (inputs -> (append normals natives))
 776                        (sexp     (gexp->sexp exp
 777                                              #:system system
 778                                              #:target target))
 779                        (extensions -> (gexp-extensions exp))
 780                        (exts     (mapm %store-monad
 781                                        (lambda (obj)
 782                                          (lower-object obj system))
 783                                        extensions))
 784                        (modules+compiled (imported+compiled-modules
 785                                           %modules system
 786                                           #:extensions extensions
 787                                           #:deprecation-warnings
 788                                           deprecation-warnings
 789                                           #:guile guile
 790                                           #:module-path module-path))
 791                        (modules ->  (car modules+compiled))
 792                        (compiled -> (cdr modules+compiled)))
 793     (define load-path
 794       (search-path modules exts
 795                    (string-append "/share/guile/site/" effective-version)))
 796
 797     (define load-compiled-path
 798       (search-path compiled exts
 799                    (string-append "/lib/guile/" effective-version
 800                                   "/site-ccache")))
 801
 802     (mbegin %store-monad
 803       (set-grafting graft?)                       ;restore the initial setting
 804       (return (lowered-gexp sexp
 805                             `(,@(if (derivation? modules)
 806                                     (list (derivation-input modules))
 807                                     '())
 808                               ,@(if compiled
 809                                     (list (derivation-input compiled))
 810                                     '())
 811                               ,@(map derivation-input exts)
 812                               ,@(filter derivation-input? inputs))
 813                             (filter string? (cons modules inputs))
 814                             (derivation-input guile '("out"))
 815                             load-path
 816                             load-compiled-path)))))
 817
 818 (define* (gexp->derivation name exp
 819                            #:key
 820                            system (target 'current)
 821                            hash hash-algo recursive?
 822                            (env-vars '())
 823                            (modules '())
 824                            (module-path %load-path)
 825                            (guile-for-build (%guile-for-build))
 826                            (effective-version "2.2")
 827                            (graft? (%graft?))
 828                            references-graphs
 829                            allowed-references disallowed-references
 830                            leaked-env-vars
 831                            local-build? (substitutable? #t)
 832                            (properties '())
 833                            deprecation-warnings
 834                            (script-name (string-append name "-builder")))
 835   "Return a derivation NAME that runs EXP (a gexp) with GUILE-FOR-BUILD (a
 836 derivation) on SYSTEM; EXP is stored in a file called SCRIPT-NAME.  When
 837 TARGET is true, it is used as the cross-compilation target triplet for
 838 packages referred to by EXP.
 839
 840 MODULES is deprecated in favor of 'with-imported-modules'.  Its meaning is to
 841 make MODULES available in the evaluation context of EXP; MODULES is a list of
 842 names of Guile modules searched in MODULE-PATH to be copied in the store,
 843 compiled, and made available in the load path during the execution of
 844 EXP---e.g., '((guix build utils) (guix build gnu-build-system)).
 845
 846 EFFECTIVE-VERSION determines the string to use when adding extensions of
 847 EXP (see 'with-extensions') to the search path---e.g., \"2.2\".
 848
 849 GRAFT? determines whether packages referred to by EXP should be grafted when
 850 applicable.
 851
 852 When REFERENCES-GRAPHS is true, it must be a list of tuples of one of the
 853 following forms:
 854
 855   (FILE-NAME PACKAGE)
 856   (FILE-NAME PACKAGE OUTPUT)
 857   (FILE-NAME DERIVATION)
 858   (FILE-NAME DERIVATION OUTPUT)
 859   (FILE-NAME STORE-ITEM)
 860
 861 The right-hand-side of each element of REFERENCES-GRAPHS is automatically made
 862 an input of the build process of EXP.  In the build environment, each
 863 FILE-NAME contains the reference graph of the corresponding item, in a simple
 864 text format.
 865
 866 ALLOWED-REFERENCES must be either #f or a list of output names and packages.
 867 In the latter case, the list denotes store items that the result is allowed to
 868 refer to.  Any reference to another store item will lead to a build error.
 869 Similarly for DISALLOWED-REFERENCES, which can list items that must not be
 870 referenced by the outputs.
 871
 872 DEPRECATION-WARNINGS determines whether to show deprecation warnings while
 873 compiling modules.  It can be #f, #t, or 'detailed.
 874
 875 The other arguments are as for 'derivation'."
 876   (define outputs (gexp-outputs exp))
 877   (define requested-graft? graft?)
 878
 879   (define (graphs-file-names graphs)
 880     ;; Return a list of (FILE-NAME . STORE-PATH) pairs made from GRAPHS.
 881     (map (match-lambda
 882            ((file-name . (? derivation-input? input))
 883             (cons file-name (first (derivation-input-output-paths input))))
 884            ((file-name . (? string? item))
 885             (cons file-name item)))
 886          graphs))
 887
 888   (define (add-modules exp modules)
 889     (if (null? modules)
 890         exp
 891         (make-gexp (gexp-references exp)
 892                    (append modules (gexp-self-modules exp))
 893                    (gexp-self-extensions exp)
 894                    (gexp-proc exp))))
 895
 896   (mlet* %store-monad ( ;; The following binding forces '%current-system' and
 897                        ;; '%current-target-system' to be looked up at >>=
 898                        ;; time.
 899                        (graft?    (set-grafting graft?))
 900
 901                        (system -> (or system (%current-system)))
 902                        (target -> (if (eq? target 'current)
 903                                       (%current-target-system)
 904                                       target))
 905                        (exp ->    (add-modules exp modules))
 906                        (lowered   (lower-gexp exp
 907                                               #:module-path module-path
 908                                               #:system system
 909                                               #:target target
 910                                               #:graft? requested-graft?
 911                                               #:guile-for-build
 912                                               guile-for-build
 913                                               #:effective-version
 914                                               effective-version
 915                                               #:deprecation-warnings
 916                                               deprecation-warnings))
 917
 918                        (graphs   (if references-graphs
 919                                      (lower-reference-graphs references-graphs
 920                                                              #:system system
 921                                                              #:target target)
 922                                      (return #f)))
 923                        (allowed  (if allowed-references
 924                                      (lower-references allowed-references
 925                                                        #:system system
 926                                                        #:target target)
 927                                      (return #f)))
 928                        (disallowed (if disallowed-references
 929                                        (lower-references disallowed-references
 930                                                          #:system system
 931                                                          #:target target)
 932                                        (return #f)))
 933                        (guile -> (lowered-gexp-guile lowered))
 934                        (builder  (text-file script-name
 935                                             (object->string
 936                                              (lowered-gexp-sexp lowered)))))
 937     (mbegin %store-monad
 938       (set-grafting graft?)                       ;restore the initial setting
 939       (raw-derivation name
 940                       (string-append (derivation-input-output-path guile)
 941                                      "/bin/guile")
 942                       `("--no-auto-compile"
 943                         ,@(append-map (lambda (directory)
 944                                         `("-L" ,directory))
 945                                       (lowered-gexp-load-path lowered))
 946                         ,@(append-map (lambda (directory)
 947                                         `("-C" ,directory))
 948                                       (lowered-gexp-load-compiled-path lowered))
 949                         ,builder)
 950                       #:outputs outputs
 951                       #:env-vars env-vars
 952                       #:system system
 953                       #:inputs `(,guile
 954                                  ,@(lowered-gexp-inputs lowered)
 955                                  ,@(match graphs
 956                                      (((_ . inputs) ...)
 957                                       (filter derivation-input? inputs))
 958                                      (#f '())))
 959                       #:sources `(,builder
 960                                   ,@(if (and (string? modules)
 961                                              (store-path? modules))
 962                                         (list modules)
 963                                         '())
 964                                   ,@(lowered-gexp-sources lowered)
 965                                   ,@(match graphs
 966                                       (((_ . inputs) ...)
 967                                        (filter string? inputs))
 968                                       (#f '())))
 969
 970                       #:hash hash #:hash-algo hash-algo #:recursive? recursive?
 971                       #:references-graphs (and=> graphs graphs-file-names)
 972                       #:allowed-references allowed
 973                       #:disallowed-references disallowed
 974                       #:leaked-env-vars leaked-env-vars
 975                       #:local-build? local-build?
 976                       #:substitutable? substitutable?
 977                       #:properties properties))))
 978
 979 (define* (gexp-inputs exp #:key native?)
 980   "Return the input list for EXP.  When NATIVE? is true, return only native
 981 references; otherwise, return only non-native references."
 982   ;; TODO: Return <gexp-input> records instead of tuples.
 983   (define (add-reference-inputs ref result)
 984     (match ref
 985       (($ <gexp-input> (? gexp? exp) _ #t)
 986        (if native?
 987            (append (gexp-inputs exp)
 988                    (gexp-inputs exp #:native? #t)
 989                    result)
 990            result))
 991       (($ <gexp-input> (? gexp? exp) _ #f)
 992        (append (gexp-inputs exp #:native? native?)
 993                result))
 994       (($ <gexp-input> (? string? str))
 995        (if (direct-store-path? str)
 996            (cons `(,str) result)
 997            result))
 998       (($ <gexp-input> (? struct? thing) output n?)
 999        (if (and (eqv? n? native?) (lookup-compiler thing))
1000            ;; THING is a derivation, or a package, or an origin, etc.
1001            (cons `(,thing ,output) result)
1002            result))
1003       (($ <gexp-input> (lst ...) output n?)
1004        (fold-right add-reference-inputs result
1005                    ;; XXX: For now, automatically convert LST to a list of
1006                    ;; gexp-inputs.  Inherit N?.
1007                    (map (match-lambda
1008                           ((? gexp-input? x)
1009                            (%gexp-input (gexp-input-thing x)
1010                                         (gexp-input-output x)
1011                                         n?))
1012                           (x
1013                            (%gexp-input x "out" n?)))
1014                         lst)))
1015       (_
1016        ;; Ignore references to other kinds of objects.
1017        result)))
1018
1019   (fold-right add-reference-inputs
1020               '()
1021               (gexp-references exp)))
1022
1023 (define gexp-native-inputs
1024   (cut gexp-inputs <> #:native? #t))
1025
1026 (define (gexp-outputs exp)
1027   "Return the outputs referred to by EXP as a list of strings."
1028   (define (add-reference-output ref result)
1029     (match ref
1030       (($ <gexp-output> name)
1031        (cons name result))
1032       (($ <gexp-input> (? gexp? exp))
1033        (append (gexp-outputs exp) result))
1034       (($ <gexp-input> (lst ...) output native?)
1035        ;; XXX: Automatically convert LST.
1036        (add-reference-output (map (match-lambda
1037                                    ((? gexp-input? x) x)
1038                                    (x (%gexp-input x "out" native?)))
1039                                   lst)
1040                              result))
1041       ((lst ...)
1042        (fold-right add-reference-output result lst))
1043       (_
1044        result)))
1045
1046   (delete-duplicates
1047    (add-reference-output (gexp-references exp) '())))
1048
1049 (define* (gexp->sexp exp #:key
1050                      (system (%current-system))
1051                      (target (%current-target-system)))
1052   "Return (monadically) the sexp corresponding to EXP for the given OUTPUT,
1053 and in the current monad setting (system type, etc.)"
1054   (define (self-quoting? x)
1055     (letrec-syntax ((one-of (syntax-rules ()
1056                               ((_) #f)
1057                               ((_ pred rest ...)
1058                                (or (pred x)
1059                                    (one-of rest ...))))))
1060       (one-of symbol? string? keyword? pair? null? array?
1061               number? boolean? char?)))
1062
1063   (define* (reference->sexp ref #:optional native?)
1064     (with-monad %store-monad
1065       (match ref
1066         (($ <gexp-output> output)
1067          ;; Output file names are not known in advance but the daemon defines
1068          ;; an environment variable for each of them at build time, so use
1069          ;; that trick.
1070          (return `((@ (guile) getenv) ,output)))
1071         (($ <gexp-input> (? gexp? exp) output n?)
1072          (gexp->sexp exp
1073                      #:system system
1074                      #:target (if (or n? native?) #f target)))
1075         (($ <gexp-input> (refs ...) output n?)
1076          (mapm %store-monad
1077                (lambda (ref)
1078                  ;; XXX: Automatically convert REF to an gexp-input.
1079                  (reference->sexp
1080                   (if (gexp-input? ref)
1081                       ref
1082                       (%gexp-input ref "out" n?))
1083                   (or n? native?)))
1084                refs))
1085         (($ <gexp-input> (? struct? thing) output n?)
1086          (let ((target (if (or n? native?) #f target))
1087                (expand (lookup-expander thing)))
1088            (mlet %store-monad ((obj (lower-object thing system
1089                                                   #:target target)))
1090              ;; OBJ must be either a derivation or a store file name.
1091              (return (expand thing obj output)))))
1092         (($ <gexp-input> (? self-quoting? x))
1093          (return x))
1094         (($ <gexp-input> x)
1095          (raise (condition (&gexp-input-error (input x)))))
1096         (x
1097          (return x)))))
1098
1099   (mlet %store-monad
1100       ((args (mapm %store-monad
1101                    reference->sexp (gexp-references exp))))
1102     (return (apply (gexp-proc exp) args))))
1103
1104 (define-syntax-rule (define-syntax-parameter-once name proc)
1105   ;; Like 'define-syntax-parameter' but ensure the top-level binding for NAME
1106   ;; does not get redefined.  This works around a race condition in a
1107   ;; multi-threaded context with Guile <= 2.2.4: <https://bugs.gnu.org/27476>.
1108   (eval-when (load eval expand compile)
1109     (define name
1110       (if (module-locally-bound? (current-module) 'name)
1111           (module-ref (current-module) 'name)
1112           (make-syntax-transformer 'name 'syntax-parameter
1113                                    (list proc))))))
1114
1115 (define-syntax-parameter-once current-imported-modules
1116   ;; Current list of imported modules.
1117   (identifier-syntax '()))
1118
1119 (define-syntax-rule (with-imported-modules modules body ...)
1120   "Mark the gexps defined in BODY... as requiring MODULES in their execution
1121 environment."
1122   (syntax-parameterize ((current-imported-modules
1123                          (identifier-syntax modules)))
1124     body ...))
1125
1126 (define-syntax-parameter-once current-imported-extensions
1127   ;; Current list of extensions.
1128   (identifier-syntax '()))
1129
1130 (define-syntax-rule (with-extensions extensions body ...)
1131   "Mark the gexps defined in BODY... as requiring EXTENSIONS in their
1132 execution environment."
1133   (syntax-parameterize ((current-imported-extensions
1134                          (identifier-syntax extensions)))
1135     body ...))
1136
1137 (define-syntax gexp
1138   (lambda (s)
1139     (define (collect-escapes exp)
1140       ;; Return all the 'ungexp' present in EXP.
1141       (let loop ((exp    exp)
1142                  (result '()))
1143         (syntax-case exp (ungexp
1144                           ungexp-splicing
1145                           ungexp-native
1146                           ungexp-native-splicing)
1147           ((ungexp _)
1148            (cons exp result))
1149           ((ungexp _ _)
1150            (cons exp result))
1151           ((ungexp-splicing _ ...)
1152            (cons exp result))
1153           ((ungexp-native _ ...)
1154            (cons exp result))
1155           ((ungexp-native-splicing _ ...)
1156            (cons exp result))
1157           ((exp0 . exp)
1158            (let ((result (loop #'exp0 result)))
1159              (loop  #'exp result)))
1160           (_
1161            result))))
1162
1163     (define (escape->ref exp)
1164       ;; Turn 'ungexp' form EXP into a "reference".
1165       (syntax-case exp (ungexp ungexp-splicing
1166                         ungexp-native ungexp-native-splicing
1167                         output)
1168         ((ungexp output)
1169          #'(gexp-output "out"))
1170         ((ungexp output name)
1171          #'(gexp-output name))
1172         ((ungexp thing)
1173          #'(%gexp-input thing "out" #f))
1174         ((ungexp drv-or-pkg out)
1175          #'(%gexp-input drv-or-pkg out #f))
1176         ((ungexp-splicing lst)
1177          #'(%gexp-input lst "out" #f))
1178         ((ungexp-native thing)
1179          #'(%gexp-input thing "out" #t))
1180         ((ungexp-native drv-or-pkg out)
1181          #'(%gexp-input drv-or-pkg out #t))
1182         ((ungexp-native-splicing lst)
1183          #'(%gexp-input lst "out" #t))))
1184
1185     (define (substitute-ungexp exp substs)
1186       ;; Given EXP, an 'ungexp' or 'ungexp-native' form, substitute it with
1187       ;; the corresponding form in SUBSTS.
1188       (match (assoc exp substs)
1189         ((_ id)
1190          id)
1191         (_                                        ;internal error
1192          (with-syntax ((exp exp))
1193            #'(syntax-error "error: no 'ungexp' substitution" exp)))))
1194
1195     (define (substitute-ungexp-splicing exp substs)
1196       (syntax-case exp ()
1197         ((exp rest ...)
1198          (match (assoc #'exp substs)
1199            ((_ id)
1200             (with-syntax ((id id))
1201               #`(append id
1202                         #,(substitute-references #'(rest ...) substs))))
1203            (_
1204             #'(syntax-error "error: no 'ungexp-splicing' substitution"
1205                             exp))))))
1206
1207     (define (substitute-references exp substs)
1208       ;; Return a variant of EXP where all the cars of SUBSTS have been
1209       ;; replaced by the corresponding cdr.
1210       (syntax-case exp (ungexp ungexp-native
1211                         ungexp-splicing ungexp-native-splicing)
1212         ((ungexp _ ...)
1213          (substitute-ungexp exp substs))
1214         ((ungexp-native _ ...)
1215          (substitute-ungexp exp substs))
1216         (((ungexp-splicing _ ...) rest ...)
1217          (substitute-ungexp-splicing exp substs))
1218         (((ungexp-native-splicing _ ...) rest ...)
1219          (substitute-ungexp-splicing exp substs))
1220         ((exp0 . exp)
1221          #`(cons #,(substitute-references #'exp0 substs)
1222                  #,(substitute-references #'exp substs)))
1223         (x #''x)))
1224
1225     (syntax-case s (ungexp output)
1226       ((_ exp)
1227        (let* ((escapes (delete-duplicates (collect-escapes #'exp)))
1228               (formals (generate-temporaries escapes))
1229               (sexp    (substitute-references #'exp (zip escapes formals)))
1230               (refs    (map escape->ref escapes)))
1231          #`(make-gexp (list #,@refs)
1232                       current-imported-modules
1233                       current-imported-extensions
1234                       (lambda #,formals
1235                         #,sexp)))))))
1236
1237 \f
1238 ;;;
1239 ;;; Module handling.
1240 ;;;
1241
1242 (define %not-slash
1243   (char-set-complement (char-set #\/)))
1244
1245 (define (file-mapping->tree mapping)
1246   "Convert MAPPING, an alist like:
1247
1248   ((\"guix/build/utils.scm\" . \"…/utils.scm\"))
1249
1250 to a tree suitable for 'interned-file-tree'."
1251   (let ((mapping (map (match-lambda
1252                         ((destination . source)
1253                          (cons (string-tokenize destination
1254                                                 %not-slash)
1255                                source)))
1256                       mapping)))
1257     (fold (lambda (pair result)
1258             (match pair
1259               ((destination . source)
1260                (let loop ((destination destination)
1261                           (result result))
1262                  (match destination
1263                    ((file)
1264                     (let* ((mode (stat:mode (stat source)))
1265                            (type (if (zero? (logand mode #o100))
1266                                      'regular
1267                                      'executable)))
1268                       (alist-cons file
1269                                   `(,type (file ,source))
1270                                   result)))
1271                    ((file rest ...)
1272                     (let ((directory (assoc-ref result file)))
1273                       (alist-cons file
1274                                   `(directory
1275                                     ,@(loop rest
1276                                             (match directory
1277                                               (('directory . entries) entries)
1278                                               (#f '()))))
1279                                   (if directory
1280                                       (alist-delete file result)
1281                                       result)))))))))
1282           '()
1283           mapping)))
1284
1285 (define %utils-module
1286   ;; This file provides 'mkdir-p', needed to implement 'imported-files' and
1287   ;; other primitives below.  Note: We give the file name relative to this
1288   ;; file you are currently reading; 'search-path' could return a file name
1289   ;; relative to the current working directory.
1290   (local-file "build/utils.scm"
1291               "build-utils.scm"))
1292
1293 (define* (imported-files/derivation files
1294                                     #:key (name "file-import")
1295                                     (symlink? #f)
1296                                     (system (%current-system))
1297                                     (guile (%guile-for-build)))
1298   "Return a derivation that imports FILES into STORE.  FILES must be a list
1299 of (FINAL-PATH . FILE) pairs.  Each FILE is mapped to FINAL-PATH in the
1300 resulting store path.  FILE can be either a file name, or a file-like object,
1301 as returned by 'local-file' for example.  If SYMLINK? is true, create symlinks
1302 to the source files instead of copying them."
1303   (define file-pair
1304     (match-lambda
1305      ((final-path . (? string? file-name))
1306       (mlet %store-monad ((file (interned-file file-name
1307                                                (basename final-path))))
1308         (return (list final-path file))))
1309      ((final-path . file-like)
1310       (mlet %store-monad ((file (lower-object file-like system)))
1311         (return (list final-path file))))))
1312
1313   (mlet %store-monad ((files (mapm %store-monad file-pair files)))
1314     (define build
1315       (gexp
1316        (begin
1317          (primitive-load (ungexp %utils-module))  ;for 'mkdir-p'
1318          (use-modules (ice-9 match))
1319
1320          (mkdir (ungexp output)) (chdir (ungexp output))
1321          (for-each (match-lambda
1322                     ((final-path store-path)
1323                      (mkdir-p (dirname final-path))
1324                      ((ungexp (if symlink? 'symlink 'copy-file))
1325                       store-path final-path)))
1326                    '(ungexp files)))))
1327
1328     ;; TODO: Pass FILES as an environment variable so that BUILD remains
1329     ;; exactly the same regardless of FILES: less disk space, and fewer
1330     ;; 'add-to-store' RPCs.
1331     (gexp->derivation name build
1332                       #:system system
1333                       #:guile-for-build guile
1334                       #:local-build? #t
1335
1336                       ;; Avoid deprecation warnings about the use of the _IO*
1337                       ;; constants in (guix build utils).
1338                       #:env-vars
1339                       '(("GUILE_WARN_DEPRECATED" . "no")))))
1340
1341 (define* (imported-files files
1342                          #:key (name "file-import")
1343                          ;; The following parameters make sense when creating
1344                          ;; an actual derivation.
1345                          (system (%current-system))
1346                          (guile (%guile-for-build)))
1347   "Import FILES into the store and return the resulting derivation or store
1348 file name (a derivation is created if and only if some elements of FILES are
1349 file-like objects and not local file names.)  FILES must be a list
1350 of (FINAL-PATH . FILE) pairs.  Each FILE is mapped to FINAL-PATH in the
1351 resulting store path.  FILE can be either a file name, or a file-like object,
1352 as returned by 'local-file' for example."
1353   (if (any (match-lambda
1354              ((_ . (? struct? source)) #t)
1355              (_ #f))
1356            files)
1357       (imported-files/derivation files #:name name
1358                                  #:symlink? derivation?
1359                                  #:system system #:guile guile)
1360       (interned-file-tree `(,name directory
1361                                   ,@(file-mapping->tree files)))))
1362
1363 (define* (imported-modules modules
1364                            #:key (name "module-import")
1365                            (system (%current-system))
1366                            (guile (%guile-for-build))
1367                            (module-path %load-path))
1368   "Return a derivation that contains the source files of MODULES, a list of
1369 module names such as `(ice-9 q)'.  All of MODULES must be either names of
1370 modules to be found in the MODULE-PATH search path, or a module name followed
1371 by an arrow followed by a file-like object.  For example:
1372
1373   (imported-modules `((guix build utils)
1374                       (guix gcrypt)
1375                       ((guix config) => ,(scheme-file …))))
1376
1377 In this example, the first two modules are taken from MODULE-PATH, and the
1378 last one is created from the given <scheme-file> object."
1379   (let ((files (map (match-lambda
1380                       (((module ...) '=> file)
1381                        (cons (module->source-file-name module)
1382                              file))
1383                       ((module ...)
1384                        (let ((f (module->source-file-name module)))
1385                          (cons f (search-path* module-path f)))))
1386                     modules)))
1387     (imported-files files #:name name
1388                     #:system system
1389                     #:guile guile)))
1390
1391 (define* (compiled-modules modules
1392                            #:key (name "module-import-compiled")
1393                            (system (%current-system))
1394                            target
1395                            (guile (%guile-for-build))
1396                            (module-path %load-path)
1397                            (extensions '())
1398                            (deprecation-warnings #f))
1399   "Return a derivation that builds a tree containing the `.go' files
1400 corresponding to MODULES.  All the MODULES are built in a context where
1401 they can refer to each other.  When TARGET is true, cross-compile MODULES for
1402 TARGET, a GNU triplet."
1403   (define total (length modules))
1404
1405   (mlet %store-monad ((modules (imported-modules modules
1406                                                  #:system system
1407                                                  #:guile guile
1408                                                  #:module-path
1409                                                  module-path)))
1410     (define build
1411       (gexp
1412        (begin
1413          (primitive-load (ungexp %utils-module))  ;for 'mkdir-p'
1414
1415          (use-modules (ice-9 ftw)
1416                       (ice-9 format)
1417                       (srfi srfi-1)
1418                       (srfi srfi-26)
1419                       (system base target)
1420                       (system base compile))
1421
1422          (define (regular? file)
1423            (not (member file '("." ".."))))
1424
1425          (define (process-entry entry output processed)
1426            (if (file-is-directory? entry)
1427                (let ((output (string-append output "/" (basename entry))))
1428                  (mkdir-p output)
1429                  (process-directory entry output processed))
1430                (let* ((base   (basename entry ".scm"))
1431                       (output (string-append output "/" base ".go")))
1432                  (format #t "[~2@a/~2@a] Compiling '~a'...~%"
1433                          (+ 1 processed (ungexp total))
1434                          (ungexp (* total 2))
1435                          entry)
1436
1437                  (ungexp-splicing
1438                   (if target
1439                       (gexp ((with-target (ungexp target)
1440                                (lambda ()
1441                                  (compile-file entry
1442                                                #:output-file output
1443                                                #:opts
1444                                                %auto-compilation-options)))))
1445                       (gexp ((compile-file entry
1446                                            #:output-file output
1447                                            #:opts %auto-compilation-options)))))
1448
1449                  (+ 1 processed))))
1450
1451          (define (process-directory directory output processed)
1452            (let ((entries (map (cut string-append directory "/" <>)
1453                                (scandir directory regular?))))
1454              (fold (cut process-entry <> output <>)
1455                    processed
1456                    entries)))
1457
1458          (define* (load-from-directory directory
1459                                        #:optional (loaded 0))
1460            "Load all the source files found in DIRECTORY."
1461            ;; XXX: This works around <https://bugs.gnu.org/15602>.
1462            (let ((entries (map (cut string-append directory "/" <>)
1463                                (scandir directory regular?))))
1464              (fold (lambda (file loaded)
1465                      (if (file-is-directory? file)
1466                          (load-from-directory file loaded)
1467                          (begin
1468                            (format #t "[~2@a/~2@a] Loading '~a'...~%"
1469                                    (+ 1 loaded) (ungexp (* 2 total))
1470                                    file)
1471                            (save-module-excursion
1472                             (lambda ()
1473                               (primitive-load file)))
1474                            (+ 1 loaded))))
1475                    loaded
1476                    entries)))
1477
1478          (setvbuf (current-output-port)
1479                   (cond-expand (guile-2.2 'line) (else _IOLBF)))
1480
1481          (define mkdir-p
1482            ;; Capture 'mkdir-p'.
1483            (@ (guix build utils) mkdir-p))
1484
1485          ;; Add EXTENSIONS to the search path.
1486          (set! %load-path
1487            (append (map (lambda (extension)
1488                           (string-append extension
1489                                          "/share/guile/site/"
1490                                          (effective-version)))
1491                         '((ungexp-native-splicing extensions)))
1492                    %load-path))
1493          (set! %load-compiled-path
1494            (append (map (lambda (extension)
1495                           (string-append extension "/lib/guile/"
1496                                          (effective-version)
1497                                          "/site-ccache"))
1498                         '((ungexp-native-splicing extensions)))
1499                    %load-compiled-path))
1500
1501          (set! %load-path (cons (ungexp modules) %load-path))
1502
1503          ;; Above we loaded our own (guix build utils) but now we may need to
1504          ;; load a compile a different one.  Thus, force a reload.
1505          (let ((utils (string-append (ungexp modules)
1506                                      "/guix/build/utils.scm")))
1507            (when (file-exists? utils)
1508              (load utils)))
1509
1510          (mkdir (ungexp output))
1511          (chdir (ungexp modules))
1512
1513          (load-from-directory ".")
1514          (process-directory "." (ungexp output) 0))))
1515
1516     ;; TODO: Pass MODULES as an environment variable.
1517     (gexp->derivation name build
1518                       #:system system
1519                       #:guile-for-build guile
1520                       #:local-build? #t
1521                       #:env-vars
1522                       (case deprecation-warnings
1523                         ((#f)
1524                          '(("GUILE_WARN_DEPRECATED" . "no")))
1525                         ((detailed)
1526                          '(("GUILE_WARN_DEPRECATED" . "detailed")))
1527                         (else
1528                          '())))))
1529
1530 \f
1531 ;;;
1532 ;;; Convenience procedures.
1533 ;;;
1534
1535 (define (default-guile)
1536   ;; Lazily resolve 'guile-2.2' (not 'guile-final' because this is for
1537   ;; programs returned by 'program-file' and we don't want to keep references
1538   ;; to several Guile packages).  This module must not refer to (gnu …)
1539   ;; modules directly, to avoid circular dependencies, hence this hack.
1540   (module-ref (resolve-interface '(gnu packages guile))
1541               'guile-2.2))
1542
1543 (define* (load-path-expression modules #:optional (path %load-path)
1544                                #:key (extensions '()) system target)
1545   "Return as a monadic value a gexp that sets '%load-path' and
1546 '%load-compiled-path' to point to MODULES, a list of module names.  MODULES
1547 are searched for in PATH.  Return #f when MODULES and EXTENSIONS are empty."
1548   (if (and (null? modules) (null? extensions))
1549       (with-monad %store-monad
1550         (return #f))
1551       (mlet %store-monad ((modules  (imported-modules modules
1552                                                       #:module-path path
1553                                                       #:system system))
1554                           (compiled (compiled-modules modules
1555                                                       #:extensions extensions
1556                                                       #:module-path path
1557                                                       #:system system
1558                                                       #:target target)))
1559         (return
1560          (gexp (eval-when (expand load eval)
1561                  ;; Augment the load paths and delete duplicates.  Do that
1562                  ;; without loading (srfi srfi-1) or anything.
1563                  (let ((extensions '((ungexp-splicing extensions)))
1564                        (prepend (lambda (items lst)
1565                                   ;; This is O(N²) but N is typically small.
1566                                   (let loop ((items items)
1567                                              (lst lst))
1568                                     (if (null? items)
1569                                         lst
1570                                         (loop (cdr items)
1571                                               (cons (car items)
1572                                                     (delete (car items) lst))))))))
1573                    (set! %load-path
1574                      (prepend (cons (ungexp modules)
1575                                     (map (lambda (extension)
1576                                            (string-append extension
1577                                                           "/share/guile/site/"
1578                                                           (effective-version)))
1579                                          extensions))
1580                               %load-path))
1581                    (set! %load-compiled-path
1582                      (prepend (cons (ungexp compiled)
1583                                     (map (lambda (extension)
1584                                            (string-append extension
1585                                                           "/lib/guile/"
1586                                                           (effective-version)
1587                                                           "/site-ccache"))
1588                                          extensions))
1589                               %load-compiled-path)))))))))
1590
1591 (define* (gexp->script name exp
1592                        #:key (guile (default-guile))
1593                        (module-path %load-path)
1594                        (system (%current-system))
1595                        target)
1596   "Return an executable script NAME that runs EXP using GUILE, with EXP's
1597 imported modules in its search path.  Look up EXP's modules in MODULE-PATH."
1598   (mlet %store-monad ((set-load-path
1599                        (load-path-expression (gexp-modules exp)
1600                                              module-path
1601                                              #:extensions
1602                                              (gexp-extensions exp)
1603                                              #:system system
1604                                              #:target target)))
1605     (gexp->derivation name
1606                       (gexp
1607                        (call-with-output-file (ungexp output)
1608                          (lambda (port)
1609                            ;; Note: that makes a long shebang.  When the store
1610                            ;; is /gnu/store, that fits within the 128-byte
1611                            ;; limit imposed by Linux, but that may go beyond
1612                            ;; when running tests.
1613                            (format port
1614                                    "#!~a/bin/guile --no-auto-compile~%!#~%"
1615                                    (ungexp guile))
1616
1617                            (ungexp-splicing
1618                             (if set-load-path
1619                                 (gexp ((write '(ungexp set-load-path) port)))
1620                                 (gexp ())))
1621
1622                            (write '(ungexp exp) port)
1623                            (chmod port #o555))))
1624                       #:system system
1625                       #:target target
1626                       #:module-path module-path
1627
1628                       ;; These derivations are not worth offloading or
1629                       ;; substituting.
1630                       #:local-build? #t
1631                       #:substitutable? #f)))
1632
1633 (define* (gexp->file name exp #:key
1634                      (set-load-path? #t)
1635                      (module-path %load-path)
1636                      (splice? #f)
1637                      (system (%current-system))
1638                      target)
1639   "Return a derivation that builds a file NAME containing EXP.  When SPLICE?
1640 is true, EXP is considered to be a list of expressions that will be spliced in
1641 the resulting file.
1642
1643 When SET-LOAD-PATH? is true, emit code in the resulting file to set
1644 '%load-path' and '%load-compiled-path' to honor EXP's imported modules.
1645 Lookup EXP's modules in MODULE-PATH."
1646   (define modules (gexp-modules exp))
1647   (define extensions (gexp-extensions exp))
1648
1649   (if (or (not set-load-path?)
1650           (and (null? modules) (null? extensions)))
1651       (gexp->derivation name
1652                         (gexp
1653                          (call-with-output-file (ungexp output)
1654                            (lambda (port)
1655                              (for-each (lambda (exp)
1656                                          (write exp port))
1657                                        '(ungexp (if splice?
1658                                                     exp
1659                                                     (gexp ((ungexp exp)))))))))
1660                         #:local-build? #t
1661                         #:substitutable? #f
1662                         #:system system
1663                         #:target target)
1664       (mlet %store-monad ((set-load-path
1665                            (load-path-expression modules module-path
1666                                                  #:extensions extensions
1667                                                  #:system system
1668                                                  #:target target)))
1669         (gexp->derivation name
1670                           (gexp
1671                            (call-with-output-file (ungexp output)
1672                              (lambda (port)
1673                                (write '(ungexp set-load-path) port)
1674                                (for-each (lambda (exp)
1675                                            (write exp port))
1676                                          '(ungexp (if splice?
1677                                                       exp
1678                                                       (gexp ((ungexp exp)))))))))
1679                           #:module-path module-path
1680                           #:local-build? #t
1681                           #:substitutable? #f
1682                           #:system system
1683                           #:target target))))
1684
1685 (define* (text-file* name #:rest text)
1686   "Return as a monadic value a derivation that builds a text file containing
1687 all of TEXT.  TEXT may list, in addition to strings, objects of any type that
1688 can be used in a gexp: packages, derivations, local file objects, etc.  The
1689 resulting store file holds references to all these."
1690   (define builder
1691     (gexp (call-with-output-file (ungexp output "out")
1692             (lambda (port)
1693               (display (string-append (ungexp-splicing text)) port)))))
1694
1695   (gexp->derivation name builder
1696                     #:local-build? #t
1697                     #:substitutable? #f))
1698
1699 (define* (mixed-text-file name #:rest text)
1700   "Return an object representing store file NAME containing TEXT.  TEXT is a
1701 sequence of strings and file-like objects, as in:
1702
1703   (mixed-text-file \"profile\"
1704                    \"export PATH=\" coreutils \"/bin:\" grep \"/bin\")
1705
1706 This is the declarative counterpart of 'text-file*'."
1707   (define build
1708     (gexp (call-with-output-file (ungexp output "out")
1709             (lambda (port)
1710               (display (string-append (ungexp-splicing text)) port)))))
1711
1712   (computed-file name build))
1713
1714 (define (file-union name files)
1715   "Return a <computed-file> that builds a directory containing all of FILES.
1716 Each item in FILES must be a two-element list where the first element is the
1717 file name to use in the new directory, and the second element is a gexp
1718 denoting the target file.  Here's an example:
1719
1720   (file-union \"etc\"
1721               `((\"hosts\" ,(plain-file \"hosts\"
1722                                         \"127.0.0.1 localhost\"))
1723                 (\"bashrc\" ,(plain-file \"bashrc\"
1724                                          \"alias ls='ls --color'\"))
1725                 (\"libvirt/qemu.conf\" ,(plain-file \"qemu.conf\" \"\"))))
1726
1727 This yields an 'etc' directory containing these two files."
1728   (computed-file name
1729                  (with-imported-modules '((guix build utils))
1730                    (gexp
1731                     (begin
1732                       (use-modules (guix build utils))
1733
1734                       (mkdir (ungexp output))
1735                       (chdir (ungexp output))
1736                       (ungexp-splicing
1737                        (map (match-lambda
1738                               ((target source)
1739                                (gexp
1740                                 (begin
1741                                   ;; Stat the source to abort early if it does
1742                                   ;; not exist.
1743                                   (stat (ungexp source))
1744
1745                                   (mkdir-p (dirname (ungexp target)))
1746                                   (symlink (ungexp source)
1747                                            (ungexp target))))))
1748                             files)))))))
1749
1750 (define* (directory-union name things
1751                           #:key (copy? #f) (quiet? #f)
1752                           (resolve-collision 'warn-about-collision))
1753   "Return a directory that is the union of THINGS, where THINGS is a list of
1754 file-like objects denoting directories.  For example:
1755
1756   (directory-union \"guile+emacs\" (list guile emacs))
1757
1758 yields a directory that is the union of the 'guile' and 'emacs' packages.
1759
1760 Call RESOLVE-COLLISION when several files collide, passing it the list of
1761 colliding files.  RESOLVE-COLLISION must return the chosen file or #f, in
1762 which case the colliding entry is skipped altogether.
1763
1764 When HARD-LINKS? is true, create hard links instead of symlinks.  When QUIET?
1765 is true, the derivation will not print anything."
1766   (define symlink
1767     (if copy?
1768         (gexp (lambda (old new)
1769                 (if (file-is-directory? old)
1770                     (symlink old new)
1771                     (copy-file old new))))
1772         (gexp symlink)))
1773
1774   (define log-port
1775     (if quiet?
1776         (gexp (%make-void-port "w"))
1777         (gexp (current-error-port))))
1778
1779   (match things
1780     ((one)
1781      ;; Only one thing; return it.
1782      one)
1783     (_
1784      (computed-file name
1785                     (with-imported-modules '((guix build union))
1786                       (gexp (begin
1787                               (use-modules (guix build union)
1788                                            (srfi srfi-1)) ;for 'first' and 'last'
1789
1790                               (union-build (ungexp output)
1791                                            '(ungexp things)
1792
1793                                            #:log-port (ungexp log-port)
1794                                            #:symlink (ungexp symlink)
1795                                            #:resolve-collision
1796                                            (ungexp resolve-collision)))))))))
1797
1798 \f
1799 ;;;
1800 ;;; Syntactic sugar.
1801 ;;;
1802
1803 (eval-when (expand load eval)
1804   (define* (read-ungexp chr port #:optional native?)
1805     "Read an 'ungexp' or 'ungexp-splicing' form from PORT.  When NATIVE? is
1806 true, use 'ungexp-native' and 'ungexp-native-splicing' instead."
1807     (define unquote-symbol
1808       (match (peek-char port)
1809         (#\@
1810          (read-char port)
1811          (if native?
1812              'ungexp-native-splicing
1813              'ungexp-splicing))
1814         (_
1815          (if native?
1816              'ungexp-native
1817              'ungexp))))
1818
1819     (match (read port)
1820       ((? symbol? symbol)
1821        (let ((str (symbol->string symbol)))
1822          (match (string-index-right str #\:)
1823            (#f
1824             `(,unquote-symbol ,symbol))
1825            (colon
1826             (let ((name   (string->symbol (substring str 0 colon)))
1827                   (output (substring str (+ colon 1))))
1828               `(,unquote-symbol ,name ,output))))))
1829       (x
1830        `(,unquote-symbol ,x))))
1831
1832   (define (read-gexp chr port)
1833     "Read a 'gexp' form from PORT."
1834     `(gexp ,(read port)))
1835
1836   ;; Extend the reader
1837   (read-hash-extend #\~ read-gexp)
1838   (read-hash-extend #\$ read-ungexp)
1839   (read-hash-extend #\+ (cut read-ungexp <> <> #t)))
1840
1841 ;;; gexp.scm ends here