guix/gexp.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019 Ludovic Courtès <ludo@gnu.org>
   3 ;;; Copyright © 2018 Clément Lassieur <clement@lassieur.org>
   4 ;;; Copyright © 2018 Jan Nieuwenhuizen <janneke@gnu.org>
   5 ;;;
   6 ;;; This file is part of GNU Guix.
   7 ;;;
   8 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   9 ;;; under the terms of the GNU General Public License as published by
  10 ;;; the Free Software Foundation; either version 3 of the License, or (at
  11 ;;; your option) any later version.
  12 ;;;
  13 ;;; GNU Guix is distributed in the hope that it will be useful, but
  14 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  15 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 ;;; GNU General Public License for more details.
  17 ;;;
  18 ;;; You should have received a copy of the GNU General Public License
  19 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 (define-module (guix gexp)
  22   #:use-module (guix store)
  23   #:use-module (guix monads)
  24   #:use-module (guix derivations)
  25   #:use-module (guix grafts)
  26   #:use-module (guix utils)
  27   #:use-module (rnrs bytevectors)
  28   #:use-module (srfi srfi-1)
  29   #:use-module (srfi srfi-9)
  30   #:use-module (srfi srfi-9 gnu)
  31   #:use-module (srfi srfi-26)
  32   #:use-module (srfi srfi-34)
  33   #:use-module (srfi srfi-35)
  34   #:use-module (ice-9 match)
  35   #:export (gexp
  36             gexp?
  37             with-imported-modules
  38             with-extensions
  39
  40             gexp-input
  41             gexp-input?
  42             gexp-input-thing
  43             gexp-input-output
  44             gexp-input-native?
  45
  46             local-file
  47             local-file?
  48             local-file-file
  49             local-file-absolute-file-name
  50             local-file-name
  51             local-file-recursive?
  52
  53             plain-file
  54             plain-file?
  55             plain-file-name
  56             plain-file-content
  57
  58             computed-file
  59             computed-file?
  60             computed-file-name
  61             computed-file-gexp
  62             computed-file-options
  63
  64             program-file
  65             program-file?
  66             program-file-name
  67             program-file-gexp
  68             program-file-guile
  69             program-file-module-path
  70
  71             scheme-file
  72             scheme-file?
  73             scheme-file-name
  74             scheme-file-gexp
  75
  76             file-append
  77             file-append?
  78             file-append-base
  79             file-append-suffix
  80
  81             load-path-expression
  82             gexp-modules
  83
  84             lower-gexp
  85             lowered-gexp?
  86             lowered-gexp-sexp
  87             lowered-gexp-inputs
  88             lowered-gexp-sources
  89             lowered-gexp-guile
  90             lowered-gexp-load-path
  91             lowered-gexp-load-compiled-path
  92
  93             gexp->derivation
  94             gexp->file
  95             gexp->script
  96             text-file*
  97             mixed-text-file
  98             file-union
  99             directory-union
 100             imported-files
 101             imported-modules
 102             compiled-modules
 103
 104             define-gexp-compiler
 105             gexp-compiler?
 106             file-like?
 107             lower-object
 108
 109             lower-inputs
 110
 111             &gexp-error
 112             gexp-error?
 113             &gexp-input-error
 114             gexp-input-error?
 115             gexp-error-invalid-input))
 116
 117 ;;; Commentary:
 118 ;;;
 119 ;;; This module implements "G-expressions", or "gexps".  Gexps are like
 120 ;;; S-expressions (sexps), with two differences:
 121 ;;;
 122 ;;;   1. References (un-quotations) to derivations or packages in a gexp are
 123 ;;;      replaced by the corresponding output file name; in addition, the
 124 ;;;      'ungexp-native' unquote-like form allows code to explicitly refer to
 125 ;;;      the native code of a given package, in case of cross-compilation;
 126 ;;;
 127 ;;;   2. Gexps embed information about the derivations they refer to.
 128 ;;;
 129 ;;; Gexps make it easy to write to files Scheme code that refers to store
 130 ;;; items, or to write Scheme code to build derivations.
 131 ;;;
 132 ;;; Code:
 133
 134 ;; "G expressions".
 135 (define-record-type <gexp>
 136   (make-gexp references modules extensions proc)
 137   gexp?
 138   (references gexp-references)                    ;list of <gexp-input>
 139   (modules    gexp-self-modules)                  ;list of module names
 140   (extensions gexp-self-extensions)               ;list of lowerable things
 141   (proc       gexp-proc))                         ;procedure
 142
 143 (define (write-gexp gexp port)
 144   "Write GEXP on PORT."
 145   (display "#<gexp " port)
 146
 147   ;; Try to write the underlying sexp.  Now, this trick doesn't work when
 148   ;; doing things like (ungexp-splicing (gexp ())) because GEXP's procedure
 149   ;; tries to use 'append' on that, which fails with wrong-type-arg.
 150   (false-if-exception
 151    (write (apply (gexp-proc gexp)
 152                  (gexp-references gexp))
 153           port))
 154   (format port " ~a>"
 155           (number->string (object-address gexp) 16)))
 156
 157 (set-record-type-printer! <gexp> write-gexp)
 158
 159 \f
 160 ;;;
 161 ;;; Methods.
 162 ;;;
 163
 164 ;; Compiler for a type of objects that may be introduced in a gexp.
 165 (define-record-type <gexp-compiler>
 166   (gexp-compiler type lower expand)
 167   gexp-compiler?
 168   (type       gexp-compiler-type)                 ;record type descriptor
 169   (lower      gexp-compiler-lower)
 170   (expand     gexp-compiler-expand))              ;#f | DRV -> sexp
 171
 172 (define-condition-type &gexp-error &error
 173   gexp-error?)
 174
 175 (define-condition-type &gexp-input-error &gexp-error
 176   gexp-input-error?
 177   (input gexp-error-invalid-input))
 178
 179
 180 (define %gexp-compilers
 181   ;; 'eq?' mapping of record type descriptor to <gexp-compiler>.
 182   (make-hash-table 20))
 183
 184 (define (default-expander thing obj output)
 185   "This is the default expander for \"things\" that appear in gexps.  It
 186 returns its output file name of OBJ's OUTPUT."
 187   (match obj
 188     ((? derivation? drv)
 189      (derivation->output-path drv output))
 190     ((? string? file)
 191      file)))
 192
 193 (define (register-compiler! compiler)
 194   "Register COMPILER as a gexp compiler."
 195   (hashq-set! %gexp-compilers
 196               (gexp-compiler-type compiler) compiler))
 197
 198 (define (lookup-compiler object)
 199   "Search for a compiler for OBJECT.  Upon success, return the three argument
 200 procedure to lower it; otherwise return #f."
 201   (and=> (hashq-ref %gexp-compilers (struct-vtable object))
 202          gexp-compiler-lower))
 203
 204 (define (file-like? object)
 205   "Return #t if OBJECT leads to a file in the store once unquoted in a
 206 G-expression; otherwise return #f."
 207   (and (struct? object) (->bool (lookup-compiler object))))
 208
 209 (define (lookup-expander object)
 210   "Search for an expander for OBJECT.  Upon success, return the three argument
 211 procedure to expand it; otherwise return #f."
 212   (and=> (hashq-ref %gexp-compilers (struct-vtable object))
 213          gexp-compiler-expand))
 214
 215 (define* (lower-object obj
 216                        #:optional (system (%current-system))
 217                        #:key target)
 218   "Return as a value in %STORE-MONAD the derivation or store item
 219 corresponding to OBJ for SYSTEM, cross-compiling for TARGET if TARGET is true.
 220 OBJ must be an object that has an associated gexp compiler, such as a
 221 <package>."
 222   (match (lookup-compiler obj)
 223     (#f
 224      (raise (condition (&gexp-input-error (input obj)))))
 225     (lower
 226      ;; Cache in STORE the result of lowering OBJ.
 227      (mlet %store-monad ((graft? (grafting?)))
 228        (mcached (let ((lower (lookup-compiler obj)))
 229                   (lower obj system target))
 230                 obj
 231                 system target graft?)))))
 232
 233 (define-syntax define-gexp-compiler
 234   (syntax-rules (=> compiler expander)
 235     "Define NAME as a compiler for objects matching PREDICATE encountered in
 236 gexps.
 237
 238 In the simplest form of the macro, BODY must return a derivation for PARAM, an
 239 object that matches PREDICATE, for SYSTEM and TARGET (the latter of which is
 240 #f except when cross-compiling.)
 241
 242 The more elaborate form allows you to specify an expander:
 243
 244   (define-gexp-compiler something something?
 245     compiler => (lambda (param system target) ...)
 246     expander => (lambda (param drv output) ...))
 247
 248 The expander specifies how an object is converted to its sexp representation."
 249     ((_ (name (param record-type) system target) body ...)
 250      (define-gexp-compiler name record-type
 251        compiler => (lambda (param system target) body ...)
 252        expander => default-expander))
 253     ((_ name record-type
 254         compiler => compile
 255         expander => expand)
 256      (begin
 257        (define name
 258          (gexp-compiler record-type compile expand))
 259        (register-compiler! name)))))
 260
 261 (define-gexp-compiler (derivation-compiler (drv <derivation>) system target)
 262   ;; Derivations are the lowest-level representation, so this is the identity
 263   ;; compiler.
 264   (with-monad %store-monad
 265     (return drv)))
 266
 267 \f
 268 ;;;
 269 ;;; File declarations.
 270 ;;;
 271
 272 ;; A local file name.  FILE is the file name the user entered, which can be a
 273 ;; relative file name, and ABSOLUTE is a promise that computes its canonical
 274 ;; absolute file name.  We keep it in a promise to compute it lazily and avoid
 275 ;; repeated 'stat' calls.
 276 (define-record-type <local-file>
 277   (%%local-file file absolute name recursive? select?)
 278   local-file?
 279   (file       local-file-file)                    ;string
 280   (absolute   %local-file-absolute-file-name)     ;promise string
 281   (name       local-file-name)                    ;string
 282   (recursive? local-file-recursive?)              ;Boolean
 283   (select?    local-file-select?))                ;string stat -> Boolean
 284
 285 (define (true file stat) #t)
 286
 287 (define* (%local-file file promise #:optional (name (basename file))
 288                       #:key recursive? (select? true))
 289   ;; This intermediate procedure is part of our ABI, but the underlying
 290   ;; %%LOCAL-FILE is not.
 291   (%%local-file file promise name recursive? select?))
 292
 293 (define (absolute-file-name file directory)
 294   "Return the canonical absolute file name for FILE, which lives in the
 295 vicinity of DIRECTORY."
 296   (canonicalize-path
 297    (cond ((string-prefix? "/" file) file)
 298          ((not directory) file)
 299          ((string-prefix? "/" directory)
 300           (string-append directory "/" file))
 301          (else file))))
 302
 303 (define-syntax local-file
 304   (lambda (s)
 305     "Return an object representing local file FILE to add to the store; this
 306 object can be used in a gexp.  If FILE is a relative file name, it is looked
 307 up relative to the source file where this form appears.  FILE will be added to
 308 the store under NAME--by default the base name of FILE.
 309
 310 When RECURSIVE? is true, the contents of FILE are added recursively; if FILE
 311 designates a flat file and RECURSIVE? is true, its contents are added, and its
 312 permission bits are kept.
 313
 314 When RECURSIVE? is true, call (SELECT?  FILE STAT) for each directory entry,
 315 where FILE is the entry's absolute file name and STAT is the result of
 316 'lstat'; exclude entries for which SELECT? does not return true.
 317
 318 This is the declarative counterpart of the 'interned-file' monadic procedure.
 319 It is implemented as a macro to capture the current source directory where it
 320 appears."
 321     (syntax-case s ()
 322       ((_ file rest ...)
 323        (string? (syntax->datum #'file))
 324        ;; FILE is a literal, so resolve it relative to the source directory.
 325        #'(%local-file file
 326                       (delay (absolute-file-name file (current-source-directory)))
 327                       rest ...))
 328       ((_ file rest ...)
 329        ;; Resolve FILE relative to the current directory.
 330        #'(%local-file file
 331                       (delay (absolute-file-name file (getcwd)))
 332                       rest ...))
 333       ((_)
 334        #'(syntax-error "missing file name"))
 335       (id
 336        (identifier? #'id)
 337        ;; XXX: We could return #'(lambda (file . rest) ...).  However,
 338        ;; (syntax-source #'id) is #f so (current-source-directory) would not
 339        ;; work.  Thus, simply forbid this form.
 340        #'(syntax-error
 341           "'local-file' is a macro and cannot be used like this")))))
 342
 343 (define (local-file-absolute-file-name file)
 344   "Return the absolute file name for FILE, a <local-file> instance.  A
 345 'system-error' exception is raised if FILE could not be found."
 346   (force (%local-file-absolute-file-name file)))
 347
 348 (define-gexp-compiler (local-file-compiler (file <local-file>) system target)
 349   ;; "Compile" FILE by adding it to the store.
 350   (match file
 351     (($ <local-file> file (= force absolute) name recursive? select?)
 352      ;; Canonicalize FILE so that if it's a symlink, it is resolved.  Failing
 353      ;; to do that, when RECURSIVE? is #t, we could end up creating a dangling
 354      ;; symlink in the store, and when RECURSIVE? is #f 'add-to-store' would
 355      ;; just throw an error, both of which are inconvenient.
 356      (interned-file absolute name
 357                     #:recursive? recursive? #:select? select?))))
 358
 359 (define-record-type <plain-file>
 360   (%plain-file name content references)
 361   plain-file?
 362   (name        plain-file-name)                   ;string
 363   (content     plain-file-content)                ;string or bytevector
 364   (references  plain-file-references))            ;list (currently unused)
 365
 366 (define (plain-file name content)
 367   "Return an object representing a text file called NAME with the given
 368 CONTENT (a string) to be added to the store.
 369
 370 This is the declarative counterpart of 'text-file'."
 371   ;; XXX: For now just ignore 'references' because it's not clear how to use
 372   ;; them in a declarative context.
 373   (%plain-file name content '()))
 374
 375 (define-gexp-compiler (plain-file-compiler (file <plain-file>) system target)
 376   ;; "Compile" FILE by adding it to the store.
 377   (match file
 378     (($ <plain-file> name (and (? string?) content) references)
 379      (text-file name content references))
 380     (($ <plain-file> name (and (? bytevector?) content) references)
 381      (binary-file name content references))))
 382
 383 (define-record-type <computed-file>
 384   (%computed-file name gexp guile options)
 385   computed-file?
 386   (name       computed-file-name)                 ;string
 387   (gexp       computed-file-gexp)                 ;gexp
 388   (guile      computed-file-guile)                ;<package>
 389   (options    computed-file-options))             ;list of arguments
 390
 391 (define* (computed-file name gexp
 392                         #:key guile (options '(#:local-build? #t)))
 393   "Return an object representing the store item NAME, a file or directory
 394 computed by GEXP.  OPTIONS is a list of additional arguments to pass
 395 to 'gexp->derivation'.
 396
 397 This is the declarative counterpart of 'gexp->derivation'."
 398   (%computed-file name gexp guile options))
 399
 400 (define-gexp-compiler (computed-file-compiler (file <computed-file>)
 401                                               system target)
 402   ;; Compile FILE by returning a derivation whose build expression is its
 403   ;; gexp.
 404   (match file
 405     (($ <computed-file> name gexp guile options)
 406      (if guile
 407          (mlet %store-monad ((guile (lower-object guile system
 408                                                   #:target target)))
 409            (apply gexp->derivation name gexp #:guile-for-build guile
 410                   #:system system #:target target options))
 411          (apply gexp->derivation name gexp
 412                 #:system system #:target target options)))))
 413
 414 (define-record-type <program-file>
 415   (%program-file name gexp guile path)
 416   program-file?
 417   (name       program-file-name)                  ;string
 418   (gexp       program-file-gexp)                  ;gexp
 419   (guile      program-file-guile)                 ;package
 420   (path       program-file-module-path))          ;list of strings
 421
 422 (define* (program-file name gexp #:key (guile #f) (module-path %load-path))
 423   "Return an object representing the executable store item NAME that runs
 424 GEXP.  GUILE is the Guile package used to execute that script.  Imported
 425 modules of GEXP are looked up in MODULE-PATH.
 426
 427 This is the declarative counterpart of 'gexp->script'."
 428   (%program-file name gexp guile module-path))
 429
 430 (define-gexp-compiler (program-file-compiler (file <program-file>)
 431                                              system target)
 432   ;; Compile FILE by returning a derivation that builds the script.
 433   (match file
 434     (($ <program-file> name gexp guile module-path)
 435      (gexp->script name gexp
 436                    #:module-path module-path
 437                    #:guile (or guile (default-guile))
 438                    #:system system
 439                    #:target target))))
 440
 441 (define-record-type <scheme-file>
 442   (%scheme-file name gexp splice?)
 443   scheme-file?
 444   (name       scheme-file-name)                  ;string
 445   (gexp       scheme-file-gexp)                  ;gexp
 446   (splice?    scheme-file-splice?))              ;Boolean
 447
 448 (define* (scheme-file name gexp #:key splice?)
 449   "Return an object representing the Scheme file NAME that contains GEXP.
 450
 451 This is the declarative counterpart of 'gexp->file'."
 452   (%scheme-file name gexp splice?))
 453
 454 (define-gexp-compiler (scheme-file-compiler (file <scheme-file>)
 455                                             system target)
 456   ;; Compile FILE by returning a derivation that builds the file.
 457   (match file
 458     (($ <scheme-file> name gexp splice?)
 459      (gexp->file name gexp #:splice? splice?))))
 460
 461 ;; Appending SUFFIX to BASE's output file name.
 462 (define-record-type <file-append>
 463   (%file-append base suffix)
 464   file-append?
 465   (base   file-append-base)                    ;<package> | <derivation> | ...
 466   (suffix file-append-suffix))                 ;list of strings
 467
 468 (define (write-file-append file port)
 469   (match file
 470     (($ <file-append> base suffix)
 471      (format port "#<file-append ~s ~s>" base
 472              (string-join suffix)))))
 473
 474 (set-record-type-printer! <file-append> write-file-append)
 475
 476 (define (file-append base . suffix)
 477   "Return a <file-append> object that expands to the concatenation of BASE and
 478 SUFFIX."
 479   (%file-append base suffix))
 480
 481 (define-gexp-compiler file-append-compiler <file-append>
 482   compiler => (lambda (obj system target)
 483                 (match obj
 484                   (($ <file-append> base _)
 485                    (lower-object base system #:target target))))
 486   expander => (lambda (obj lowered output)
 487                 (match obj
 488                   (($ <file-append> base suffix)
 489                    (let* ((expand (lookup-expander base))
 490                           (base   (expand base lowered output)))
 491                      (string-append base (string-concatenate suffix)))))))
 492
 493 \f
 494 ;;;
 495 ;;; Inputs & outputs.
 496 ;;;
 497
 498 ;; The input of a gexp.
 499 (define-record-type <gexp-input>
 500   (%gexp-input thing output native?)
 501   gexp-input?
 502   (thing     gexp-input-thing)       ;<package> | <origin> | <derivation> | ...
 503   (output    gexp-input-output)      ;string
 504   (native?   gexp-input-native?))    ;Boolean
 505
 506 (define (write-gexp-input input port)
 507   (match input
 508     (($ <gexp-input> thing output #f)
 509      (format port "#<gexp-input ~s:~a>" thing output))
 510     (($ <gexp-input> thing output #t)
 511      (format port "#<gexp-input native ~s:~a>" thing output))))
 512
 513 (set-record-type-printer! <gexp-input> write-gexp-input)
 514
 515 (define* (gexp-input thing                        ;convenience procedure
 516                      #:optional (output "out")
 517                      #:key native?)
 518   "Return a new <gexp-input> for the OUTPUT of THING; NATIVE? determines
 519 whether this should be considered a \"native\" input or not."
 520   (%gexp-input thing output native?))
 521
 522 ;; Reference to one of the derivation's outputs, for gexps used in
 523 ;; derivations.
 524 (define-record-type <gexp-output>
 525   (gexp-output name)
 526   gexp-output?
 527   (name gexp-output-name))
 528
 529 (define (write-gexp-output output port)
 530   (match output
 531     (($ <gexp-output> name)
 532      (format port "#<gexp-output ~a>" name))))
 533
 534 (set-record-type-printer! <gexp-output> write-gexp-output)
 535
 536 (define* (gexp-attribute gexp self-attribute #:optional (equal? equal?))
 537   "Recurse on GEXP and the expressions it refers to, summing the items
 538 returned by SELF-ATTRIBUTE, a procedure that takes a gexp.  Use EQUAL? as the
 539 second argument to 'delete-duplicates'."
 540   (if (gexp? gexp)
 541       (delete-duplicates
 542        (append (self-attribute gexp)
 543                (append-map (match-lambda
 544                              (($ <gexp-input> (? gexp? exp))
 545                               (gexp-attribute exp self-attribute))
 546                              (($ <gexp-input> (lst ...))
 547                               (append-map (lambda (item)
 548                                             (if (gexp? item)
 549                                                 (gexp-attribute item
 550                                                                 self-attribute)
 551                                                 '()))
 552                                           lst))
 553                              (_
 554                               '()))
 555                            (gexp-references gexp)))
 556        equal?)
 557       '()))                                       ;plain Scheme data type
 558
 559 (define (gexp-modules gexp)
 560   "Return the list of Guile module names GEXP relies on.  If (gexp? GEXP) is
 561 false, meaning that GEXP is a plain Scheme object, return the empty list."
 562   (define (module=? m1 m2)
 563     ;; Return #t when M1 equals M2.  Special-case '=>' specs because their
 564     ;; right-hand side may not be comparable with 'equal?': it's typically a
 565     ;; file-like object that embeds a gexp, which in turn embeds closure;
 566     ;; those closures may be 'eq?' when running compiled code but are unlikely
 567     ;; to be 'eq?' when running on 'eval'.  Ignore the right-hand side to
 568     ;; avoid this discrepancy.
 569     (match m1
 570       (((name1 ...) '=> _)
 571        (match m2
 572          (((name2 ...) '=> _) (equal? name1 name2))
 573          (_ #f)))
 574       (_
 575        (equal? m1 m2))))
 576
 577   (gexp-attribute gexp gexp-self-modules module=?))
 578
 579 (define (gexp-extensions gexp)
 580   "Return the list of Guile extensions (packages) GEXP relies on.  If (gexp?
 581 GEXP) is false, meaning that GEXP is a plain Scheme object, return the empty
 582 list."
 583   (gexp-attribute gexp gexp-self-extensions))
 584
 585 (define* (lower-inputs inputs
 586                        #:key system target)
 587   "Turn any object from INPUTS into a derivation input for SYSTEM or a store
 588 item (a \"source\"); return the corresponding input list as a monadic value.
 589 When TARGET is true, use it as the cross-compilation target triplet."
 590   (define (store-item? obj)
 591     (and (string? obj) (store-path? obj)))
 592
 593   (with-monad %store-monad
 594     (mapm %store-monad
 595           (match-lambda
 596             (((? struct? thing) sub-drv ...)
 597              (mlet %store-monad ((obj (lower-object
 598                                        thing system #:target target)))
 599                (return (match obj
 600                          ((? derivation? drv)
 601                           (let ((outputs (if (null? sub-drv)
 602                                              '("out")
 603                                              sub-drv)))
 604                             (derivation-input drv outputs)))
 605                          ((? store-item? item)
 606                           item)))))
 607             (((? store-item? item))
 608              (return item)))
 609           inputs)))
 610
 611 (define* (lower-reference-graphs graphs #:key system target)
 612   "Given GRAPHS, a list of (FILE-NAME INPUT ...) lists for use as a
 613 #:reference-graphs argument, lower it such that each INPUT is replaced by the
 614 corresponding <derivation-input> or store item."
 615   (match graphs
 616     (((file-names . inputs) ...)
 617      (mlet %store-monad ((inputs (lower-inputs inputs
 618                                                #:system system
 619                                                #:target target)))
 620        (return (map cons file-names inputs))))))
 621
 622 (define* (lower-references lst #:key system target)
 623   "Based on LST, a list of output names and packages, return a list of output
 624 names and file names suitable for the #:allowed-references argument to
 625 'derivation'."
 626   (with-monad %store-monad
 627     (define lower
 628       (match-lambda
 629        ((? string? output)
 630         (return output))
 631        (($ <gexp-input> thing output native?)
 632         (mlet %store-monad ((drv (lower-object thing system
 633                                                #:target (if native?
 634                                                             #f target))))
 635           (return (derivation->output-path drv output))))
 636        (thing
 637         (mlet %store-monad ((drv (lower-object thing system
 638                                                #:target target)))
 639           (return (derivation->output-path drv))))))
 640
 641     (mapm %store-monad lower lst)))
 642
 643 (define default-guile-derivation
 644   ;; Here we break the abstraction by talking to the higher-level layer.
 645   ;; Thus, do the resolution lazily to hide the circular dependency.
 646   (let ((proc (delay
 647                 (let ((iface (resolve-interface '(guix packages))))
 648                   (module-ref iface 'default-guile-derivation)))))
 649     (lambda (system)
 650       ((force proc) system))))
 651
 652 ;; Representation of a gexp instantiated for a given target and system.
 653 ;; It's an intermediate representation between <gexp> and <derivation>.
 654 (define-record-type <lowered-gexp>
 655   (lowered-gexp sexp inputs sources guile load-path load-compiled-path)
 656   lowered-gexp?
 657   (sexp                lowered-gexp-sexp)         ;sexp
 658   (inputs              lowered-gexp-inputs)       ;list of <derivation-input>
 659   (sources             lowered-gexp-sources)      ;list of store items
 660   (guile               lowered-gexp-guile)        ;<derivation-input> | #f
 661   (load-path           lowered-gexp-load-path)    ;list of store items
 662   (load-compiled-path  lowered-gexp-load-compiled-path)) ;list of store items
 663
 664 (define* (imported+compiled-modules modules system
 665                                     #:key (extensions '())
 666                                     deprecation-warnings guile
 667                                     (module-path %load-path))
 668   "Return a pair where the first element is the imported MODULES and the
 669 second element is the derivation to compile them."
 670   (mcached equal?
 671            (mlet %store-monad ((modules  (if (pair? modules)
 672                                              (imported-modules modules
 673                                                                #:system system
 674                                                                #:module-path module-path)
 675                                              (return #f)))
 676                                (compiled (if (pair? modules)
 677                                              (compiled-modules modules
 678                                                                #:system system
 679                                                                #:module-path module-path
 680                                                                #:extensions extensions
 681                                                                #:guile guile
 682                                                                #:deprecation-warnings
 683                                                                deprecation-warnings)
 684                                              (return #f))))
 685              (return (cons modules compiled)))
 686            modules
 687            system extensions guile deprecation-warnings module-path))
 688
 689 (define* (lower-gexp exp
 690                      #:key
 691                      (module-path %load-path)
 692                      (system (%current-system))
 693                      (target 'current)
 694                      (graft? (%graft?))
 695                      (guile-for-build (%guile-for-build))
 696                      (effective-version "2.2")
 697
 698                      deprecation-warnings)
 699   "*Note: This API is subject to change; use at your own risk!*
 700
 701 Lower EXP, a gexp, instantiating it for SYSTEM and TARGET.  Return a
 702 <lowered-gexp> ready to be used.
 703
 704 Lowered gexps are an intermediate representation that's useful for
 705 applications that deal with gexps outside in a way that is disconnected from
 706 derivations--e.g., code evaluated for its side effects."
 707   (define %modules
 708     (delete-duplicates (gexp-modules exp)))
 709
 710   (define (search-path modules extensions suffix)
 711     (append (match modules
 712               ((? derivation? drv)
 713                (list (derivation->output-path drv)))
 714               (#f
 715                '())
 716               ((? store-path? item)
 717                (list item)))
 718             (map (lambda (extension)
 719                    (string-append (match extension
 720                                     ((? derivation? drv)
 721                                      (derivation->output-path drv))
 722                                     ((? store-path? item)
 723                                      item))
 724                                   suffix))
 725                  extensions)))
 726
 727   (mlet* %store-monad ( ;; The following binding forces '%current-system' and
 728                        ;; '%current-target-system' to be looked up at >>=
 729                        ;; time.
 730                        (graft?    (set-grafting graft?))
 731
 732                        (system -> (or system (%current-system)))
 733                        (target -> (if (eq? target 'current)
 734                                       (%current-target-system)
 735                                       target))
 736                        (guile     (if guile-for-build
 737                                       (return guile-for-build)
 738                                       (default-guile-derivation system)))
 739                        (normals  (lower-inputs (gexp-inputs exp)
 740                                                #:system system
 741                                                #:target target))
 742                        (natives  (lower-inputs (gexp-native-inputs exp)
 743                                                #:system system
 744                                                #:target #f))
 745                        (inputs -> (append normals natives))
 746                        (sexp     (gexp->sexp exp
 747                                              #:system system
 748                                              #:target target))
 749                        (extensions -> (gexp-extensions exp))
 750                        (exts     (mapm %store-monad
 751                                        (lambda (obj)
 752                                          (lower-object obj system))
 753                                        extensions))
 754                        (modules+compiled (imported+compiled-modules
 755                                           %modules system
 756                                           #:extensions extensions
 757                                           #:deprecation-warnings
 758                                           deprecation-warnings
 759                                           #:guile guile
 760                                           #:module-path module-path))
 761                        (modules ->  (car modules+compiled))
 762                        (compiled -> (cdr modules+compiled)))
 763     (define load-path
 764       (search-path modules exts
 765                    (string-append "/share/guile/site/" effective-version)))
 766
 767     (define load-compiled-path
 768       (search-path compiled exts
 769                    (string-append "/lib/guile/" effective-version
 770                                   "/site-ccache")))
 771
 772     (mbegin %store-monad
 773       (set-grafting graft?)                       ;restore the initial setting
 774       (return (lowered-gexp sexp
 775                             `(,@(if (derivation? modules)
 776                                     (list (derivation-input modules))
 777                                     '())
 778                               ,@(if compiled
 779                                     (list (derivation-input compiled))
 780                                     '())
 781                               ,@(map derivation-input exts)
 782                               ,@(filter derivation-input? inputs))
 783                             (filter string? (cons modules inputs))
 784                             (derivation-input guile '("out"))
 785                             load-path
 786                             load-compiled-path)))))
 787
 788 (define* (gexp->derivation name exp
 789                            #:key
 790                            system (target 'current)
 791                            hash hash-algo recursive?
 792                            (env-vars '())
 793                            (modules '())
 794                            (module-path %load-path)
 795                            (guile-for-build (%guile-for-build))
 796                            (effective-version "2.2")
 797                            (graft? (%graft?))
 798                            references-graphs
 799                            allowed-references disallowed-references
 800                            leaked-env-vars
 801                            local-build? (substitutable? #t)
 802                            (properties '())
 803                            deprecation-warnings
 804                            (script-name (string-append name "-builder")))
 805   "Return a derivation NAME that runs EXP (a gexp) with GUILE-FOR-BUILD (a
 806 derivation) on SYSTEM; EXP is stored in a file called SCRIPT-NAME.  When
 807 TARGET is true, it is used as the cross-compilation target triplet for
 808 packages referred to by EXP.
 809
 810 MODULES is deprecated in favor of 'with-imported-modules'.  Its meaning is to
 811 make MODULES available in the evaluation context of EXP; MODULES is a list of
 812 names of Guile modules searched in MODULE-PATH to be copied in the store,
 813 compiled, and made available in the load path during the execution of
 814 EXP---e.g., '((guix build utils) (guix build gnu-build-system)).
 815
 816 EFFECTIVE-VERSION determines the string to use when adding extensions of
 817 EXP (see 'with-extensions') to the search path---e.g., \"2.2\".
 818
 819 GRAFT? determines whether packages referred to by EXP should be grafted when
 820 applicable.
 821
 822 When REFERENCES-GRAPHS is true, it must be a list of tuples of one of the
 823 following forms:
 824
 825   (FILE-NAME PACKAGE)
 826   (FILE-NAME PACKAGE OUTPUT)
 827   (FILE-NAME DERIVATION)
 828   (FILE-NAME DERIVATION OUTPUT)
 829   (FILE-NAME STORE-ITEM)
 830
 831 The right-hand-side of each element of REFERENCES-GRAPHS is automatically made
 832 an input of the build process of EXP.  In the build environment, each
 833 FILE-NAME contains the reference graph of the corresponding item, in a simple
 834 text format.
 835
 836 ALLOWED-REFERENCES must be either #f or a list of output names and packages.
 837 In the latter case, the list denotes store items that the result is allowed to
 838 refer to.  Any reference to another store item will lead to a build error.
 839 Similarly for DISALLOWED-REFERENCES, which can list items that must not be
 840 referenced by the outputs.
 841
 842 DEPRECATION-WARNINGS determines whether to show deprecation warnings while
 843 compiling modules.  It can be #f, #t, or 'detailed.
 844
 845 The other arguments are as for 'derivation'."
 846   (define outputs (gexp-outputs exp))
 847   (define requested-graft? graft?)
 848
 849   (define (graphs-file-names graphs)
 850     ;; Return a list of (FILE-NAME . STORE-PATH) pairs made from GRAPHS.
 851     (map (match-lambda
 852            ((file-name . (? derivation-input? input))
 853             (cons file-name (first (derivation-input-output-paths input))))
 854            ((file-name . (? string? item))
 855             (cons file-name item)))
 856          graphs))
 857
 858   (define (add-modules exp modules)
 859     (if (null? modules)
 860         exp
 861         (make-gexp (gexp-references exp)
 862                    (append modules (gexp-self-modules exp))
 863                    (gexp-self-extensions exp)
 864                    (gexp-proc exp))))
 865
 866   (mlet* %store-monad ( ;; The following binding forces '%current-system' and
 867                        ;; '%current-target-system' to be looked up at >>=
 868                        ;; time.
 869                        (graft?    (set-grafting graft?))
 870
 871                        (system -> (or system (%current-system)))
 872                        (target -> (if (eq? target 'current)
 873                                       (%current-target-system)
 874                                       target))
 875                        (exp ->    (add-modules exp modules))
 876                        (lowered   (lower-gexp exp
 877                                               #:module-path module-path
 878                                               #:system system
 879                                               #:target target
 880                                               #:graft? requested-graft?
 881                                               #:guile-for-build
 882                                               guile-for-build
 883                                               #:effective-version
 884                                               effective-version
 885                                               #:deprecation-warnings
 886                                               deprecation-warnings))
 887
 888                        (graphs   (if references-graphs
 889                                      (lower-reference-graphs references-graphs
 890                                                              #:system system
 891                                                              #:target target)
 892                                      (return #f)))
 893                        (allowed  (if allowed-references
 894                                      (lower-references allowed-references
 895                                                        #:system system
 896                                                        #:target target)
 897                                      (return #f)))
 898                        (disallowed (if disallowed-references
 899                                        (lower-references disallowed-references
 900                                                          #:system system
 901                                                          #:target target)
 902                                        (return #f)))
 903                        (guile -> (lowered-gexp-guile lowered))
 904                        (builder  (text-file script-name
 905                                             (object->string
 906                                              (lowered-gexp-sexp lowered)))))
 907     (mbegin %store-monad
 908       (set-grafting graft?)                       ;restore the initial setting
 909       (raw-derivation name
 910                       (string-append (derivation-input-output-path guile)
 911                                      "/bin/guile")
 912                       `("--no-auto-compile"
 913                         ,@(append-map (lambda (directory)
 914                                         `("-L" ,directory))
 915                                       (lowered-gexp-load-path lowered))
 916                         ,@(append-map (lambda (directory)
 917                                         `("-C" ,directory))
 918                                       (lowered-gexp-load-compiled-path lowered))
 919                         ,builder)
 920                       #:outputs outputs
 921                       #:env-vars env-vars
 922                       #:system system
 923                       #:inputs `(,guile
 924                                  ,@(lowered-gexp-inputs lowered)
 925                                  ,@(match graphs
 926                                      (((_ . inputs) ...)
 927                                       (filter derivation-input? inputs))
 928                                      (#f '())))
 929                       #:sources `(,builder
 930                                   ,@(if (and (string? modules)
 931                                              (store-path? modules))
 932                                         (list modules)
 933                                         '())
 934                                   ,@(lowered-gexp-sources lowered)
 935                                   ,@(match graphs
 936                                       (((_ . inputs) ...)
 937                                        (filter string? inputs))
 938                                       (#f '())))
 939
 940                       #:hash hash #:hash-algo hash-algo #:recursive? recursive?
 941                       #:references-graphs (and=> graphs graphs-file-names)
 942                       #:allowed-references allowed
 943                       #:disallowed-references disallowed
 944                       #:leaked-env-vars leaked-env-vars
 945                       #:local-build? local-build?
 946                       #:substitutable? substitutable?
 947                       #:properties properties))))
 948
 949 (define* (gexp-inputs exp #:key native?)
 950   "Return the input list for EXP.  When NATIVE? is true, return only native
 951 references; otherwise, return only non-native references."
 952   ;; TODO: Return <gexp-input> records instead of tuples.
 953   (define (add-reference-inputs ref result)
 954     (match ref
 955       (($ <gexp-input> (? gexp? exp) _ #t)
 956        (if native?
 957            (append (gexp-inputs exp)
 958                    (gexp-inputs exp #:native? #t)
 959                    result)
 960            result))
 961       (($ <gexp-input> (? gexp? exp) _ #f)
 962        (append (gexp-inputs exp #:native? native?)
 963                result))
 964       (($ <gexp-input> (? string? str))
 965        (if (direct-store-path? str)
 966            (cons `(,str) result)
 967            result))
 968       (($ <gexp-input> (? struct? thing) output n?)
 969        (if (and (eqv? n? native?) (lookup-compiler thing))
 970            ;; THING is a derivation, or a package, or an origin, etc.
 971            (cons `(,thing ,output) result)
 972            result))
 973       (($ <gexp-input> (lst ...) output n?)
 974        (fold-right add-reference-inputs result
 975                    ;; XXX: For now, automatically convert LST to a list of
 976                    ;; gexp-inputs.  Inherit N?.
 977                    (map (match-lambda
 978                           ((? gexp-input? x)
 979                            (%gexp-input (gexp-input-thing x)
 980                                         (gexp-input-output x)
 981                                         n?))
 982                           (x
 983                            (%gexp-input x "out" n?)))
 984                         lst)))
 985       (_
 986        ;; Ignore references to other kinds of objects.
 987        result)))
 988
 989   (fold-right add-reference-inputs
 990               '()
 991               (gexp-references exp)))
 992
 993 (define gexp-native-inputs
 994   (cut gexp-inputs <> #:native? #t))
 995
 996 (define (gexp-outputs exp)
 997   "Return the outputs referred to by EXP as a list of strings."
 998   (define (add-reference-output ref result)
 999     (match ref
1000       (($ <gexp-output> name)
1001        (cons name result))
1002       (($ <gexp-input> (? gexp? exp))
1003        (append (gexp-outputs exp) result))
1004       (($ <gexp-input> (lst ...) output native?)
1005        ;; XXX: Automatically convert LST.
1006        (add-reference-output (map (match-lambda
1007                                    ((? gexp-input? x) x)
1008                                    (x (%gexp-input x "out" native?)))
1009                                   lst)
1010                              result))
1011       ((lst ...)
1012        (fold-right add-reference-output result lst))
1013       (_
1014        result)))
1015
1016   (delete-duplicates
1017    (add-reference-output (gexp-references exp) '())))
1018
1019 (define* (gexp->sexp exp #:key
1020                      (system (%current-system))
1021                      (target (%current-target-system)))
1022   "Return (monadically) the sexp corresponding to EXP for the given OUTPUT,
1023 and in the current monad setting (system type, etc.)"
1024   (define (self-quoting? x)
1025     (letrec-syntax ((one-of (syntax-rules ()
1026                               ((_) #f)
1027                               ((_ pred rest ...)
1028                                (or (pred x)
1029                                    (one-of rest ...))))))
1030       (one-of symbol? string? keyword? pair? null? array?
1031               number? boolean?)))
1032
1033   (define* (reference->sexp ref #:optional native?)
1034     (with-monad %store-monad
1035       (match ref
1036         (($ <gexp-output> output)
1037          ;; Output file names are not known in advance but the daemon defines
1038          ;; an environment variable for each of them at build time, so use
1039          ;; that trick.
1040          (return `((@ (guile) getenv) ,output)))
1041         (($ <gexp-input> (? gexp? exp) output n?)
1042          (gexp->sexp exp
1043                      #:system system
1044                      #:target (if (or n? native?) #f target)))
1045         (($ <gexp-input> (refs ...) output n?)
1046          (mapm %store-monad
1047                (lambda (ref)
1048                  ;; XXX: Automatically convert REF to an gexp-input.
1049                  (reference->sexp
1050                   (if (gexp-input? ref)
1051                       ref
1052                       (%gexp-input ref "out" n?))
1053                   (or n? native?)))
1054                refs))
1055         (($ <gexp-input> (? struct? thing) output n?)
1056          (let ((target (if (or n? native?) #f target))
1057                (expand (lookup-expander thing)))
1058            (mlet %store-monad ((obj (lower-object thing system
1059                                                   #:target target)))
1060              ;; OBJ must be either a derivation or a store file name.
1061              (return (expand thing obj output)))))
1062         (($ <gexp-input> (? self-quoting? x))
1063          (return x))
1064         (($ <gexp-input> x)
1065          (raise (condition (&gexp-input-error (input x)))))
1066         (x
1067          (return x)))))
1068
1069   (mlet %store-monad
1070       ((args (mapm %store-monad
1071                    reference->sexp (gexp-references exp))))
1072     (return (apply (gexp-proc exp) args))))
1073
1074 (define-syntax-rule (define-syntax-parameter-once name proc)
1075   ;; Like 'define-syntax-parameter' but ensure the top-level binding for NAME
1076   ;; does not get redefined.  This works around a race condition in a
1077   ;; multi-threaded context with Guile <= 2.2.4: <https://bugs.gnu.org/27476>.
1078   (eval-when (load eval expand compile)
1079     (define name
1080       (if (module-locally-bound? (current-module) 'name)
1081           (module-ref (current-module) 'name)
1082           (make-syntax-transformer 'name 'syntax-parameter
1083                                    (list proc))))))
1084
1085 (define-syntax-parameter-once current-imported-modules
1086   ;; Current list of imported modules.
1087   (identifier-syntax '()))
1088
1089 (define-syntax-rule (with-imported-modules modules body ...)
1090   "Mark the gexps defined in BODY... as requiring MODULES in their execution
1091 environment."
1092   (syntax-parameterize ((current-imported-modules
1093                          (identifier-syntax modules)))
1094     body ...))
1095
1096 (define-syntax-parameter-once current-imported-extensions
1097   ;; Current list of extensions.
1098   (identifier-syntax '()))
1099
1100 (define-syntax-rule (with-extensions extensions body ...)
1101   "Mark the gexps defined in BODY... as requiring EXTENSIONS in their
1102 execution environment."
1103   (syntax-parameterize ((current-imported-extensions
1104                          (identifier-syntax extensions)))
1105     body ...))
1106
1107 (define-syntax gexp
1108   (lambda (s)
1109     (define (collect-escapes exp)
1110       ;; Return all the 'ungexp' present in EXP.
1111       (let loop ((exp    exp)
1112                  (result '()))
1113         (syntax-case exp (ungexp
1114                           ungexp-splicing
1115                           ungexp-native
1116                           ungexp-native-splicing)
1117           ((ungexp _)
1118            (cons exp result))
1119           ((ungexp _ _)
1120            (cons exp result))
1121           ((ungexp-splicing _ ...)
1122            (cons exp result))
1123           ((ungexp-native _ ...)
1124            (cons exp result))
1125           ((ungexp-native-splicing _ ...)
1126            (cons exp result))
1127           ((exp0 . exp)
1128            (let ((result (loop #'exp0 result)))
1129              (loop  #'exp result)))
1130           (_
1131            result))))
1132
1133     (define (escape->ref exp)
1134       ;; Turn 'ungexp' form EXP into a "reference".
1135       (syntax-case exp (ungexp ungexp-splicing
1136                         ungexp-native ungexp-native-splicing
1137                         output)
1138         ((ungexp output)
1139          #'(gexp-output "out"))
1140         ((ungexp output name)
1141          #'(gexp-output name))
1142         ((ungexp thing)
1143          #'(%gexp-input thing "out" #f))
1144         ((ungexp drv-or-pkg out)
1145          #'(%gexp-input drv-or-pkg out #f))
1146         ((ungexp-splicing lst)
1147          #'(%gexp-input lst "out" #f))
1148         ((ungexp-native thing)
1149          #'(%gexp-input thing "out" #t))
1150         ((ungexp-native drv-or-pkg out)
1151          #'(%gexp-input drv-or-pkg out #t))
1152         ((ungexp-native-splicing lst)
1153          #'(%gexp-input lst "out" #t))))
1154
1155     (define (substitute-ungexp exp substs)
1156       ;; Given EXP, an 'ungexp' or 'ungexp-native' form, substitute it with
1157       ;; the corresponding form in SUBSTS.
1158       (match (assoc exp substs)
1159         ((_ id)
1160          id)
1161         (_                                        ;internal error
1162          (with-syntax ((exp exp))
1163            #'(syntax-error "error: no 'ungexp' substitution" exp)))))
1164
1165     (define (substitute-ungexp-splicing exp substs)
1166       (syntax-case exp ()
1167         ((exp rest ...)
1168          (match (assoc #'exp substs)
1169            ((_ id)
1170             (with-syntax ((id id))
1171               #`(append id
1172                         #,(substitute-references #'(rest ...) substs))))
1173            (_
1174             #'(syntax-error "error: no 'ungexp-splicing' substitution"
1175                             exp))))))
1176
1177     (define (substitute-references exp substs)
1178       ;; Return a variant of EXP where all the cars of SUBSTS have been
1179       ;; replaced by the corresponding cdr.
1180       (syntax-case exp (ungexp ungexp-native
1181                         ungexp-splicing ungexp-native-splicing)
1182         ((ungexp _ ...)
1183          (substitute-ungexp exp substs))
1184         ((ungexp-native _ ...)
1185          (substitute-ungexp exp substs))
1186         (((ungexp-splicing _ ...) rest ...)
1187          (substitute-ungexp-splicing exp substs))
1188         (((ungexp-native-splicing _ ...) rest ...)
1189          (substitute-ungexp-splicing exp substs))
1190         ((exp0 . exp)
1191          #`(cons #,(substitute-references #'exp0 substs)
1192                  #,(substitute-references #'exp substs)))
1193         (x #''x)))
1194
1195     (syntax-case s (ungexp output)
1196       ((_ exp)
1197        (let* ((escapes (delete-duplicates (collect-escapes #'exp)))
1198               (formals (generate-temporaries escapes))
1199               (sexp    (substitute-references #'exp (zip escapes formals)))
1200               (refs    (map escape->ref escapes)))
1201          #`(make-gexp (list #,@refs)
1202                       current-imported-modules
1203                       current-imported-extensions
1204                       (lambda #,formals
1205                         #,sexp)))))))
1206
1207 \f
1208 ;;;
1209 ;;; Module handling.
1210 ;;;
1211
1212 (define %not-slash
1213   (char-set-complement (char-set #\/)))
1214
1215 (define (file-mapping->tree mapping)
1216   "Convert MAPPING, an alist like:
1217
1218   ((\"guix/build/utils.scm\" . \"…/utils.scm\"))
1219
1220 to a tree suitable for 'interned-file-tree'."
1221   (let ((mapping (map (match-lambda
1222                         ((destination . source)
1223                          (cons (string-tokenize destination
1224                                                 %not-slash)
1225                                source)))
1226                       mapping)))
1227     (fold (lambda (pair result)
1228             (match pair
1229               ((destination . source)
1230                (let loop ((destination destination)
1231                           (result result))
1232                  (match destination
1233                    ((file)
1234                     (let* ((mode (stat:mode (stat source)))
1235                            (type (if (zero? (logand mode #o100))
1236                                      'regular
1237                                      'executable)))
1238                       (alist-cons file
1239                                   `(,type (file ,source))
1240                                   result)))
1241                    ((file rest ...)
1242                     (let ((directory (assoc-ref result file)))
1243                       (alist-cons file
1244                                   `(directory
1245                                     ,@(loop rest
1246                                             (match directory
1247                                               (('directory . entries) entries)
1248                                               (#f '()))))
1249                                   (if directory
1250                                       (alist-delete file result)
1251                                       result)))))))))
1252           '()
1253           mapping)))
1254
1255 (define %utils-module
1256   ;; This file provides 'mkdir-p', needed to implement 'imported-files' and
1257   ;; other primitives below.  Note: We give the file name relative to this
1258   ;; file you are currently reading; 'search-path' could return a file name
1259   ;; relative to the current working directory.
1260   (local-file "build/utils.scm"
1261               "build-utils.scm"))
1262
1263 (define* (imported-files/derivation files
1264                                     #:key (name "file-import")
1265                                     (symlink? #f)
1266                                     (system (%current-system))
1267                                     (guile (%guile-for-build)))
1268   "Return a derivation that imports FILES into STORE.  FILES must be a list
1269 of (FINAL-PATH . FILE) pairs.  Each FILE is mapped to FINAL-PATH in the
1270 resulting store path.  FILE can be either a file name, or a file-like object,
1271 as returned by 'local-file' for example.  If SYMLINK? is true, create symlinks
1272 to the source files instead of copying them."
1273   (define file-pair
1274     (match-lambda
1275      ((final-path . (? string? file-name))
1276       (mlet %store-monad ((file (interned-file file-name
1277                                                (basename final-path))))
1278         (return (list final-path file))))
1279      ((final-path . file-like)
1280       (mlet %store-monad ((file (lower-object file-like system)))
1281         (return (list final-path file))))))
1282
1283   (mlet %store-monad ((files (mapm %store-monad file-pair files)))
1284     (define build
1285       (gexp
1286        (begin
1287          (primitive-load (ungexp %utils-module))  ;for 'mkdir-p'
1288          (use-modules (ice-9 match))
1289
1290          (mkdir (ungexp output)) (chdir (ungexp output))
1291          (for-each (match-lambda
1292                     ((final-path store-path)
1293                      (mkdir-p (dirname final-path))
1294                      ((ungexp (if symlink? 'symlink 'copy-file))
1295                       store-path final-path)))
1296                    '(ungexp files)))))
1297
1298     ;; TODO: Pass FILES as an environment variable so that BUILD remains
1299     ;; exactly the same regardless of FILES: less disk space, and fewer
1300     ;; 'add-to-store' RPCs.
1301     (gexp->derivation name build
1302                       #:system system
1303                       #:guile-for-build guile
1304                       #:local-build? #t
1305
1306                       ;; Avoid deprecation warnings about the use of the _IO*
1307                       ;; constants in (guix build utils).
1308                       #:env-vars
1309                       '(("GUILE_WARN_DEPRECATED" . "no")))))
1310
1311 (define* (imported-files files
1312                          #:key (name "file-import")
1313                          ;; The following parameters make sense when creating
1314                          ;; an actual derivation.
1315                          (system (%current-system))
1316                          (guile (%guile-for-build)))
1317   "Import FILES into the store and return the resulting derivation or store
1318 file name (a derivation is created if and only if some elements of FILES are
1319 file-like objects and not local file names.)  FILES must be a list
1320 of (FINAL-PATH . FILE) pairs.  Each FILE is mapped to FINAL-PATH in the
1321 resulting store path.  FILE can be either a file name, or a file-like object,
1322 as returned by 'local-file' for example."
1323   (if (any (match-lambda
1324              ((_ . (? struct? source)) #t)
1325              (_ #f))
1326            files)
1327       (imported-files/derivation files #:name name
1328                                  #:symlink? derivation?
1329                                  #:system system #:guile guile)
1330       (interned-file-tree `(,name directory
1331                                   ,@(file-mapping->tree files)))))
1332
1333 (define* (imported-modules modules
1334                            #:key (name "module-import")
1335                            (system (%current-system))
1336                            (guile (%guile-for-build))
1337                            (module-path %load-path))
1338   "Return a derivation that contains the source files of MODULES, a list of
1339 module names such as `(ice-9 q)'.  All of MODULES must be either names of
1340 modules to be found in the MODULE-PATH search path, or a module name followed
1341 by an arrow followed by a file-like object.  For example:
1342
1343   (imported-modules `((guix build utils)
1344                       (guix gcrypt)
1345                       ((guix config) => ,(scheme-file …))))
1346
1347 In this example, the first two modules are taken from MODULE-PATH, and the
1348 last one is created from the given <scheme-file> object."
1349   (let ((files (map (match-lambda
1350                       (((module ...) '=> file)
1351                        (cons (module->source-file-name module)
1352                              file))
1353                       ((module ...)
1354                        (let ((f (module->source-file-name module)))
1355                          (cons f (search-path* module-path f)))))
1356                     modules)))
1357     (imported-files files #:name name
1358                     #:system system
1359                     #:guile guile)))
1360
1361 (define* (compiled-modules modules
1362                            #:key (name "module-import-compiled")
1363                            (system (%current-system))
1364                            target
1365                            (guile (%guile-for-build))
1366                            (module-path %load-path)
1367                            (extensions '())
1368                            (deprecation-warnings #f))
1369   "Return a derivation that builds a tree containing the `.go' files
1370 corresponding to MODULES.  All the MODULES are built in a context where
1371 they can refer to each other.  When TARGET is true, cross-compile MODULES for
1372 TARGET, a GNU triplet."
1373   (define total (length modules))
1374
1375   (mlet %store-monad ((modules (imported-modules modules
1376                                                  #:system system
1377                                                  #:guile guile
1378                                                  #:module-path
1379                                                  module-path)))
1380     (define build
1381       (gexp
1382        (begin
1383          (primitive-load (ungexp %utils-module))  ;for 'mkdir-p'
1384
1385          (use-modules (ice-9 ftw)
1386                       (ice-9 format)
1387                       (srfi srfi-1)
1388                       (srfi srfi-26)
1389                       (system base compile))
1390
1391          ;; TODO: Inline this on the next rebuild cycle.
1392          (ungexp-splicing
1393           (if target
1394               (gexp ((use-modules (system base target))))
1395               (gexp ())))
1396
1397          (define (regular? file)
1398            (not (member file '("." ".."))))
1399
1400          (define (process-entry entry output processed)
1401            (if (file-is-directory? entry)
1402                (let ((output (string-append output "/" (basename entry))))
1403                  (mkdir-p output)
1404                  (process-directory entry output processed))
1405                (let* ((base   (basename entry ".scm"))
1406                       (output (string-append output "/" base ".go")))
1407                  (format #t "[~2@a/~2@a] Compiling '~a'...~%"
1408                          (+ 1 processed (ungexp total))
1409                          (ungexp (* total 2))
1410                          entry)
1411
1412                  (ungexp-splicing
1413                   (if target
1414                       (gexp ((with-target (ungexp target)
1415                                (lambda ()
1416                                  (compile-file entry
1417                                                #:output-file output
1418                                                #:opts
1419                                                %auto-compilation-options)))))
1420                       (gexp ((compile-file entry
1421                                            #:output-file output
1422                                            #:opts %auto-compilation-options)))))
1423
1424                  (+ 1 processed))))
1425
1426          (define (process-directory directory output processed)
1427            (let ((entries (map (cut string-append directory "/" <>)
1428                                (scandir directory regular?))))
1429              (fold (cut process-entry <> output <>)
1430                    processed
1431                    entries)))
1432
1433          (define* (load-from-directory directory
1434                                        #:optional (loaded 0))
1435            "Load all the source files found in DIRECTORY."
1436            ;; XXX: This works around <https://bugs.gnu.org/15602>.
1437            (let ((entries (map (cut string-append directory "/" <>)
1438                                (scandir directory regular?))))
1439              (fold (lambda (file loaded)
1440                      (if (file-is-directory? file)
1441                          (load-from-directory file loaded)
1442                          (begin
1443                            (format #t "[~2@a/~2@a] Loading '~a'...~%"
1444                                    (+ 1 loaded) (ungexp (* 2 total))
1445                                    file)
1446                            (save-module-excursion
1447                             (lambda ()
1448                               (primitive-load file)))
1449                            (+ 1 loaded))))
1450                    loaded
1451                    entries)))
1452
1453          (setvbuf (current-output-port)
1454                   (cond-expand (guile-2.2 'line) (else _IOLBF)))
1455
1456          (define mkdir-p
1457            ;; Capture 'mkdir-p'.
1458            (@ (guix build utils) mkdir-p))
1459
1460          ;; Add EXTENSIONS to the search path.
1461          (set! %load-path
1462            (append (map (lambda (extension)
1463                           (string-append extension
1464                                          "/share/guile/site/"
1465                                          (effective-version)))
1466                         '((ungexp-native-splicing extensions)))
1467                    %load-path))
1468          (set! %load-compiled-path
1469            (append (map (lambda (extension)
1470                           (string-append extension "/lib/guile/"
1471                                          (effective-version)
1472                                          "/site-ccache"))
1473                         '((ungexp-native-splicing extensions)))
1474                    %load-compiled-path))
1475
1476          (set! %load-path (cons (ungexp modules) %load-path))
1477
1478          ;; Above we loaded our own (guix build utils) but now we may need to
1479          ;; load a compile a different one.  Thus, force a reload.
1480          (let ((utils (string-append (ungexp modules)
1481                                      "/guix/build/utils.scm")))
1482            (when (file-exists? utils)
1483              (load utils)))
1484
1485          (mkdir (ungexp output))
1486          (chdir (ungexp modules))
1487
1488          (load-from-directory ".")
1489          (process-directory "." (ungexp output) 0))))
1490
1491     ;; TODO: Pass MODULES as an environment variable.
1492     (gexp->derivation name build
1493                       #:system system
1494                       #:guile-for-build guile
1495                       #:local-build? #t
1496                       #:env-vars
1497                       (case deprecation-warnings
1498                         ((#f)
1499                          '(("GUILE_WARN_DEPRECATED" . "no")))
1500                         ((detailed)
1501                          '(("GUILE_WARN_DEPRECATED" . "detailed")))
1502                         (else
1503                          '())))))
1504
1505 \f
1506 ;;;
1507 ;;; Convenience procedures.
1508 ;;;
1509
1510 (define (default-guile)
1511   ;; Lazily resolve 'guile-2.2' (not 'guile-final' because this is for
1512   ;; programs returned by 'program-file' and we don't want to keep references
1513   ;; to several Guile packages).  This module must not refer to (gnu …)
1514   ;; modules directly, to avoid circular dependencies, hence this hack.
1515   (module-ref (resolve-interface '(gnu packages guile))
1516               'guile-2.2))
1517
1518 (define* (load-path-expression modules #:optional (path %load-path)
1519                                #:key (extensions '()) system target)
1520   "Return as a monadic value a gexp that sets '%load-path' and
1521 '%load-compiled-path' to point to MODULES, a list of module names.  MODULES
1522 are searched for in PATH.  Return #f when MODULES and EXTENSIONS are empty."
1523   (if (and (null? modules) (null? extensions))
1524       (with-monad %store-monad
1525         (return #f))
1526       (mlet %store-monad ((modules  (imported-modules modules
1527                                                       #:module-path path
1528                                                       #:system system))
1529                           (compiled (compiled-modules modules
1530                                                       #:extensions extensions
1531                                                       #:module-path path
1532                                                       #:system system
1533                                                       #:target target)))
1534         (return
1535          (gexp (eval-when (expand load eval)
1536                  ;; Augment the load paths and delete duplicates.  Do that
1537                  ;; without loading (srfi srfi-1) or anything.
1538                  (let ((extensions '((ungexp-splicing extensions)))
1539                        (prepend (lambda (items lst)
1540                                   ;; This is O(N²) but N is typically small.
1541                                   (let loop ((items items)
1542                                              (lst lst))
1543                                     (if (null? items)
1544                                         lst
1545                                         (loop (cdr items)
1546                                               (cons (car items)
1547                                                     (delete (car items) lst))))))))
1548                    (set! %load-path
1549                      (prepend (cons (ungexp modules)
1550                                     (map (lambda (extension)
1551                                            (string-append extension
1552                                                           "/share/guile/site/"
1553                                                           (effective-version)))
1554                                          extensions))
1555                               %load-path))
1556                    (set! %load-compiled-path
1557                      (prepend (cons (ungexp compiled)
1558                                     (map (lambda (extension)
1559                                            (string-append extension
1560                                                           "/lib/guile/"
1561                                                           (effective-version)
1562                                                           "/site-ccache"))
1563                                          extensions))
1564                               %load-compiled-path)))))))))
1565
1566 (define* (gexp->script name exp
1567                        #:key (guile (default-guile))
1568                        (module-path %load-path)
1569                        (system (%current-system))
1570                        target)
1571   "Return an executable script NAME that runs EXP using GUILE, with EXP's
1572 imported modules in its search path.  Look up EXP's modules in MODULE-PATH."
1573   (mlet %store-monad ((set-load-path
1574                        (load-path-expression (gexp-modules exp)
1575                                              module-path
1576                                              #:extensions
1577                                              (gexp-extensions exp)
1578                                              #:system system
1579                                              #:target target)))
1580     (gexp->derivation name
1581                       (gexp
1582                        (call-with-output-file (ungexp output)
1583                          (lambda (port)
1584                            ;; Note: that makes a long shebang.  When the store
1585                            ;; is /gnu/store, that fits within the 128-byte
1586                            ;; limit imposed by Linux, but that may go beyond
1587                            ;; when running tests.
1588                            (format port
1589                                    "#!~a/bin/guile --no-auto-compile~%!#~%"
1590                                    (ungexp guile))
1591
1592                            (ungexp-splicing
1593                             (if set-load-path
1594                                 (gexp ((write '(ungexp set-load-path) port)))
1595                                 (gexp ())))
1596
1597                            (write '(ungexp exp) port)
1598                            (chmod port #o555))))
1599                       #:system system
1600                       #:target target
1601                       #:module-path module-path)))
1602
1603 (define* (gexp->file name exp #:key
1604                      (set-load-path? #t)
1605                      (module-path %load-path)
1606                      (splice? #f))
1607   "Return a derivation that builds a file NAME containing EXP.  When SPLICE?
1608 is true, EXP is considered to be a list of expressions that will be spliced in
1609 the resulting file.
1610
1611 When SET-LOAD-PATH? is true, emit code in the resulting file to set
1612 '%load-path' and '%load-compiled-path' to honor EXP's imported modules.
1613 Lookup EXP's modules in MODULE-PATH."
1614   (define modules (gexp-modules exp))
1615   (define extensions (gexp-extensions exp))
1616
1617   (if (or (not set-load-path?)
1618           (and (null? modules) (null? extensions)))
1619       (gexp->derivation name
1620                         (gexp
1621                          (call-with-output-file (ungexp output)
1622                            (lambda (port)
1623                              (for-each (lambda (exp)
1624                                          (write exp port))
1625                                        '(ungexp (if splice?
1626                                                     exp
1627                                                     (gexp ((ungexp exp)))))))))
1628                         #:local-build? #t
1629                         #:substitutable? #f)
1630       (mlet %store-monad ((set-load-path
1631                            (load-path-expression modules module-path
1632                                                  #:extensions extensions)))
1633         (gexp->derivation name
1634                           (gexp
1635                            (call-with-output-file (ungexp output)
1636                              (lambda (port)
1637                                (write '(ungexp set-load-path) port)
1638                                (for-each (lambda (exp)
1639                                            (write exp port))
1640                                          '(ungexp (if splice?
1641                                                       exp
1642                                                       (gexp ((ungexp exp)))))))))
1643                           #:module-path module-path
1644                           #:local-build? #t
1645                           #:substitutable? #f))))
1646
1647 (define* (text-file* name #:rest text)
1648   "Return as a monadic value a derivation that builds a text file containing
1649 all of TEXT.  TEXT may list, in addition to strings, objects of any type that
1650 can be used in a gexp: packages, derivations, local file objects, etc.  The
1651 resulting store file holds references to all these."
1652   (define builder
1653     (gexp (call-with-output-file (ungexp output "out")
1654             (lambda (port)
1655               (display (string-append (ungexp-splicing text)) port)))))
1656
1657   (gexp->derivation name builder
1658                     #:local-build? #t
1659                     #:substitutable? #f))
1660
1661 (define* (mixed-text-file name #:rest text)
1662   "Return an object representing store file NAME containing TEXT.  TEXT is a
1663 sequence of strings and file-like objects, as in:
1664
1665   (mixed-text-file \"profile\"
1666                    \"export PATH=\" coreutils \"/bin:\" grep \"/bin\")
1667
1668 This is the declarative counterpart of 'text-file*'."
1669   (define build
1670     (gexp (call-with-output-file (ungexp output "out")
1671             (lambda (port)
1672               (display (string-append (ungexp-splicing text)) port)))))
1673
1674   (computed-file name build))
1675
1676 (define (file-union name files)
1677   "Return a <computed-file> that builds a directory containing all of FILES.
1678 Each item in FILES must be a two-element list where the first element is the
1679 file name to use in the new directory, and the second element is a gexp
1680 denoting the target file.  Here's an example:
1681
1682   (file-union \"etc\"
1683               `((\"hosts\" ,(plain-file \"hosts\"
1684                                         \"127.0.0.1 localhost\"))
1685                 (\"bashrc\" ,(plain-file \"bashrc\"
1686                                          \"alias ls='ls --color'\"))
1687                 (\"libvirt/qemu.conf\" ,(plain-file \"qemu.conf\" \"\"))))
1688
1689 This yields an 'etc' directory containing these two files."
1690   (computed-file name
1691                  (with-imported-modules '((guix build utils))
1692                    (gexp
1693                     (begin
1694                       (use-modules (guix build utils))
1695
1696                       (mkdir (ungexp output))
1697                       (chdir (ungexp output))
1698                       (ungexp-splicing
1699                        (map (match-lambda
1700                               ((target source)
1701                                (gexp
1702                                 (begin
1703                                   ;; Stat the source to abort early if it does
1704                                   ;; not exist.
1705                                   (stat (ungexp source))
1706
1707                                   (mkdir-p (dirname (ungexp target)))
1708                                   (symlink (ungexp source)
1709                                            (ungexp target))))))
1710                             files)))))))
1711
1712 (define* (directory-union name things
1713                           #:key (copy? #f) (quiet? #f)
1714                           (resolve-collision 'warn-about-collision))
1715   "Return a directory that is the union of THINGS, where THINGS is a list of
1716 file-like objects denoting directories.  For example:
1717
1718   (directory-union \"guile+emacs\" (list guile emacs))
1719
1720 yields a directory that is the union of the 'guile' and 'emacs' packages.
1721
1722 Call RESOLVE-COLLISION when several files collide, passing it the list of
1723 colliding files.  RESOLVE-COLLISION must return the chosen file or #f, in
1724 which case the colliding entry is skipped altogether.
1725
1726 When HARD-LINKS? is true, create hard links instead of symlinks.  When QUIET?
1727 is true, the derivation will not print anything."
1728   (define symlink
1729     (if copy?
1730         (gexp (lambda (old new)
1731                 (if (file-is-directory? old)
1732                     (symlink old new)
1733                     (copy-file old new))))
1734         (gexp symlink)))
1735
1736   (define log-port
1737     (if quiet?
1738         (gexp (%make-void-port "w"))
1739         (gexp (current-error-port))))
1740
1741   (match things
1742     ((one)
1743      ;; Only one thing; return it.
1744      one)
1745     (_
1746      (computed-file name
1747                     (with-imported-modules '((guix build union))
1748                       (gexp (begin
1749                               (use-modules (guix build union)
1750                                            (srfi srfi-1)) ;for 'first' and 'last'
1751
1752                               (union-build (ungexp output)
1753                                            '(ungexp things)
1754
1755                                            #:log-port (ungexp log-port)
1756                                            #:symlink (ungexp symlink)
1757                                            #:resolve-collision
1758                                            (ungexp resolve-collision)))))))))
1759
1760 \f
1761 ;;;
1762 ;;; Syntactic sugar.
1763 ;;;
1764
1765 (eval-when (expand load eval)
1766   (define* (read-ungexp chr port #:optional native?)
1767     "Read an 'ungexp' or 'ungexp-splicing' form from PORT.  When NATIVE? is
1768 true, use 'ungexp-native' and 'ungexp-native-splicing' instead."
1769     (define unquote-symbol
1770       (match (peek-char port)
1771         (#\@
1772          (read-char port)
1773          (if native?
1774              'ungexp-native-splicing
1775              'ungexp-splicing))
1776         (_
1777          (if native?
1778              'ungexp-native
1779              'ungexp))))
1780
1781     (match (read port)
1782       ((? symbol? symbol)
1783        (let ((str (symbol->string symbol)))
1784          (match (string-index-right str #\:)
1785            (#f
1786             `(,unquote-symbol ,symbol))
1787            (colon
1788             (let ((name   (string->symbol (substring str 0 colon)))
1789                   (output (substring str (+ colon 1))))
1790               `(,unquote-symbol ,name ,output))))))
1791       (x
1792        `(,unquote-symbol ,x))))
1793
1794   (define (read-gexp chr port)
1795     "Read a 'gexp' form from PORT."
1796     `(gexp ,(read port)))
1797
1798   ;; Extend the reader
1799   (read-hash-extend #\~ read-gexp)
1800   (read-hash-extend #\$ read-ungexp)
1801   (read-hash-extend #\+ (cut read-ungexp <> <> #t)))
1802
1803 ;;; gexp.scm ends here