guix/gexp.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019 Ludovic Courtès <ludo@gnu.org>
   3 ;;; Copyright © 2018 Clément Lassieur <clement@lassieur.org>
   4 ;;; Copyright © 2018 Jan Nieuwenhuizen <janneke@gnu.org>
   5 ;;;
   6 ;;; This file is part of GNU Guix.
   7 ;;;
   8 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   9 ;;; under the terms of the GNU General Public License as published by
  10 ;;; the Free Software Foundation; either version 3 of the License, or (at
  11 ;;; your option) any later version.
  12 ;;;
  13 ;;; GNU Guix is distributed in the hope that it will be useful, but
  14 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  15 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 ;;; GNU General Public License for more details.
  17 ;;;
  18 ;;; You should have received a copy of the GNU General Public License
  19 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 (define-module (guix gexp)
  22   #:use-module (guix store)
  23   #:use-module (guix monads)
  24   #:use-module (guix derivations)
  25   #:use-module (guix grafts)
  26   #:use-module (guix utils)
  27   #:use-module (rnrs bytevectors)
  28   #:use-module (srfi srfi-1)
  29   #:use-module (srfi srfi-9)
  30   #:use-module (srfi srfi-9 gnu)
  31   #:use-module (srfi srfi-26)
  32   #:use-module (srfi srfi-34)
  33   #:use-module (srfi srfi-35)
  34   #:use-module (ice-9 match)
  35   #:export (gexp
  36             gexp?
  37             with-imported-modules
  38             with-extensions
  39
  40             gexp-input
  41             gexp-input?
  42             gexp-input-thing
  43             gexp-input-output
  44             gexp-input-native?
  45
  46             local-file
  47             local-file?
  48             local-file-file
  49             local-file-absolute-file-name
  50             local-file-name
  51             local-file-recursive?
  52
  53             plain-file
  54             plain-file?
  55             plain-file-name
  56             plain-file-content
  57
  58             computed-file
  59             computed-file?
  60             computed-file-name
  61             computed-file-gexp
  62             computed-file-options
  63
  64             program-file
  65             program-file?
  66             program-file-name
  67             program-file-gexp
  68             program-file-guile
  69             program-file-module-path
  70
  71             scheme-file
  72             scheme-file?
  73             scheme-file-name
  74             scheme-file-gexp
  75
  76             file-append
  77             file-append?
  78             file-append-base
  79             file-append-suffix
  80
  81             load-path-expression
  82             gexp-modules
  83
  84             lower-gexp
  85             lowered-gexp?
  86             lowered-gexp-sexp
  87             lowered-gexp-inputs
  88             lowered-gexp-sources
  89             lowered-gexp-guile
  90             lowered-gexp-load-path
  91             lowered-gexp-load-compiled-path
  92
  93             gexp->derivation
  94             gexp->file
  95             gexp->script
  96             text-file*
  97             mixed-text-file
  98             file-union
  99             directory-union
 100             imported-files
 101             imported-modules
 102             compiled-modules
 103
 104             define-gexp-compiler
 105             gexp-compiler?
 106             file-like?
 107             lower-object
 108
 109             lower-inputs
 110
 111             &gexp-error
 112             gexp-error?
 113             &gexp-input-error
 114             gexp-input-error?
 115             gexp-error-invalid-input))
 116
 117 ;;; Commentary:
 118 ;;;
 119 ;;; This module implements "G-expressions", or "gexps".  Gexps are like
 120 ;;; S-expressions (sexps), with two differences:
 121 ;;;
 122 ;;;   1. References (un-quotations) to derivations or packages in a gexp are
 123 ;;;      replaced by the corresponding output file name; in addition, the
 124 ;;;      'ungexp-native' unquote-like form allows code to explicitly refer to
 125 ;;;      the native code of a given package, in case of cross-compilation;
 126 ;;;
 127 ;;;   2. Gexps embed information about the derivations they refer to.
 128 ;;;
 129 ;;; Gexps make it easy to write to files Scheme code that refers to store
 130 ;;; items, or to write Scheme code to build derivations.
 131 ;;;
 132 ;;; Code:
 133
 134 ;; "G expressions".
 135 (define-record-type <gexp>
 136   (make-gexp references modules extensions proc)
 137   gexp?
 138   (references gexp-references)                    ;list of <gexp-input>
 139   (modules    gexp-self-modules)                  ;list of module names
 140   (extensions gexp-self-extensions)               ;list of lowerable things
 141   (proc       gexp-proc))                         ;procedure
 142
 143 (define (write-gexp gexp port)
 144   "Write GEXP on PORT."
 145   (display "#<gexp " port)
 146
 147   ;; Try to write the underlying sexp.  Now, this trick doesn't work when
 148   ;; doing things like (ungexp-splicing (gexp ())) because GEXP's procedure
 149   ;; tries to use 'append' on that, which fails with wrong-type-arg.
 150   (false-if-exception
 151    (write (apply (gexp-proc gexp)
 152                  (gexp-references gexp))
 153           port))
 154   (format port " ~a>"
 155           (number->string (object-address gexp) 16)))
 156
 157 (set-record-type-printer! <gexp> write-gexp)
 158
 159 \f
 160 ;;;
 161 ;;; Methods.
 162 ;;;
 163
 164 ;; Compiler for a type of objects that may be introduced in a gexp.
 165 (define-record-type <gexp-compiler>
 166   (gexp-compiler type lower expand)
 167   gexp-compiler?
 168   (type       gexp-compiler-type)                 ;record type descriptor
 169   (lower      gexp-compiler-lower)
 170   (expand     gexp-compiler-expand))              ;#f | DRV -> sexp
 171
 172 (define-condition-type &gexp-error &error
 173   gexp-error?)
 174
 175 (define-condition-type &gexp-input-error &gexp-error
 176   gexp-input-error?
 177   (input gexp-error-invalid-input))
 178
 179
 180 (define %gexp-compilers
 181   ;; 'eq?' mapping of record type descriptor to <gexp-compiler>.
 182   (make-hash-table 20))
 183
 184 (define (default-expander thing obj output)
 185   "This is the default expander for \"things\" that appear in gexps.  It
 186 returns its output file name of OBJ's OUTPUT."
 187   (match obj
 188     ((? derivation? drv)
 189      (derivation->output-path drv output))
 190     ((? string? file)
 191      file)))
 192
 193 (define (register-compiler! compiler)
 194   "Register COMPILER as a gexp compiler."
 195   (hashq-set! %gexp-compilers
 196               (gexp-compiler-type compiler) compiler))
 197
 198 (define (lookup-compiler object)
 199   "Search for a compiler for OBJECT.  Upon success, return the three argument
 200 procedure to lower it; otherwise return #f."
 201   (and=> (hashq-ref %gexp-compilers (struct-vtable object))
 202          gexp-compiler-lower))
 203
 204 (define (file-like? object)
 205   "Return #t if OBJECT leads to a file in the store once unquoted in a
 206 G-expression; otherwise return #f."
 207   (and (struct? object) (->bool (lookup-compiler object))))
 208
 209 (define (lookup-expander object)
 210   "Search for an expander for OBJECT.  Upon success, return the three argument
 211 procedure to expand it; otherwise return #f."
 212   (and=> (hashq-ref %gexp-compilers (struct-vtable object))
 213          gexp-compiler-expand))
 214
 215 (define* (lower-object obj
 216                        #:optional (system (%current-system))
 217                        #:key target)
 218   "Return as a value in %STORE-MONAD the derivation or store item
 219 corresponding to OBJ for SYSTEM, cross-compiling for TARGET if TARGET is true.
 220 OBJ must be an object that has an associated gexp compiler, such as a
 221 <package>."
 222   (match (lookup-compiler obj)
 223     (#f
 224      (raise (condition (&gexp-input-error (input obj)))))
 225     (lower
 226      ;; Cache in STORE the result of lowering OBJ.
 227      (mlet %store-monad ((graft? (grafting?)))
 228        (mcached (let ((lower (lookup-compiler obj)))
 229                   (lower obj system target))
 230                 obj
 231                 system target graft?)))))
 232
 233 (define-syntax define-gexp-compiler
 234   (syntax-rules (=> compiler expander)
 235     "Define NAME as a compiler for objects matching PREDICATE encountered in
 236 gexps.
 237
 238 In the simplest form of the macro, BODY must return a derivation for PARAM, an
 239 object that matches PREDICATE, for SYSTEM and TARGET (the latter of which is
 240 #f except when cross-compiling.)
 241
 242 The more elaborate form allows you to specify an expander:
 243
 244   (define-gexp-compiler something something?
 245     compiler => (lambda (param system target) ...)
 246     expander => (lambda (param drv output) ...))
 247
 248 The expander specifies how an object is converted to its sexp representation."
 249     ((_ (name (param record-type) system target) body ...)
 250      (define-gexp-compiler name record-type
 251        compiler => (lambda (param system target) body ...)
 252        expander => default-expander))
 253     ((_ name record-type
 254         compiler => compile
 255         expander => expand)
 256      (begin
 257        (define name
 258          (gexp-compiler record-type compile expand))
 259        (register-compiler! name)))))
 260
 261 (define-gexp-compiler (derivation-compiler (drv <derivation>) system target)
 262   ;; Derivations are the lowest-level representation, so this is the identity
 263   ;; compiler.
 264   (with-monad %store-monad
 265     (return drv)))
 266
 267 \f
 268 ;;;
 269 ;;; File declarations.
 270 ;;;
 271
 272 ;; A local file name.  FILE is the file name the user entered, which can be a
 273 ;; relative file name, and ABSOLUTE is a promise that computes its canonical
 274 ;; absolute file name.  We keep it in a promise to compute it lazily and avoid
 275 ;; repeated 'stat' calls.
 276 (define-record-type <local-file>
 277   (%%local-file file absolute name recursive? select?)
 278   local-file?
 279   (file       local-file-file)                    ;string
 280   (absolute   %local-file-absolute-file-name)     ;promise string
 281   (name       local-file-name)                    ;string
 282   (recursive? local-file-recursive?)              ;Boolean
 283   (select?    local-file-select?))                ;string stat -> Boolean
 284
 285 (define (true file stat) #t)
 286
 287 (define* (%local-file file promise #:optional (name (basename file))
 288                       #:key recursive? (select? true))
 289   ;; This intermediate procedure is part of our ABI, but the underlying
 290   ;; %%LOCAL-FILE is not.
 291   (%%local-file file promise name recursive? select?))
 292
 293 (define (absolute-file-name file directory)
 294   "Return the canonical absolute file name for FILE, which lives in the
 295 vicinity of DIRECTORY."
 296   (canonicalize-path
 297    (cond ((string-prefix? "/" file) file)
 298          ((not directory) file)
 299          ((string-prefix? "/" directory)
 300           (string-append directory "/" file))
 301          (else file))))
 302
 303 (define-syntax local-file
 304   (lambda (s)
 305     "Return an object representing local file FILE to add to the store; this
 306 object can be used in a gexp.  If FILE is a relative file name, it is looked
 307 up relative to the source file where this form appears.  FILE will be added to
 308 the store under NAME--by default the base name of FILE.
 309
 310 When RECURSIVE? is true, the contents of FILE are added recursively; if FILE
 311 designates a flat file and RECURSIVE? is true, its contents are added, and its
 312 permission bits are kept.
 313
 314 When RECURSIVE? is true, call (SELECT?  FILE STAT) for each directory entry,
 315 where FILE is the entry's absolute file name and STAT is the result of
 316 'lstat'; exclude entries for which SELECT? does not return true.
 317
 318 This is the declarative counterpart of the 'interned-file' monadic procedure.
 319 It is implemented as a macro to capture the current source directory where it
 320 appears."
 321     (syntax-case s ()
 322       ((_ file rest ...)
 323        #'(%local-file file
 324                       (delay (absolute-file-name file (current-source-directory)))
 325                       rest ...))
 326       ((_)
 327        #'(syntax-error "missing file name"))
 328       (id
 329        (identifier? #'id)
 330        ;; XXX: We could return #'(lambda (file . rest) ...).  However,
 331        ;; (syntax-source #'id) is #f so (current-source-directory) would not
 332        ;; work.  Thus, simply forbid this form.
 333        #'(syntax-error
 334           "'local-file' is a macro and cannot be used like this")))))
 335
 336 (define (local-file-absolute-file-name file)
 337   "Return the absolute file name for FILE, a <local-file> instance.  A
 338 'system-error' exception is raised if FILE could not be found."
 339   (force (%local-file-absolute-file-name file)))
 340
 341 (define-gexp-compiler (local-file-compiler (file <local-file>) system target)
 342   ;; "Compile" FILE by adding it to the store.
 343   (match file
 344     (($ <local-file> file (= force absolute) name recursive? select?)
 345      ;; Canonicalize FILE so that if it's a symlink, it is resolved.  Failing
 346      ;; to do that, when RECURSIVE? is #t, we could end up creating a dangling
 347      ;; symlink in the store, and when RECURSIVE? is #f 'add-to-store' would
 348      ;; just throw an error, both of which are inconvenient.
 349      (interned-file absolute name
 350                     #:recursive? recursive? #:select? select?))))
 351
 352 (define-record-type <plain-file>
 353   (%plain-file name content references)
 354   plain-file?
 355   (name        plain-file-name)                   ;string
 356   (content     plain-file-content)                ;string or bytevector
 357   (references  plain-file-references))            ;list (currently unused)
 358
 359 (define (plain-file name content)
 360   "Return an object representing a text file called NAME with the given
 361 CONTENT (a string) to be added to the store.
 362
 363 This is the declarative counterpart of 'text-file'."
 364   ;; XXX: For now just ignore 'references' because it's not clear how to use
 365   ;; them in a declarative context.
 366   (%plain-file name content '()))
 367
 368 (define-gexp-compiler (plain-file-compiler (file <plain-file>) system target)
 369   ;; "Compile" FILE by adding it to the store.
 370   (match file
 371     (($ <plain-file> name (and (? string?) content) references)
 372      (text-file name content references))
 373     (($ <plain-file> name (and (? bytevector?) content) references)
 374      (binary-file name content references))))
 375
 376 (define-record-type <computed-file>
 377   (%computed-file name gexp guile options)
 378   computed-file?
 379   (name       computed-file-name)                 ;string
 380   (gexp       computed-file-gexp)                 ;gexp
 381   (guile      computed-file-guile)                ;<package>
 382   (options    computed-file-options))             ;list of arguments
 383
 384 (define* (computed-file name gexp
 385                         #:key guile (options '(#:local-build? #t)))
 386   "Return an object representing the store item NAME, a file or directory
 387 computed by GEXP.  OPTIONS is a list of additional arguments to pass
 388 to 'gexp->derivation'.
 389
 390 This is the declarative counterpart of 'gexp->derivation'."
 391   (%computed-file name gexp guile options))
 392
 393 (define-gexp-compiler (computed-file-compiler (file <computed-file>)
 394                                               system target)
 395   ;; Compile FILE by returning a derivation whose build expression is its
 396   ;; gexp.
 397   (match file
 398     (($ <computed-file> name gexp guile options)
 399      (if guile
 400          (mlet %store-monad ((guile (lower-object guile system
 401                                                   #:target target)))
 402            (apply gexp->derivation name gexp #:guile-for-build guile
 403                   #:system system #:target target options))
 404          (apply gexp->derivation name gexp
 405                 #:system system #:target target options)))))
 406
 407 (define-record-type <program-file>
 408   (%program-file name gexp guile path)
 409   program-file?
 410   (name       program-file-name)                  ;string
 411   (gexp       program-file-gexp)                  ;gexp
 412   (guile      program-file-guile)                 ;package
 413   (path       program-file-module-path))          ;list of strings
 414
 415 (define* (program-file name gexp #:key (guile #f) (module-path %load-path))
 416   "Return an object representing the executable store item NAME that runs
 417 GEXP.  GUILE is the Guile package used to execute that script.  Imported
 418 modules of GEXP are looked up in MODULE-PATH.
 419
 420 This is the declarative counterpart of 'gexp->script'."
 421   (%program-file name gexp guile module-path))
 422
 423 (define-gexp-compiler (program-file-compiler (file <program-file>)
 424                                              system target)
 425   ;; Compile FILE by returning a derivation that builds the script.
 426   (match file
 427     (($ <program-file> name gexp guile module-path)
 428      (gexp->script name gexp
 429                    #:module-path module-path
 430                    #:guile (or guile (default-guile))
 431                    #:system system
 432                    #:target target))))
 433
 434 (define-record-type <scheme-file>
 435   (%scheme-file name gexp splice?)
 436   scheme-file?
 437   (name       scheme-file-name)                  ;string
 438   (gexp       scheme-file-gexp)                  ;gexp
 439   (splice?    scheme-file-splice?))              ;Boolean
 440
 441 (define* (scheme-file name gexp #:key splice?)
 442   "Return an object representing the Scheme file NAME that contains GEXP.
 443
 444 This is the declarative counterpart of 'gexp->file'."
 445   (%scheme-file name gexp splice?))
 446
 447 (define-gexp-compiler (scheme-file-compiler (file <scheme-file>)
 448                                             system target)
 449   ;; Compile FILE by returning a derivation that builds the file.
 450   (match file
 451     (($ <scheme-file> name gexp splice?)
 452      (gexp->file name gexp #:splice? splice?))))
 453
 454 ;; Appending SUFFIX to BASE's output file name.
 455 (define-record-type <file-append>
 456   (%file-append base suffix)
 457   file-append?
 458   (base   file-append-base)                    ;<package> | <derivation> | ...
 459   (suffix file-append-suffix))                 ;list of strings
 460
 461 (define (write-file-append file port)
 462   (match file
 463     (($ <file-append> base suffix)
 464      (format port "#<file-append ~s ~s>" base
 465              (string-join suffix)))))
 466
 467 (set-record-type-printer! <file-append> write-file-append)
 468
 469 (define (file-append base . suffix)
 470   "Return a <file-append> object that expands to the concatenation of BASE and
 471 SUFFIX."
 472   (%file-append base suffix))
 473
 474 (define-gexp-compiler file-append-compiler <file-append>
 475   compiler => (lambda (obj system target)
 476                 (match obj
 477                   (($ <file-append> base _)
 478                    (lower-object base system #:target target))))
 479   expander => (lambda (obj lowered output)
 480                 (match obj
 481                   (($ <file-append> base suffix)
 482                    (let* ((expand (lookup-expander base))
 483                           (base   (expand base lowered output)))
 484                      (string-append base (string-concatenate suffix)))))))
 485
 486 \f
 487 ;;;
 488 ;;; Inputs & outputs.
 489 ;;;
 490
 491 ;; The input of a gexp.
 492 (define-record-type <gexp-input>
 493   (%gexp-input thing output native?)
 494   gexp-input?
 495   (thing     gexp-input-thing)       ;<package> | <origin> | <derivation> | ...
 496   (output    gexp-input-output)      ;string
 497   (native?   gexp-input-native?))    ;Boolean
 498
 499 (define (write-gexp-input input port)
 500   (match input
 501     (($ <gexp-input> thing output #f)
 502      (format port "#<gexp-input ~s:~a>" thing output))
 503     (($ <gexp-input> thing output #t)
 504      (format port "#<gexp-input native ~s:~a>" thing output))))
 505
 506 (set-record-type-printer! <gexp-input> write-gexp-input)
 507
 508 (define* (gexp-input thing                        ;convenience procedure
 509                      #:optional (output "out")
 510                      #:key native?)
 511   "Return a new <gexp-input> for the OUTPUT of THING; NATIVE? determines
 512 whether this should be considered a \"native\" input or not."
 513   (%gexp-input thing output native?))
 514
 515 ;; Reference to one of the derivation's outputs, for gexps used in
 516 ;; derivations.
 517 (define-record-type <gexp-output>
 518   (gexp-output name)
 519   gexp-output?
 520   (name gexp-output-name))
 521
 522 (define (write-gexp-output output port)
 523   (match output
 524     (($ <gexp-output> name)
 525      (format port "#<gexp-output ~a>" name))))
 526
 527 (set-record-type-printer! <gexp-output> write-gexp-output)
 528
 529 (define* (gexp-attribute gexp self-attribute #:optional (equal? equal?))
 530   "Recurse on GEXP and the expressions it refers to, summing the items
 531 returned by SELF-ATTRIBUTE, a procedure that takes a gexp.  Use EQUAL? as the
 532 second argument to 'delete-duplicates'."
 533   (if (gexp? gexp)
 534       (delete-duplicates
 535        (append (self-attribute gexp)
 536                (append-map (match-lambda
 537                              (($ <gexp-input> (? gexp? exp))
 538                               (gexp-attribute exp self-attribute))
 539                              (($ <gexp-input> (lst ...))
 540                               (append-map (lambda (item)
 541                                             (if (gexp? item)
 542                                                 (gexp-attribute item
 543                                                                 self-attribute)
 544                                                 '()))
 545                                           lst))
 546                              (_
 547                               '()))
 548                            (gexp-references gexp)))
 549        equal?)
 550       '()))                                       ;plain Scheme data type
 551
 552 (define (gexp-modules gexp)
 553   "Return the list of Guile module names GEXP relies on.  If (gexp? GEXP) is
 554 false, meaning that GEXP is a plain Scheme object, return the empty list."
 555   (define (module=? m1 m2)
 556     ;; Return #t when M1 equals M2.  Special-case '=>' specs because their
 557     ;; right-hand side may not be comparable with 'equal?': it's typically a
 558     ;; file-like object that embeds a gexp, which in turn embeds closure;
 559     ;; those closures may be 'eq?' when running compiled code but are unlikely
 560     ;; to be 'eq?' when running on 'eval'.  Ignore the right-hand side to
 561     ;; avoid this discrepancy.
 562     (match m1
 563       (((name1 ...) '=> _)
 564        (match m2
 565          (((name2 ...) '=> _) (equal? name1 name2))
 566          (_ #f)))
 567       (_
 568        (equal? m1 m2))))
 569
 570   (gexp-attribute gexp gexp-self-modules module=?))
 571
 572 (define (gexp-extensions gexp)
 573   "Return the list of Guile extensions (packages) GEXP relies on.  If (gexp?
 574 GEXP) is false, meaning that GEXP is a plain Scheme object, return the empty
 575 list."
 576   (gexp-attribute gexp gexp-self-extensions))
 577
 578 (define* (lower-inputs inputs
 579                        #:key system target)
 580   "Turn any object from INPUTS into a derivation input for SYSTEM or a store
 581 item (a \"source\"); return the corresponding input list as a monadic value.
 582 When TARGET is true, use it as the cross-compilation target triplet."
 583   (define (store-item? obj)
 584     (and (string? obj) (store-path? obj)))
 585
 586   (with-monad %store-monad
 587     (mapm %store-monad
 588           (match-lambda
 589             (((? struct? thing) sub-drv ...)
 590              (mlet %store-monad ((obj (lower-object
 591                                        thing system #:target target)))
 592                (return (match obj
 593                          ((? derivation? drv)
 594                           (let ((outputs (if (null? sub-drv)
 595                                              '("out")
 596                                              sub-drv)))
 597                             (derivation-input drv outputs)))
 598                          ((? store-item? item)
 599                           item)))))
 600             (((? store-item? item))
 601              (return item)))
 602           inputs)))
 603
 604 (define* (lower-reference-graphs graphs #:key system target)
 605   "Given GRAPHS, a list of (FILE-NAME INPUT ...) lists for use as a
 606 #:reference-graphs argument, lower it such that each INPUT is replaced by the
 607 corresponding <derivation-input> or store item."
 608   (match graphs
 609     (((file-names . inputs) ...)
 610      (mlet %store-monad ((inputs (lower-inputs inputs
 611                                                #:system system
 612                                                #:target target)))
 613        (return (map cons file-names inputs))))))
 614
 615 (define* (lower-references lst #:key system target)
 616   "Based on LST, a list of output names and packages, return a list of output
 617 names and file names suitable for the #:allowed-references argument to
 618 'derivation'."
 619   (with-monad %store-monad
 620     (define lower
 621       (match-lambda
 622        ((? string? output)
 623         (return output))
 624        (($ <gexp-input> thing output native?)
 625         (mlet %store-monad ((drv (lower-object thing system
 626                                                #:target (if native?
 627                                                             #f target))))
 628           (return (derivation->output-path drv output))))
 629        (thing
 630         (mlet %store-monad ((drv (lower-object thing system
 631                                                #:target target)))
 632           (return (derivation->output-path drv))))))
 633
 634     (mapm %store-monad lower lst)))
 635
 636 (define default-guile-derivation
 637   ;; Here we break the abstraction by talking to the higher-level layer.
 638   ;; Thus, do the resolution lazily to hide the circular dependency.
 639   (let ((proc (delay
 640                 (let ((iface (resolve-interface '(guix packages))))
 641                   (module-ref iface 'default-guile-derivation)))))
 642     (lambda (system)
 643       ((force proc) system))))
 644
 645 ;; Representation of a gexp instantiated for a given target and system.
 646 ;; It's an intermediate representation between <gexp> and <derivation>.
 647 (define-record-type <lowered-gexp>
 648   (lowered-gexp sexp inputs sources guile load-path load-compiled-path)
 649   lowered-gexp?
 650   (sexp                lowered-gexp-sexp)         ;sexp
 651   (inputs              lowered-gexp-inputs)       ;list of <derivation-input>
 652   (sources             lowered-gexp-sources)      ;list of store items
 653   (guile               lowered-gexp-guile)        ;<derivation-input> | #f
 654   (load-path           lowered-gexp-load-path)    ;list of store items
 655   (load-compiled-path  lowered-gexp-load-compiled-path)) ;list of store items
 656
 657 (define* (lower-gexp exp
 658                      #:key
 659                      (module-path %load-path)
 660                      (system (%current-system))
 661                      (target 'current)
 662                      (graft? (%graft?))
 663                      (guile-for-build (%guile-for-build))
 664                      (effective-version "2.2")
 665
 666                      deprecation-warnings)
 667   "*Note: This API is subject to change; use at your own risk!*
 668
 669 Lower EXP, a gexp, instantiating it for SYSTEM and TARGET.  Return a
 670 <lowered-gexp> ready to be used.
 671
 672 Lowered gexps are an intermediate representation that's useful for
 673 applications that deal with gexps outside in a way that is disconnected from
 674 derivations--e.g., code evaluated for its side effects."
 675   (define %modules
 676     (delete-duplicates (gexp-modules exp)))
 677
 678   (define (search-path modules extensions suffix)
 679     (append (match modules
 680               ((? derivation? drv)
 681                (list (derivation->output-path drv)))
 682               (#f
 683                '())
 684               ((? store-path? item)
 685                (list item)))
 686             (map (lambda (extension)
 687                    (string-append (match extension
 688                                     ((? derivation? drv)
 689                                      (derivation->output-path drv))
 690                                     ((? store-path? item)
 691                                      item))
 692                                   suffix))
 693                  extensions)))
 694
 695   (mlet* %store-monad ( ;; The following binding forces '%current-system' and
 696                        ;; '%current-target-system' to be looked up at >>=
 697                        ;; time.
 698                        (graft?    (set-grafting graft?))
 699
 700                        (system -> (or system (%current-system)))
 701                        (target -> (if (eq? target 'current)
 702                                       (%current-target-system)
 703                                       target))
 704                        (guile     (if guile-for-build
 705                                       (return guile-for-build)
 706                                       (default-guile-derivation system)))
 707                        (normals  (lower-inputs (gexp-inputs exp)
 708                                                #:system system
 709                                                #:target target))
 710                        (natives  (lower-inputs (gexp-native-inputs exp)
 711                                                #:system system
 712                                                #:target #f))
 713                        (inputs -> (append normals natives))
 714                        (sexp     (gexp->sexp exp
 715                                              #:system system
 716                                              #:target target))
 717                        (extensions -> (gexp-extensions exp))
 718                        (exts     (mapm %store-monad
 719                                        (lambda (obj)
 720                                          (lower-object obj system))
 721                                        extensions))
 722                        (modules  (if (pair? %modules)
 723                                      (imported-modules %modules
 724                                                        #:system system
 725                                                        #:module-path module-path)
 726                                      (return #f)))
 727                        (compiled (if (pair? %modules)
 728                                      (compiled-modules %modules
 729                                                        #:system system
 730                                                        #:module-path module-path
 731                                                        #:extensions extensions
 732                                                        #:guile guile
 733                                                        #:deprecation-warnings
 734                                                        deprecation-warnings)
 735                                      (return #f))))
 736     (define load-path
 737       (search-path modules exts
 738                    (string-append "/share/guile/site/" effective-version)))
 739
 740     (define load-compiled-path
 741       (search-path compiled exts
 742                    (string-append "/lib/guile/" effective-version
 743                                   "/site-ccache")))
 744
 745     (mbegin %store-monad
 746       (set-grafting graft?)                       ;restore the initial setting
 747       (return (lowered-gexp sexp
 748                             `(,@(if (derivation? modules)
 749                                     (list (derivation-input modules))
 750                                     '())
 751                               ,@(if compiled
 752                                     (list (derivation-input compiled))
 753                                     '())
 754                               ,@(map derivation-input exts)
 755                               ,@(filter derivation-input? inputs))
 756                             (filter string? (cons modules inputs))
 757                             (derivation-input guile '("out"))
 758                             load-path
 759                             load-compiled-path)))))
 760
 761 (define* (gexp->derivation name exp
 762                            #:key
 763                            system (target 'current)
 764                            hash hash-algo recursive?
 765                            (env-vars '())
 766                            (modules '())
 767                            (module-path %load-path)
 768                            (guile-for-build (%guile-for-build))
 769                            (effective-version "2.2")
 770                            (graft? (%graft?))
 771                            references-graphs
 772                            allowed-references disallowed-references
 773                            leaked-env-vars
 774                            local-build? (substitutable? #t)
 775                            (properties '())
 776                            deprecation-warnings
 777                            (script-name (string-append name "-builder")))
 778   "Return a derivation NAME that runs EXP (a gexp) with GUILE-FOR-BUILD (a
 779 derivation) on SYSTEM; EXP is stored in a file called SCRIPT-NAME.  When
 780 TARGET is true, it is used as the cross-compilation target triplet for
 781 packages referred to by EXP.
 782
 783 MODULES is deprecated in favor of 'with-imported-modules'.  Its meaning is to
 784 make MODULES available in the evaluation context of EXP; MODULES is a list of
 785 names of Guile modules searched in MODULE-PATH to be copied in the store,
 786 compiled, and made available in the load path during the execution of
 787 EXP---e.g., '((guix build utils) (guix build gnu-build-system)).
 788
 789 EFFECTIVE-VERSION determines the string to use when adding extensions of
 790 EXP (see 'with-extensions') to the search path---e.g., \"2.2\".
 791
 792 GRAFT? determines whether packages referred to by EXP should be grafted when
 793 applicable.
 794
 795 When REFERENCES-GRAPHS is true, it must be a list of tuples of one of the
 796 following forms:
 797
 798   (FILE-NAME PACKAGE)
 799   (FILE-NAME PACKAGE OUTPUT)
 800   (FILE-NAME DERIVATION)
 801   (FILE-NAME DERIVATION OUTPUT)
 802   (FILE-NAME STORE-ITEM)
 803
 804 The right-hand-side of each element of REFERENCES-GRAPHS is automatically made
 805 an input of the build process of EXP.  In the build environment, each
 806 FILE-NAME contains the reference graph of the corresponding item, in a simple
 807 text format.
 808
 809 ALLOWED-REFERENCES must be either #f or a list of output names and packages.
 810 In the latter case, the list denotes store items that the result is allowed to
 811 refer to.  Any reference to another store item will lead to a build error.
 812 Similarly for DISALLOWED-REFERENCES, which can list items that must not be
 813 referenced by the outputs.
 814
 815 DEPRECATION-WARNINGS determines whether to show deprecation warnings while
 816 compiling modules.  It can be #f, #t, or 'detailed.
 817
 818 The other arguments are as for 'derivation'."
 819   (define outputs (gexp-outputs exp))
 820   (define requested-graft? graft?)
 821
 822   (define (graphs-file-names graphs)
 823     ;; Return a list of (FILE-NAME . STORE-PATH) pairs made from GRAPHS.
 824     (map (match-lambda
 825            ((file-name . (? derivation-input? input))
 826             (cons file-name (first (derivation-input-output-paths input))))
 827            ((file-name . (? string? item))
 828             (cons file-name item)))
 829          graphs))
 830
 831   (define (add-modules exp modules)
 832     (if (null? modules)
 833         exp
 834         (make-gexp (gexp-references exp)
 835                    (append modules (gexp-self-modules exp))
 836                    (gexp-self-extensions exp)
 837                    (gexp-proc exp))))
 838
 839   (mlet* %store-monad ( ;; The following binding forces '%current-system' and
 840                        ;; '%current-target-system' to be looked up at >>=
 841                        ;; time.
 842                        (graft?    (set-grafting graft?))
 843
 844                        (system -> (or system (%current-system)))
 845                        (target -> (if (eq? target 'current)
 846                                       (%current-target-system)
 847                                       target))
 848                        (exp ->    (add-modules exp modules))
 849                        (lowered   (lower-gexp exp
 850                                               #:module-path module-path
 851                                               #:system system
 852                                               #:target target
 853                                               #:graft? requested-graft?
 854                                               #:guile-for-build
 855                                               guile-for-build
 856                                               #:effective-version
 857                                               effective-version
 858                                               #:deprecation-warnings
 859                                               deprecation-warnings))
 860
 861                        (graphs   (if references-graphs
 862                                      (lower-reference-graphs references-graphs
 863                                                              #:system system
 864                                                              #:target target)
 865                                      (return #f)))
 866                        (allowed  (if allowed-references
 867                                      (lower-references allowed-references
 868                                                        #:system system
 869                                                        #:target target)
 870                                      (return #f)))
 871                        (disallowed (if disallowed-references
 872                                        (lower-references disallowed-references
 873                                                          #:system system
 874                                                          #:target target)
 875                                        (return #f)))
 876                        (guile -> (lowered-gexp-guile lowered))
 877                        (builder  (text-file script-name
 878                                             (object->string
 879                                              (lowered-gexp-sexp lowered)))))
 880     (mbegin %store-monad
 881       (set-grafting graft?)                       ;restore the initial setting
 882       (raw-derivation name
 883                       (string-append (derivation-input-output-path guile)
 884                                      "/bin/guile")
 885                       `("--no-auto-compile"
 886                         ,@(append-map (lambda (directory)
 887                                         `("-L" ,directory))
 888                                       (lowered-gexp-load-path lowered))
 889                         ,@(append-map (lambda (directory)
 890                                         `("-C" ,directory))
 891                                       (lowered-gexp-load-compiled-path lowered))
 892                         ,builder)
 893                       #:outputs outputs
 894                       #:env-vars env-vars
 895                       #:system system
 896                       #:inputs `(,guile
 897                                  ,@(lowered-gexp-inputs lowered)
 898                                  ,@(match graphs
 899                                      (((_ . inputs) ...)
 900                                       (filter derivation-input? inputs))
 901                                      (#f '())))
 902                       #:sources `(,builder
 903                                   ,@(if (and (string? modules)
 904                                              (store-path? modules))
 905                                         (list modules)
 906                                         '())
 907                                   ,@(lowered-gexp-sources lowered)
 908                                   ,@(match graphs
 909                                       (((_ . inputs) ...)
 910                                        (filter string? inputs))
 911                                       (#f '())))
 912
 913                       #:hash hash #:hash-algo hash-algo #:recursive? recursive?
 914                       #:references-graphs (and=> graphs graphs-file-names)
 915                       #:allowed-references allowed
 916                       #:disallowed-references disallowed
 917                       #:leaked-env-vars leaked-env-vars
 918                       #:local-build? local-build?
 919                       #:substitutable? substitutable?
 920                       #:properties properties))))
 921
 922 (define* (gexp-inputs exp #:key native?)
 923   "Return the input list for EXP.  When NATIVE? is true, return only native
 924 references; otherwise, return only non-native references."
 925   ;; TODO: Return <gexp-input> records instead of tuples.
 926   (define (add-reference-inputs ref result)
 927     (match ref
 928       (($ <gexp-input> (? gexp? exp) _ #t)
 929        (if native?
 930            (append (gexp-inputs exp)
 931                    (gexp-inputs exp #:native? #t)
 932                    result)
 933            result))
 934       (($ <gexp-input> (? gexp? exp) _ #f)
 935        (append (gexp-inputs exp #:native? native?)
 936                result))
 937       (($ <gexp-input> (? string? str))
 938        (if (direct-store-path? str)
 939            (cons `(,str) result)
 940            result))
 941       (($ <gexp-input> (? struct? thing) output n?)
 942        (if (and (eqv? n? native?) (lookup-compiler thing))
 943            ;; THING is a derivation, or a package, or an origin, etc.
 944            (cons `(,thing ,output) result)
 945            result))
 946       (($ <gexp-input> (lst ...) output n?)
 947        (fold-right add-reference-inputs result
 948                    ;; XXX: For now, automatically convert LST to a list of
 949                    ;; gexp-inputs.  Inherit N?.
 950                    (map (match-lambda
 951                           ((? gexp-input? x)
 952                            (%gexp-input (gexp-input-thing x)
 953                                         (gexp-input-output x)
 954                                         n?))
 955                           (x
 956                            (%gexp-input x "out" n?)))
 957                         lst)))
 958       (_
 959        ;; Ignore references to other kinds of objects.
 960        result)))
 961
 962   (fold-right add-reference-inputs
 963               '()
 964               (gexp-references exp)))
 965
 966 (define gexp-native-inputs
 967   (cut gexp-inputs <> #:native? #t))
 968
 969 (define (gexp-outputs exp)
 970   "Return the outputs referred to by EXP as a list of strings."
 971   (define (add-reference-output ref result)
 972     (match ref
 973       (($ <gexp-output> name)
 974        (cons name result))
 975       (($ <gexp-input> (? gexp? exp))
 976        (append (gexp-outputs exp) result))
 977       (($ <gexp-input> (lst ...) output native?)
 978        ;; XXX: Automatically convert LST.
 979        (add-reference-output (map (match-lambda
 980                                    ((? gexp-input? x) x)
 981                                    (x (%gexp-input x "out" native?)))
 982                                   lst)
 983                              result))
 984       ((lst ...)
 985        (fold-right add-reference-output result lst))
 986       (_
 987        result)))
 988
 989   (delete-duplicates
 990    (add-reference-output (gexp-references exp) '())))
 991
 992 (define* (gexp->sexp exp #:key
 993                      (system (%current-system))
 994                      (target (%current-target-system)))
 995   "Return (monadically) the sexp corresponding to EXP for the given OUTPUT,
 996 and in the current monad setting (system type, etc.)"
 997   (define (self-quoting? x)
 998     (letrec-syntax ((one-of (syntax-rules ()
 999                               ((_) #f)
1000                               ((_ pred rest ...)
1001                                (or (pred x)
1002                                    (one-of rest ...))))))
1003       (one-of symbol? string? keyword? pair? null? array?
1004               number? boolean?)))
1005
1006   (define* (reference->sexp ref #:optional native?)
1007     (with-monad %store-monad
1008       (match ref
1009         (($ <gexp-output> output)
1010          ;; Output file names are not known in advance but the daemon defines
1011          ;; an environment variable for each of them at build time, so use
1012          ;; that trick.
1013          (return `((@ (guile) getenv) ,output)))
1014         (($ <gexp-input> (? gexp? exp) output n?)
1015          (gexp->sexp exp
1016                      #:system system
1017                      #:target (if (or n? native?) #f target)))
1018         (($ <gexp-input> (refs ...) output n?)
1019          (mapm %store-monad
1020                (lambda (ref)
1021                  ;; XXX: Automatically convert REF to an gexp-input.
1022                  (reference->sexp
1023                   (if (gexp-input? ref)
1024                       ref
1025                       (%gexp-input ref "out" n?))
1026                   (or n? native?)))
1027                refs))
1028         (($ <gexp-input> (? struct? thing) output n?)
1029          (let ((target (if (or n? native?) #f target))
1030                (expand (lookup-expander thing)))
1031            (mlet %store-monad ((obj (lower-object thing system
1032                                                   #:target target)))
1033              ;; OBJ must be either a derivation or a store file name.
1034              (return (expand thing obj output)))))
1035         (($ <gexp-input> (? self-quoting? x))
1036          (return x))
1037         (($ <gexp-input> x)
1038          (raise (condition (&gexp-input-error (input x)))))
1039         (x
1040          (return x)))))
1041
1042   (mlet %store-monad
1043       ((args (mapm %store-monad
1044                    reference->sexp (gexp-references exp))))
1045     (return (apply (gexp-proc exp) args))))
1046
1047 (define-syntax-rule (define-syntax-parameter-once name proc)
1048   ;; Like 'define-syntax-parameter' but ensure the top-level binding for NAME
1049   ;; does not get redefined.  This works around a race condition in a
1050   ;; multi-threaded context with Guile <= 2.2.4: <https://bugs.gnu.org/27476>.
1051   (eval-when (load eval expand compile)
1052     (define name
1053       (if (module-locally-bound? (current-module) 'name)
1054           (module-ref (current-module) 'name)
1055           (make-syntax-transformer 'name 'syntax-parameter
1056                                    (list proc))))))
1057
1058 (define-syntax-parameter-once current-imported-modules
1059   ;; Current list of imported modules.
1060   (identifier-syntax '()))
1061
1062 (define-syntax-rule (with-imported-modules modules body ...)
1063   "Mark the gexps defined in BODY... as requiring MODULES in their execution
1064 environment."
1065   (syntax-parameterize ((current-imported-modules
1066                          (identifier-syntax modules)))
1067     body ...))
1068
1069 (define-syntax-parameter-once current-imported-extensions
1070   ;; Current list of extensions.
1071   (identifier-syntax '()))
1072
1073 (define-syntax-rule (with-extensions extensions body ...)
1074   "Mark the gexps defined in BODY... as requiring EXTENSIONS in their
1075 execution environment."
1076   (syntax-parameterize ((current-imported-extensions
1077                          (identifier-syntax extensions)))
1078     body ...))
1079
1080 (define-syntax gexp
1081   (lambda (s)
1082     (define (collect-escapes exp)
1083       ;; Return all the 'ungexp' present in EXP.
1084       (let loop ((exp    exp)
1085                  (result '()))
1086         (syntax-case exp (ungexp
1087                           ungexp-splicing
1088                           ungexp-native
1089                           ungexp-native-splicing)
1090           ((ungexp _)
1091            (cons exp result))
1092           ((ungexp _ _)
1093            (cons exp result))
1094           ((ungexp-splicing _ ...)
1095            (cons exp result))
1096           ((ungexp-native _ ...)
1097            (cons exp result))
1098           ((ungexp-native-splicing _ ...)
1099            (cons exp result))
1100           ((exp0 . exp)
1101            (let ((result (loop #'exp0 result)))
1102              (loop  #'exp result)))
1103           (_
1104            result))))
1105
1106     (define (escape->ref exp)
1107       ;; Turn 'ungexp' form EXP into a "reference".
1108       (syntax-case exp (ungexp ungexp-splicing
1109                         ungexp-native ungexp-native-splicing
1110                         output)
1111         ((ungexp output)
1112          #'(gexp-output "out"))
1113         ((ungexp output name)
1114          #'(gexp-output name))
1115         ((ungexp thing)
1116          #'(%gexp-input thing "out" #f))
1117         ((ungexp drv-or-pkg out)
1118          #'(%gexp-input drv-or-pkg out #f))
1119         ((ungexp-splicing lst)
1120          #'(%gexp-input lst "out" #f))
1121         ((ungexp-native thing)
1122          #'(%gexp-input thing "out" #t))
1123         ((ungexp-native drv-or-pkg out)
1124          #'(%gexp-input drv-or-pkg out #t))
1125         ((ungexp-native-splicing lst)
1126          #'(%gexp-input lst "out" #t))))
1127
1128     (define (substitute-ungexp exp substs)
1129       ;; Given EXP, an 'ungexp' or 'ungexp-native' form, substitute it with
1130       ;; the corresponding form in SUBSTS.
1131       (match (assoc exp substs)
1132         ((_ id)
1133          id)
1134         (_                                        ;internal error
1135          (with-syntax ((exp exp))
1136            #'(syntax-error "error: no 'ungexp' substitution" exp)))))
1137
1138     (define (substitute-ungexp-splicing exp substs)
1139       (syntax-case exp ()
1140         ((exp rest ...)
1141          (match (assoc #'exp substs)
1142            ((_ id)
1143             (with-syntax ((id id))
1144               #`(append id
1145                         #,(substitute-references #'(rest ...) substs))))
1146            (_
1147             #'(syntax-error "error: no 'ungexp-splicing' substitution"
1148                             exp))))))
1149
1150     (define (substitute-references exp substs)
1151       ;; Return a variant of EXP where all the cars of SUBSTS have been
1152       ;; replaced by the corresponding cdr.
1153       (syntax-case exp (ungexp ungexp-native
1154                         ungexp-splicing ungexp-native-splicing)
1155         ((ungexp _ ...)
1156          (substitute-ungexp exp substs))
1157         ((ungexp-native _ ...)
1158          (substitute-ungexp exp substs))
1159         (((ungexp-splicing _ ...) rest ...)
1160          (substitute-ungexp-splicing exp substs))
1161         (((ungexp-native-splicing _ ...) rest ...)
1162          (substitute-ungexp-splicing exp substs))
1163         ((exp0 . exp)
1164          #`(cons #,(substitute-references #'exp0 substs)
1165                  #,(substitute-references #'exp substs)))
1166         (x #''x)))
1167
1168     (syntax-case s (ungexp output)
1169       ((_ exp)
1170        (let* ((escapes (delete-duplicates (collect-escapes #'exp)))
1171               (formals (generate-temporaries escapes))
1172               (sexp    (substitute-references #'exp (zip escapes formals)))
1173               (refs    (map escape->ref escapes)))
1174          #`(make-gexp (list #,@refs)
1175                       current-imported-modules
1176                       current-imported-extensions
1177                       (lambda #,formals
1178                         #,sexp)))))))
1179
1180 \f
1181 ;;;
1182 ;;; Module handling.
1183 ;;;
1184
1185 (define %not-slash
1186   (char-set-complement (char-set #\/)))
1187
1188 (define (file-mapping->tree mapping)
1189   "Convert MAPPING, an alist like:
1190
1191   ((\"guix/build/utils.scm\" . \"…/utils.scm\"))
1192
1193 to a tree suitable for 'interned-file-tree'."
1194   (let ((mapping (map (match-lambda
1195                         ((destination . source)
1196                          (cons (string-tokenize destination
1197                                                 %not-slash)
1198                                source)))
1199                       mapping)))
1200     (fold (lambda (pair result)
1201             (match pair
1202               ((destination . source)
1203                (let loop ((destination destination)
1204                           (result result))
1205                  (match destination
1206                    ((file)
1207                     (let* ((mode (stat:mode (stat source)))
1208                            (type (if (zero? (logand mode #o100))
1209                                      'regular
1210                                      'executable)))
1211                       (alist-cons file
1212                                   `(,type (file ,source))
1213                                   result)))
1214                    ((file rest ...)
1215                     (let ((directory (assoc-ref result file)))
1216                       (alist-cons file
1217                                   `(directory
1218                                     ,@(loop rest
1219                                             (match directory
1220                                               (('directory . entries) entries)
1221                                               (#f '()))))
1222                                   (if directory
1223                                       (alist-delete file result)
1224                                       result)))))))))
1225           '()
1226           mapping)))
1227
1228 (define %utils-module
1229   ;; This file provides 'mkdir-p', needed to implement 'imported-files' and
1230   ;; other primitives below.  Note: We give the file name relative to this
1231   ;; file you are currently reading; 'search-path' could return a file name
1232   ;; relative to the current working directory.
1233   (local-file "build/utils.scm"
1234               "build-utils.scm"))
1235
1236 (define* (imported-files/derivation files
1237                                     #:key (name "file-import")
1238                                     (symlink? #f)
1239                                     (system (%current-system))
1240                                     (guile (%guile-for-build)))
1241   "Return a derivation that imports FILES into STORE.  FILES must be a list
1242 of (FINAL-PATH . FILE) pairs.  Each FILE is mapped to FINAL-PATH in the
1243 resulting store path.  FILE can be either a file name, or a file-like object,
1244 as returned by 'local-file' for example.  If SYMLINK? is true, create symlinks
1245 to the source files instead of copying them."
1246   (define file-pair
1247     (match-lambda
1248      ((final-path . (? string? file-name))
1249       (mlet %store-monad ((file (interned-file file-name
1250                                                (basename final-path))))
1251         (return (list final-path file))))
1252      ((final-path . file-like)
1253       (mlet %store-monad ((file (lower-object file-like system)))
1254         (return (list final-path file))))))
1255
1256   (mlet %store-monad ((files (mapm %store-monad file-pair files)))
1257     (define build
1258       (gexp
1259        (begin
1260          (primitive-load (ungexp %utils-module))  ;for 'mkdir-p'
1261          (use-modules (ice-9 match))
1262
1263          (mkdir (ungexp output)) (chdir (ungexp output))
1264          (for-each (match-lambda
1265                     ((final-path store-path)
1266                      (mkdir-p (dirname final-path))
1267                      ((ungexp (if symlink? 'symlink 'copy-file))
1268                       store-path final-path)))
1269                    '(ungexp files)))))
1270
1271     ;; TODO: Pass FILES as an environment variable so that BUILD remains
1272     ;; exactly the same regardless of FILES: less disk space, and fewer
1273     ;; 'add-to-store' RPCs.
1274     (gexp->derivation name build
1275                       #:system system
1276                       #:guile-for-build guile
1277                       #:local-build? #t
1278
1279                       ;; Avoid deprecation warnings about the use of the _IO*
1280                       ;; constants in (guix build utils).
1281                       #:env-vars
1282                       '(("GUILE_WARN_DEPRECATED" . "no")))))
1283
1284 (define* (imported-files files
1285                          #:key (name "file-import")
1286                          ;; The following parameters make sense when creating
1287                          ;; an actual derivation.
1288                          (system (%current-system))
1289                          (guile (%guile-for-build)))
1290   "Import FILES into the store and return the resulting derivation or store
1291 file name (a derivation is created if and only if some elements of FILES are
1292 file-like objects and not local file names.)  FILES must be a list
1293 of (FINAL-PATH . FILE) pairs.  Each FILE is mapped to FINAL-PATH in the
1294 resulting store path.  FILE can be either a file name, or a file-like object,
1295 as returned by 'local-file' for example."
1296   (if (any (match-lambda
1297              ((_ . (? struct? source)) #t)
1298              (_ #f))
1299            files)
1300       (imported-files/derivation files #:name name
1301                                  #:symlink? derivation?
1302                                  #:system system #:guile guile)
1303       (interned-file-tree `(,name directory
1304                                   ,@(file-mapping->tree files)))))
1305
1306 (define* (imported-modules modules
1307                            #:key (name "module-import")
1308                            (system (%current-system))
1309                            (guile (%guile-for-build))
1310                            (module-path %load-path))
1311   "Return a derivation that contains the source files of MODULES, a list of
1312 module names such as `(ice-9 q)'.  All of MODULES must be either names of
1313 modules to be found in the MODULE-PATH search path, or a module name followed
1314 by an arrow followed by a file-like object.  For example:
1315
1316   (imported-modules `((guix build utils)
1317                       (guix gcrypt)
1318                       ((guix config) => ,(scheme-file …))))
1319
1320 In this example, the first two modules are taken from MODULE-PATH, and the
1321 last one is created from the given <scheme-file> object."
1322   (let ((files (map (match-lambda
1323                       (((module ...) '=> file)
1324                        (cons (module->source-file-name module)
1325                              file))
1326                       ((module ...)
1327                        (let ((f (module->source-file-name module)))
1328                          (cons f (search-path* module-path f)))))
1329                     modules)))
1330     (imported-files files #:name name
1331                     #:system system
1332                     #:guile guile)))
1333
1334 (define* (compiled-modules modules
1335                            #:key (name "module-import-compiled")
1336                            (system (%current-system))
1337                            target
1338                            (guile (%guile-for-build))
1339                            (module-path %load-path)
1340                            (extensions '())
1341                            (deprecation-warnings #f))
1342   "Return a derivation that builds a tree containing the `.go' files
1343 corresponding to MODULES.  All the MODULES are built in a context where
1344 they can refer to each other.  When TARGET is true, cross-compile MODULES for
1345 TARGET, a GNU triplet."
1346   (define total (length modules))
1347
1348   (mlet %store-monad ((modules (imported-modules modules
1349                                                  #:system system
1350                                                  #:guile guile
1351                                                  #:module-path
1352                                                  module-path)))
1353     (define build
1354       (gexp
1355        (begin
1356          (primitive-load (ungexp %utils-module))  ;for 'mkdir-p'
1357
1358          (use-modules (ice-9 ftw)
1359                       (ice-9 format)
1360                       (srfi srfi-1)
1361                       (srfi srfi-26)
1362                       (system base compile))
1363
1364          ;; TODO: Inline this on the next rebuild cycle.
1365          (ungexp-splicing
1366           (if target
1367               (gexp ((use-modules (system base target))))
1368               (gexp ())))
1369
1370          (define (regular? file)
1371            (not (member file '("." ".."))))
1372
1373          (define (process-entry entry output processed)
1374            (if (file-is-directory? entry)
1375                (let ((output (string-append output "/" (basename entry))))
1376                  (mkdir-p output)
1377                  (process-directory entry output processed))
1378                (let* ((base   (basename entry ".scm"))
1379                       (output (string-append output "/" base ".go")))
1380                  (format #t "[~2@a/~2@a] Compiling '~a'...~%"
1381                          (+ 1 processed (ungexp total))
1382                          (ungexp (* total 2))
1383                          entry)
1384
1385                  (ungexp-splicing
1386                   (if target
1387                       (gexp ((with-target (ungexp target)
1388                                (lambda ()
1389                                  (compile-file entry
1390                                                #:output-file output
1391                                                #:opts
1392                                                %auto-compilation-options)))))
1393                       (gexp ((compile-file entry
1394                                            #:output-file output
1395                                            #:opts %auto-compilation-options)))))
1396
1397                  (+ 1 processed))))
1398
1399          (define (process-directory directory output processed)
1400            (let ((entries (map (cut string-append directory "/" <>)
1401                                (scandir directory regular?))))
1402              (fold (cut process-entry <> output <>)
1403                    processed
1404                    entries)))
1405
1406          (define* (load-from-directory directory
1407                                        #:optional (loaded 0))
1408            "Load all the source files found in DIRECTORY."
1409            ;; XXX: This works around <https://bugs.gnu.org/15602>.
1410            (let ((entries (map (cut string-append directory "/" <>)
1411                                (scandir directory regular?))))
1412              (fold (lambda (file loaded)
1413                      (if (file-is-directory? file)
1414                          (load-from-directory file loaded)
1415                          (begin
1416                            (format #t "[~2@a/~2@a] Loading '~a'...~%"
1417                                    (+ 1 loaded) (ungexp (* 2 total))
1418                                    file)
1419                            (save-module-excursion
1420                             (lambda ()
1421                               (primitive-load file)))
1422                            (+ 1 loaded))))
1423                    loaded
1424                    entries)))
1425
1426          (setvbuf (current-output-port)
1427                   (cond-expand (guile-2.2 'line) (else _IOLBF)))
1428
1429          (define mkdir-p
1430            ;; Capture 'mkdir-p'.
1431            (@ (guix build utils) mkdir-p))
1432
1433          ;; Add EXTENSIONS to the search path.
1434          (set! %load-path
1435            (append (map (lambda (extension)
1436                           (string-append extension
1437                                          "/share/guile/site/"
1438                                          (effective-version)))
1439                         '((ungexp-native-splicing extensions)))
1440                    %load-path))
1441          (set! %load-compiled-path
1442            (append (map (lambda (extension)
1443                           (string-append extension "/lib/guile/"
1444                                          (effective-version)
1445                                          "/site-ccache"))
1446                         '((ungexp-native-splicing extensions)))
1447                    %load-compiled-path))
1448
1449          (set! %load-path (cons (ungexp modules) %load-path))
1450
1451          ;; Above we loaded our own (guix build utils) but now we may need to
1452          ;; load a compile a different one.  Thus, force a reload.
1453          (let ((utils (string-append (ungexp modules)
1454                                      "/guix/build/utils.scm")))
1455            (when (file-exists? utils)
1456              (load utils)))
1457
1458          (mkdir (ungexp output))
1459          (chdir (ungexp modules))
1460
1461          (load-from-directory ".")
1462          (process-directory "." (ungexp output) 0))))
1463
1464     ;; TODO: Pass MODULES as an environment variable.
1465     (gexp->derivation name build
1466                       #:system system
1467                       #:guile-for-build guile
1468                       #:local-build? #t
1469                       #:env-vars
1470                       (case deprecation-warnings
1471                         ((#f)
1472                          '(("GUILE_WARN_DEPRECATED" . "no")))
1473                         ((detailed)
1474                          '(("GUILE_WARN_DEPRECATED" . "detailed")))
1475                         (else
1476                          '())))))
1477
1478 \f
1479 ;;;
1480 ;;; Convenience procedures.
1481 ;;;
1482
1483 (define (default-guile)
1484   ;; Lazily resolve 'guile-2.2' (not 'guile-final' because this is for
1485   ;; programs returned by 'program-file' and we don't want to keep references
1486   ;; to several Guile packages).  This module must not refer to (gnu …)
1487   ;; modules directly, to avoid circular dependencies, hence this hack.
1488   (module-ref (resolve-interface '(gnu packages guile))
1489               'guile-2.2))
1490
1491 (define* (load-path-expression modules #:optional (path %load-path)
1492                                #:key (extensions '()) system target)
1493   "Return as a monadic value a gexp that sets '%load-path' and
1494 '%load-compiled-path' to point to MODULES, a list of module names.  MODULES
1495 are searched for in PATH.  Return #f when MODULES and EXTENSIONS are empty."
1496   (if (and (null? modules) (null? extensions))
1497       (with-monad %store-monad
1498         (return #f))
1499       (mlet %store-monad ((modules  (imported-modules modules
1500                                                       #:module-path path
1501                                                       #:system system))
1502                           (compiled (compiled-modules modules
1503                                                       #:extensions extensions
1504                                                       #:module-path path
1505                                                       #:system system
1506                                                       #:target target)))
1507         (return (gexp (eval-when (expand load eval)
1508                         (set! %load-path
1509                           (cons (ungexp modules)
1510                                 (append (map (lambda (extension)
1511                                                (string-append extension
1512                                                               "/share/guile/site/"
1513                                                               (effective-version)))
1514                                              '((ungexp-native-splicing extensions)))
1515                                         %load-path)))
1516                         (set! %load-compiled-path
1517                           (cons (ungexp compiled)
1518                                 (append (map (lambda (extension)
1519                                                (string-append extension
1520                                                               "/lib/guile/"
1521                                                               (effective-version)
1522                                                               "/site-ccache"))
1523                                              '((ungexp-native-splicing extensions)))
1524                                         %load-compiled-path)))))))))
1525
1526 (define* (gexp->script name exp
1527                        #:key (guile (default-guile))
1528                        (module-path %load-path)
1529                        (system (%current-system))
1530                        target)
1531   "Return an executable script NAME that runs EXP using GUILE, with EXP's
1532 imported modules in its search path.  Look up EXP's modules in MODULE-PATH."
1533   (mlet %store-monad ((set-load-path
1534                        (load-path-expression (gexp-modules exp)
1535                                              module-path
1536                                              #:extensions
1537                                              (gexp-extensions exp)
1538                                              #:system system
1539                                              #:target target)))
1540     (gexp->derivation name
1541                       (gexp
1542                        (call-with-output-file (ungexp output)
1543                          (lambda (port)
1544                            ;; Note: that makes a long shebang.  When the store
1545                            ;; is /gnu/store, that fits within the 128-byte
1546                            ;; limit imposed by Linux, but that may go beyond
1547                            ;; when running tests.
1548                            (format port
1549                                    "#!~a/bin/guile --no-auto-compile~%!#~%"
1550                                    (ungexp guile))
1551
1552                            (ungexp-splicing
1553                             (if set-load-path
1554                                 (gexp ((write '(ungexp set-load-path) port)))
1555                                 (gexp ())))
1556
1557                            (write '(ungexp exp) port)
1558                            (chmod port #o555))))
1559                       #:system system
1560                       #:target target
1561                       #:module-path module-path)))
1562
1563 (define* (gexp->file name exp #:key
1564                      (set-load-path? #t)
1565                      (module-path %load-path)
1566                      (splice? #f))
1567   "Return a derivation that builds a file NAME containing EXP.  When SPLICE?
1568 is true, EXP is considered to be a list of expressions that will be spliced in
1569 the resulting file.
1570
1571 When SET-LOAD-PATH? is true, emit code in the resulting file to set
1572 '%load-path' and '%load-compiled-path' to honor EXP's imported modules.
1573 Lookup EXP's modules in MODULE-PATH."
1574   (define modules (gexp-modules exp))
1575   (define extensions (gexp-extensions exp))
1576
1577   (if (or (not set-load-path?)
1578           (and (null? modules) (null? extensions)))
1579       (gexp->derivation name
1580                         (gexp
1581                          (call-with-output-file (ungexp output)
1582                            (lambda (port)
1583                              (for-each (lambda (exp)
1584                                          (write exp port))
1585                                        '(ungexp (if splice?
1586                                                     exp
1587                                                     (gexp ((ungexp exp)))))))))
1588                         #:local-build? #t
1589                         #:substitutable? #f)
1590       (mlet %store-monad ((set-load-path
1591                            (load-path-expression modules module-path
1592                                                  #:extensions extensions)))
1593         (gexp->derivation name
1594                           (gexp
1595                            (call-with-output-file (ungexp output)
1596                              (lambda (port)
1597                                (write '(ungexp set-load-path) port)
1598                                (for-each (lambda (exp)
1599                                            (write exp port))
1600                                          '(ungexp (if splice?
1601                                                       exp
1602                                                       (gexp ((ungexp exp)))))))))
1603                           #:module-path module-path
1604                           #:local-build? #t
1605                           #:substitutable? #f))))
1606
1607 (define* (text-file* name #:rest text)
1608   "Return as a monadic value a derivation that builds a text file containing
1609 all of TEXT.  TEXT may list, in addition to strings, objects of any type that
1610 can be used in a gexp: packages, derivations, local file objects, etc.  The
1611 resulting store file holds references to all these."
1612   (define builder
1613     (gexp (call-with-output-file (ungexp output "out")
1614             (lambda (port)
1615               (display (string-append (ungexp-splicing text)) port)))))
1616
1617   (gexp->derivation name builder
1618                     #:local-build? #t
1619                     #:substitutable? #f))
1620
1621 (define* (mixed-text-file name #:rest text)
1622   "Return an object representing store file NAME containing TEXT.  TEXT is a
1623 sequence of strings and file-like objects, as in:
1624
1625   (mixed-text-file \"profile\"
1626                    \"export PATH=\" coreutils \"/bin:\" grep \"/bin\")
1627
1628 This is the declarative counterpart of 'text-file*'."
1629   (define build
1630     (gexp (call-with-output-file (ungexp output "out")
1631             (lambda (port)
1632               (display (string-append (ungexp-splicing text)) port)))))
1633
1634   (computed-file name build))
1635
1636 (define (file-union name files)
1637   "Return a <computed-file> that builds a directory containing all of FILES.
1638 Each item in FILES must be a two-element list where the first element is the
1639 file name to use in the new directory, and the second element is a gexp
1640 denoting the target file.  Here's an example:
1641
1642   (file-union \"etc\"
1643               `((\"hosts\" ,(plain-file \"hosts\"
1644                                         \"127.0.0.1 localhost\"))
1645                 (\"bashrc\" ,(plain-file \"bashrc\"
1646                                          \"alias ls='ls --color'\"))
1647                 (\"libvirt/qemu.conf\" ,(plain-file \"qemu.conf\" \"\"))))
1648
1649 This yields an 'etc' directory containing these two files."
1650   (computed-file name
1651                  (with-imported-modules '((guix build utils))
1652                    (gexp
1653                     (begin
1654                       (use-modules (guix build utils))
1655
1656                       (mkdir (ungexp output))
1657                       (chdir (ungexp output))
1658                       (ungexp-splicing
1659                        (map (match-lambda
1660                               ((target source)
1661                                (gexp
1662                                 (begin
1663                                   ;; Stat the source to abort early if it does
1664                                   ;; not exist.
1665                                   (stat (ungexp source))
1666
1667                                   (mkdir-p (dirname (ungexp target)))
1668                                   (symlink (ungexp source)
1669                                            (ungexp target))))))
1670                             files)))))))
1671
1672 (define* (directory-union name things
1673                           #:key (copy? #f) (quiet? #f)
1674                           (resolve-collision 'warn-about-collision))
1675   "Return a directory that is the union of THINGS, where THINGS is a list of
1676 file-like objects denoting directories.  For example:
1677
1678   (directory-union \"guile+emacs\" (list guile emacs))
1679
1680 yields a directory that is the union of the 'guile' and 'emacs' packages.
1681
1682 Call RESOLVE-COLLISION when several files collide, passing it the list of
1683 colliding files.  RESOLVE-COLLISION must return the chosen file or #f, in
1684 which case the colliding entry is skipped altogether.
1685
1686 When HARD-LINKS? is true, create hard links instead of symlinks.  When QUIET?
1687 is true, the derivation will not print anything."
1688   (define symlink
1689     (if copy?
1690         (gexp (lambda (old new)
1691                 (if (file-is-directory? old)
1692                     (symlink old new)
1693                     (copy-file old new))))
1694         (gexp symlink)))
1695
1696   (define log-port
1697     (if quiet?
1698         (gexp (%make-void-port "w"))
1699         (gexp (current-error-port))))
1700
1701   (match things
1702     ((one)
1703      ;; Only one thing; return it.
1704      one)
1705     (_
1706      (computed-file name
1707                     (with-imported-modules '((guix build union))
1708                       (gexp (begin
1709                               (use-modules (guix build union)
1710                                            (srfi srfi-1)) ;for 'first' and 'last'
1711
1712                               (union-build (ungexp output)
1713                                            '(ungexp things)
1714
1715                                            #:log-port (ungexp log-port)
1716                                            #:symlink (ungexp symlink)
1717                                            #:resolve-collision
1718                                            (ungexp resolve-collision)))))))))
1719
1720 \f
1721 ;;;
1722 ;;; Syntactic sugar.
1723 ;;;
1724
1725 (eval-when (expand load eval)
1726   (define* (read-ungexp chr port #:optional native?)
1727     "Read an 'ungexp' or 'ungexp-splicing' form from PORT.  When NATIVE? is
1728 true, use 'ungexp-native' and 'ungexp-native-splicing' instead."
1729     (define unquote-symbol
1730       (match (peek-char port)
1731         (#\@
1732          (read-char port)
1733          (if native?
1734              'ungexp-native-splicing
1735              'ungexp-splicing))
1736         (_
1737          (if native?
1738              'ungexp-native
1739              'ungexp))))
1740
1741     (match (read port)
1742       ((? symbol? symbol)
1743        (let ((str (symbol->string symbol)))
1744          (match (string-index-right str #\:)
1745            (#f
1746             `(,unquote-symbol ,symbol))
1747            (colon
1748             (let ((name   (string->symbol (substring str 0 colon)))
1749                   (output (substring str (+ colon 1))))
1750               `(,unquote-symbol ,name ,output))))))
1751       (x
1752        `(,unquote-symbol ,x))))
1753
1754   (define (read-gexp chr port)
1755     "Read a 'gexp' form from PORT."
1756     `(gexp ,(read port)))
1757
1758   ;; Extend the reader
1759   (read-hash-extend #\~ read-gexp)
1760   (read-hash-extend #\$ read-ungexp)
1761   (read-hash-extend #\+ (cut read-ungexp <> <> #t)))
1762
1763 ;;; gexp.scm ends here