guix/gexp.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2014, 2015, 2016, 2017, 2018, 2019 Ludovic Courtès <ludo@gnu.org>
   3 ;;; Copyright © 2018 Clément Lassieur <clement@lassieur.org>
   4 ;;; Copyright © 2018 Jan Nieuwenhuizen <janneke@gnu.org>
   5 ;;;
   6 ;;; This file is part of GNU Guix.
   7 ;;;
   8 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   9 ;;; under the terms of the GNU General Public License as published by
  10 ;;; the Free Software Foundation; either version 3 of the License, or (at
  11 ;;; your option) any later version.
  12 ;;;
  13 ;;; GNU Guix is distributed in the hope that it will be useful, but
  14 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  15 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 ;;; GNU General Public License for more details.
  17 ;;;
  18 ;;; You should have received a copy of the GNU General Public License
  19 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  20
  21 (define-module (guix gexp)
  22   #:use-module (guix store)
  23   #:use-module (guix monads)
  24   #:use-module (guix derivations)
  25   #:use-module (guix grafts)
  26   #:use-module (guix utils)
  27   #:use-module (rnrs bytevectors)
  28   #:use-module (srfi srfi-1)
  29   #:use-module (srfi srfi-9)
  30   #:use-module (srfi srfi-9 gnu)
  31   #:use-module (srfi srfi-26)
  32   #:use-module (srfi srfi-34)
  33   #:use-module (srfi srfi-35)
  34   #:use-module (ice-9 match)
  35   #:export (gexp
  36             gexp?
  37             with-imported-modules
  38             with-extensions
  39
  40             gexp-input
  41             gexp-input?
  42             gexp-input-thing
  43             gexp-input-output
  44             gexp-input-native?
  45
  46             local-file
  47             local-file?
  48             local-file-file
  49             local-file-absolute-file-name
  50             local-file-name
  51             local-file-recursive?
  52
  53             plain-file
  54             plain-file?
  55             plain-file-name
  56             plain-file-content
  57
  58             computed-file
  59             computed-file?
  60             computed-file-name
  61             computed-file-gexp
  62             computed-file-options
  63
  64             program-file
  65             program-file?
  66             program-file-name
  67             program-file-gexp
  68             program-file-guile
  69             program-file-module-path
  70
  71             scheme-file
  72             scheme-file?
  73             scheme-file-name
  74             scheme-file-gexp
  75
  76             file-append
  77             file-append?
  78             file-append-base
  79             file-append-suffix
  80
  81             load-path-expression
  82             gexp-modules
  83
  84             lower-gexp
  85             lowered-gexp?
  86             lowered-gexp-sexp
  87             lowered-gexp-inputs
  88             lowered-gexp-sources
  89             lowered-gexp-guile
  90             lowered-gexp-load-path
  91             lowered-gexp-load-compiled-path
  92
  93             gexp->derivation
  94             gexp->file
  95             gexp->script
  96             text-file*
  97             mixed-text-file
  98             file-union
  99             directory-union
 100             imported-files
 101             imported-modules
 102             compiled-modules
 103
 104             define-gexp-compiler
 105             gexp-compiler?
 106             file-like?
 107             lower-object
 108
 109             lower-inputs
 110
 111             &gexp-error
 112             gexp-error?
 113             &gexp-input-error
 114             gexp-input-error?
 115             gexp-error-invalid-input))
 116
 117 ;;; Commentary:
 118 ;;;
 119 ;;; This module implements "G-expressions", or "gexps".  Gexps are like
 120 ;;; S-expressions (sexps), with two differences:
 121 ;;;
 122 ;;;   1. References (un-quotations) to derivations or packages in a gexp are
 123 ;;;      replaced by the corresponding output file name; in addition, the
 124 ;;;      'ungexp-native' unquote-like form allows code to explicitly refer to
 125 ;;;      the native code of a given package, in case of cross-compilation;
 126 ;;;
 127 ;;;   2. Gexps embed information about the derivations they refer to.
 128 ;;;
 129 ;;; Gexps make it easy to write to files Scheme code that refers to store
 130 ;;; items, or to write Scheme code to build derivations.
 131 ;;;
 132 ;;; Code:
 133
 134 ;; "G expressions".
 135 (define-record-type <gexp>
 136   (make-gexp references modules extensions proc)
 137   gexp?
 138   (references gexp-references)                    ;list of <gexp-input>
 139   (modules    gexp-self-modules)                  ;list of module names
 140   (extensions gexp-self-extensions)               ;list of lowerable things
 141   (proc       gexp-proc))                         ;procedure
 142
 143 (define (write-gexp gexp port)
 144   "Write GEXP on PORT."
 145   (display "#<gexp " port)
 146
 147   ;; Try to write the underlying sexp.  Now, this trick doesn't work when
 148   ;; doing things like (ungexp-splicing (gexp ())) because GEXP's procedure
 149   ;; tries to use 'append' on that, which fails with wrong-type-arg.
 150   (false-if-exception
 151    (write (apply (gexp-proc gexp)
 152                  (gexp-references gexp))
 153           port))
 154   (format port " ~a>"
 155           (number->string (object-address gexp) 16)))
 156
 157 (set-record-type-printer! <gexp> write-gexp)
 158
 159 \f
 160 ;;;
 161 ;;; Methods.
 162 ;;;
 163
 164 ;; Compiler for a type of objects that may be introduced in a gexp.
 165 (define-record-type <gexp-compiler>
 166   (gexp-compiler type lower expand)
 167   gexp-compiler?
 168   (type       gexp-compiler-type)                 ;record type descriptor
 169   (lower      gexp-compiler-lower)
 170   (expand     gexp-compiler-expand))              ;#f | DRV -> sexp
 171
 172 (define-condition-type &gexp-error &error
 173   gexp-error?)
 174
 175 (define-condition-type &gexp-input-error &gexp-error
 176   gexp-input-error?
 177   (input gexp-error-invalid-input))
 178
 179
 180 (define %gexp-compilers
 181   ;; 'eq?' mapping of record type descriptor to <gexp-compiler>.
 182   (make-hash-table 20))
 183
 184 (define (default-expander thing obj output)
 185   "This is the default expander for \"things\" that appear in gexps.  It
 186 returns its output file name of OBJ's OUTPUT."
 187   (match obj
 188     ((? derivation? drv)
 189      (derivation->output-path drv output))
 190     ((? string? file)
 191      file)))
 192
 193 (define (register-compiler! compiler)
 194   "Register COMPILER as a gexp compiler."
 195   (hashq-set! %gexp-compilers
 196               (gexp-compiler-type compiler) compiler))
 197
 198 (define (lookup-compiler object)
 199   "Search for a compiler for OBJECT.  Upon success, return the three argument
 200 procedure to lower it; otherwise return #f."
 201   (and=> (hashq-ref %gexp-compilers (struct-vtable object))
 202          gexp-compiler-lower))
 203
 204 (define (file-like? object)
 205   "Return #t if OBJECT leads to a file in the store once unquoted in a
 206 G-expression; otherwise return #f."
 207   (and (struct? object) (->bool (lookup-compiler object))))
 208
 209 (define (lookup-expander object)
 210   "Search for an expander for OBJECT.  Upon success, return the three argument
 211 procedure to expand it; otherwise return #f."
 212   (and=> (hashq-ref %gexp-compilers (struct-vtable object))
 213          gexp-compiler-expand))
 214
 215 (define* (lower-object obj
 216                        #:optional (system (%current-system))
 217                        #:key target)
 218   "Return as a value in %STORE-MONAD the derivation or store item
 219 corresponding to OBJ for SYSTEM, cross-compiling for TARGET if TARGET is true.
 220 OBJ must be an object that has an associated gexp compiler, such as a
 221 <package>."
 222   (match (lookup-compiler obj)
 223     (#f
 224      (raise (condition (&gexp-input-error (input obj)))))
 225     (lower
 226      ;; Cache in STORE the result of lowering OBJ.
 227      (mlet %store-monad ((graft? (grafting?)))
 228        (mcached (let ((lower (lookup-compiler obj)))
 229                   (lower obj system target))
 230                 obj
 231                 system target graft?)))))
 232
 233 (define-syntax define-gexp-compiler
 234   (syntax-rules (=> compiler expander)
 235     "Define NAME as a compiler for objects matching PREDICATE encountered in
 236 gexps.
 237
 238 In the simplest form of the macro, BODY must return a derivation for PARAM, an
 239 object that matches PREDICATE, for SYSTEM and TARGET (the latter of which is
 240 #f except when cross-compiling.)
 241
 242 The more elaborate form allows you to specify an expander:
 243
 244   (define-gexp-compiler something something?
 245     compiler => (lambda (param system target) ...)
 246     expander => (lambda (param drv output) ...))
 247
 248 The expander specifies how an object is converted to its sexp representation."
 249     ((_ (name (param record-type) system target) body ...)
 250      (define-gexp-compiler name record-type
 251        compiler => (lambda (param system target) body ...)
 252        expander => default-expander))
 253     ((_ name record-type
 254         compiler => compile
 255         expander => expand)
 256      (begin
 257        (define name
 258          (gexp-compiler record-type compile expand))
 259        (register-compiler! name)))))
 260
 261 (define-gexp-compiler (derivation-compiler (drv <derivation>) system target)
 262   ;; Derivations are the lowest-level representation, so this is the identity
 263   ;; compiler.
 264   (with-monad %store-monad
 265     (return drv)))
 266
 267 \f
 268 ;;;
 269 ;;; File declarations.
 270 ;;;
 271
 272 ;; A local file name.  FILE is the file name the user entered, which can be a
 273 ;; relative file name, and ABSOLUTE is a promise that computes its canonical
 274 ;; absolute file name.  We keep it in a promise to compute it lazily and avoid
 275 ;; repeated 'stat' calls.
 276 (define-record-type <local-file>
 277   (%%local-file file absolute name recursive? select?)
 278   local-file?
 279   (file       local-file-file)                    ;string
 280   (absolute   %local-file-absolute-file-name)     ;promise string
 281   (name       local-file-name)                    ;string
 282   (recursive? local-file-recursive?)              ;Boolean
 283   (select?    local-file-select?))                ;string stat -> Boolean
 284
 285 (define (true file stat) #t)
 286
 287 (define* (%local-file file promise #:optional (name (basename file))
 288                       #:key recursive? (select? true))
 289   ;; This intermediate procedure is part of our ABI, but the underlying
 290   ;; %%LOCAL-FILE is not.
 291   (%%local-file file promise name recursive? select?))
 292
 293 (define (absolute-file-name file directory)
 294   "Return the canonical absolute file name for FILE, which lives in the
 295 vicinity of DIRECTORY."
 296   (canonicalize-path
 297    (cond ((string-prefix? "/" file) file)
 298          ((not directory) file)
 299          ((string-prefix? "/" directory)
 300           (string-append directory "/" file))
 301          (else file))))
 302
 303 (define-syntax local-file
 304   (lambda (s)
 305     "Return an object representing local file FILE to add to the store; this
 306 object can be used in a gexp.  If FILE is a relative file name, it is looked
 307 up relative to the source file where this form appears.  FILE will be added to
 308 the store under NAME--by default the base name of FILE.
 309
 310 When RECURSIVE? is true, the contents of FILE are added recursively; if FILE
 311 designates a flat file and RECURSIVE? is true, its contents are added, and its
 312 permission bits are kept.
 313
 314 When RECURSIVE? is true, call (SELECT?  FILE STAT) for each directory entry,
 315 where FILE is the entry's absolute file name and STAT is the result of
 316 'lstat'; exclude entries for which SELECT? does not return true.
 317
 318 This is the declarative counterpart of the 'interned-file' monadic procedure.
 319 It is implemented as a macro to capture the current source directory where it
 320 appears."
 321     (syntax-case s ()
 322       ((_ file rest ...)
 323        #'(%local-file file
 324                       (delay (absolute-file-name file (current-source-directory)))
 325                       rest ...))
 326       ((_)
 327        #'(syntax-error "missing file name"))
 328       (id
 329        (identifier? #'id)
 330        ;; XXX: We could return #'(lambda (file . rest) ...).  However,
 331        ;; (syntax-source #'id) is #f so (current-source-directory) would not
 332        ;; work.  Thus, simply forbid this form.
 333        #'(syntax-error
 334           "'local-file' is a macro and cannot be used like this")))))
 335
 336 (define (local-file-absolute-file-name file)
 337   "Return the absolute file name for FILE, a <local-file> instance.  A
 338 'system-error' exception is raised if FILE could not be found."
 339   (force (%local-file-absolute-file-name file)))
 340
 341 (define-gexp-compiler (local-file-compiler (file <local-file>) system target)
 342   ;; "Compile" FILE by adding it to the store.
 343   (match file
 344     (($ <local-file> file (= force absolute) name recursive? select?)
 345      ;; Canonicalize FILE so that if it's a symlink, it is resolved.  Failing
 346      ;; to do that, when RECURSIVE? is #t, we could end up creating a dangling
 347      ;; symlink in the store, and when RECURSIVE? is #f 'add-to-store' would
 348      ;; just throw an error, both of which are inconvenient.
 349      (interned-file absolute name
 350                     #:recursive? recursive? #:select? select?))))
 351
 352 (define-record-type <plain-file>
 353   (%plain-file name content references)
 354   plain-file?
 355   (name        plain-file-name)                   ;string
 356   (content     plain-file-content)                ;string or bytevector
 357   (references  plain-file-references))            ;list (currently unused)
 358
 359 (define (plain-file name content)
 360   "Return an object representing a text file called NAME with the given
 361 CONTENT (a string) to be added to the store.
 362
 363 This is the declarative counterpart of 'text-file'."
 364   ;; XXX: For now just ignore 'references' because it's not clear how to use
 365   ;; them in a declarative context.
 366   (%plain-file name content '()))
 367
 368 (define-gexp-compiler (plain-file-compiler (file <plain-file>) system target)
 369   ;; "Compile" FILE by adding it to the store.
 370   (match file
 371     (($ <plain-file> name (and (? string?) content) references)
 372      (text-file name content references))
 373     (($ <plain-file> name (and (? bytevector?) content) references)
 374      (binary-file name content references))))
 375
 376 (define-record-type <computed-file>
 377   (%computed-file name gexp guile options)
 378   computed-file?
 379   (name       computed-file-name)                 ;string
 380   (gexp       computed-file-gexp)                 ;gexp
 381   (guile      computed-file-guile)                ;<package>
 382   (options    computed-file-options))             ;list of arguments
 383
 384 (define* (computed-file name gexp
 385                         #:key guile (options '(#:local-build? #t)))
 386   "Return an object representing the store item NAME, a file or directory
 387 computed by GEXP.  OPTIONS is a list of additional arguments to pass
 388 to 'gexp->derivation'.
 389
 390 This is the declarative counterpart of 'gexp->derivation'."
 391   (%computed-file name gexp guile options))
 392
 393 (define-gexp-compiler (computed-file-compiler (file <computed-file>)
 394                                               system target)
 395   ;; Compile FILE by returning a derivation whose build expression is its
 396   ;; gexp.
 397   (match file
 398     (($ <computed-file> name gexp guile options)
 399      (if guile
 400          (mlet %store-monad ((guile (lower-object guile system
 401                                                   #:target target)))
 402            (apply gexp->derivation name gexp #:guile-for-build guile
 403                   #:system system #:target target options))
 404          (apply gexp->derivation name gexp
 405                 #:system system #:target target options)))))
 406
 407 (define-record-type <program-file>
 408   (%program-file name gexp guile path)
 409   program-file?
 410   (name       program-file-name)                  ;string
 411   (gexp       program-file-gexp)                  ;gexp
 412   (guile      program-file-guile)                 ;package
 413   (path       program-file-module-path))          ;list of strings
 414
 415 (define* (program-file name gexp #:key (guile #f) (module-path %load-path))
 416   "Return an object representing the executable store item NAME that runs
 417 GEXP.  GUILE is the Guile package used to execute that script.  Imported
 418 modules of GEXP are looked up in MODULE-PATH.
 419
 420 This is the declarative counterpart of 'gexp->script'."
 421   (%program-file name gexp guile module-path))
 422
 423 (define-gexp-compiler (program-file-compiler (file <program-file>)
 424                                              system target)
 425   ;; Compile FILE by returning a derivation that builds the script.
 426   (match file
 427     (($ <program-file> name gexp guile module-path)
 428      (gexp->script name gexp
 429                    #:module-path module-path
 430                    #:guile (or guile (default-guile))
 431                    #:system system
 432                    #:target target))))
 433
 434 (define-record-type <scheme-file>
 435   (%scheme-file name gexp splice?)
 436   scheme-file?
 437   (name       scheme-file-name)                  ;string
 438   (gexp       scheme-file-gexp)                  ;gexp
 439   (splice?    scheme-file-splice?))              ;Boolean
 440
 441 (define* (scheme-file name gexp #:key splice?)
 442   "Return an object representing the Scheme file NAME that contains GEXP.
 443
 444 This is the declarative counterpart of 'gexp->file'."
 445   (%scheme-file name gexp splice?))
 446
 447 (define-gexp-compiler (scheme-file-compiler (file <scheme-file>)
 448                                             system target)
 449   ;; Compile FILE by returning a derivation that builds the file.
 450   (match file
 451     (($ <scheme-file> name gexp splice?)
 452      (gexp->file name gexp #:splice? splice?))))
 453
 454 ;; Appending SUFFIX to BASE's output file name.
 455 (define-record-type <file-append>
 456   (%file-append base suffix)
 457   file-append?
 458   (base   file-append-base)                    ;<package> | <derivation> | ...
 459   (suffix file-append-suffix))                 ;list of strings
 460
 461 (define (write-file-append file port)
 462   (match file
 463     (($ <file-append> base suffix)
 464      (format port "#<file-append ~s ~s>" base
 465              (string-join suffix)))))
 466
 467 (set-record-type-printer! <file-append> write-file-append)
 468
 469 (define (file-append base . suffix)
 470   "Return a <file-append> object that expands to the concatenation of BASE and
 471 SUFFIX."
 472   (%file-append base suffix))
 473
 474 (define-gexp-compiler file-append-compiler <file-append>
 475   compiler => (lambda (obj system target)
 476                 (match obj
 477                   (($ <file-append> base _)
 478                    (lower-object base system #:target target))))
 479   expander => (lambda (obj lowered output)
 480                 (match obj
 481                   (($ <file-append> base suffix)
 482                    (let* ((expand (lookup-expander base))
 483                           (base   (expand base lowered output)))
 484                      (string-append base (string-concatenate suffix)))))))
 485
 486 \f
 487 ;;;
 488 ;;; Inputs & outputs.
 489 ;;;
 490
 491 ;; The input of a gexp.
 492 (define-record-type <gexp-input>
 493   (%gexp-input thing output native?)
 494   gexp-input?
 495   (thing     gexp-input-thing)       ;<package> | <origin> | <derivation> | ...
 496   (output    gexp-input-output)      ;string
 497   (native?   gexp-input-native?))    ;Boolean
 498
 499 (define (write-gexp-input input port)
 500   (match input
 501     (($ <gexp-input> thing output #f)
 502      (format port "#<gexp-input ~s:~a>" thing output))
 503     (($ <gexp-input> thing output #t)
 504      (format port "#<gexp-input native ~s:~a>" thing output))))
 505
 506 (set-record-type-printer! <gexp-input> write-gexp-input)
 507
 508 (define* (gexp-input thing                        ;convenience procedure
 509                      #:optional (output "out")
 510                      #:key native?)
 511   "Return a new <gexp-input> for the OUTPUT of THING; NATIVE? determines
 512 whether this should be considered a \"native\" input or not."
 513   (%gexp-input thing output native?))
 514
 515 ;; Reference to one of the derivation's outputs, for gexps used in
 516 ;; derivations.
 517 (define-record-type <gexp-output>
 518   (gexp-output name)
 519   gexp-output?
 520   (name gexp-output-name))
 521
 522 (define (write-gexp-output output port)
 523   (match output
 524     (($ <gexp-output> name)
 525      (format port "#<gexp-output ~a>" name))))
 526
 527 (set-record-type-printer! <gexp-output> write-gexp-output)
 528
 529 (define* (gexp-attribute gexp self-attribute #:optional (equal? equal?))
 530   "Recurse on GEXP and the expressions it refers to, summing the items
 531 returned by SELF-ATTRIBUTE, a procedure that takes a gexp.  Use EQUAL? as the
 532 second argument to 'delete-duplicates'."
 533   (if (gexp? gexp)
 534       (delete-duplicates
 535        (append (self-attribute gexp)
 536                (append-map (match-lambda
 537                              (($ <gexp-input> (? gexp? exp))
 538                               (gexp-attribute exp self-attribute))
 539                              (($ <gexp-input> (lst ...))
 540                               (append-map (lambda (item)
 541                                             (if (gexp? item)
 542                                                 (gexp-attribute item
 543                                                                 self-attribute)
 544                                                 '()))
 545                                           lst))
 546                              (_
 547                               '()))
 548                            (gexp-references gexp)))
 549        equal?)
 550       '()))                                       ;plain Scheme data type
 551
 552 (define (gexp-modules gexp)
 553   "Return the list of Guile module names GEXP relies on.  If (gexp? GEXP) is
 554 false, meaning that GEXP is a plain Scheme object, return the empty list."
 555   (define (module=? m1 m2)
 556     ;; Return #t when M1 equals M2.  Special-case '=>' specs because their
 557     ;; right-hand side may not be comparable with 'equal?': it's typically a
 558     ;; file-like object that embeds a gexp, which in turn embeds closure;
 559     ;; those closures may be 'eq?' when running compiled code but are unlikely
 560     ;; to be 'eq?' when running on 'eval'.  Ignore the right-hand side to
 561     ;; avoid this discrepancy.
 562     (match m1
 563       (((name1 ...) '=> _)
 564        (match m2
 565          (((name2 ...) '=> _) (equal? name1 name2))
 566          (_ #f)))
 567       (_
 568        (equal? m1 m2))))
 569
 570   (gexp-attribute gexp gexp-self-modules module=?))
 571
 572 (define (gexp-extensions gexp)
 573   "Return the list of Guile extensions (packages) GEXP relies on.  If (gexp?
 574 GEXP) is false, meaning that GEXP is a plain Scheme object, return the empty
 575 list."
 576   (gexp-attribute gexp gexp-self-extensions))
 577
 578 (define* (lower-inputs inputs
 579                        #:key system target)
 580   "Turn any object from INPUTS into a derivation input for SYSTEM or a store
 581 item (a \"source\"); return the corresponding input list as a monadic value.
 582 When TARGET is true, use it as the cross-compilation target triplet."
 583   (define (store-item? obj)
 584     (and (string? obj) (store-path? obj)))
 585
 586   (with-monad %store-monad
 587     (mapm %store-monad
 588           (match-lambda
 589             (((? struct? thing) sub-drv ...)
 590              (mlet %store-monad ((obj (lower-object
 591                                        thing system #:target target)))
 592                (return (match obj
 593                          ((? derivation? drv)
 594                           (let ((outputs (if (null? sub-drv)
 595                                              '("out")
 596                                              sub-drv)))
 597                             (derivation-input drv outputs)))
 598                          ((? store-item? item)
 599                           item)))))
 600             (((? store-item? item))
 601              (return item)))
 602           inputs)))
 603
 604 (define* (lower-reference-graphs graphs #:key system target)
 605   "Given GRAPHS, a list of (FILE-NAME INPUT ...) lists for use as a
 606 #:reference-graphs argument, lower it such that each INPUT is replaced by the
 607 corresponding <derivation-input> or store item."
 608   (match graphs
 609     (((file-names . inputs) ...)
 610      (mlet %store-monad ((inputs (lower-inputs inputs
 611                                                #:system system
 612                                                #:target target)))
 613        (return (map cons file-names inputs))))))
 614
 615 (define* (lower-references lst #:key system target)
 616   "Based on LST, a list of output names and packages, return a list of output
 617 names and file names suitable for the #:allowed-references argument to
 618 'derivation'."
 619   (with-monad %store-monad
 620     (define lower
 621       (match-lambda
 622        ((? string? output)
 623         (return output))
 624        (($ <gexp-input> thing output native?)
 625         (mlet %store-monad ((drv (lower-object thing system
 626                                                #:target (if native?
 627                                                             #f target))))
 628           (return (derivation->output-path drv output))))
 629        (thing
 630         (mlet %store-monad ((drv (lower-object thing system
 631                                                #:target target)))
 632           (return (derivation->output-path drv))))))
 633
 634     (mapm %store-monad lower lst)))
 635
 636 (define default-guile-derivation
 637   ;; Here we break the abstraction by talking to the higher-level layer.
 638   ;; Thus, do the resolution lazily to hide the circular dependency.
 639   (let ((proc (delay
 640                 (let ((iface (resolve-interface '(guix packages))))
 641                   (module-ref iface 'default-guile-derivation)))))
 642     (lambda (system)
 643       ((force proc) system))))
 644
 645 ;; Representation of a gexp instantiated for a given target and system.
 646 ;; It's an intermediate representation between <gexp> and <derivation>.
 647 (define-record-type <lowered-gexp>
 648   (lowered-gexp sexp inputs sources guile load-path load-compiled-path)
 649   lowered-gexp?
 650   (sexp                lowered-gexp-sexp)         ;sexp
 651   (inputs              lowered-gexp-inputs)       ;list of <derivation-input>
 652   (sources             lowered-gexp-sources)      ;list of store items
 653   (guile               lowered-gexp-guile)        ;<derivation-input> | #f
 654   (load-path           lowered-gexp-load-path)    ;list of store items
 655   (load-compiled-path  lowered-gexp-load-compiled-path)) ;list of store items
 656
 657 (define* (lower-gexp exp
 658                      #:key
 659                      (module-path %load-path)
 660                      (system (%current-system))
 661                      (target 'current)
 662                      (graft? (%graft?))
 663                      (guile-for-build (%guile-for-build))
 664                      (effective-version "2.2")
 665
 666                      deprecation-warnings
 667                      (pre-load-modules? #t))      ;transitional
 668   "*Note: This API is subject to change; use at your own risk!*
 669
 670 Lower EXP, a gexp, instantiating it for SYSTEM and TARGET.  Return a
 671 <lowered-gexp> ready to be used.
 672
 673 Lowered gexps are an intermediate representation that's useful for
 674 applications that deal with gexps outside in a way that is disconnected from
 675 derivations--e.g., code evaluated for its side effects."
 676   (define %modules
 677     (delete-duplicates (gexp-modules exp)))
 678
 679   (define (search-path modules extensions suffix)
 680     (append (match modules
 681               ((? derivation? drv)
 682                (list (derivation->output-path drv)))
 683               (#f
 684                '())
 685               ((? store-path? item)
 686                (list item)))
 687             (map (lambda (extension)
 688                    (string-append (match extension
 689                                     ((? derivation? drv)
 690                                      (derivation->output-path drv))
 691                                     ((? store-path? item)
 692                                      item))
 693                                   suffix))
 694                  extensions)))
 695
 696   (mlet* %store-monad ( ;; The following binding forces '%current-system' and
 697                        ;; '%current-target-system' to be looked up at >>=
 698                        ;; time.
 699                        (graft?    (set-grafting graft?))
 700
 701                        (system -> (or system (%current-system)))
 702                        (target -> (if (eq? target 'current)
 703                                       (%current-target-system)
 704                                       target))
 705                        (guile     (if guile-for-build
 706                                       (return guile-for-build)
 707                                       (default-guile-derivation system)))
 708                        (normals  (lower-inputs (gexp-inputs exp)
 709                                                #:system system
 710                                                #:target target))
 711                        (natives  (lower-inputs (gexp-native-inputs exp)
 712                                                #:system system
 713                                                #:target #f))
 714                        (inputs -> (append normals natives))
 715                        (sexp     (gexp->sexp exp
 716                                              #:system system
 717                                              #:target target))
 718                        (extensions -> (gexp-extensions exp))
 719                        (exts     (mapm %store-monad
 720                                        (lambda (obj)
 721                                          (lower-object obj system))
 722                                        extensions))
 723                        (modules  (if (pair? %modules)
 724                                      (imported-modules %modules
 725                                                        #:system system
 726                                                        #:module-path module-path)
 727                                      (return #f)))
 728                        (compiled (if (pair? %modules)
 729                                      (compiled-modules %modules
 730                                                        #:system system
 731                                                        #:module-path module-path
 732                                                        #:extensions extensions
 733                                                        #:guile guile
 734                                                        #:pre-load-modules?
 735                                                        pre-load-modules?
 736                                                        #:deprecation-warnings
 737                                                        deprecation-warnings)
 738                                      (return #f))))
 739     (define load-path
 740       (search-path modules exts
 741                    (string-append "/share/guile/site/" effective-version)))
 742
 743     (define load-compiled-path
 744       (search-path compiled exts
 745                    (string-append "/lib/guile/" effective-version
 746                                   "/site-ccache")))
 747
 748     (mbegin %store-monad
 749       (set-grafting graft?)                       ;restore the initial setting
 750       (return (lowered-gexp sexp
 751                             `(,@(if (derivation? modules)
 752                                     (list (derivation-input modules))
 753                                     '())
 754                               ,@(if compiled
 755                                     (list (derivation-input compiled))
 756                                     '())
 757                               ,@(map derivation-input exts)
 758                               ,@(filter derivation-input? inputs))
 759                             (filter string? (cons modules inputs))
 760                             (derivation-input guile '("out"))
 761                             load-path
 762                             load-compiled-path)))))
 763
 764 (define* (gexp->derivation name exp
 765                            #:key
 766                            system (target 'current)
 767                            hash hash-algo recursive?
 768                            (env-vars '())
 769                            (modules '())
 770                            (module-path %load-path)
 771                            (guile-for-build (%guile-for-build))
 772                            (effective-version "2.2")
 773                            (graft? (%graft?))
 774                            references-graphs
 775                            allowed-references disallowed-references
 776                            leaked-env-vars
 777                            local-build? (substitutable? #t)
 778                            (properties '())
 779
 780                            ;; TODO: This parameter is transitional; it's here
 781                            ;; to avoid a full rebuild.  Remove it on the next
 782                            ;; rebuild cycle.
 783                            (pre-load-modules? #t)
 784
 785                            deprecation-warnings
 786                            (script-name (string-append name "-builder")))
 787   "Return a derivation NAME that runs EXP (a gexp) with GUILE-FOR-BUILD (a
 788 derivation) on SYSTEM; EXP is stored in a file called SCRIPT-NAME.  When
 789 TARGET is true, it is used as the cross-compilation target triplet for
 790 packages referred to by EXP.
 791
 792 MODULES is deprecated in favor of 'with-imported-modules'.  Its meaning is to
 793 make MODULES available in the evaluation context of EXP; MODULES is a list of
 794 names of Guile modules searched in MODULE-PATH to be copied in the store,
 795 compiled, and made available in the load path during the execution of
 796 EXP---e.g., '((guix build utils) (guix build gnu-build-system)).
 797
 798 EFFECTIVE-VERSION determines the string to use when adding extensions of
 799 EXP (see 'with-extensions') to the search path---e.g., \"2.2\".
 800
 801 GRAFT? determines whether packages referred to by EXP should be grafted when
 802 applicable.
 803
 804 When REFERENCES-GRAPHS is true, it must be a list of tuples of one of the
 805 following forms:
 806
 807   (FILE-NAME PACKAGE)
 808   (FILE-NAME PACKAGE OUTPUT)
 809   (FILE-NAME DERIVATION)
 810   (FILE-NAME DERIVATION OUTPUT)
 811   (FILE-NAME STORE-ITEM)
 812
 813 The right-hand-side of each element of REFERENCES-GRAPHS is automatically made
 814 an input of the build process of EXP.  In the build environment, each
 815 FILE-NAME contains the reference graph of the corresponding item, in a simple
 816 text format.
 817
 818 ALLOWED-REFERENCES must be either #f or a list of output names and packages.
 819 In the latter case, the list denotes store items that the result is allowed to
 820 refer to.  Any reference to another store item will lead to a build error.
 821 Similarly for DISALLOWED-REFERENCES, which can list items that must not be
 822 referenced by the outputs.
 823
 824 DEPRECATION-WARNINGS determines whether to show deprecation warnings while
 825 compiling modules.  It can be #f, #t, or 'detailed.
 826
 827 The other arguments are as for 'derivation'."
 828   (define outputs (gexp-outputs exp))
 829   (define requested-graft? graft?)
 830
 831   (define (graphs-file-names graphs)
 832     ;; Return a list of (FILE-NAME . STORE-PATH) pairs made from GRAPHS.
 833     (map (match-lambda
 834            ((file-name . (? derivation-input? input))
 835             (cons file-name (first (derivation-input-output-paths input))))
 836            ((file-name . (? string? item))
 837             (cons file-name item)))
 838          graphs))
 839
 840   (define (add-modules exp modules)
 841     (if (null? modules)
 842         exp
 843         (make-gexp (gexp-references exp)
 844                    (append modules (gexp-self-modules exp))
 845                    (gexp-self-extensions exp)
 846                    (gexp-proc exp))))
 847
 848   (mlet* %store-monad ( ;; The following binding forces '%current-system' and
 849                        ;; '%current-target-system' to be looked up at >>=
 850                        ;; time.
 851                        (graft?    (set-grafting graft?))
 852
 853                        (system -> (or system (%current-system)))
 854                        (target -> (if (eq? target 'current)
 855                                       (%current-target-system)
 856                                       target))
 857                        (exp ->    (add-modules exp modules))
 858                        (lowered   (lower-gexp exp
 859                                               #:module-path module-path
 860                                               #:system system
 861                                               #:target target
 862                                               #:graft? requested-graft?
 863                                               #:guile-for-build
 864                                               guile-for-build
 865                                               #:effective-version
 866                                               effective-version
 867                                               #:deprecation-warnings
 868                                               deprecation-warnings
 869                                               #:pre-load-modules?
 870                                               pre-load-modules?))
 871
 872                        (graphs   (if references-graphs
 873                                      (lower-reference-graphs references-graphs
 874                                                              #:system system
 875                                                              #:target target)
 876                                      (return #f)))
 877                        (allowed  (if allowed-references
 878                                      (lower-references allowed-references
 879                                                        #:system system
 880                                                        #:target target)
 881                                      (return #f)))
 882                        (disallowed (if disallowed-references
 883                                        (lower-references disallowed-references
 884                                                          #:system system
 885                                                          #:target target)
 886                                        (return #f)))
 887                        (guile -> (lowered-gexp-guile lowered))
 888                        (builder  (text-file script-name
 889                                             (object->string
 890                                              (lowered-gexp-sexp lowered)))))
 891     (mbegin %store-monad
 892       (set-grafting graft?)                       ;restore the initial setting
 893       (raw-derivation name
 894                       (string-append (derivation-input-output-path guile)
 895                                      "/bin/guile")
 896                       `("--no-auto-compile"
 897                         ,@(append-map (lambda (directory)
 898                                         `("-L" ,directory))
 899                                       (lowered-gexp-load-path lowered))
 900                         ,@(append-map (lambda (directory)
 901                                         `("-C" ,directory))
 902                                       (lowered-gexp-load-compiled-path lowered))
 903                         ,builder)
 904                       #:outputs outputs
 905                       #:env-vars env-vars
 906                       #:system system
 907                       #:inputs `(,guile
 908                                  ,@(lowered-gexp-inputs lowered)
 909                                  ,@(match graphs
 910                                      (((_ . inputs) ...)
 911                                       (filter derivation-input? inputs))
 912                                      (#f '())))
 913                       #:sources `(,builder
 914                                   ,@(if (and (string? modules)
 915                                              (store-path? modules))
 916                                         (list modules)
 917                                         '())
 918                                   ,@(lowered-gexp-sources lowered)
 919                                   ,@(match graphs
 920                                       (((_ . inputs) ...)
 921                                        (filter string? inputs))
 922                                       (#f '())))
 923
 924                       #:hash hash #:hash-algo hash-algo #:recursive? recursive?
 925                       #:references-graphs (and=> graphs graphs-file-names)
 926                       #:allowed-references allowed
 927                       #:disallowed-references disallowed
 928                       #:leaked-env-vars leaked-env-vars
 929                       #:local-build? local-build?
 930                       #:substitutable? substitutable?
 931                       #:properties properties))))
 932
 933 (define* (gexp-inputs exp #:key native?)
 934   "Return the input list for EXP.  When NATIVE? is true, return only native
 935 references; otherwise, return only non-native references."
 936   ;; TODO: Return <gexp-input> records instead of tuples.
 937   (define (add-reference-inputs ref result)
 938     (match ref
 939       (($ <gexp-input> (? gexp? exp) _ #t)
 940        (if native?
 941            (append (gexp-inputs exp)
 942                    (gexp-inputs exp #:native? #t)
 943                    result)
 944            result))
 945       (($ <gexp-input> (? gexp? exp) _ #f)
 946        (append (gexp-inputs exp #:native? native?)
 947                result))
 948       (($ <gexp-input> (? string? str))
 949        (if (direct-store-path? str)
 950            (cons `(,str) result)
 951            result))
 952       (($ <gexp-input> (? struct? thing) output n?)
 953        (if (and (eqv? n? native?) (lookup-compiler thing))
 954            ;; THING is a derivation, or a package, or an origin, etc.
 955            (cons `(,thing ,output) result)
 956            result))
 957       (($ <gexp-input> (lst ...) output n?)
 958        (fold-right add-reference-inputs result
 959                    ;; XXX: For now, automatically convert LST to a list of
 960                    ;; gexp-inputs.  Inherit N?.
 961                    (map (match-lambda
 962                           ((? gexp-input? x)
 963                            (%gexp-input (gexp-input-thing x)
 964                                         (gexp-input-output x)
 965                                         n?))
 966                           (x
 967                            (%gexp-input x "out" n?)))
 968                         lst)))
 969       (_
 970        ;; Ignore references to other kinds of objects.
 971        result)))
 972
 973   (fold-right add-reference-inputs
 974               '()
 975               (gexp-references exp)))
 976
 977 (define gexp-native-inputs
 978   (cut gexp-inputs <> #:native? #t))
 979
 980 (define (gexp-outputs exp)
 981   "Return the outputs referred to by EXP as a list of strings."
 982   (define (add-reference-output ref result)
 983     (match ref
 984       (($ <gexp-output> name)
 985        (cons name result))
 986       (($ <gexp-input> (? gexp? exp))
 987        (append (gexp-outputs exp) result))
 988       (($ <gexp-input> (lst ...) output native?)
 989        ;; XXX: Automatically convert LST.
 990        (add-reference-output (map (match-lambda
 991                                    ((? gexp-input? x) x)
 992                                    (x (%gexp-input x "out" native?)))
 993                                   lst)
 994                              result))
 995       ((lst ...)
 996        (fold-right add-reference-output result lst))
 997       (_
 998        result)))
 999
1000   (delete-duplicates
1001    (add-reference-output (gexp-references exp) '())))
1002
1003 (define* (gexp->sexp exp #:key
1004                      (system (%current-system))
1005                      (target (%current-target-system)))
1006   "Return (monadically) the sexp corresponding to EXP for the given OUTPUT,
1007 and in the current monad setting (system type, etc.)"
1008   (define* (reference->sexp ref #:optional native?)
1009     (with-monad %store-monad
1010       (match ref
1011         (($ <gexp-output> output)
1012          ;; Output file names are not known in advance but the daemon defines
1013          ;; an environment variable for each of them at build time, so use
1014          ;; that trick.
1015          (return `((@ (guile) getenv) ,output)))
1016         (($ <gexp-input> (? gexp? exp) output n?)
1017          (gexp->sexp exp
1018                      #:system system
1019                      #:target (if (or n? native?) #f target)))
1020         (($ <gexp-input> (refs ...) output n?)
1021          (mapm %store-monad
1022                (lambda (ref)
1023                  ;; XXX: Automatically convert REF to an gexp-input.
1024                  (reference->sexp
1025                   (if (gexp-input? ref)
1026                       ref
1027                       (%gexp-input ref "out" n?))
1028                   (or n? native?)))
1029                refs))
1030         (($ <gexp-input> (? struct? thing) output n?)
1031          (let ((target (if (or n? native?) #f target))
1032                (expand (lookup-expander thing)))
1033            (mlet %store-monad ((obj (lower-object thing system
1034                                                   #:target target)))
1035              ;; OBJ must be either a derivation or a store file name.
1036              (return (expand thing obj output)))))
1037         (($ <gexp-input> x)
1038          (return x))
1039         (x
1040          (return x)))))
1041
1042   (mlet %store-monad
1043       ((args (mapm %store-monad
1044                    reference->sexp (gexp-references exp))))
1045     (return (apply (gexp-proc exp) args))))
1046
1047 (define (syntax-location-string s)
1048   "Return a string representing the source code location of S."
1049   (let ((props (syntax-source s)))
1050     (if props
1051         (let ((file   (assoc-ref props 'filename))
1052               (line   (and=> (assoc-ref props 'line) 1+))
1053               (column (assoc-ref props 'column)))
1054           (if file
1055               (simple-format #f "~a:~a:~a"
1056                              file line column)
1057               (simple-format #f "~a:~a" line column)))
1058         "<unknown location>")))
1059
1060 (define-syntax-rule (define-syntax-parameter-once name proc)
1061   ;; Like 'define-syntax-parameter' but ensure the top-level binding for NAME
1062   ;; does not get redefined.  This works around a race condition in a
1063   ;; multi-threaded context with Guile <= 2.2.4: <https://bugs.gnu.org/27476>.
1064   (eval-when (load eval expand compile)
1065     (define name
1066       (if (module-locally-bound? (current-module) 'name)
1067           (module-ref (current-module) 'name)
1068           (make-syntax-transformer 'name 'syntax-parameter
1069                                    (list proc))))))
1070
1071 (define-syntax-parameter-once current-imported-modules
1072   ;; Current list of imported modules.
1073   (identifier-syntax '()))
1074
1075 (define-syntax-rule (with-imported-modules modules body ...)
1076   "Mark the gexps defined in BODY... as requiring MODULES in their execution
1077 environment."
1078   (syntax-parameterize ((current-imported-modules
1079                          (identifier-syntax modules)))
1080     body ...))
1081
1082 (define-syntax-parameter-once current-imported-extensions
1083   ;; Current list of extensions.
1084   (identifier-syntax '()))
1085
1086 (define-syntax-rule (with-extensions extensions body ...)
1087   "Mark the gexps defined in BODY... as requiring EXTENSIONS in their
1088 execution environment."
1089   (syntax-parameterize ((current-imported-extensions
1090                          (identifier-syntax extensions)))
1091     body ...))
1092
1093 (define-syntax gexp
1094   (lambda (s)
1095     (define (collect-escapes exp)
1096       ;; Return all the 'ungexp' present in EXP.
1097       (let loop ((exp    exp)
1098                  (result '()))
1099         (syntax-case exp (ungexp
1100                           ungexp-splicing
1101                           ungexp-native
1102                           ungexp-native-splicing)
1103           ((ungexp _)
1104            (cons exp result))
1105           ((ungexp _ _)
1106            (cons exp result))
1107           ((ungexp-splicing _ ...)
1108            (cons exp result))
1109           ((ungexp-native _ ...)
1110            (cons exp result))
1111           ((ungexp-native-splicing _ ...)
1112            (cons exp result))
1113           ((exp0 . exp)
1114            (let ((result (loop #'exp0 result)))
1115              (loop  #'exp result)))
1116           (_
1117            result))))
1118
1119     (define (escape->ref exp)
1120       ;; Turn 'ungexp' form EXP into a "reference".
1121       (syntax-case exp (ungexp ungexp-splicing
1122                         ungexp-native ungexp-native-splicing
1123                         output)
1124         ((ungexp output)
1125          #'(gexp-output "out"))
1126         ((ungexp output name)
1127          #'(gexp-output name))
1128         ((ungexp thing)
1129          #'(%gexp-input thing "out" #f))
1130         ((ungexp drv-or-pkg out)
1131          #'(%gexp-input drv-or-pkg out #f))
1132         ((ungexp-splicing lst)
1133          #'(%gexp-input lst "out" #f))
1134         ((ungexp-native thing)
1135          #'(%gexp-input thing "out" #t))
1136         ((ungexp-native drv-or-pkg out)
1137          #'(%gexp-input drv-or-pkg out #t))
1138         ((ungexp-native-splicing lst)
1139          #'(%gexp-input lst "out" #t))))
1140
1141     (define (substitute-ungexp exp substs)
1142       ;; Given EXP, an 'ungexp' or 'ungexp-native' form, substitute it with
1143       ;; the corresponding form in SUBSTS.
1144       (match (assoc exp substs)
1145         ((_ id)
1146          id)
1147         (_                                        ;internal error
1148          (with-syntax ((exp exp))
1149            #'(syntax-error "error: no 'ungexp' substitution" exp)))))
1150
1151     (define (substitute-ungexp-splicing exp substs)
1152       (syntax-case exp ()
1153         ((exp rest ...)
1154          (match (assoc #'exp substs)
1155            ((_ id)
1156             (with-syntax ((id id))
1157               #`(append id
1158                         #,(substitute-references #'(rest ...) substs))))
1159            (_
1160             #'(syntax-error "error: no 'ungexp-splicing' substitution"
1161                             exp))))))
1162
1163     (define (substitute-references exp substs)
1164       ;; Return a variant of EXP where all the cars of SUBSTS have been
1165       ;; replaced by the corresponding cdr.
1166       (syntax-case exp (ungexp ungexp-native
1167                         ungexp-splicing ungexp-native-splicing)
1168         ((ungexp _ ...)
1169          (substitute-ungexp exp substs))
1170         ((ungexp-native _ ...)
1171          (substitute-ungexp exp substs))
1172         (((ungexp-splicing _ ...) rest ...)
1173          (substitute-ungexp-splicing exp substs))
1174         (((ungexp-native-splicing _ ...) rest ...)
1175          (substitute-ungexp-splicing exp substs))
1176         ((exp0 . exp)
1177          #`(cons #,(substitute-references #'exp0 substs)
1178                  #,(substitute-references #'exp substs)))
1179         (x #''x)))
1180
1181     (syntax-case s (ungexp output)
1182       ((_ exp)
1183        (let* ((escapes (delete-duplicates (collect-escapes #'exp)))
1184               (formals (generate-temporaries escapes))
1185               (sexp    (substitute-references #'exp (zip escapes formals)))
1186               (refs    (map escape->ref escapes)))
1187          #`(make-gexp (list #,@refs)
1188                       current-imported-modules
1189                       current-imported-extensions
1190                       (lambda #,formals
1191                         #,sexp)))))))
1192
1193 \f
1194 ;;;
1195 ;;; Module handling.
1196 ;;;
1197
1198 (define %not-slash
1199   (char-set-complement (char-set #\/)))
1200
1201 (define (file-mapping->tree mapping)
1202   "Convert MAPPING, an alist like:
1203
1204   ((\"guix/build/utils.scm\" . \"…/utils.scm\"))
1205
1206 to a tree suitable for 'interned-file-tree'."
1207   (let ((mapping (map (match-lambda
1208                         ((destination . source)
1209                          (cons (string-tokenize destination
1210                                                 %not-slash)
1211                                source)))
1212                       mapping)))
1213     (fold (lambda (pair result)
1214             (match pair
1215               ((destination . source)
1216                (let loop ((destination destination)
1217                           (result result))
1218                  (match destination
1219                    ((file)
1220                     (let* ((mode (stat:mode (stat source)))
1221                            (type (if (zero? (logand mode #o100))
1222                                      'regular
1223                                      'executable)))
1224                       (alist-cons file
1225                                   `(,type (file ,source))
1226                                   result)))
1227                    ((file rest ...)
1228                     (let ((directory (assoc-ref result file)))
1229                       (alist-cons file
1230                                   `(directory
1231                                     ,@(loop rest
1232                                             (match directory
1233                                               (('directory . entries) entries)
1234                                               (#f '()))))
1235                                   (if directory
1236                                       (alist-delete file result)
1237                                       result)))))))))
1238           '()
1239           mapping)))
1240
1241 (define %utils-module
1242   ;; This file provides 'mkdir-p', needed to implement 'imported-files' and
1243   ;; other primitives below.  Note: We give the file name relative to this
1244   ;; file you are currently reading; 'search-path' could return a file name
1245   ;; relative to the current working directory.
1246   (local-file "build/utils.scm"
1247               "build-utils.scm"))
1248
1249 (define* (imported-files/derivation files
1250                                     #:key (name "file-import")
1251                                     (symlink? #f)
1252                                     (system (%current-system))
1253                                     (guile (%guile-for-build)))
1254   "Return a derivation that imports FILES into STORE.  FILES must be a list
1255 of (FINAL-PATH . FILE) pairs.  Each FILE is mapped to FINAL-PATH in the
1256 resulting store path.  FILE can be either a file name, or a file-like object,
1257 as returned by 'local-file' for example.  If SYMLINK? is true, create symlinks
1258 to the source files instead of copying them."
1259   (define file-pair
1260     (match-lambda
1261      ((final-path . (? string? file-name))
1262       (mlet %store-monad ((file (interned-file file-name
1263                                                (basename final-path))))
1264         (return (list final-path file))))
1265      ((final-path . file-like)
1266       (mlet %store-monad ((file (lower-object file-like system)))
1267         (return (list final-path file))))))
1268
1269   (mlet %store-monad ((files (mapm %store-monad file-pair files)))
1270     (define build
1271       (gexp
1272        (begin
1273          (primitive-load (ungexp %utils-module))  ;for 'mkdir-p'
1274          (use-modules (ice-9 match))
1275
1276          (mkdir (ungexp output)) (chdir (ungexp output))
1277          (for-each (match-lambda
1278                     ((final-path store-path)
1279                      (mkdir-p (dirname final-path))
1280                      ((ungexp (if symlink? 'symlink 'copy-file))
1281                       store-path final-path)))
1282                    '(ungexp files)))))
1283
1284     ;; TODO: Pass FILES as an environment variable so that BUILD remains
1285     ;; exactly the same regardless of FILES: less disk space, and fewer
1286     ;; 'add-to-store' RPCs.
1287     (gexp->derivation name build
1288                       #:system system
1289                       #:guile-for-build guile
1290                       #:local-build? #t
1291
1292                       ;; Avoid deprecation warnings about the use of the _IO*
1293                       ;; constants in (guix build utils).
1294                       #:env-vars
1295                       '(("GUILE_WARN_DEPRECATED" . "no")))))
1296
1297 (define* (imported-files files
1298                          #:key (name "file-import")
1299                          ;; The following parameters make sense when creating
1300                          ;; an actual derivation.
1301                          (system (%current-system))
1302                          (guile (%guile-for-build)))
1303   "Import FILES into the store and return the resulting derivation or store
1304 file name (a derivation is created if and only if some elements of FILES are
1305 file-like objects and not local file names.)  FILES must be a list
1306 of (FINAL-PATH . FILE) pairs.  Each FILE is mapped to FINAL-PATH in the
1307 resulting store path.  FILE can be either a file name, or a file-like object,
1308 as returned by 'local-file' for example."
1309   (if (any (match-lambda
1310              ((_ . (? struct? source)) #t)
1311              (_ #f))
1312            files)
1313       (imported-files/derivation files #:name name
1314                                  #:symlink? derivation?
1315                                  #:system system #:guile guile)
1316       (interned-file-tree `(,name directory
1317                                   ,@(file-mapping->tree files)))))
1318
1319 (define* (imported-modules modules
1320                            #:key (name "module-import")
1321                            (system (%current-system))
1322                            (guile (%guile-for-build))
1323                            (module-path %load-path))
1324   "Return a derivation that contains the source files of MODULES, a list of
1325 module names such as `(ice-9 q)'.  All of MODULES must be either names of
1326 modules to be found in the MODULE-PATH search path, or a module name followed
1327 by an arrow followed by a file-like object.  For example:
1328
1329   (imported-modules `((guix build utils)
1330                       (guix gcrypt)
1331                       ((guix config) => ,(scheme-file …))))
1332
1333 In this example, the first two modules are taken from MODULE-PATH, and the
1334 last one is created from the given <scheme-file> object."
1335   (let ((files (map (match-lambda
1336                       (((module ...) '=> file)
1337                        (cons (module->source-file-name module)
1338                              file))
1339                       ((module ...)
1340                        (let ((f (module->source-file-name module)))
1341                          (cons f (search-path* module-path f)))))
1342                     modules)))
1343     (imported-files files #:name name
1344                     #:system system
1345                     #:guile guile)))
1346
1347 (define* (compiled-modules modules
1348                            #:key (name "module-import-compiled")
1349                            (system (%current-system))
1350                            target
1351                            (guile (%guile-for-build))
1352                            (module-path %load-path)
1353                            (extensions '())
1354                            (deprecation-warnings #f)
1355
1356                            ;; TODO: This flag is here to prevent a full
1357                            ;; rebuild.  Remove it on the next rebuild cycle.
1358                            (pre-load-modules? #t))
1359   "Return a derivation that builds a tree containing the `.go' files
1360 corresponding to MODULES.  All the MODULES are built in a context where
1361 they can refer to each other.  When TARGET is true, cross-compile MODULES for
1362 TARGET, a GNU triplet."
1363   (define total (length modules))
1364
1365   (mlet %store-monad ((modules (imported-modules modules
1366                                                  #:system system
1367                                                  #:guile guile
1368                                                  #:module-path
1369                                                  module-path)))
1370     (define build
1371       (gexp
1372        (begin
1373          (primitive-load (ungexp %utils-module))  ;for 'mkdir-p'
1374
1375          (use-modules (ice-9 ftw)
1376                       (ice-9 format)
1377                       (srfi srfi-1)
1378                       (srfi srfi-26)
1379                       (system base compile))
1380
1381          ;; TODO: Inline this on the next rebuild cycle.
1382          (ungexp-splicing
1383           (if target
1384               (gexp ((use-modules (system base target))))
1385               (gexp ())))
1386
1387          (define (regular? file)
1388            (not (member file '("." ".."))))
1389
1390          (define (process-entry entry output processed)
1391            (if (file-is-directory? entry)
1392                (let ((output (string-append output "/" (basename entry))))
1393                  (mkdir-p output)
1394                  (process-directory entry output processed))
1395                (let* ((base   (basename entry ".scm"))
1396                       (output (string-append output "/" base ".go")))
1397                  (format #t "[~2@a/~2@a] Compiling '~a'...~%"
1398                          (+ 1 processed
1399                               (ungexp-splicing (if pre-load-modules?
1400                                                    (gexp ((ungexp total)))
1401                                                    (gexp ()))))
1402                          (ungexp (* total (if pre-load-modules? 2 1)))
1403                          entry)
1404
1405                  (ungexp-splicing
1406                   (if target
1407                       (gexp ((with-target (ungexp target)
1408                                (lambda ()
1409                                  (compile-file entry
1410                                                #:output-file output
1411                                                #:opts
1412                                                %auto-compilation-options)))))
1413                       (gexp ((compile-file entry
1414                                            #:output-file output
1415                                            #:opts %auto-compilation-options)))))
1416
1417                  (+ 1 processed))))
1418
1419          (define (process-directory directory output processed)
1420            (let ((entries (map (cut string-append directory "/" <>)
1421                                (scandir directory regular?))))
1422              (fold (cut process-entry <> output <>)
1423                    processed
1424                    entries)))
1425
1426          (setvbuf (current-output-port)
1427                   (cond-expand (guile-2.2 'line) (else _IOLBF)))
1428
1429          (define mkdir-p
1430            ;; Capture 'mkdir-p'.
1431            (@ (guix build utils) mkdir-p))
1432
1433          ;; Add EXTENSIONS to the search path.
1434          (set! %load-path
1435            (append (map (lambda (extension)
1436                           (string-append extension
1437                                          "/share/guile/site/"
1438                                          (effective-version)))
1439                         '((ungexp-native-splicing extensions)))
1440                    %load-path))
1441          (set! %load-compiled-path
1442            (append (map (lambda (extension)
1443                           (string-append extension "/lib/guile/"
1444                                          (effective-version)
1445                                          "/site-ccache"))
1446                         '((ungexp-native-splicing extensions)))
1447                    %load-compiled-path))
1448
1449          (set! %load-path (cons (ungexp modules) %load-path))
1450
1451          ;; Above we loaded our own (guix build utils) but now we may need to
1452          ;; load a compile a different one.  Thus, force a reload.
1453          (let ((utils (string-append (ungexp modules)
1454                                      "/guix/build/utils.scm")))
1455            (when (file-exists? utils)
1456              (load utils)))
1457
1458          (mkdir (ungexp output))
1459          (chdir (ungexp modules))
1460
1461          (ungexp-splicing
1462           (if pre-load-modules?
1463               (gexp ((define* (load-from-directory directory
1464                                                    #:optional (loaded 0))
1465                        "Load all the source files found in DIRECTORY."
1466                        ;; XXX: This works around <https://bugs.gnu.org/15602>.
1467                        (let ((entries (map (cut string-append directory "/" <>)
1468                                            (scandir directory regular?))))
1469                          (fold (lambda (file loaded)
1470                                  (if (file-is-directory? file)
1471                                      (load-from-directory file loaded)
1472                                      (begin
1473                                        (format #t "[~2@a/~2@a] Loading '~a'...~%"
1474                                                (+ 1 loaded)
1475                                                (ungexp (* 2 total))
1476                                                file)
1477                                        (save-module-excursion
1478                                         (lambda ()
1479                                           (primitive-load file)))
1480                                        (+ 1 loaded))))
1481                                loaded
1482                                entries)))
1483
1484                      (load-from-directory ".")))
1485               (gexp ())))
1486
1487          (process-directory "." (ungexp output) 0))))
1488
1489     ;; TODO: Pass MODULES as an environment variable.
1490     (gexp->derivation name build
1491                       #:system system
1492                       #:guile-for-build guile
1493                       #:local-build? #t
1494                       #:env-vars
1495                       (case deprecation-warnings
1496                         ((#f)
1497                          '(("GUILE_WARN_DEPRECATED" . "no")))
1498                         ((detailed)
1499                          '(("GUILE_WARN_DEPRECATED" . "detailed")))
1500                         (else
1501                          '())))))
1502
1503 \f
1504 ;;;
1505 ;;; Convenience procedures.
1506 ;;;
1507
1508 (define (default-guile)
1509   ;; Lazily resolve 'guile-2.2' (not 'guile-final' because this is for
1510   ;; programs returned by 'program-file' and we don't want to keep references
1511   ;; to several Guile packages).  This module must not refer to (gnu …)
1512   ;; modules directly, to avoid circular dependencies, hence this hack.
1513   (module-ref (resolve-interface '(gnu packages guile))
1514               'guile-2.2))
1515
1516 (define* (load-path-expression modules #:optional (path %load-path)
1517                                #:key (extensions '()) system target)
1518   "Return as a monadic value a gexp that sets '%load-path' and
1519 '%load-compiled-path' to point to MODULES, a list of module names.  MODULES
1520 are searched for in PATH.  Return #f when MODULES and EXTENSIONS are empty."
1521   (if (and (null? modules) (null? extensions))
1522       (with-monad %store-monad
1523         (return #f))
1524       (mlet %store-monad ((modules  (imported-modules modules
1525                                                       #:module-path path
1526                                                       #:system system))
1527                           (compiled (compiled-modules modules
1528                                                       #:extensions extensions
1529                                                       #:module-path path
1530                                                       #:system system
1531                                                       #:target target)))
1532         (return (gexp (eval-when (expand load eval)
1533                         (set! %load-path
1534                           (cons (ungexp modules)
1535                                 (append (map (lambda (extension)
1536                                                (string-append extension
1537                                                               "/share/guile/site/"
1538                                                               (effective-version)))
1539                                              '((ungexp-native-splicing extensions)))
1540                                         %load-path)))
1541                         (set! %load-compiled-path
1542                           (cons (ungexp compiled)
1543                                 (append (map (lambda (extension)
1544                                                (string-append extension
1545                                                               "/lib/guile/"
1546                                                               (effective-version)
1547                                                               "/site-ccache"))
1548                                              '((ungexp-native-splicing extensions)))
1549                                         %load-compiled-path)))))))))
1550
1551 (define* (gexp->script name exp
1552                        #:key (guile (default-guile))
1553                        (module-path %load-path)
1554                        (system (%current-system))
1555                        target)
1556   "Return an executable script NAME that runs EXP using GUILE, with EXP's
1557 imported modules in its search path.  Look up EXP's modules in MODULE-PATH."
1558   (mlet %store-monad ((set-load-path
1559                        (load-path-expression (gexp-modules exp)
1560                                              module-path
1561                                              #:extensions
1562                                              (gexp-extensions exp)
1563                                              #:system system
1564                                              #:target target)))
1565     (gexp->derivation name
1566                       (gexp
1567                        (call-with-output-file (ungexp output)
1568                          (lambda (port)
1569                            ;; Note: that makes a long shebang.  When the store
1570                            ;; is /gnu/store, that fits within the 128-byte
1571                            ;; limit imposed by Linux, but that may go beyond
1572                            ;; when running tests.
1573                            (format port
1574                                    "#!~a/bin/guile --no-auto-compile~%!#~%"
1575                                    (ungexp guile))
1576
1577                            (ungexp-splicing
1578                             (if set-load-path
1579                                 (gexp ((write '(ungexp set-load-path) port)))
1580                                 (gexp ())))
1581
1582                            (write '(ungexp exp) port)
1583                            (chmod port #o555))))
1584                       #:system system
1585                       #:target target
1586                       #:module-path module-path)))
1587
1588 (define* (gexp->file name exp #:key
1589                      (set-load-path? #t)
1590                      (module-path %load-path)
1591                      (splice? #f))
1592   "Return a derivation that builds a file NAME containing EXP.  When SPLICE?
1593 is true, EXP is considered to be a list of expressions that will be spliced in
1594 the resulting file.
1595
1596 When SET-LOAD-PATH? is true, emit code in the resulting file to set
1597 '%load-path' and '%load-compiled-path' to honor EXP's imported modules.
1598 Lookup EXP's modules in MODULE-PATH."
1599   (define modules (gexp-modules exp))
1600   (define extensions (gexp-extensions exp))
1601
1602   (if (or (not set-load-path?)
1603           (and (null? modules) (null? extensions)))
1604       (gexp->derivation name
1605                         (gexp
1606                          (call-with-output-file (ungexp output)
1607                            (lambda (port)
1608                              (for-each (lambda (exp)
1609                                          (write exp port))
1610                                        '(ungexp (if splice?
1611                                                     exp
1612                                                     (gexp ((ungexp exp)))))))))
1613                         #:local-build? #t
1614                         #:substitutable? #f)
1615       (mlet %store-monad ((set-load-path
1616                            (load-path-expression modules module-path
1617                                                  #:extensions extensions)))
1618         (gexp->derivation name
1619                           (gexp
1620                            (call-with-output-file (ungexp output)
1621                              (lambda (port)
1622                                (write '(ungexp set-load-path) port)
1623                                (for-each (lambda (exp)
1624                                            (write exp port))
1625                                          '(ungexp (if splice?
1626                                                       exp
1627                                                       (gexp ((ungexp exp)))))))))
1628                           #:module-path module-path
1629                           #:local-build? #t
1630                           #:substitutable? #f))))
1631
1632 (define* (text-file* name #:rest text)
1633   "Return as a monadic value a derivation that builds a text file containing
1634 all of TEXT.  TEXT may list, in addition to strings, objects of any type that
1635 can be used in a gexp: packages, derivations, local file objects, etc.  The
1636 resulting store file holds references to all these."
1637   (define builder
1638     (gexp (call-with-output-file (ungexp output "out")
1639             (lambda (port)
1640               (display (string-append (ungexp-splicing text)) port)))))
1641
1642   (gexp->derivation name builder
1643                     #:local-build? #t
1644                     #:substitutable? #f))
1645
1646 (define* (mixed-text-file name #:rest text)
1647   "Return an object representing store file NAME containing TEXT.  TEXT is a
1648 sequence of strings and file-like objects, as in:
1649
1650   (mixed-text-file \"profile\"
1651                    \"export PATH=\" coreutils \"/bin:\" grep \"/bin\")
1652
1653 This is the declarative counterpart of 'text-file*'."
1654   (define build
1655     (gexp (call-with-output-file (ungexp output "out")
1656             (lambda (port)
1657               (display (string-append (ungexp-splicing text)) port)))))
1658
1659   (computed-file name build))
1660
1661 (define (file-union name files)
1662   "Return a <computed-file> that builds a directory containing all of FILES.
1663 Each item in FILES must be a two-element list where the first element is the
1664 file name to use in the new directory, and the second element is a gexp
1665 denoting the target file.  Here's an example:
1666
1667   (file-union \"etc\"
1668               `((\"hosts\" ,(plain-file \"hosts\"
1669                                         \"127.0.0.1 localhost\"))
1670                 (\"bashrc\" ,(plain-file \"bashrc\"
1671                                          \"alias ls='ls --color'\"))
1672                 (\"libvirt/qemu.conf\" ,(plain-file \"qemu.conf\" \"\"))))
1673
1674 This yields an 'etc' directory containing these two files."
1675   (computed-file name
1676                  (with-imported-modules '((guix build utils))
1677                    (gexp
1678                     (begin
1679                       (use-modules (guix build utils))
1680
1681                       (mkdir (ungexp output))
1682                       (chdir (ungexp output))
1683                       (ungexp-splicing
1684                        (map (match-lambda
1685                               ((target source)
1686                                (gexp
1687                                 (begin
1688                                   ;; Stat the source to abort early if it does
1689                                   ;; not exist.
1690                                   (stat (ungexp source))
1691
1692                                   (mkdir-p (dirname (ungexp target)))
1693                                   (symlink (ungexp source)
1694                                            (ungexp target))))))
1695                             files)))))))
1696
1697 (define* (directory-union name things
1698                           #:key (copy? #f) (quiet? #f)
1699                           (resolve-collision 'warn-about-collision))
1700   "Return a directory that is the union of THINGS, where THINGS is a list of
1701 file-like objects denoting directories.  For example:
1702
1703   (directory-union \"guile+emacs\" (list guile emacs))
1704
1705 yields a directory that is the union of the 'guile' and 'emacs' packages.
1706
1707 Call RESOLVE-COLLISION when several files collide, passing it the list of
1708 colliding files.  RESOLVE-COLLISION must return the chosen file or #f, in
1709 which case the colliding entry is skipped altogether.
1710
1711 When HARD-LINKS? is true, create hard links instead of symlinks.  When QUIET?
1712 is true, the derivation will not print anything."
1713   (define symlink
1714     (if copy?
1715         (gexp (lambda (old new)
1716                 (if (file-is-directory? old)
1717                     (symlink old new)
1718                     (copy-file old new))))
1719         (gexp symlink)))
1720
1721   (define log-port
1722     (if quiet?
1723         (gexp (%make-void-port "w"))
1724         (gexp (current-error-port))))
1725
1726   (match things
1727     ((one)
1728      ;; Only one thing; return it.
1729      one)
1730     (_
1731      (computed-file name
1732                     (with-imported-modules '((guix build union))
1733                       (gexp (begin
1734                               (use-modules (guix build union)
1735                                            (srfi srfi-1)) ;for 'first' and 'last'
1736
1737                               (union-build (ungexp output)
1738                                            '(ungexp things)
1739
1740                                            #:log-port (ungexp log-port)
1741                                            #:symlink (ungexp symlink)
1742                                            #:resolve-collision
1743                                            (ungexp resolve-collision)))))))))
1744
1745 \f
1746 ;;;
1747 ;;; Syntactic sugar.
1748 ;;;
1749
1750 (eval-when (expand load eval)
1751   (define* (read-ungexp chr port #:optional native?)
1752     "Read an 'ungexp' or 'ungexp-splicing' form from PORT.  When NATIVE? is
1753 true, use 'ungexp-native' and 'ungexp-native-splicing' instead."
1754     (define unquote-symbol
1755       (match (peek-char port)
1756         (#\@
1757          (read-char port)
1758          (if native?
1759              'ungexp-native-splicing
1760              'ungexp-splicing))
1761         (_
1762          (if native?
1763              'ungexp-native
1764              'ungexp))))
1765
1766     (match (read port)
1767       ((? symbol? symbol)
1768        (let ((str (symbol->string symbol)))
1769          (match (string-index-right str #\:)
1770            (#f
1771             `(,unquote-symbol ,symbol))
1772            (colon
1773             (let ((name   (string->symbol (substring str 0 colon)))
1774                   (output (substring str (+ colon 1))))
1775               `(,unquote-symbol ,name ,output))))))
1776       (x
1777        `(,unquote-symbol ,x))))
1778
1779   (define (read-gexp chr port)
1780     "Read a 'gexp' form from PORT."
1781     `(gexp ,(read port)))
1782
1783   ;; Extend the reader
1784   (read-hash-extend #\~ read-gexp)
1785   (read-hash-extend #\$ read-ungexp)
1786   (read-hash-extend #\+ (cut read-ungexp <> <> #t)))
1787
1788 ;;; gexp.scm ends here