Merge branch 'master' into core-updates
[jackhill/guix/guix.git] / build-aux / hydra / gnu-system.scm
index 4c27a9d..19371be 100644 (file)
@@ -1,5 +1,6 @@
 ;;; GNU Guix --- Functional package management for GNU
-;;; Copyright © 2012, 2013, 2014, 2015, 2016 Ludovic Courtès <ludo@gnu.org>
+;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org>
+;;; Copyright © 2017 Jan Nieuwenhuizen <janneke@gnu.org>
 ;;;
 ;;; This file is part of GNU Guix.
 ;;;
 ;;; tool.
 ;;;
 
-;; Attempt to use our very own Guix modules.
+(use-modules (system base compile))
+
 (eval-when (compile load eval)
 
-  ;; Ignore any available .go, and force recompilation.  This is because our
-  ;; checkout in the store has mtime set to the epoch, and thus .go files look
-  ;; newer, even though they may not correspond.
+  ;; Pre-load the compiler so we don't end up auto-compiling it.
+  (compile #t)
+
+  ;; Use our very own Guix modules.
   (set! %fresh-auto-compile #t)
 
   (and=> (assoc-ref (current-source-location) 'filename)
 (use-modules (guix config)
              (guix store)
              (guix grafts)
+             (guix profiles)
              (guix packages)
              (guix derivations)
              (guix monads)
              ((guix licenses) #:select (gpl3+))
              ((guix utils) #:select (%current-system))
              ((guix scripts system) #:select (read-operating-system))
+             ((guix scripts pack)
+              #:select (lookup-compressor self-contained-tarball))
              (gnu packages)
              (gnu packages gcc)
              (gnu packages base)
@@ -54,7 +60,6 @@
              (gnu packages compression)
              (gnu packages multiprecision)
              (gnu packages make-bootstrap)
-             (gnu packages commencement)
              (gnu packages package-management)
              (gnu system)
              (gnu system vm)
@@ -109,10 +114,10 @@ SYSTEM."
   ;; chain.)
   (list gcc-4.8 gcc-4.9 gcc-5 glibc binutils
         gmp mpfr mpc coreutils findutils diffutils patch sed grep
-        gawk gnu-gettext hello guile-2.0 zlib gzip xz
+        gawk gnu-gettext hello guile-2.0 guile-2.2 zlib gzip xz
         %bootstrap-binaries-tarball
         %binutils-bootstrap-tarball
-        %glibc-bootstrap-tarball
+        (%glibc-bootstrap-tarball)
         %gcc-bootstrap-tarball
         %guile-bootstrap-tarball
         %bootstrap-tarballs))
@@ -122,13 +127,12 @@ SYSTEM."
 
 (define %cross-targets
   '("mips64el-linux-gnu"
-    "mips64el-linux-gnuabi64"))
-
-(define (demo-os)
-  "Return the \"demo\" 'operating-system' structure."
-  (let* ((dir  (dirname (assoc-ref (current-source-location) 'filename)))
-         (file (string-append dir "/demo-os.scm")))
-    (read-operating-system file)))
+    "mips64el-linux-gnuabi64"
+    "arm-linux-gnueabihf"
+    "aarch64-linux-gnu"
+    "powerpc-linux-gnu"
+    "i586-pc-gnu"                                 ;aka. GNU/Hurd
+    "i686-w64-mingw32"))
 
 (define %guixsd-supported-systems
   '("x86_64-linux" "i686-linux"))
@@ -155,34 +159,47 @@ system.")
     (expt 2 20))
 
   (if (member system %guixsd-supported-systems)
-      (list (->job 'qemu-image
+      (list (->job 'usb-image
                    (run-with-store store
                      (mbegin %store-monad
                        (set-guile-for-build (default-guile))
-                       (system-qemu-image (demo-os)
+                       (system-disk-image installation-os
                                           #:disk-image-size
-                                          (* 1400 MiB))))) ; 1.4 GiB
-            (->job 'usb-image
+                                          (* 1024 MiB)))))
+            (->job 'iso9660-image
                    (run-with-store store
                      (mbegin %store-monad
                        (set-guile-for-build (default-guile))
                        (system-disk-image installation-os
-                                          #:disk-image-size
-                                          (* 1024 MiB))))))
+                                          #:file-system-type
+                                          "iso9660")))))
       '()))
 
 (define (system-test-jobs store system)
   "Return a list of jobs for the system tests."
+  (define (test->thunk test)
+    (lambda ()
+      (define drv
+        (run-with-store store
+          (mbegin %store-monad
+            (set-current-system system)
+            (set-grafting #f)
+            (set-guile-for-build (default-guile))
+            (system-test-value test))))
+
+      `((derivation . ,(derivation-file-name drv))
+        (description . ,(format #f "GuixSD '~a' system test"
+                                (system-test-name test)))
+        (long-description . ,(system-test-description test))
+        (license . ,gpl3+)
+        (home-page . ,%guix-home-page-url)
+        (maintainers . ("bug-guix@gnu.org")))))
+
   (define (->job test)
     (let ((name (string->symbol
                  (string-append "test." (system-test-name test)
                                 "." system))))
-      `(,name . ,(lambda ()
-                   (run-with-store store
-                     (mbegin %store-monad
-                       (set-current-system system)
-                       (set-grafting #f)
-                       (system-test-value test)))))))
+      (cons name (test->thunk test))))
 
   (if (member system %guixsd-supported-systems)
       (map ->job (all-system-tests))
@@ -211,7 +228,12 @@ all its dependencies, and ready to be installed on non-GuixSD distributions.")
                (run-with-store store
                  (mbegin %store-monad
                    (set-guile-for-build (default-guile))
-                   (self-contained-tarball))
+                   (>>= (profile-derivation (packages->manifest (list guix)))
+                        (lambda (profile)
+                          (self-contained-tarball "guix-binary" profile
+                                                  #:localstatedir? #t
+                                                  #:compressor
+                                                  (lookup-compressor "xz")))))
                  #:system system))))
 
 (define job-name
@@ -226,17 +248,41 @@ all its dependencies, and ready to be installed on non-GuixSD distributions.")
                         (match (package-transitive-inputs package)
                           (((_ inputs _ ...) ...)
                            inputs))))
-                      %final-inputs))))
+                      (%final-inputs)))))
     (lambda (store package system)
       "Return a job for PACKAGE on SYSTEM, or #f if this combination is not
 valid."
       (cond ((member package base-packages)
-             #f)
+             (package-job store (symbol-append 'base. (job-name package))
+                          package system))
             ((supported-package? package system)
-             (package-job store (job-name package) package system))
+             (let ((drv (package-derivation store package system
+                                            #:graft? #f)))
+               (and (substitutable-derivation? drv)
+                    (package-job store (job-name package)
+                                 package system))))
             (else
              #f)))))
 
+(define (all-packages)
+  "Return the list of packages to build."
+  (define (adjust package result)
+    (cond ((package-replacement package)
+           (cons* package                         ;build both
+                  (package-replacement package)
+                  result))
+          ((package-superseded package)
+           result)                                ;don't build it
+          (else
+           (cons package result))))
+
+  (fold-packages adjust
+                 (fold adjust '()                 ;include base packages
+                       (match (%final-inputs)
+                         (((labels packages _ ...) ...)
+                          packages)))
+                 #:select? (const #t)))           ;include hidden packages
+
 \f
 ;;;
 ;;; Hydra entry point.
@@ -247,6 +293,8 @@ valid."
   (define subset
     (match (assoc-ref arguments 'subset)
       ("core" 'core)                              ; only build core packages
+      ("hello" 'hello)                            ; only build hello
+      (((? string?) (? string?) ...) 'list)       ; only build selected list of packages
       (_ 'all)))                                  ; build everything
 
   (define (cross-jobs system)
@@ -255,41 +303,57 @@ valid."
       ;; prevents known-to-fail cross-builds from i686-linux or armhf-linux to
       ;; mips64el-linux-gnuabi64.
       (and (or (string-prefix? "i686-" system)
+               (string-prefix? "i586-" system)
                (string-prefix? "armhf-" system))
-           (string-suffix? "64" target)))
+           (string-contains target "64")))    ;x86_64, mips64el, aarch64, etc.
 
     (define (same? target)
       ;; Return true if SYSTEM and TARGET are the same thing.  This is so we
       ;; don't try to cross-compile to 'mips64el-linux-gnu' from
       ;; 'mips64el-linux'.
-      (string-contains target system))
-
-    (define (either proc1 proc2)
+      (or (string-contains target system)
+          (and (string-prefix? "armhf" system)    ;armhf-linux
+               (string-prefix? "arm" target))))   ;arm-linux-gnueabihf
+
+    (define (pointless? target)
+      ;; Return #t if it makes no sense to cross-build to TARGET from SYSTEM.
+      (match system
+        ((or "x86_64-linux" "i686-linux")
+         (if (string-contains target "mingw")
+             (not (string=? "x86_64-linux" system))
+             #f))
+        (_
+         ;; Don't try to cross-compile from non-Intel platforms: this isn't
+         ;; very useful and these are often brittle configurations.
+         #t)))
+
+    (define (either proc1 proc2 proc3)
       (lambda (x)
-        (or (proc1 x) (proc2 x))))
+        (or (proc1 x) (proc2 x) (proc3 x))))
 
     (append-map (lambda (target)
                   (map (lambda (package)
                          (package-cross-job store (job-name package)
                                             package target system))
                        %packages-to-cross-build))
-                (remove (either from-32-to-64? same?) %cross-targets)))
+                (remove (either from-32-to-64? same? pointless?)
+                        %cross-targets)))
 
   ;; Turn off grafts.  Grafting is meant to happen on the user's machines.
   (parameterize ((%graft? #f))
     ;; Return one job for each package, except bootstrap packages.
     (append-map (lambda (system)
+                  (format (current-error-port)
+                          "evaluating for '~a' (heap size: ~a MiB)...~%"
+                          system
+                          (round
+                           (/ (assoc-ref (gc-stats) 'heap-size)
+                              (expt 2. 20))))
+                  (invalidate-derivation-caches!)
                   (case subset
                     ((all)
                      ;; Build everything, including replacements.
-                     (let ((all (fold-packages
-                                 (lambda (package result)
-                                   (if (package-replacement package)
-                                       (cons* package
-                                              (package-replacement package)
-                                              result)
-                                       (cons package result)))
-                                 '()))
+                     (let ((all (all-packages))
                            (job (lambda (package)
                                   (package->job store package
                                                 system))))
@@ -305,6 +369,22 @@ valid."
                                                  package system))
                                   %core-packages)
                              (cross-jobs system)))
+                    ((hello)
+                     ;; Build hello package only.
+                     (if (string=? system (%current-system))
+                         (let ((hello (specification->package "hello")))
+                           (list (package-job store (job-name hello) hello system)))
+                         '()))
+                    ((list)
+                     ;; Build selected list of packages only.
+                     (if (string=? system (%current-system))
+                         (let* ((names (assoc-ref arguments 'subset))
+                                (packages (map specification->package names)))
+                           (map (lambda (package)
+                                    (package-job store (job-name package)
+                                                 package system))
+                                  packages))
+                         '()))
                     (else
                      (error "unknown subset" subset))))
                 %hydra-supported-systems)))