gnu/system/vm.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2013, 2014, 2015, 2016, 2017, 2018, 2019 Ludovic Courtès <ludo@gnu.org>
   3 ;;; Copyright © 2016 Christopher Allan Webber <cwebber@dustycloud.org>
   4 ;;; Copyright © 2016, 2017 Leo Famulari <leo@famulari.name>
   5 ;;; Copyright © 2017 Mathieu Othacehe <m.othacehe@gmail.com>
   6 ;;; Copyright © 2017 Marius Bakke <mbakke@fastmail.com>
   7 ;;; Copyright © 2018 Chris Marusich <cmmarusich@gmail.com>
   8 ;;;
   9 ;;; This file is part of GNU Guix.
  10 ;;;
  11 ;;; GNU Guix is free software; you can redistribute it and/or modify it
  12 ;;; under the terms of the GNU General Public License as published by
  13 ;;; the Free Software Foundation; either version 3 of the License, or (at
  14 ;;; your option) any later version.
  15 ;;;
  16 ;;; GNU Guix is distributed in the hope that it will be useful, but
  17 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  18 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 ;;; GNU General Public License for more details.
  20 ;;;
  21 ;;; You should have received a copy of the GNU General Public License
  22 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  23
  24 (define-module (gnu system vm)
  25   #:use-module (guix config)
  26   #:use-module (guix store)
  27   #:use-module (guix gexp)
  28   #:use-module (guix derivations)
  29   #:use-module (guix packages)
  30   #:use-module (guix monads)
  31   #:use-module (guix records)
  32   #:use-module (guix modules)
  33   #:use-module (guix utils)
  34   #:use-module (gcrypt hash)
  35   #:use-module (guix base32)
  36   #:use-module ((guix self) #:select (make-config.scm))
  37
  38   #:use-module ((gnu build vm)
  39                 #:select (qemu-command))
  40   #:use-module (gnu packages base)
  41   #:use-module (gnu packages bootloaders)
  42   #:use-module (gnu packages cdrom)
  43   #:use-module (gnu packages compression)
  44   #:use-module (gnu packages guile)
  45   #:autoload   (gnu packages gnupg) (guile-gcrypt)
  46   #:use-module (gnu packages gawk)
  47   #:use-module (gnu packages bash)
  48   #:use-module (gnu packages less)
  49   #:use-module (gnu packages virtualization)
  50   #:use-module (gnu packages disk)
  51   #:use-module (gnu packages zile)
  52   #:use-module (gnu packages linux)
  53   #:use-module ((gnu packages make-bootstrap)
  54                 #:select (%guile-static-stripped))
  55   #:use-module (gnu packages admin)
  56
  57   #:use-module (gnu bootloader)
  58   #:use-module (gnu bootloader grub)
  59   #:use-module (gnu system shadow)
  60   #:use-module (gnu system pam)
  61   #:use-module (gnu system linux-container)
  62   #:use-module (gnu system linux-initrd)
  63   #:use-module (gnu bootloader)
  64   #:use-module (gnu system file-systems)
  65   #:use-module (gnu system)
  66   #:use-module (gnu services)
  67   #:use-module (gnu services base)
  68   #:use-module (gnu system uuid)
  69
  70   #:use-module (srfi srfi-1)
  71   #:use-module (srfi srfi-26)
  72   #:use-module (rnrs bytevectors)
  73   #:use-module (ice-9 match)
  74
  75   #:export (expression->derivation-in-linux-vm
  76             qemu-image
  77             virtualized-operating-system
  78             system-qemu-image
  79
  80             system-qemu-image/shared-store
  81             system-qemu-image/shared-store-script
  82             system-disk-image
  83             system-docker-image
  84
  85             virtual-machine
  86             virtual-machine?))
  87
  88 \f
  89 ;;; Commentary:
  90 ;;;
  91 ;;; Tools to evaluate build expressions within virtual machines.
  92 ;;;
  93 ;;; Code:
  94
  95 (define %linux-vm-file-systems
  96   ;; File systems mounted for 'derivation-in-linux-vm'.  These are shared with
  97   ;; the host over 9p.
  98   ;;
  99   ;; The 9p documentation says that cache=loose is "intended for exclusive,
 100   ;; read-only mounts", without additional details.  It's much faster than the
 101   ;; default cache=none, especially when copying and registering store items.
 102   ;; Thus, use cache=loose, except for /xchg where we want to ensure
 103   ;; consistency.
 104   (list (file-system
 105           (mount-point (%store-prefix))
 106           (device "store")
 107           (type "9p")
 108           (needed-for-boot? #t)
 109           (flags '(read-only))
 110           (options "trans=virtio,cache=loose")
 111           (check? #f))
 112         (file-system
 113           (mount-point "/xchg")
 114           (device "xchg")
 115           (type "9p")
 116           (needed-for-boot? #t)
 117           (options "trans=virtio")
 118           (check? #f))
 119         (file-system
 120           (mount-point "/tmp")
 121           (device "tmp")
 122           (type "9p")
 123           (needed-for-boot? #t)
 124           (options "trans=virtio,cache=loose")
 125           (check? #f))))
 126
 127 (define not-config?
 128   ;; Select (guix …) and (gnu …) modules, except (guix config).
 129   (match-lambda
 130     (('guix 'config) #f)
 131     (('guix rest ...) #t)
 132     (('gnu rest ...) #t)
 133     (rest #f)))
 134
 135 (define gcrypt-sqlite3&co
 136   ;; Guile-Gcrypt, Guile-SQLite3, and their propagated inputs.
 137   (append-map (lambda (package)
 138                 (cons package
 139                       (match (package-transitive-propagated-inputs package)
 140                         (((labels packages) ...)
 141                          packages))))
 142               (list guile-gcrypt guile-sqlite3)))
 143
 144 (define* (expression->derivation-in-linux-vm name exp
 145                                              #:key
 146                                              (system (%current-system)) target
 147                                              (linux linux-libre)
 148                                              initrd
 149                                              (qemu qemu-minimal)
 150                                              (env-vars '())
 151                                              (guile-for-build
 152                                               (%guile-for-build))
 153                                              (file-systems
 154                                               %linux-vm-file-systems)
 155
 156                                              (single-file-output? #f)
 157                                              (make-disk-image? #f)
 158                                              (references-graphs #f)
 159                                              (memory-size 256)
 160                                              (disk-image-format "qcow2")
 161                                              (disk-image-size 'guess))
 162   "Evaluate EXP in a QEMU virtual machine running LINUX with INITRD (a
 163 derivation).  The virtual machine runs with MEMORY-SIZE MiB of memory.  In the
 164 virtual machine, EXP has access to FILE-SYSTEMS, which, by default, includes a
 165 9p share of the store, the '/xchg' where EXP should put its output file(s),
 166 and a 9p share of /tmp.
 167
 168 If SINGLE-FILE-OUTPUT? is true, copy a single file from '/xchg' to OUTPUT.
 169 Otherwise, copy the contents of /xchg to a new directory OUTPUT.
 170
 171 When MAKE-DISK-IMAGE? is true, then create a QEMU disk image of type
 172 DISK-IMAGE-FORMAT (e.g., 'qcow2' or 'raw'), of DISK-IMAGE-SIZE bytes and
 173 return it.  When DISK-IMAGE-SIZE is 'guess, estimate the image size based
 174 based on the size of the closure of REFERENCES-GRAPHS.
 175
 176 When REFERENCES-GRAPHS is true, it must be a list of file name/store path
 177 pairs, as for `derivation'.  The files containing the reference graphs are
 178 made available under the /xchg CIFS share."
 179   (define user-builder
 180     (program-file "builder-in-linux-vm" exp))
 181
 182   (define loader
 183     ;; Invoke USER-BUILDER instead using 'primitive-load'.  The reason for
 184     ;; this is to allow USER-BUILDER to dlopen stuff by using a full-featured
 185     ;; Guile, which it couldn't do using the statically-linked guile used in
 186     ;; the initrd.  See example at
 187     ;; <https://lists.gnu.org/archive/html/guix-devel/2017-10/msg00233.html>.
 188     (program-file "linux-vm-loader"
 189                   ;; When USER-BUILDER succeeds, reboot (indicating a
 190                   ;; success), otherwise die, which causes a kernel panic
 191                   ;; ("Attempted to kill init!").
 192                   #~(if (zero? (system* #$user-builder))
 193                         (reboot)
 194                         (exit 1))))
 195
 196   (let ((initrd (or initrd
 197                     (base-initrd file-systems
 198                                  #:on-error 'backtrace
 199                                  #:linux linux
 200                                  #:linux-modules %base-initrd-modules
 201                                  #:qemu-networking? #t))))
 202
 203     (define builder
 204       ;; Code that launches the VM that evaluates EXP.
 205       (with-extensions gcrypt-sqlite3&co
 206         (with-imported-modules `(,@(source-module-closure
 207                                     '((guix build utils)
 208                                       (gnu build vm))
 209                                     #:select? not-config?)
 210
 211                                  ;; For consumption by (gnu store database).
 212                                  ((guix config) => ,(make-config.scm)))
 213           #~(begin
 214               (use-modules (guix build utils)
 215                            (gnu build vm))
 216
 217               (let* ((native-inputs
 218                       '#+(list qemu (canonical-package coreutils)))
 219                      (linux   (string-append #$linux "/"
 220                                              #$(system-linux-image-file-name)))
 221                      (initrd  #$initrd)
 222                      (loader  #$loader)
 223                      (graphs  '#$(match references-graphs
 224                                    (((graph-files . _) ...) graph-files)
 225                                    (_ #f)))
 226                      (target  #$(or (%current-target-system) (%current-system)))
 227                      (size    #$(if (eq? 'guess disk-image-size)
 228                                     #~(+ (* 70 (expt 2 20)) ;ESP
 229                                          (estimated-partition-size graphs))
 230                                     disk-image-size)))
 231
 232                 (set-path-environment-variable "PATH" '("bin") native-inputs)
 233
 234                 (load-in-linux-vm loader
 235                                   #:output #$output
 236                                   #:linux linux #:initrd initrd
 237                                   #:qemu (qemu-command target)
 238                                   #:memory-size #$memory-size
 239                                   #:make-disk-image? #$make-disk-image?
 240                                   #:single-file-output? #$single-file-output?
 241                                   ;; FIXME: ‘target-arm32?’ and
 242                                   ;; ‘target-aarch64?’ may not operate on the
 243                                   ;; right system/target values.  Rewrite
 244                                   ;; using ‘let-system’ when available.
 245                                   #:target-arm32? #$(target-arm32?)
 246                                   #:target-aarch64? #$(target-aarch64?)
 247                                   #:disk-image-format #$disk-image-format
 248                                   #:disk-image-size size
 249                                   #:references-graphs graphs))))))
 250
 251     (gexp->derivation name builder
 252                       ;; TODO: Require the "kvm" feature.
 253                       #:system system
 254                       #:target target
 255                       #:env-vars env-vars
 256                       #:guile-for-build guile-for-build
 257                       #:references-graphs references-graphs)))
 258
 259 (define (has-guix-service-type? os)
 260   "Return true if OS contains a service of the type GUIX-SERVICE-TYPE."
 261   (not (not (find (lambda (service)
 262                      (eq? (service-kind service) guix-service-type))
 263                    (operating-system-services os)))))
 264
 265 (define* (iso9660-image #:key
 266                         (name "iso9660-image")
 267                         file-system-label
 268                         file-system-uuid
 269                         (system (%current-system))
 270                         (target (%current-target-system))
 271                         (qemu qemu-minimal)
 272                         os
 273                         bootcfg-drv
 274                         bootloader
 275                         (register-closures? (has-guix-service-type? os))
 276                         (inputs '())
 277                         (grub-mkrescue-environment '()))
 278   "Return a bootable, stand-alone iso9660 image.
 279
 280 INPUTS is a list of inputs (as for packages)."
 281   (define schema
 282     (and register-closures?
 283          (local-file (search-path %load-path
 284                                   "guix/store/schema.sql"))))
 285
 286   (expression->derivation-in-linux-vm
 287    name
 288    (with-extensions gcrypt-sqlite3&co
 289      (with-imported-modules `(,@(source-module-closure '((gnu build vm)
 290                                                          (guix store database)
 291                                                          (guix build utils))
 292                                                        #:select? not-config?)
 293                               ((guix config) => ,(make-config.scm)))
 294        #~(begin
 295            (use-modules (gnu build vm)
 296                         (guix store database)
 297                         (guix build utils))
 298
 299            (sql-schema #$schema)
 300
 301            ;; Allow non-ASCII file names--e.g., 'nss-certs'--to be decoded.
 302            (setenv "GUIX_LOCPATH"
 303                    #+(file-append glibc-utf8-locales "/lib/locale"))
 304            (setlocale LC_ALL "en_US.utf8")
 305
 306            (let ((inputs
 307                   '#$(append (list parted e2fsprogs dosfstools xorriso)
 308                              (map canonical-package
 309                                   (list sed grep coreutils findutils gawk))))
 310
 311
 312                  (graphs     '#$(match inputs
 313                                   (((names . _) ...)
 314                                    names)))
 315                  ;; This variable is unused but allows us to add INPUTS-TO-COPY
 316                  ;; as inputs.
 317                  (to-register
 318                   '#$(map (match-lambda
 319                             ((name thing) thing)
 320                             ((name thing output) `(,thing ,output)))
 321                           inputs)))
 322
 323              (set-path-environment-variable "PATH" '("bin" "sbin") inputs)
 324              (make-iso9660-image #$xorriso
 325                                  '#$grub-mkrescue-environment
 326                                  #$(bootloader-package bootloader)
 327                                  #$bootcfg-drv
 328                                  #$os
 329                                  "/xchg/guixsd.iso"
 330                                  #:register-closures? #$register-closures?
 331                                  #:closures graphs
 332                                  #:volume-id #$file-system-label
 333                                  #:volume-uuid #$(and=> file-system-uuid
 334                                                         uuid-bytevector))))))
 335    #:system system
 336    #:target target
 337
 338    ;; Keep a local file system for /tmp so that we can populate it directly as
 339    ;; root and have files owned by root.  See <https://bugs.gnu.org/31752>.
 340    #:file-systems (remove (lambda (file-system)
 341                             (string=? (file-system-mount-point file-system)
 342                                       "/tmp"))
 343                           %linux-vm-file-systems)
 344
 345    #:make-disk-image? #f
 346    #:single-file-output? #t
 347    #:references-graphs inputs
 348
 349    ;; Xorriso seems to be quite memory-hungry, so increase the VM's RAM size.
 350    #:memory-size 512))
 351
 352 (define* (qemu-image #:key
 353                      (name "qemu-image")
 354                      (system (%current-system))
 355                      (target (%current-target-system))
 356                      (qemu qemu-minimal)
 357                      (disk-image-size 'guess)
 358                      (disk-image-format "qcow2")
 359                      (file-system-type "ext4")
 360                      file-system-label
 361                      file-system-uuid
 362                      os
 363                      bootcfg-drv
 364                      bootloader
 365                      (register-closures? (has-guix-service-type? os))
 366                      (inputs '())
 367                      copy-inputs?)
 368   "Return a bootable, stand-alone QEMU image of type DISK-IMAGE-FORMAT (e.g.,
 369 'qcow2' or 'raw'), with a root partition of type FILE-SYSTEM-TYPE.
 370 Optionally, FILE-SYSTEM-LABEL can be specified as the volume name for the root
 371 partition; likewise FILE-SYSTEM-UUID, if true, specifies the UUID of the root
 372 partition (a UUID object).
 373
 374 The returned image is a full disk image that runs OS-DERIVATION,
 375 with a GRUB installation that uses GRUB-CONFIGURATION as its configuration
 376 file (GRUB-CONFIGURATION must be the name of a file in the VM.)
 377
 378 INPUTS is a list of inputs (as for packages).  When COPY-INPUTS? is true, copy
 379 all of INPUTS into the image being built.  When REGISTER-CLOSURES? is true,
 380 register INPUTS in the store database of the image so that Guix can be used in
 381 the image.  By default, REGISTER-CLOSURES? is set to true only if a service of
 382 type GUIX-SERVICE-TYPE is present in the services definition of the operating
 383 system."
 384   (define schema
 385     (and register-closures?
 386          (local-file (search-path %load-path
 387                                   "guix/store/schema.sql"))))
 388
 389   (expression->derivation-in-linux-vm
 390    name
 391    (with-extensions gcrypt-sqlite3&co
 392      (with-imported-modules `(,@(source-module-closure '((gnu build vm)
 393                                                          (gnu build bootloader)
 394                                                          (guix store database)
 395                                                          (guix build utils))
 396                                                        #:select? not-config?)
 397                               ((guix config) => ,(make-config.scm)))
 398        #~(begin
 399            (use-modules (gnu build bootloader)
 400                         (gnu build vm)
 401                         (guix store database)
 402                         (guix build utils)
 403                         (srfi srfi-26)
 404                         (ice-9 binary-ports))
 405
 406            (sql-schema #$schema)
 407
 408            ;; Allow non-ASCII file names--e.g., 'nss-certs'--to be decoded.
 409            (setenv "GUIX_LOCPATH"
 410                    #+(file-append glibc-utf8-locales "/lib/locale"))
 411            (setlocale LC_ALL "en_US.utf8")
 412
 413            (let ((inputs
 414                   '#$(append (list parted e2fsprogs dosfstools)
 415                              (map canonical-package
 416                                   (list sed grep coreutils findutils gawk))))
 417
 418                  ;; This variable is unused but allows us to add INPUTS-TO-COPY
 419                  ;; as inputs.
 420                  (to-register
 421                   '#$(map (match-lambda
 422                             ((name thing) thing)
 423                             ((name thing output) `(,thing ,output)))
 424                           inputs)))
 425
 426              (set-path-environment-variable "PATH" '("bin" "sbin") inputs)
 427
 428              (let* ((graphs     '#$(match inputs
 429                                      (((names . _) ...)
 430                                       names)))
 431                     (initialize (root-partition-initializer
 432                                  #:closures graphs
 433                                  #:copy-closures? #$copy-inputs?
 434                                  #:register-closures? #$register-closures?
 435                                  #:system-directory #$os
 436
 437                                  ;; Disable deduplication to speed things up,
 438                                  ;; and because it doesn't help much for a
 439                                  ;; single system generation.
 440                                  #:deduplicate? #f))
 441                     (root-size  #$(if (eq? 'guess disk-image-size)
 442                                       #~(max
 443                                          ;; Minimum 20 MiB root size
 444                                          (* 20 (expt 2 20))
 445                                          (estimated-partition-size
 446                                           (map (cut string-append "/xchg/" <>)
 447                                                graphs)))
 448                                       (- disk-image-size
 449                                          (* 50 (expt 2 20)))))
 450                     (partitions
 451                      (append
 452                       (list (partition
 453                              (size root-size)
 454                              (label #$file-system-label)
 455                              (uuid #$(and=> file-system-uuid
 456                                             uuid-bytevector))
 457                              (file-system #$file-system-type)
 458                              (flags '(boot))
 459                              (initializer initialize)))
 460                       ;; Append a small EFI System Partition for use with UEFI
 461                       ;; bootloaders if we are not targeting ARM because UEFI
 462                       ;; support in U-Boot is experimental.
 463                       ;;
 464                       ;; FIXME: ‘target-arm?’ may be not operate on the right
 465                       ;; system/target values.  Rewrite using ‘let-system’ when
 466                       ;; available.
 467                       (if #$(target-arm?)
 468                           '()
 469                           (list (partition
 470                                  ;; The standalone grub image is about 10MiB, but
 471                                  ;; leave some room for custom or multiple images.
 472                                  (size (* 40 (expt 2 20)))
 473                                  (label "GNU-ESP") ;cosmetic only
 474                                  ;; Use "vfat" here since this property is used
 475                                  ;; when mounting. The actual FAT-ness is based
 476                                  ;; on file system size (16 in this case).
 477                                  (file-system "vfat")
 478                                  (flags '(esp)))))))
 479                     (grub-efi #$(and (not (target-arm?)) grub-efi)))
 480                (initialize-hard-disk "/dev/vda"
 481                                      #:partitions partitions
 482                                      #:grub-efi grub-efi
 483                                      #:bootloader-package
 484                                      #$(bootloader-package bootloader)
 485                                      #:bootcfg #$bootcfg-drv
 486                                      #:bootcfg-location
 487                                      #$(bootloader-configuration-file bootloader)
 488                                      #:bootloader-installer
 489                                      #$(bootloader-installer bootloader)))))))
 490    #:system system
 491    #:target target
 492    #:make-disk-image? #t
 493    #:disk-image-size disk-image-size
 494    #:disk-image-format disk-image-format
 495    #:references-graphs inputs))
 496
 497 (define* (system-docker-image os
 498                               #:key
 499                               (name "guix-docker-image")
 500                               (register-closures? (has-guix-service-type? os)))
 501   "Build a docker image.  OS is the desired <operating-system>.  NAME is the
 502 base name to use for the output file.  When REGISTER-CLOSURES? is true,
 503 register the closure of OS with Guix in the resulting Docker image.  By
 504 default, REGISTER-CLOSURES? is set to true only if a service of type
 505 GUIX-SERVICE-TYPE is present in the services definition of the operating
 506 system."
 507   (define schema
 508     (and register-closures?
 509          (local-file (search-path %load-path
 510                                   "guix/store/schema.sql"))))
 511
 512   (define boot-program
 513     ;; Program that runs the boot script of OS, which in turn starts shepherd.
 514     (program-file "boot-program"
 515                   #~(let ((system (cadr (command-line))))
 516                       (setenv "GUIX_NEW_SYSTEM" system)
 517                       (execl #$(file-append guile-2.2 "/bin/guile")
 518                              "guile" "--no-auto-compile"
 519                              (string-append system "/boot")))))
 520
 521
 522   (let ((os    (operating-system-with-gc-roots
 523                 (containerized-operating-system os '())
 524                 (list boot-program)))
 525         (name  (string-append name ".tar.gz"))
 526         (graph "system-graph"))
 527     (define build
 528       (with-extensions (cons guile-json-3         ;for (guix docker)
 529                              gcrypt-sqlite3&co)   ;for (guix store database)
 530         (with-imported-modules `(,@(source-module-closure
 531                                     '((guix docker)
 532                                       (guix store database)
 533                                       (guix build utils)
 534                                       (guix build store-copy)
 535                                       (gnu build vm))
 536                                     #:select? not-config?)
 537                                  ((guix config) => ,(make-config.scm)))
 538           #~(begin
 539               (use-modules (guix docker)
 540                            (guix build utils)
 541                            (gnu build vm)
 542                            (srfi srfi-19)
 543                            (guix build store-copy)
 544                            (guix store database))
 545
 546               ;; Set the SQL schema location.
 547               (sql-schema #$schema)
 548
 549               ;; Allow non-ASCII file names--e.g., 'nss-certs'--to be decoded.
 550               (setenv "GUIX_LOCPATH"
 551                       #+(file-append glibc-utf8-locales "/lib/locale"))
 552               (setlocale LC_ALL "en_US.utf8")
 553
 554               (let* (;; This initializer requires elevated privileges that are
 555                      ;; not normally available in the build environment (e.g.,
 556                      ;; it needs to create device nodes).  In order to obtain
 557                      ;; such privileges, we run it as root in a VM.
 558                      (initialize (root-partition-initializer
 559                                   #:closures '(#$graph)
 560                                   #:register-closures? #$register-closures?
 561                                   #:system-directory #$os
 562                                   ;; De-duplication would fail due to
 563                                   ;; cross-device link errors, so don't do it.
 564                                   #:deduplicate? #f))
 565                      ;; Even as root in a VM, the initializer would fail due to
 566                      ;; lack of privileges if we use a root-directory that is on
 567                      ;; a file system that is shared with the host (e.g., /tmp).
 568                      (root-directory "/guixsd-system-root"))
 569                 (set-path-environment-variable "PATH" '("bin" "sbin") '(#+tar))
 570                 (mkdir root-directory)
 571                 (initialize root-directory)
 572                 (build-docker-image
 573                  (string-append "/xchg/" #$name) ;; The output file.
 574                  (cons* root-directory
 575                         (map store-info-item
 576                              (call-with-input-file
 577                                  (string-append "/xchg/" #$graph)
 578                                read-reference-graph)))
 579                  #$os
 580                  #:entry-point '(#$boot-program #$os)
 581                  #:compressor '(#+(file-append gzip "/bin/gzip") "-9n")
 582                  #:creation-time (make-time time-utc 0 1)
 583                  #:transformations `((,root-directory -> ""))))))))
 584
 585     (expression->derivation-in-linux-vm
 586      name build
 587      #:make-disk-image? #f
 588      #:single-file-output? #t
 589      #:references-graphs `((,graph ,os)))))
 590
 591 \f
 592 ;;;
 593 ;;; VM and disk images.
 594 ;;;
 595
 596 (define* (operating-system-uuid os #:optional (type 'dce))
 597   "Compute UUID object with a deterministic \"UUID\" for OS, of the given
 598 TYPE (one of 'iso9660 or 'dce).  Return a UUID object."
 599   ;; Note: For this to be deterministic, we must not hash things that contains
 600   ;; (directly or indirectly) procedures, for example.  That rules out
 601   ;; anything that contains gexps, thunk or delayed record fields, etc.
 602
 603   (define service-name
 604     (compose service-type-name service-kind))
 605
 606   (define (file-system-digest fs)
 607     ;; Return a hashable digest that does not contain 'dependencies' since
 608     ;; this field can contain procedures.
 609     (let ((device (file-system-device fs)))
 610       (list (file-system-mount-point fs)
 611             (file-system-type fs)
 612             (cond ((file-system-label? device)
 613                    (file-system-label->string device))
 614                   ((uuid? device)
 615                    (uuid->string device))
 616                   ((string? device)
 617                    device)
 618                   (else #f))
 619             (file-system-options fs))))
 620
 621   (if (eq? type 'iso9660)
 622       (let ((pad (compose (cut string-pad <> 2 #\0)
 623                           number->string))
 624             (h   (hash (map service-name (operating-system-services os))
 625                        3600)))
 626         (bytevector->uuid
 627          (string->iso9660-uuid
 628           (string-append "1970-01-01-"
 629                          (pad (hash (operating-system-host-name os) 24)) "-"
 630                          (pad (quotient h 60)) "-"
 631                          (pad (modulo h 60)) "-"
 632                          (pad (hash (map file-system-digest
 633                                          (operating-system-file-systems os))
 634                                     100))))
 635          'iso9660))
 636       (bytevector->uuid
 637        (uint-list->bytevector
 638         (list (hash file-system-type
 639                     (- (expt 2 32) 1))
 640               (hash (operating-system-host-name os)
 641                     (- (expt 2 32) 1))
 642               (hash (map service-name (operating-system-services os))
 643                     (- (expt 2 32) 1))
 644               (hash (map file-system-digest (operating-system-file-systems os))
 645                     (- (expt 2 32) 1)))
 646         (endianness little)
 647         4)
 648        type)))
 649
 650 (define* (system-disk-image os
 651                             #:key
 652                             (name "disk-image")
 653                             (file-system-type "ext4")
 654                             (disk-image-size (* 900 (expt 2 20)))
 655                             (volatile? #t))
 656   "Return the derivation of a disk image of DISK-IMAGE-SIZE bytes of the
 657 system described by OS.  Said image can be copied on a USB stick as is.  When
 658 VOLATILE? is true, the root file system is made volatile; this is useful
 659 to USB sticks meant to be read-only."
 660   (define normalize-label
 661     ;; ISO labels are all-caps (case-insensitive), but since
 662     ;; 'find-partition-by-label' is case-sensitive, make it all-caps here.
 663     (if (string=? "iso9660" file-system-type)
 664         string-upcase
 665         identity))
 666
 667   (define root-label
 668     ;; Volume name of the root file system.
 669     (normalize-label "Guix_image"))
 670
 671   (define (root-uuid os)
 672     ;; UUID of the root file system, computed in a deterministic fashion.
 673     ;; This is what we use to locate the root file system so it has to be
 674     ;; different from the user's own file system UUIDs.
 675     (operating-system-uuid os
 676                            (if (string=? file-system-type "iso9660")
 677                                'iso9660
 678                                'dce)))
 679
 680   (define file-systems-to-keep
 681     (remove (lambda (fs)
 682               (string=? (file-system-mount-point fs) "/"))
 683             (operating-system-file-systems os)))
 684
 685   (let* ((os (operating-system (inherit os)
 686                ;; Since this is meant to be used on real hardware, don't
 687                ;; install QEMU networking or anything like that.  Assume USB
 688                ;; mass storage devices (usb-storage.ko) are available.
 689                (initrd (lambda (file-systems . rest)
 690                          (apply (operating-system-initrd os)
 691                                 file-systems
 692                                 #:volatile-root? #t
 693                                 rest)))
 694
 695                (bootloader (if (string=? "iso9660" file-system-type)
 696                                (bootloader-configuration
 697                                  (inherit (operating-system-bootloader os))
 698                                  (bootloader grub-mkrescue-bootloader))
 699                                (operating-system-bootloader os)))
 700
 701                ;; Force our own root file system.  (We need a "/" file system
 702                ;; to call 'root-uuid'.)
 703                (file-systems (cons (file-system
 704                                      (mount-point "/")
 705                                      (device "/dev/placeholder")
 706                                      (type file-system-type))
 707                                    file-systems-to-keep))))
 708          (uuid (root-uuid os))
 709          (os (operating-system
 710                (inherit os)
 711                (file-systems (cons (file-system
 712                                      (mount-point "/")
 713                                      (device uuid)
 714                                      (type file-system-type))
 715                                    file-systems-to-keep))))
 716         (bootcfg (operating-system-bootcfg os)))
 717     (if (string=? "iso9660" file-system-type)
 718         (iso9660-image #:name name
 719                        #:file-system-label root-label
 720                        #:file-system-uuid uuid
 721                        #:os os
 722                        #:bootcfg-drv bootcfg
 723                        #:bootloader (bootloader-configuration-bootloader
 724                                      (operating-system-bootloader os))
 725                        #:inputs `(("system" ,os)
 726                                   ("bootcfg" ,bootcfg))
 727                        #:grub-mkrescue-environment
 728                        '(("MKRESCUE_SED_MODE" . "mbr_hfs")))
 729         (qemu-image #:name name
 730                     #:os os
 731                     #:bootcfg-drv bootcfg
 732                     #:bootloader (bootloader-configuration-bootloader
 733                                   (operating-system-bootloader os))
 734                     #:disk-image-size disk-image-size
 735                     #:disk-image-format "raw"
 736                     #:file-system-type file-system-type
 737                     #:file-system-label root-label
 738                     #:file-system-uuid uuid
 739                     #:copy-inputs? #t
 740                     #:inputs `(("system" ,os)
 741                                ("bootcfg" ,bootcfg))))))
 742
 743 (define* (system-qemu-image os
 744                             #:key
 745                             (file-system-type "ext4")
 746                             (disk-image-size (* 900 (expt 2 20))))
 747   "Return the derivation of a freestanding QEMU image of DISK-IMAGE-SIZE bytes
 748 of the GNU system as described by OS."
 749   (define file-systems-to-keep
 750     ;; Keep only file systems other than root and not normally bound to real
 751     ;; devices.
 752     (remove (lambda (fs)
 753               (let ((target (file-system-mount-point fs))
 754                     (source (file-system-device fs)))
 755                 (or (string=? target "/")
 756                     (string-prefix? "/dev/" source))))
 757             (operating-system-file-systems os)))
 758
 759   (define root-uuid
 760     ;; UUID of the root file system.
 761     (operating-system-uuid os
 762                            (if (string=? file-system-type "iso9660")
 763                                'iso9660
 764                                'dce)))
 765
 766
 767   (let* ((os (operating-system (inherit os)
 768                ;; Assume we have an initrd with the whole QEMU shebang.
 769
 770                ;; Force our own root file system.  Refer to it by UUID so that
 771                ;; it works regardless of how the image is used ("qemu -hda",
 772                ;; Xen, etc.).
 773                (file-systems (cons (file-system
 774                                      (mount-point "/")
 775                                      (device root-uuid)
 776                                      (type file-system-type))
 777                                    file-systems-to-keep))))
 778          (bootcfg (operating-system-bootcfg os)))
 779     (qemu-image  #:os os
 780                  #:bootcfg-drv bootcfg
 781                  #:bootloader (bootloader-configuration-bootloader
 782                                (operating-system-bootloader os))
 783                  #:disk-image-size disk-image-size
 784                  #:file-system-type file-system-type
 785                  #:file-system-uuid root-uuid
 786                  #:inputs `(("system" ,os)
 787                             ("bootcfg" ,bootcfg))
 788                  #:copy-inputs? #t)))
 789
 790 \f
 791 ;;;
 792 ;;; VMs that share file systems with the host.
 793 ;;;
 794
 795 (define (file-system->mount-tag fs)
 796   "Return a 9p mount tag for host file system FS."
 797   ;; QEMU mount tags must be ASCII, at most 31-byte long, cannot contain
 798   ;; slashes, and cannot start with '_'.  Compute an identifier that
 799   ;; corresponds to the rules.
 800   (string-append "TAG"
 801                  (string-drop (bytevector->base32-string
 802                                (sha1 (string->utf8 fs)))
 803                               4)))
 804
 805 (define (mapping->file-system mapping)
 806   "Return a 9p file system that realizes MAPPING."
 807   (match mapping
 808     (($ <file-system-mapping> source target writable?)
 809      (file-system
 810        (mount-point target)
 811        (device (file-system->mount-tag source))
 812        (type "9p")
 813        (flags (if writable? '() '(read-only)))
 814        (options "trans=virtio,cache=loose")
 815        (check? #f)
 816        (create-mount-point? #t)))))
 817
 818 (define* (virtualized-operating-system os mappings #:optional (full-boot? #f))
 819   "Return an operating system based on OS suitable for use in a virtualized
 820 environment with the store shared with the host.  MAPPINGS is a list of
 821 <file-system-mapping> to realize in the virtualized OS."
 822   (define user-file-systems
 823     ;; Remove file systems that conflict with those added below, or that are
 824     ;; normally bound to real devices.
 825     (remove (lambda (fs)
 826               (let ((target (file-system-mount-point fs))
 827                     (source (file-system-device fs)))
 828                 (or (string=? target (%store-prefix))
 829                     (string=? target "/")
 830                     (and (string? source)
 831                          (string-prefix? "/dev/" source))
 832
 833                     ;; Labels and UUIDs are necessarily invalid in the VM.
 834                     (and (file-system-mount? fs)
 835                          (or (file-system-label? source)
 836                              (uuid? source))))))
 837             (operating-system-file-systems os)))
 838
 839   (define virtual-file-systems
 840     (cons (file-system
 841             (mount-point "/")
 842             (device "/dev/vda1")
 843             (type "ext4"))
 844
 845           (append (map mapping->file-system mappings)
 846                   user-file-systems)))
 847
 848   (operating-system (inherit os)
 849
 850     ;; XXX: Until we run QEMU with UEFI support (with the OVMF firmware),
 851     ;; force the traditional i386/BIOS method.
 852     ;; See <https://bugs.gnu.org/28768>.
 853     (bootloader (bootloader-configuration
 854                   (inherit (operating-system-bootloader os))
 855                   (bootloader grub-bootloader)
 856                   (target "/dev/vda")))
 857
 858     (initrd (lambda (file-systems . rest)
 859               (apply (operating-system-initrd os)
 860                      file-systems
 861                      #:volatile-root? #t
 862                      rest)))
 863
 864     ;; Disable swap.
 865     (swap-devices '())
 866
 867     ;; XXX: When FULL-BOOT? is true, do not add a 9p mount for /gnu/store
 868     ;; since that would lead the bootloader config to look for the kernel and
 869     ;; initrd in it.
 870     (file-systems (if full-boot?
 871                       virtual-file-systems
 872                       (cons
 873                        (file-system
 874                          (inherit (mapping->file-system %store-mapping))
 875                          (needed-for-boot? #t))
 876                        virtual-file-systems)))))
 877
 878 (define* (system-qemu-image/shared-store
 879           os
 880           #:key
 881           full-boot?
 882           (disk-image-size (* (if full-boot? 500 30) (expt 2 20))))
 883   "Return a derivation that builds a QEMU image of OS that shares its store
 884 with the host.
 885
 886 When FULL-BOOT? is true, return an image that does a complete boot sequence,
 887 bootloaded included; thus, make a disk image that contains everything the
 888 bootloader refers to: OS kernel, initrd, bootloader data, etc."
 889   (define root-uuid
 890     ;; Use a fixed UUID to improve determinism.
 891     (operating-system-uuid os 'dce))
 892
 893   (define bootcfg
 894     (operating-system-bootcfg os))
 895
 896   ;; XXX: When FULL-BOOT? is true, we end up creating an image that contains
 897   ;; BOOTCFG and all its dependencies, including the output of OS.
 898   ;; This is more than needed (we only need the kernel, initrd, GRUB for its
 899   ;; font, and the background image), but it's hard to filter that.
 900   (qemu-image #:os os
 901               #:bootcfg-drv bootcfg
 902               #:bootloader (bootloader-configuration-bootloader
 903                             (operating-system-bootloader os))
 904               #:disk-image-size disk-image-size
 905               #:file-system-uuid root-uuid
 906               #:inputs (if full-boot?
 907                            `(("bootcfg" ,bootcfg))
 908                            '())
 909
 910               ;; XXX: Passing #t here is too slow, so let it off by default.
 911               #:register-closures? #f
 912               #:copy-inputs? full-boot?))
 913
 914 (define* (common-qemu-options image shared-fs)
 915   "Return the a string-value gexp with the common QEMU options to boot IMAGE,
 916 with '-virtfs' options for the host file systems listed in SHARED-FS."
 917
 918   (define (virtfs-option fs)
 919     #~(format #f "-virtfs local,path=~s,security_model=none,mount_tag=~s"
 920               #$fs #$(file-system->mount-tag fs)))
 921
 922   #~(;; Only enable kvm if we see /dev/kvm exists.
 923      ;; This allows users without hardware virtualization to still use these
 924      ;; commands.
 925      #$@(if (file-exists? "/dev/kvm")
 926             '("-enable-kvm")
 927             '())
 928
 929      "-no-reboot"
 930      "-net nic,model=virtio"
 931      "-object" "rng-random,filename=/dev/urandom,id=guixsd-vm-rng"
 932      "-device" "virtio-rng-pci,rng=guixsd-vm-rng"
 933
 934      #$@(map virtfs-option shared-fs)
 935      "-vga std"
 936      (format #f "-drive file=~a,if=virtio,cache=writeback,werror=report,readonly"
 937              #$image)))
 938
 939 (define* (system-qemu-image/shared-store-script os
 940                                                 #:key
 941                                                 (qemu qemu)
 942                                                 (graphic? #t)
 943                                                 (memory-size 256)
 944                                                 (mappings '())
 945                                                 full-boot?
 946                                                 (disk-image-size
 947                                                  (* (if full-boot? 500 70)
 948                                                     (expt 2 20)))
 949                                                 (options '()))
 950   "Return a derivation that builds a script to run a virtual machine image of
 951 OS that shares its store with the host.  The virtual machine runs with
 952 MEMORY-SIZE MiB of memory.
 953
 954 MAPPINGS is a list of <file-system-mapping> specifying mapping of host file
 955 systems into the guest.
 956
 957 When FULL-BOOT? is true, the returned script runs everything starting from the
 958 bootloader; otherwise it directly starts the operating system kernel.  The
 959 DISK-IMAGE-SIZE parameter specifies the size in bytes of the root disk image;
 960 it is mostly useful when FULL-BOOT?  is true."
 961   (mlet* %store-monad ((os ->  (virtualized-operating-system os mappings full-boot?))
 962                        (image  (system-qemu-image/shared-store
 963                                 os
 964                                 #:full-boot? full-boot?
 965                                 #:disk-image-size disk-image-size)))
 966     (define kernel-arguments
 967       #~(list #$@(if graphic? #~() #~("console=ttyS0"))
 968               #+@(operating-system-kernel-arguments os "/dev/vda1")))
 969
 970     (define qemu-exec
 971       #~(list (string-append #$qemu "/bin/" #$(qemu-command (%current-system)))
 972               #$@(if full-boot?
 973                      #~()
 974                      #~("-kernel" #$(operating-system-kernel-file os)
 975                         "-initrd" #$(file-append os "/initrd")
 976                         (format #f "-append ~s"
 977                                 (string-join #$kernel-arguments " "))))
 978               #$@(common-qemu-options image
 979                                       (map file-system-mapping-source
 980                                            (cons %store-mapping mappings)))
 981               "-m " (number->string #$memory-size)
 982               #$@options))
 983
 984     (define builder
 985       #~(call-with-output-file #$output
 986           (lambda (port)
 987             (format port "#!~a~% exec ~a \"$@\"~%"
 988                     #$(file-append bash "/bin/sh")
 989                     (string-join #$qemu-exec " "))
 990             (chmod port #o555))))
 991
 992     (gexp->derivation "run-vm.sh" builder)))
 993
 994 \f
 995 ;;;
 996 ;;; High-level abstraction.
 997 ;;;
 998
 999 (define-record-type* <virtual-machine> %virtual-machine
1000   make-virtual-machine
1001   virtual-machine?
1002   (operating-system virtual-machine-operating-system) ;<operating-system>
1003   (qemu             virtual-machine-qemu              ;<package>
1004                     (default qemu))
1005   (graphic?         virtual-machine-graphic?      ;Boolean
1006                     (default #f))
1007   (memory-size      virtual-machine-memory-size   ;integer (MiB)
1008                     (default 256))
1009   (disk-image-size  virtual-machine-disk-image-size   ;integer (bytes)
1010                     (default 'guess))
1011   (port-forwardings virtual-machine-port-forwardings ;list of integer pairs
1012                     (default '())))
1013
1014 (define-syntax virtual-machine
1015   (syntax-rules ()
1016     "Declare a virtual machine running the specified OS, with the given
1017 options."
1018     ((_ os)                                       ;shortcut
1019      (%virtual-machine (operating-system os)))
1020     ((_ fields ...)
1021      (%virtual-machine fields ...))))
1022
1023 (define (port-forwardings->qemu-options forwardings)
1024   "Return the QEMU option for the given port FORWARDINGS as a string, where
1025 FORWARDINGS is a list of host-port/guest-port pairs."
1026   (string-join
1027    (map (match-lambda
1028           ((host-port . guest-port)
1029            (string-append "hostfwd=tcp::"
1030                           (number->string host-port)
1031                           "-:" (number->string guest-port))))
1032         forwardings)
1033    ","))
1034
1035 (define-gexp-compiler (virtual-machine-compiler (vm <virtual-machine>)
1036                                                 system target)
1037   ;; XXX: SYSTEM and TARGET are ignored.
1038   (match vm
1039     (($ <virtual-machine> os qemu graphic? memory-size disk-image-size ())
1040      (system-qemu-image/shared-store-script os
1041                                             #:qemu qemu
1042                                             #:graphic? graphic?
1043                                             #:memory-size memory-size
1044                                             #:disk-image-size
1045                                             disk-image-size))
1046     (($ <virtual-machine> os qemu graphic? memory-size disk-image-size
1047                           forwardings)
1048      (let ((options
1049             `("-net" ,(string-append
1050                        "user,"
1051                        (port-forwardings->qemu-options forwardings)))))
1052        (system-qemu-image/shared-store-script os
1053                                               #:qemu qemu
1054                                               #:graphic? graphic?
1055                                               #:memory-size memory-size
1056                                               #:disk-image-size
1057                                               disk-image-size
1058                                               #:options options)))))
1059
1060 ;;; vm.scm ends here