gnu/system/vm.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Ludovic Courtès <ludo@gnu.org>
   3 ;;; Copyright © 2016 Christopher Allan Webber <cwebber@dustycloud.org>
   4 ;;; Copyright © 2016, 2017 Leo Famulari <leo@famulari.name>
   5 ;;; Copyright © 2017 Mathieu Othacehe <m.othacehe@gmail.com>
   6 ;;; Copyright © 2017 Marius Bakke <mbakke@fastmail.com>
   7 ;;; Copyright © 2018 Chris Marusich <cmmarusich@gmail.com>
   8 ;;;
   9 ;;; This file is part of GNU Guix.
  10 ;;;
  11 ;;; GNU Guix is free software; you can redistribute it and/or modify it
  12 ;;; under the terms of the GNU General Public License as published by
  13 ;;; the Free Software Foundation; either version 3 of the License, or (at
  14 ;;; your option) any later version.
  15 ;;;
  16 ;;; GNU Guix is distributed in the hope that it will be useful, but
  17 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  18 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 ;;; GNU General Public License for more details.
  20 ;;;
  21 ;;; You should have received a copy of the GNU General Public License
  22 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  23
  24 (define-module (gnu system vm)
  25   #:use-module (guix config)
  26   #:use-module (guix store)
  27   #:use-module (guix gexp)
  28   #:use-module (guix derivations)
  29   #:use-module (guix packages)
  30   #:use-module (guix monads)
  31   #:use-module (guix records)
  32   #:use-module (guix modules)
  33   #:use-module (guix utils)
  34   #:use-module (gcrypt hash)
  35   #:use-module (guix base32)
  36   #:use-module ((guix self) #:select (make-config.scm))
  37
  38   #:use-module ((gnu build vm)
  39                 #:select (qemu-command))
  40   #:use-module (gnu packages base)
  41   #:use-module (gnu packages bootloaders)
  42   #:use-module (gnu packages cdrom)
  43   #:use-module (gnu packages compression)
  44   #:use-module (gnu packages guile)
  45   #:autoload   (gnu packages gnupg) (guile-gcrypt)
  46   #:use-module (gnu packages gawk)
  47   #:use-module (gnu packages bash)
  48   #:use-module (gnu packages less)
  49   #:use-module (gnu packages virtualization)
  50   #:use-module (gnu packages disk)
  51   #:use-module (gnu packages zile)
  52   #:use-module (gnu packages linux)
  53   #:use-module (gnu packages admin)
  54
  55   #:use-module (gnu bootloader)
  56   #:use-module (gnu bootloader grub)
  57   #:use-module (gnu system shadow)
  58   #:use-module (gnu system pam)
  59   #:use-module (gnu system linux-container)
  60   #:use-module (gnu system linux-initrd)
  61   #:use-module (gnu bootloader)
  62   #:use-module (gnu system file-systems)
  63   #:use-module (gnu system)
  64   #:use-module (gnu services)
  65   #:use-module (gnu services base)
  66   #:use-module (gnu system uuid)
  67
  68   #:use-module (srfi srfi-1)
  69   #:use-module (srfi srfi-26)
  70   #:use-module (rnrs bytevectors)
  71   #:use-module (ice-9 match)
  72
  73   #:export (expression->derivation-in-linux-vm
  74             qemu-image
  75             virtualized-operating-system
  76             system-qemu-image
  77
  78             system-qemu-image/shared-store
  79             system-qemu-image/shared-store-script
  80             system-disk-image-in-vm
  81             system-docker-image
  82
  83             virtual-machine
  84             virtual-machine?))
  85
  86 \f
  87 ;;; Commentary:
  88 ;;;
  89 ;;; Tools to evaluate build expressions within virtual machines.
  90 ;;;
  91 ;;; Code:
  92
  93 (define %linux-vm-file-systems
  94   ;; File systems mounted for 'derivation-in-linux-vm'.  These are shared with
  95   ;; the host over 9p.
  96   ;;
  97   ;; The 9p documentation says that cache=loose is "intended for exclusive,
  98   ;; read-only mounts", without additional details.  It's much faster than the
  99   ;; default cache=none, especially when copying and registering store items.
 100   ;; Thus, use cache=loose, except for /xchg where we want to ensure
 101   ;; consistency.
 102   (list (file-system
 103           (mount-point (%store-prefix))
 104           (device "store")
 105           (type "9p")
 106           (needed-for-boot? #t)
 107           (flags '(read-only))
 108           (options "trans=virtio,cache=loose")
 109           (check? #f))
 110         (file-system
 111           (mount-point "/xchg")
 112           (device "xchg")
 113           (type "9p")
 114           (needed-for-boot? #t)
 115           (options "trans=virtio")
 116           (check? #f))
 117         (file-system
 118           (mount-point "/tmp")
 119           (device "tmp")
 120           (type "9p")
 121           (needed-for-boot? #t)
 122           (options "trans=virtio,cache=loose")
 123           (check? #f))))
 124
 125 (define not-config?
 126   ;; Select (guix …) and (gnu …) modules, except (guix config).
 127   (match-lambda
 128     (('guix 'config) #f)
 129     (('guix rest ...) #t)
 130     (('gnu rest ...) #t)
 131     (rest #f)))
 132
 133 (define gcrypt-sqlite3&co
 134   ;; Guile-Gcrypt, Guile-SQLite3, and their propagated inputs.
 135   (append-map (lambda (package)
 136                 (cons package
 137                       (match (package-transitive-propagated-inputs package)
 138                         (((labels packages) ...)
 139                          packages))))
 140               (list guile-gcrypt guile-sqlite3)))
 141
 142 (define* (expression->derivation-in-linux-vm name exp
 143                                              #:key
 144                                              (system (%current-system)) target
 145                                              (linux linux-libre)
 146                                              initrd
 147                                              (qemu qemu-minimal)
 148                                              (env-vars '())
 149                                              (guile-for-build
 150                                               (%guile-for-build))
 151                                              (file-systems
 152                                               %linux-vm-file-systems)
 153
 154                                              (single-file-output? #f)
 155                                              (make-disk-image? #f)
 156                                              (references-graphs #f)
 157                                              (memory-size 256)
 158                                              (disk-image-format "qcow2")
 159                                              (disk-image-size 'guess)
 160
 161                                              (substitutable? #t))
 162   "Evaluate EXP in a QEMU virtual machine running LINUX with INITRD (a
 163 derivation).  The virtual machine runs with MEMORY-SIZE MiB of memory.  In the
 164 virtual machine, EXP has access to FILE-SYSTEMS, which, by default, includes a
 165 9p share of the store, the '/xchg' where EXP should put its output file(s),
 166 and a 9p share of /tmp.
 167
 168 If SINGLE-FILE-OUTPUT? is true, copy a single file from '/xchg' to OUTPUT.
 169 Otherwise, copy the contents of /xchg to a new directory OUTPUT.
 170
 171 When MAKE-DISK-IMAGE? is true, then create a QEMU disk image of type
 172 DISK-IMAGE-FORMAT (e.g., 'qcow2' or 'raw'), of DISK-IMAGE-SIZE bytes and
 173 return it.  When DISK-IMAGE-SIZE is 'guess, estimate the image size based
 174 based on the size of the closure of REFERENCES-GRAPHS.
 175
 176 When REFERENCES-GRAPHS is true, it must be a list of file name/store path
 177 pairs, as for `derivation'.  The files containing the reference graphs are
 178 made available under the /xchg CIFS share.
 179
 180 SUBSTITUTABLE? determines whether the returned derivation should be marked as
 181 substitutable."
 182   (define user-builder
 183     (program-file "builder-in-linux-vm" exp))
 184
 185   (define loader
 186     ;; Invoke USER-BUILDER instead using 'primitive-load'.  The reason for
 187     ;; this is to allow USER-BUILDER to dlopen stuff by using a full-featured
 188     ;; Guile, which it couldn't do using the statically-linked guile used in
 189     ;; the initrd.  See example at
 190     ;; <https://lists.gnu.org/archive/html/guix-devel/2017-10/msg00233.html>.
 191     (program-file "linux-vm-loader"
 192                   ;; Communicate USER-BUILDER's exit status via /xchg so that
 193                   ;; the host can distinguish between success, failure, and
 194                   ;; kernel panic.
 195                   #~(let ((status (system* #$user-builder)))
 196                       (call-with-output-file "/xchg/.exit-status"
 197                         (lambda (port)
 198                           (write status port)))
 199                       (sync)
 200                       (reboot))))
 201
 202   (let ((initrd (or initrd
 203                     (base-initrd file-systems
 204                                  #:on-error 'backtrace
 205                                  #:linux linux
 206                                  #:linux-modules %base-initrd-modules
 207                                  #:qemu-networking? #t))))
 208
 209     (define builder
 210       ;; Code that launches the VM that evaluates EXP.
 211       (with-extensions gcrypt-sqlite3&co
 212         (with-imported-modules `(,@(source-module-closure
 213                                     '((guix build utils)
 214                                       (gnu build vm))
 215                                     #:select? not-config?)
 216
 217                                  ;; For consumption by (gnu store database).
 218                                  ((guix config) => ,(make-config.scm)))
 219           #~(begin
 220               (use-modules (guix build utils)
 221                            (gnu build vm))
 222
 223               (let* ((native-inputs
 224                       '#+(list qemu (canonical-package coreutils)))
 225                      (linux   (string-append #$linux "/"
 226                                              #$(system-linux-image-file-name)))
 227                      (initrd  #$initrd)
 228                      (loader  #$loader)
 229                      (graphs  '#$(match references-graphs
 230                                    (((graph-files . _) ...) graph-files)
 231                                    (_ #f)))
 232                      (target  #$(or (%current-target-system) (%current-system)))
 233                      (size    #$(if (eq? 'guess disk-image-size)
 234                                     #~(+ (* 70 (expt 2 20)) ;ESP
 235                                          (estimated-partition-size graphs))
 236                                     disk-image-size)))
 237
 238                 (set-path-environment-variable "PATH" '("bin") native-inputs)
 239
 240                 (load-in-linux-vm loader
 241                                   #:output #$output
 242                                   #:linux linux #:initrd initrd
 243                                   #:qemu (qemu-command target)
 244                                   #:memory-size #$memory-size
 245                                   #:make-disk-image? #$make-disk-image?
 246                                   #:single-file-output? #$single-file-output?
 247                                   ;; FIXME: ‘target-arm32?’ and
 248                                   ;; ‘target-aarch64?’ may not operate on the
 249                                   ;; right system/target values.  Rewrite
 250                                   ;; using ‘let-system’ when available.
 251                                   #:target-arm32? #$(target-arm32?)
 252                                   #:target-aarch64? #$(target-aarch64?)
 253                                   #:disk-image-format #$disk-image-format
 254                                   #:disk-image-size size
 255                                   #:references-graphs graphs))))))
 256
 257     (gexp->derivation name builder
 258                       ;; TODO: Require the "kvm" feature.
 259                       #:system system
 260                       #:target target
 261                       #:env-vars env-vars
 262                       #:guile-for-build guile-for-build
 263                       #:references-graphs references-graphs
 264                       #:substitutable? substitutable?)))
 265
 266 (define (has-guix-service-type? os)
 267   "Return true if OS contains a service of the type GUIX-SERVICE-TYPE."
 268   (not (not (find (lambda (service)
 269                      (eq? (service-kind service) guix-service-type))
 270                    (operating-system-services os)))))
 271
 272 (define* (iso9660-image #:key
 273                         (name "iso9660-image")
 274                         file-system-label
 275                         file-system-uuid
 276                         (system (%current-system))
 277                         (target (%current-target-system))
 278                         (qemu qemu-minimal)
 279                         os
 280                         bootcfg-drv
 281                         bootloader
 282                         (register-closures? (has-guix-service-type? os))
 283                         (inputs '())
 284                         (grub-mkrescue-environment '())
 285                         (substitutable? #t))
 286   "Return a bootable, stand-alone iso9660 image.
 287
 288 INPUTS is a list of inputs (as for packages)."
 289   (define schema
 290     (and register-closures?
 291          (local-file (search-path %load-path
 292                                   "guix/store/schema.sql"))))
 293
 294   (expression->derivation-in-linux-vm
 295    name
 296    (with-extensions gcrypt-sqlite3&co
 297      (with-imported-modules `(,@(source-module-closure '((gnu build vm)
 298                                                          (guix store database)
 299                                                          (guix build utils))
 300                                                        #:select? not-config?)
 301                               ((guix config) => ,(make-config.scm)))
 302        #~(begin
 303            (use-modules (gnu build vm)
 304                         (guix store database)
 305                         (guix build utils))
 306
 307            (sql-schema #$schema)
 308
 309            ;; Allow non-ASCII file names--e.g., 'nss-certs'--to be decoded.
 310            (setenv "GUIX_LOCPATH"
 311                    #+(file-append glibc-utf8-locales "/lib/locale"))
 312            (setlocale LC_ALL "en_US.utf8")
 313
 314            (let ((inputs
 315                   '#$(append (list parted e2fsprogs dosfstools xorriso)
 316                              (map canonical-package
 317                                   (list sed grep coreutils findutils gawk))))
 318
 319
 320                  (graphs     '#$(match inputs
 321                                   (((names . _) ...)
 322                                    names)))
 323                  ;; This variable is unused but allows us to add INPUTS-TO-COPY
 324                  ;; as inputs.
 325                  (to-register
 326                   '#$(map (match-lambda
 327                             ((name thing) thing)
 328                             ((name thing output) `(,thing ,output)))
 329                           inputs)))
 330
 331              (set-path-environment-variable "PATH" '("bin" "sbin") inputs)
 332              (make-iso9660-image #$xorriso
 333                                  '#$grub-mkrescue-environment
 334                                  #$(bootloader-package bootloader)
 335                                  #$bootcfg-drv
 336                                  #$os
 337                                  "/xchg/guixsd.iso"
 338                                  #:register-closures? #$register-closures?
 339                                  #:closures graphs
 340                                  #:volume-id #$file-system-label
 341                                  #:volume-uuid #$(and=> file-system-uuid
 342                                                         uuid-bytevector))))))
 343    #:system system
 344    #:target target
 345
 346    ;; Keep a local file system for /tmp so that we can populate it directly as
 347    ;; root and have files owned by root.  See <https://bugs.gnu.org/31752>.
 348    #:file-systems (remove (lambda (file-system)
 349                             (string=? (file-system-mount-point file-system)
 350                                       "/tmp"))
 351                           %linux-vm-file-systems)
 352
 353    #:make-disk-image? #f
 354    #:single-file-output? #t
 355    #:references-graphs inputs
 356    #:substitutable? substitutable?
 357
 358    ;; Xorriso seems to be quite memory-hungry, so increase the VM's RAM size.
 359    #:memory-size 512))
 360
 361 (define* (qemu-image #:key
 362                      (name "qemu-image")
 363                      (system (%current-system))
 364                      (target (%current-target-system))
 365                      (qemu qemu-minimal)
 366                      (disk-image-size 'guess)
 367                      (disk-image-format "qcow2")
 368                      (file-system-type "ext4")
 369                      file-system-label
 370                      file-system-uuid
 371                      os
 372                      bootcfg-drv
 373                      bootloader
 374                      (register-closures? (has-guix-service-type? os))
 375                      (inputs '())
 376                      copy-inputs?
 377                      (substitutable? #t))
 378   "Return a bootable, stand-alone QEMU image of type DISK-IMAGE-FORMAT (e.g.,
 379 'qcow2' or 'raw'), with a root partition of type FILE-SYSTEM-TYPE.
 380 Optionally, FILE-SYSTEM-LABEL can be specified as the volume name for the root
 381 partition; likewise FILE-SYSTEM-UUID, if true, specifies the UUID of the root
 382 partition (a UUID object).
 383
 384 The returned image is a full disk image that runs OS-DERIVATION,
 385 with a GRUB installation that uses GRUB-CONFIGURATION as its configuration
 386 file (GRUB-CONFIGURATION must be the name of a file in the VM.)
 387
 388 INPUTS is a list of inputs (as for packages).  When COPY-INPUTS? is true, copy
 389 all of INPUTS into the image being built.  When REGISTER-CLOSURES? is true,
 390 register INPUTS in the store database of the image so that Guix can be used in
 391 the image.  By default, REGISTER-CLOSURES? is set to true only if a service of
 392 type GUIX-SERVICE-TYPE is present in the services definition of the operating
 393 system."
 394   (define schema
 395     (and register-closures?
 396          (local-file (search-path %load-path
 397                                   "guix/store/schema.sql"))))
 398
 399   (expression->derivation-in-linux-vm
 400    name
 401    (with-extensions gcrypt-sqlite3&co
 402      (with-imported-modules `(,@(source-module-closure '((gnu build vm)
 403                                                          (gnu build bootloader)
 404                                                          (guix store database)
 405                                                          (guix build utils))
 406                                                        #:select? not-config?)
 407                               ((guix config) => ,(make-config.scm)))
 408        #~(begin
 409            (use-modules (gnu build bootloader)
 410                         (gnu build vm)
 411                         (guix store database)
 412                         (guix build utils)
 413                         (srfi srfi-26)
 414                         (ice-9 binary-ports))
 415
 416            (sql-schema #$schema)
 417
 418            ;; Allow non-ASCII file names--e.g., 'nss-certs'--to be decoded.
 419            (setenv "GUIX_LOCPATH"
 420                    #+(file-append glibc-utf8-locales "/lib/locale"))
 421            (setlocale LC_ALL "en_US.utf8")
 422
 423            (let ((inputs
 424                   '#$(append (list parted e2fsprogs dosfstools)
 425                              (map canonical-package
 426                                   (list sed grep coreutils findutils gawk))))
 427
 428                  ;; This variable is unused but allows us to add INPUTS-TO-COPY
 429                  ;; as inputs.
 430                  (to-register
 431                   '#$(map (match-lambda
 432                             ((name thing) thing)
 433                             ((name thing output) `(,thing ,output)))
 434                           inputs)))
 435
 436              (set-path-environment-variable "PATH" '("bin" "sbin") inputs)
 437
 438              (let* ((graphs     '#$(match inputs
 439                                      (((names . _) ...)
 440                                       names)))
 441                     (initialize (root-partition-initializer
 442                                  #:closures graphs
 443                                  #:copy-closures? #$copy-inputs?
 444                                  #:register-closures? #$register-closures?
 445                                  #:system-directory #$os
 446
 447                                  ;; Disable deduplication to speed things up,
 448                                  ;; and because it doesn't help much for a
 449                                  ;; single system generation.
 450                                  #:deduplicate? #f))
 451                     (root-size  #$(if (eq? 'guess disk-image-size)
 452                                       #~(max
 453                                          ;; Minimum 20 MiB root size
 454                                          (* 20 (expt 2 20))
 455                                          (estimated-partition-size
 456                                           (map (cut string-append "/xchg/" <>)
 457                                                graphs)))
 458                                       (- disk-image-size
 459                                          (* 50 (expt 2 20)))))
 460                     (partitions
 461                      (append
 462                       (list (partition
 463                              (size root-size)
 464                              (label #$file-system-label)
 465                              (uuid #$(and=> file-system-uuid
 466                                             uuid-bytevector))
 467                              (file-system #$file-system-type)
 468                              (flags '(boot))
 469                              (initializer initialize)))
 470                       ;; Append a small EFI System Partition for use with UEFI
 471                       ;; bootloaders if we are not targeting ARM because UEFI
 472                       ;; support in U-Boot is experimental.
 473                       ;;
 474                       ;; FIXME: ‘target-arm?’ may be not operate on the right
 475                       ;; system/target values.  Rewrite using ‘let-system’ when
 476                       ;; available.
 477                       (if #$(target-arm?)
 478                           '()
 479                           (list (partition
 480                                  ;; The standalone grub image is about 10MiB, but
 481                                  ;; leave some room for custom or multiple images.
 482                                  (size (* 40 (expt 2 20)))
 483                                  (label "GNU-ESP") ;cosmetic only
 484                                  ;; Use "vfat" here since this property is used
 485                                  ;; when mounting. The actual FAT-ness is based
 486                                  ;; on file system size (16 in this case).
 487                                  (file-system "vfat")
 488                                  (flags '(esp)))))))
 489                     (grub-efi #$(and (not (target-arm?)) grub-efi)))
 490                (initialize-hard-disk "/dev/vda"
 491                                      #:partitions partitions
 492                                      #:grub-efi grub-efi
 493                                      #:bootloader-package
 494                                      #$(bootloader-package bootloader)
 495                                      #:bootcfg #$bootcfg-drv
 496                                      #:bootcfg-location
 497                                      #$(bootloader-configuration-file bootloader)
 498                                      #:bootloader-installer
 499                                      #$(bootloader-installer bootloader)))))))
 500    #:system system
 501    #:target target
 502    #:make-disk-image? #t
 503    #:disk-image-size disk-image-size
 504    #:disk-image-format disk-image-format
 505    #:references-graphs inputs
 506    #:substitutable? substitutable?))
 507
 508 (define* (system-docker-image os
 509                               #:key
 510                               (name "guix-docker-image")
 511                               (register-closures? (has-guix-service-type? os)))
 512   "Build a docker image.  OS is the desired <operating-system>.  NAME is the
 513 base name to use for the output file.  When REGISTER-CLOSURES? is true,
 514 register the closure of OS with Guix in the resulting Docker image.  By
 515 default, REGISTER-CLOSURES? is set to true only if a service of type
 516 GUIX-SERVICE-TYPE is present in the services definition of the operating
 517 system."
 518   (define schema
 519     (and register-closures?
 520          (local-file (search-path %load-path
 521                                   "guix/store/schema.sql"))))
 522
 523   (define boot-program
 524     ;; Program that runs the boot script of OS, which in turn starts shepherd.
 525     (program-file "boot-program"
 526                   #~(let ((system (cadr (command-line))))
 527                       (setenv "GUIX_NEW_SYSTEM" system)
 528                       (execl #$(file-append guile-2.2 "/bin/guile")
 529                              "guile" "--no-auto-compile"
 530                              (string-append system "/boot")))))
 531
 532
 533   (let ((os    (operating-system-with-gc-roots
 534                 (containerized-operating-system os '())
 535                 (list boot-program)))
 536         (name  (string-append name ".tar.gz"))
 537         (graph "system-graph"))
 538     (define build
 539       (with-extensions (cons guile-json-3         ;for (guix docker)
 540                              gcrypt-sqlite3&co)   ;for (guix store database)
 541         (with-imported-modules `(,@(source-module-closure
 542                                     '((guix docker)
 543                                       (guix store database)
 544                                       (guix build utils)
 545                                       (guix build store-copy)
 546                                       (gnu build vm))
 547                                     #:select? not-config?)
 548                                  ((guix config) => ,(make-config.scm)))
 549           #~(begin
 550               (use-modules (guix docker)
 551                            (guix build utils)
 552                            (gnu build vm)
 553                            (srfi srfi-19)
 554                            (guix build store-copy)
 555                            (guix store database))
 556
 557               ;; Set the SQL schema location.
 558               (sql-schema #$schema)
 559
 560               ;; Allow non-ASCII file names--e.g., 'nss-certs'--to be decoded.
 561               (setenv "GUIX_LOCPATH"
 562                       #+(file-append glibc-utf8-locales "/lib/locale"))
 563               (setlocale LC_ALL "en_US.utf8")
 564
 565               (let* (;; This initializer requires elevated privileges that are
 566                      ;; not normally available in the build environment (e.g.,
 567                      ;; it needs to create device nodes).  In order to obtain
 568                      ;; such privileges, we run it as root in a VM.
 569                      (initialize (root-partition-initializer
 570                                   #:closures '(#$graph)
 571                                   #:register-closures? #$register-closures?
 572                                   #:system-directory #$os
 573                                   ;; De-duplication would fail due to
 574                                   ;; cross-device link errors, so don't do it.
 575                                   #:deduplicate? #f))
 576                      ;; Even as root in a VM, the initializer would fail due to
 577                      ;; lack of privileges if we use a root-directory that is on
 578                      ;; a file system that is shared with the host (e.g., /tmp).
 579                      (root-directory "/guixsd-system-root"))
 580                 (set-path-environment-variable "PATH" '("bin" "sbin") '(#+tar))
 581                 (mkdir root-directory)
 582                 (initialize root-directory)
 583                 (build-docker-image
 584                  (string-append "/xchg/" #$name) ;; The output file.
 585                  (cons* root-directory
 586                         (map store-info-item
 587                              (call-with-input-file
 588                                  (string-append "/xchg/" #$graph)
 589                                read-reference-graph)))
 590                  #$os
 591                  #:entry-point '(#$boot-program #$os)
 592                  #:compressor '(#+(file-append gzip "/bin/gzip") "-9n")
 593                  #:creation-time (make-time time-utc 0 1)
 594                  #:transformations `((,root-directory -> ""))))))))
 595
 596     (expression->derivation-in-linux-vm
 597      name build
 598      #:make-disk-image? #f
 599      #:single-file-output? #t
 600      #:references-graphs `((,graph ,os)))))
 601
 602 \f
 603 ;;;
 604 ;;; VM and disk images.
 605 ;;;
 606
 607 (define* (system-disk-image-in-vm os
 608                                   #:key
 609                                   (name "disk-image")
 610                                   (file-system-type "ext4")
 611                                   (disk-image-size (* 900 (expt 2 20)))
 612                                   (volatile? #t)
 613                                   (substitutable? #t))
 614   "Return the derivation of a disk image of DISK-IMAGE-SIZE bytes of the
 615 system described by OS.  Said image can be copied on a USB stick as is.  When
 616 VOLATILE? is true, the root file system is made volatile; this is useful
 617 to USB sticks meant to be read-only.
 618
 619 SUBSTITUTABLE? determines whether the returned derivation should be marked as
 620 substitutable."
 621   (define normalize-label
 622     ;; ISO labels are all-caps (case-insensitive), but since
 623     ;; 'find-partition-by-label' is case-sensitive, make it all-caps here.
 624     (if (string=? "iso9660" file-system-type)
 625         string-upcase
 626         identity))
 627
 628   (define root-label
 629     ;; Volume name of the root file system.
 630     (normalize-label "Guix_image"))
 631
 632   (define (root-uuid os)
 633     ;; UUID of the root file system, computed in a deterministic fashion.
 634     ;; This is what we use to locate the root file system so it has to be
 635     ;; different from the user's own file system UUIDs.
 636     (operating-system-uuid os
 637                            (if (string=? file-system-type "iso9660")
 638                                'iso9660
 639                                'dce)))
 640
 641   (define file-systems-to-keep
 642     (remove (lambda (fs)
 643               (string=? (file-system-mount-point fs) "/"))
 644             (operating-system-file-systems os)))
 645
 646   (let* ((os (operating-system (inherit os)
 647                ;; Since this is meant to be used on real hardware, don't
 648                ;; install QEMU networking or anything like that.  Assume USB
 649                ;; mass storage devices (usb-storage.ko) are available.
 650                (initrd (lambda (file-systems . rest)
 651                          (apply (operating-system-initrd os)
 652                                 file-systems
 653                                 #:volatile-root? volatile?
 654                                 rest)))
 655
 656                (bootloader (if (string=? "iso9660" file-system-type)
 657                                (bootloader-configuration
 658                                  (inherit (operating-system-bootloader os))
 659                                  (bootloader grub-mkrescue-bootloader))
 660                                (operating-system-bootloader os)))
 661
 662                ;; Force our own root file system.  (We need a "/" file system
 663                ;; to call 'root-uuid'.)
 664                (file-systems (cons (file-system
 665                                      (mount-point "/")
 666                                      (device "/dev/placeholder")
 667                                      (type file-system-type))
 668                                    file-systems-to-keep))))
 669          (uuid (root-uuid os))
 670          (os (operating-system
 671                (inherit os)
 672                (file-systems (cons (file-system
 673                                      (mount-point "/")
 674                                      (device uuid)
 675                                      (type file-system-type))
 676                                    file-systems-to-keep))))
 677         (bootcfg (operating-system-bootcfg os)))
 678     (if (string=? "iso9660" file-system-type)
 679         (iso9660-image #:name name
 680                        #:file-system-label root-label
 681                        #:file-system-uuid uuid
 682                        #:os os
 683                        #:bootcfg-drv bootcfg
 684                        #:bootloader (bootloader-configuration-bootloader
 685                                      (operating-system-bootloader os))
 686                        #:inputs `(("system" ,os)
 687                                   ("bootcfg" ,bootcfg))
 688                        #:grub-mkrescue-environment
 689                        '(("MKRESCUE_SED_MODE" . "mbr_hfs"))
 690                        #:substitutable? substitutable?)
 691         (qemu-image #:name name
 692                     #:os os
 693                     #:bootcfg-drv bootcfg
 694                     #:bootloader (bootloader-configuration-bootloader
 695                                   (operating-system-bootloader os))
 696                     #:disk-image-size disk-image-size
 697                     #:disk-image-format "raw"
 698                     #:file-system-type file-system-type
 699                     #:file-system-label root-label
 700                     #:file-system-uuid uuid
 701                     #:copy-inputs? #t
 702                     #:inputs `(("system" ,os)
 703                                ("bootcfg" ,bootcfg))
 704                     #:substitutable? substitutable?))))
 705
 706 (define* (system-qemu-image os
 707                             #:key
 708                             (file-system-type "ext4")
 709                             (disk-image-size (* 900 (expt 2 20))))
 710   "Return the derivation of a freestanding QEMU image of DISK-IMAGE-SIZE bytes
 711 of the GNU system as described by OS."
 712   (define file-systems-to-keep
 713     ;; Keep only file systems other than root and not normally bound to real
 714     ;; devices.
 715     (remove (lambda (fs)
 716               (let ((target (file-system-mount-point fs))
 717                     (source (file-system-device fs)))
 718                 (or (string=? target "/")
 719                     (string-prefix? "/dev/" source))))
 720             (operating-system-file-systems os)))
 721
 722   (define root-uuid
 723     ;; UUID of the root file system.
 724     (operating-system-uuid os
 725                            (if (string=? file-system-type "iso9660")
 726                                'iso9660
 727                                'dce)))
 728
 729
 730   (let* ((os (operating-system (inherit os)
 731                ;; Assume we have an initrd with the whole QEMU shebang.
 732
 733                ;; Force our own root file system.  Refer to it by UUID so that
 734                ;; it works regardless of how the image is used ("qemu -hda",
 735                ;; Xen, etc.).
 736                (file-systems (cons (file-system
 737                                      (mount-point "/")
 738                                      (device root-uuid)
 739                                      (type file-system-type))
 740                                    file-systems-to-keep))))
 741          (bootcfg (operating-system-bootcfg os)))
 742     (qemu-image  #:os os
 743                  #:bootcfg-drv bootcfg
 744                  #:bootloader (bootloader-configuration-bootloader
 745                                (operating-system-bootloader os))
 746                  #:disk-image-size disk-image-size
 747                  #:file-system-type file-system-type
 748                  #:file-system-uuid root-uuid
 749                  #:inputs `(("system" ,os)
 750                             ("bootcfg" ,bootcfg))
 751                  #:copy-inputs? #t)))
 752
 753 \f
 754 ;;;
 755 ;;; VMs that share file systems with the host.
 756 ;;;
 757
 758 (define (file-system->mount-tag fs)
 759   "Return a 9p mount tag for host file system FS."
 760   ;; QEMU mount tags must be ASCII, at most 31-byte long, cannot contain
 761   ;; slashes, and cannot start with '_'.  Compute an identifier that
 762   ;; corresponds to the rules.
 763   (string-append "TAG"
 764                  (string-drop (bytevector->base32-string
 765                                (sha1 (string->utf8 fs)))
 766                               4)))
 767
 768 (define (mapping->file-system mapping)
 769   "Return a 9p file system that realizes MAPPING."
 770   (match mapping
 771     (($ <file-system-mapping> source target writable?)
 772      (file-system
 773        (mount-point target)
 774        (device (file-system->mount-tag source))
 775        (type "9p")
 776        (flags (if writable? '() '(read-only)))
 777        (options "trans=virtio,cache=loose")
 778        (check? #f)
 779        (create-mount-point? #t)))))
 780
 781 (define* (virtualized-operating-system os mappings #:optional (full-boot? #f))
 782   "Return an operating system based on OS suitable for use in a virtualized
 783 environment with the store shared with the host.  MAPPINGS is a list of
 784 <file-system-mapping> to realize in the virtualized OS."
 785   (define user-file-systems
 786     ;; Remove file systems that conflict with those added below, or that are
 787     ;; normally bound to real devices.
 788     (remove (lambda (fs)
 789               (let ((target (file-system-mount-point fs))
 790                     (source (file-system-device fs)))
 791                 (or (string=? target (%store-prefix))
 792                     (string=? target "/")
 793                     (and (string? source)
 794                          (string-prefix? "/dev/" source))
 795
 796                     ;; Labels and UUIDs are necessarily invalid in the VM.
 797                     (and (file-system-mount? fs)
 798                          (or (file-system-label? source)
 799                              (uuid? source))))))
 800             (operating-system-file-systems os)))
 801
 802   (define virtual-file-systems
 803     (cons (file-system
 804             (mount-point "/")
 805             (device "/dev/vda1")
 806             (type "ext4"))
 807
 808           (append (map mapping->file-system mappings)
 809                   user-file-systems)))
 810
 811   (operating-system (inherit os)
 812
 813     ;; XXX: Until we run QEMU with UEFI support (with the OVMF firmware),
 814     ;; force the traditional i386/BIOS method.
 815     ;; See <https://bugs.gnu.org/28768>.
 816     (bootloader (bootloader-configuration
 817                   (inherit (operating-system-bootloader os))
 818                   (bootloader grub-bootloader)
 819                   (target "/dev/vda")))
 820
 821     (initrd (lambda (file-systems . rest)
 822               (apply (operating-system-initrd os)
 823                      file-systems
 824                      #:volatile-root? #t
 825                      rest)))
 826
 827     ;; Disable swap.
 828     (swap-devices '())
 829
 830     ;; XXX: When FULL-BOOT? is true, do not add a 9p mount for /gnu/store
 831     ;; since that would lead the bootloader config to look for the kernel and
 832     ;; initrd in it.
 833     (file-systems (if full-boot?
 834                       virtual-file-systems
 835                       (cons
 836                        (file-system
 837                          (inherit (mapping->file-system %store-mapping))
 838                          (needed-for-boot? #t))
 839                        virtual-file-systems)))))
 840
 841 (define* (system-qemu-image/shared-store
 842           os
 843           #:key
 844           full-boot?
 845           (disk-image-size (* (if full-boot? 500 30) (expt 2 20))))
 846   "Return a derivation that builds a QEMU image of OS that shares its store
 847 with the host.
 848
 849 When FULL-BOOT? is true, return an image that does a complete boot sequence,
 850 bootloaded included; thus, make a disk image that contains everything the
 851 bootloader refers to: OS kernel, initrd, bootloader data, etc."
 852   (define root-uuid
 853     ;; Use a fixed UUID to improve determinism.
 854     (operating-system-uuid os 'dce))
 855
 856   (define bootcfg
 857     (operating-system-bootcfg os))
 858
 859   ;; XXX: When FULL-BOOT? is true, we end up creating an image that contains
 860   ;; BOOTCFG and all its dependencies, including the output of OS.
 861   ;; This is more than needed (we only need the kernel, initrd, GRUB for its
 862   ;; font, and the background image), but it's hard to filter that.
 863   (qemu-image #:os os
 864               #:bootcfg-drv bootcfg
 865               #:bootloader (bootloader-configuration-bootloader
 866                             (operating-system-bootloader os))
 867               #:disk-image-size disk-image-size
 868               #:file-system-uuid root-uuid
 869               #:inputs (if full-boot?
 870                            `(("bootcfg" ,bootcfg))
 871                            '())
 872
 873               ;; XXX: Passing #t here is too slow, so let it off by default.
 874               #:register-closures? #f
 875               #:copy-inputs? full-boot?))
 876
 877 (define* (common-qemu-options image shared-fs)
 878   "Return the a string-value gexp with the common QEMU options to boot IMAGE,
 879 with '-virtfs' options for the host file systems listed in SHARED-FS."
 880
 881   (define (virtfs-option fs)
 882     #~(format #f "-virtfs local,path=~s,security_model=none,mount_tag=~s"
 883               #$fs #$(file-system->mount-tag fs)))
 884
 885   #~(;; Only enable kvm if we see /dev/kvm exists.
 886      ;; This allows users without hardware virtualization to still use these
 887      ;; commands.
 888      #$@(if (file-exists? "/dev/kvm")
 889             '("-enable-kvm")
 890             '())
 891
 892      "-no-reboot"
 893      "-nic" "user,model=virtio-net-pci"
 894      "-object" "rng-random,filename=/dev/urandom,id=guixsd-vm-rng"
 895      "-device" "virtio-rng-pci,rng=guixsd-vm-rng"
 896
 897      #$@(map virtfs-option shared-fs)
 898      "-vga std"
 899      (format #f "-drive file=~a,if=virtio,cache=writeback,werror=report,readonly"
 900              #$image)))
 901
 902 (define* (system-qemu-image/shared-store-script os
 903                                                 #:key
 904                                                 (qemu qemu)
 905                                                 (graphic? #t)
 906                                                 (memory-size 256)
 907                                                 (mappings '())
 908                                                 full-boot?
 909                                                 (disk-image-size
 910                                                  (* (if full-boot? 500 70)
 911                                                     (expt 2 20)))
 912                                                 (options '()))
 913   "Return a derivation that builds a script to run a virtual machine image of
 914 OS that shares its store with the host.  The virtual machine runs with
 915 MEMORY-SIZE MiB of memory.
 916
 917 MAPPINGS is a list of <file-system-mapping> specifying mapping of host file
 918 systems into the guest.
 919
 920 When FULL-BOOT? is true, the returned script runs everything starting from the
 921 bootloader; otherwise it directly starts the operating system kernel.  The
 922 DISK-IMAGE-SIZE parameter specifies the size in bytes of the root disk image;
 923 it is mostly useful when FULL-BOOT?  is true."
 924   (mlet* %store-monad ((os ->  (virtualized-operating-system os mappings full-boot?))
 925                        (image  (system-qemu-image/shared-store
 926                                 os
 927                                 #:full-boot? full-boot?
 928                                 #:disk-image-size disk-image-size)))
 929     (define kernel-arguments
 930       #~(list #$@(if graphic? #~() #~("console=ttyS0"))
 931               #+@(operating-system-kernel-arguments os "/dev/vda1")))
 932
 933     (define qemu-exec
 934       #~(list (string-append #$qemu "/bin/" #$(qemu-command (%current-system)))
 935               #$@(if full-boot?
 936                      #~()
 937                      #~("-kernel" #$(operating-system-kernel-file os)
 938                         "-initrd" #$(file-append os "/initrd")
 939                         (format #f "-append ~s"
 940                                 (string-join #$kernel-arguments " "))))
 941               #$@(common-qemu-options image
 942                                       (map file-system-mapping-source
 943                                            (cons %store-mapping mappings)))
 944               "-m " (number->string #$memory-size)
 945               #$@options))
 946
 947     (define builder
 948       #~(call-with-output-file #$output
 949           (lambda (port)
 950             (format port "#!~a~% exec ~a \"$@\"~%"
 951                     #$(file-append bash "/bin/sh")
 952                     (string-join #$qemu-exec " "))
 953             (chmod port #o555))))
 954
 955     (gexp->derivation "run-vm.sh" builder)))
 956
 957 \f
 958 ;;;
 959 ;;; High-level abstraction.
 960 ;;;
 961
 962 (define-record-type* <virtual-machine> %virtual-machine
 963   make-virtual-machine
 964   virtual-machine?
 965   (operating-system virtual-machine-operating-system) ;<operating-system>
 966   (qemu             virtual-machine-qemu              ;<package>
 967                     (default qemu))
 968   (graphic?         virtual-machine-graphic?      ;Boolean
 969                     (default #f))
 970   (memory-size      virtual-machine-memory-size   ;integer (MiB)
 971                     (default 256))
 972   (disk-image-size  virtual-machine-disk-image-size   ;integer (bytes)
 973                     (default 'guess))
 974   (port-forwardings virtual-machine-port-forwardings ;list of integer pairs
 975                     (default '())))
 976
 977 (define-syntax virtual-machine
 978   (syntax-rules ()
 979     "Declare a virtual machine running the specified OS, with the given
 980 options."
 981     ((_ os)                                       ;shortcut
 982      (%virtual-machine (operating-system os)))
 983     ((_ fields ...)
 984      (%virtual-machine fields ...))))
 985
 986 (define (port-forwardings->qemu-options forwardings)
 987   "Return the QEMU option for the given port FORWARDINGS as a string, where
 988 FORWARDINGS is a list of host-port/guest-port pairs."
 989   (string-join
 990    (map (match-lambda
 991           ((host-port . guest-port)
 992            (string-append "hostfwd=tcp::"
 993                           (number->string host-port)
 994                           "-:" (number->string guest-port))))
 995         forwardings)
 996    ","))
 997
 998 (define-gexp-compiler (virtual-machine-compiler (vm <virtual-machine>)
 999                                                 system target)
1000   ;; XXX: SYSTEM and TARGET are ignored.
1001   (match vm
1002     (($ <virtual-machine> os qemu graphic? memory-size disk-image-size ())
1003      (system-qemu-image/shared-store-script os
1004                                             #:qemu qemu
1005                                             #:graphic? graphic?
1006                                             #:memory-size memory-size
1007                                             #:disk-image-size
1008                                             disk-image-size))
1009     (($ <virtual-machine> os qemu graphic? memory-size disk-image-size
1010                           forwardings)
1011      (let ((options
1012             `("-nic" ,(string-append
1013                        "user,model=virtio-net-pci,"
1014                        (port-forwardings->qemu-options forwardings)))))
1015        (system-qemu-image/shared-store-script os
1016                                               #:qemu qemu
1017                                               #:graphic? graphic?
1018                                               #:memory-size memory-size
1019                                               #:disk-image-size
1020                                               disk-image-size
1021                                               #:options options)))))
1022
1023 ;;; vm.scm ends here