gnu/system/vm.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2013, 2014, 2015, 2016, 2017, 2018, 2019 Ludovic Courtès <ludo@gnu.org>
   3 ;;; Copyright © 2016 Christopher Allan Webber <cwebber@dustycloud.org>
   4 ;;; Copyright © 2016, 2017 Leo Famulari <leo@famulari.name>
   5 ;;; Copyright © 2017 Mathieu Othacehe <m.othacehe@gmail.com>
   6 ;;; Copyright © 2017 Marius Bakke <mbakke@fastmail.com>
   7 ;;; Copyright © 2018 Chris Marusich <cmmarusich@gmail.com>
   8 ;;;
   9 ;;; This file is part of GNU Guix.
  10 ;;;
  11 ;;; GNU Guix is free software; you can redistribute it and/or modify it
  12 ;;; under the terms of the GNU General Public License as published by
  13 ;;; the Free Software Foundation; either version 3 of the License, or (at
  14 ;;; your option) any later version.
  15 ;;;
  16 ;;; GNU Guix is distributed in the hope that it will be useful, but
  17 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  18 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 ;;; GNU General Public License for more details.
  20 ;;;
  21 ;;; You should have received a copy of the GNU General Public License
  22 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  23
  24 (define-module (gnu system vm)
  25   #:use-module (guix config)
  26   #:use-module (guix store)
  27   #:use-module (guix gexp)
  28   #:use-module (guix derivations)
  29   #:use-module (guix packages)
  30   #:use-module (guix monads)
  31   #:use-module (guix records)
  32   #:use-module (guix modules)
  33   #:use-module (guix utils)
  34   #:use-module (gcrypt hash)
  35   #:use-module (guix base32)
  36   #:use-module ((guix self) #:select (make-config.scm))
  37
  38   #:use-module ((gnu build vm)
  39                 #:select (qemu-command))
  40   #:use-module (gnu packages base)
  41   #:use-module (gnu packages bootloaders)
  42   #:use-module (gnu packages cdrom)
  43   #:use-module (gnu packages compression)
  44   #:use-module (gnu packages guile)
  45   #:autoload   (gnu packages gnupg) (guile-gcrypt)
  46   #:use-module (gnu packages gawk)
  47   #:use-module (gnu packages bash)
  48   #:use-module (gnu packages less)
  49   #:use-module (gnu packages virtualization)
  50   #:use-module (gnu packages disk)
  51   #:use-module (gnu packages zile)
  52   #:use-module (gnu packages linux)
  53   #:use-module ((gnu packages make-bootstrap)
  54                 #:select (%guile-static-stripped))
  55   #:use-module (gnu packages admin)
  56
  57   #:use-module (gnu bootloader)
  58   #:use-module (gnu bootloader grub)
  59   #:use-module (gnu system shadow)
  60   #:use-module (gnu system pam)
  61   #:use-module (gnu system linux-container)
  62   #:use-module (gnu system linux-initrd)
  63   #:use-module (gnu bootloader)
  64   #:use-module (gnu system file-systems)
  65   #:use-module (gnu system)
  66   #:use-module (gnu services)
  67   #:use-module (gnu system uuid)
  68
  69   #:use-module (srfi srfi-1)
  70   #:use-module (srfi srfi-26)
  71   #:use-module (rnrs bytevectors)
  72   #:use-module (ice-9 match)
  73
  74   #:export (expression->derivation-in-linux-vm
  75             qemu-image
  76             virtualized-operating-system
  77             system-qemu-image
  78
  79             system-qemu-image/shared-store
  80             system-qemu-image/shared-store-script
  81             system-disk-image
  82             system-docker-image
  83
  84             virtual-machine
  85             virtual-machine?))
  86
  87 \f
  88 ;;; Commentary:
  89 ;;;
  90 ;;; Tools to evaluate build expressions within virtual machines.
  91 ;;;
  92 ;;; Code:
  93
  94 (define %linux-vm-file-systems
  95   ;; File systems mounted for 'derivation-in-linux-vm'.  These are shared with
  96   ;; the host over 9p.
  97   (list (file-system
  98           (mount-point (%store-prefix))
  99           (device "store")
 100           (type "9p")
 101           (needed-for-boot? #t)
 102           (flags '(read-only))
 103           (options "trans=virtio,cache=loose")
 104           (check? #f))
 105
 106         ;; The 9p documentation says that cache=loose is "intended for
 107         ;; exclusive, read-only mounts", without additional details.  In
 108         ;; practice it seems to work well for these, and it's much faster than
 109         ;; the default cache=none, especially when copying and registering
 110         ;; store items.
 111         (file-system
 112           (mount-point "/xchg")
 113           (device "xchg")
 114           (type "9p")
 115           (needed-for-boot? #t)
 116           (options "trans=virtio,cache=loose")
 117           (check? #f))
 118         (file-system
 119           (mount-point "/tmp")
 120           (device "tmp")
 121           (type "9p")
 122           (needed-for-boot? #t)
 123           (options "trans=virtio,cache=loose")
 124           (check? #f))))
 125
 126 (define not-config?
 127   ;; Select (guix …) and (gnu …) modules, except (guix config).
 128   (match-lambda
 129     (('guix 'config) #f)
 130     (('guix rest ...) #t)
 131     (('gnu rest ...) #t)
 132     (rest #f)))
 133
 134 (define gcrypt-sqlite3&co
 135   ;; Guile-Gcrypt, Guile-SQLite3, and their propagated inputs.
 136   (append-map (lambda (package)
 137                 (cons package
 138                       (match (package-transitive-propagated-inputs package)
 139                         (((labels packages) ...)
 140                          packages))))
 141               (list guile-gcrypt guile-sqlite3)))
 142
 143 (define* (expression->derivation-in-linux-vm name exp
 144                                              #:key
 145                                              (system (%current-system))
 146                                              (linux linux-libre)
 147                                              initrd
 148                                              (qemu qemu-minimal)
 149                                              (env-vars '())
 150                                              (guile-for-build
 151                                               (%guile-for-build))
 152                                              (file-systems
 153                                               %linux-vm-file-systems)
 154
 155                                              (single-file-output? #f)
 156                                              (make-disk-image? #f)
 157                                              (references-graphs #f)
 158                                              (memory-size 256)
 159                                              (disk-image-format "qcow2")
 160                                              (disk-image-size 'guess))
 161   "Evaluate EXP in a QEMU virtual machine running LINUX with INITRD (a
 162 derivation).  The virtual machine runs with MEMORY-SIZE MiB of memory.  In the
 163 virtual machine, EXP has access to FILE-SYSTEMS, which, by default, includes a
 164 9p share of the store, the '/xchg' where EXP should put its output file(s),
 165 and a 9p share of /tmp.
 166
 167 If SINGLE-FILE-OUTPUT? is true, copy a single file from '/xchg' to OUTPUT.
 168 Otherwise, copy the contents of /xchg to a new directory OUTPUT.
 169
 170 When MAKE-DISK-IMAGE? is true, then create a QEMU disk image of type
 171 DISK-IMAGE-FORMAT (e.g., 'qcow2' or 'raw'), of DISK-IMAGE-SIZE bytes and
 172 return it.  When DISK-IMAGE-SIZE is 'guess, estimate the image size based
 173 based on the size of the closure of REFERENCES-GRAPHS.
 174
 175 When REFERENCES-GRAPHS is true, it must be a list of file name/store path
 176 pairs, as for `derivation'.  The files containing the reference graphs are
 177 made available under the /xchg CIFS share."
 178   (define user-builder
 179     (program-file "builder-in-linux-vm" exp))
 180
 181   (define loader
 182     ;; Invoke USER-BUILDER instead using 'primitive-load'.  The reason for
 183     ;; this is to allow USER-BUILDER to dlopen stuff by using a full-featured
 184     ;; Guile, which it couldn't do using the statically-linked guile used in
 185     ;; the initrd.  See example at
 186     ;; <https://lists.gnu.org/archive/html/guix-devel/2017-10/msg00233.html>.
 187     (program-file "linux-vm-loader"
 188                   ;; When USER-BUILDER succeeds, reboot (indicating a
 189                   ;; success), otherwise die, which causes a kernel panic
 190                   ;; ("Attempted to kill init!").
 191                   #~(if (zero? (system* #$user-builder))
 192                         (reboot)
 193                         (exit 1))))
 194
 195   (let ((initrd (or initrd
 196                     (base-initrd file-systems
 197                                  #:on-error 'backtrace
 198                                  #:linux linux
 199                                  #:linux-modules %base-initrd-modules
 200                                  #:qemu-networking? #t))))
 201
 202     (define builder
 203       ;; Code that launches the VM that evaluates EXP.
 204       (with-extensions gcrypt-sqlite3&co
 205         (with-imported-modules `(,@(source-module-closure
 206                                     '((guix build utils)
 207                                       (gnu build vm))
 208                                     #:select? not-config?)
 209
 210                                  ;; For consumption by (gnu store database).
 211                                  ((guix config) => ,(make-config.scm)))
 212           #~(begin
 213               (use-modules (guix build utils)
 214                            (gnu build vm))
 215
 216               (let* ((inputs  '#$(list qemu (canonical-package coreutils)))
 217                      (linux   (string-append #$linux "/"
 218                                              #$(system-linux-image-file-name)))
 219                      (initrd  #$initrd)
 220                      (loader  #$loader)
 221                      (graphs  '#$(match references-graphs
 222                                    (((graph-files . _) ...) graph-files)
 223                                    (_ #f)))
 224                      (size    #$(if (eq? 'guess disk-image-size)
 225                                     #~(+ (* 70 (expt 2 20)) ;ESP
 226                                          (estimated-partition-size graphs))
 227                                     disk-image-size)))
 228
 229                 (set-path-environment-variable "PATH" '("bin") inputs)
 230
 231                 (load-in-linux-vm loader
 232                                   #:output #$output
 233                                   #:linux linux #:initrd initrd
 234                                   #:memory-size #$memory-size
 235                                   #:make-disk-image? #$make-disk-image?
 236                                   #:single-file-output? #$single-file-output?
 237                                   ;; FIXME: ‘target-arm32?’ may not operate on
 238                                   ;; the right system/target values.  Rewrite
 239                                   ;; using ‘let-system’ when available.
 240                                   #:target-arm32? #$(target-arm32?)
 241                                   #:disk-image-format #$disk-image-format
 242                                   #:disk-image-size size
 243                                   #:references-graphs graphs))))))
 244
 245     (gexp->derivation name builder
 246                       ;; TODO: Require the "kvm" feature.
 247                       #:system system
 248                       #:env-vars env-vars
 249                       #:guile-for-build guile-for-build
 250                       #:references-graphs references-graphs)))
 251
 252 (define* (iso9660-image #:key
 253                         (name "iso9660-image")
 254                         file-system-label
 255                         file-system-uuid
 256                         (system (%current-system))
 257                         (qemu qemu-minimal)
 258                         os
 259                         bootcfg-drv
 260                         bootloader
 261                         register-closures?
 262                         (inputs '()))
 263   "Return a bootable, stand-alone iso9660 image.
 264
 265 INPUTS is a list of inputs (as for packages)."
 266   (define schema
 267     (and register-closures?
 268          (local-file (search-path %load-path
 269                                   "guix/store/schema.sql"))))
 270
 271   (expression->derivation-in-linux-vm
 272    name
 273    (with-extensions gcrypt-sqlite3&co
 274      (with-imported-modules `(,@(source-module-closure '((gnu build vm)
 275                                                          (guix store database)
 276                                                          (guix build utils))
 277                                                        #:select? not-config?)
 278                               ((guix config) => ,(make-config.scm)))
 279        #~(begin
 280            (use-modules (gnu build vm)
 281                         (guix store database)
 282                         (guix build utils))
 283
 284            (sql-schema #$schema)
 285
 286            (let ((inputs
 287                   '#$(append (list qemu parted e2fsprogs dosfstools xorriso)
 288                              (map canonical-package
 289                                   (list sed grep coreutils findutils gawk))))
 290
 291
 292                  (graphs     '#$(match inputs
 293                                   (((names . _) ...)
 294                                    names)))
 295                  ;; This variable is unused but allows us to add INPUTS-TO-COPY
 296                  ;; as inputs.
 297                  (to-register
 298                   '#$(map (match-lambda
 299                             ((name thing) thing)
 300                             ((name thing output) `(,thing ,output)))
 301                           inputs)))
 302
 303              (set-path-environment-variable "PATH" '("bin" "sbin") inputs)
 304              (make-iso9660-image #$(bootloader-package bootloader)
 305                                  #$bootcfg-drv
 306                                  #$os
 307                                  "/xchg/guixsd.iso"
 308                                  #:register-closures? #$register-closures?
 309                                  #:closures graphs
 310                                  #:volume-id #$file-system-label
 311                                  #:volume-uuid #$(and=> file-system-uuid
 312                                                         uuid-bytevector))))))
 313    #:system system
 314
 315    ;; Keep a local file system for /tmp so that we can populate it directly as
 316    ;; root and have files owned by root.  See <https://bugs.gnu.org/31752>.
 317    #:file-systems (remove (lambda (file-system)
 318                             (string=? (file-system-mount-point file-system)
 319                                       "/tmp"))
 320                           %linux-vm-file-systems)
 321
 322    #:make-disk-image? #f
 323    #:single-file-output? #t
 324    #:references-graphs inputs
 325
 326    ;; Xorriso seems to be quite memory-hungry, so increase the VM's RAM size.
 327    #:memory-size 512))
 328
 329 (define* (qemu-image #:key
 330                      (name "qemu-image")
 331                      (system (%current-system))
 332                      (qemu qemu-minimal)
 333                      (disk-image-size 'guess)
 334                      (disk-image-format "qcow2")
 335                      (file-system-type "ext4")
 336                      file-system-label
 337                      file-system-uuid
 338                      os
 339                      bootcfg-drv
 340                      bootloader
 341                      (register-closures? #t)
 342                      (inputs '())
 343                      copy-inputs?)
 344   "Return a bootable, stand-alone QEMU image of type DISK-IMAGE-FORMAT (e.g.,
 345 'qcow2' or 'raw'), with a root partition of type FILE-SYSTEM-TYPE.
 346 Optionally, FILE-SYSTEM-LABEL can be specified as the volume name for the root
 347 partition; likewise FILE-SYSTEM-UUID, if true, specifies the UUID of the root
 348 partition (a UUID object).
 349
 350 The returned image is a full disk image that runs OS-DERIVATION,
 351 with a GRUB installation that uses GRUB-CONFIGURATION as its configuration
 352 file (GRUB-CONFIGURATION must be the name of a file in the VM.)
 353
 354 INPUTS is a list of inputs (as for packages).  When COPY-INPUTS? is true, copy
 355 all of INPUTS into the image being built.  When REGISTER-CLOSURES? is true,
 356 register INPUTS in the store database of the image so that Guix can be used in
 357 the image."
 358   (define schema
 359     (and register-closures?
 360          (local-file (search-path %load-path
 361                                   "guix/store/schema.sql"))))
 362
 363   (expression->derivation-in-linux-vm
 364    name
 365    (with-extensions gcrypt-sqlite3&co
 366      (with-imported-modules `(,@(source-module-closure '((gnu build vm)
 367                                                          (gnu build bootloader)
 368                                                          (guix store database)
 369                                                          (guix build utils))
 370                                                        #:select? not-config?)
 371                               ((guix config) => ,(make-config.scm)))
 372        #~(begin
 373            (use-modules (gnu build bootloader)
 374                         (gnu build vm)
 375                         (guix store database)
 376                         (guix build utils)
 377                         (srfi srfi-26)
 378                         (ice-9 binary-ports))
 379
 380            (sql-schema #$schema)
 381
 382            (let ((inputs
 383                   '#$(append (list qemu parted e2fsprogs dosfstools)
 384                              (map canonical-package
 385                                   (list sed grep coreutils findutils gawk))))
 386
 387                  ;; This variable is unused but allows us to add INPUTS-TO-COPY
 388                  ;; as inputs.
 389                  (to-register
 390                   '#$(map (match-lambda
 391                             ((name thing) thing)
 392                             ((name thing output) `(,thing ,output)))
 393                           inputs)))
 394
 395              (set-path-environment-variable "PATH" '("bin" "sbin") inputs)
 396
 397              (let* ((graphs     '#$(match inputs
 398                                      (((names . _) ...)
 399                                       names)))
 400                     (initialize (root-partition-initializer
 401                                  #:closures graphs
 402                                  #:copy-closures? #$copy-inputs?
 403                                  #:register-closures? #$register-closures?
 404                                  #:system-directory #$os
 405
 406                                  ;; Disable deduplication to speed things up,
 407                                  ;; and because it doesn't help much for a
 408                                  ;; single system generation.
 409                                  #:deduplicate? #f))
 410                     (root-size  #$(if (eq? 'guess disk-image-size)
 411                                       #~(max
 412                                          ;; Minimum 20 MiB root size
 413                                          (* 20 (expt 2 20))
 414                                          (estimated-partition-size
 415                                           (map (cut string-append "/xchg/" <>)
 416                                                graphs)))
 417                                       (- disk-image-size
 418                                          (* 50 (expt 2 20)))))
 419                     (partitions
 420                      (append
 421                       (list (partition
 422                              (size root-size)
 423                              (label #$file-system-label)
 424                              (uuid #$(and=> file-system-uuid
 425                                             uuid-bytevector))
 426                              (file-system #$file-system-type)
 427                              (flags '(boot))
 428                              (initializer initialize)))
 429                       ;; Append a small EFI System Partition for use with UEFI
 430                       ;; bootloaders if we are not targeting ARM because UEFI
 431                       ;; support in U-Boot is experimental.
 432                       ;;
 433                       ;; FIXME: ‘target-arm32?’ may be not operate on the right
 434                       ;; system/target values.  Rewrite using ‘let-system’ when
 435                       ;; available.
 436                       (if #$(target-arm32?)
 437                           '()
 438                           (list (partition
 439                                  ;; The standalone grub image is about 10MiB, but
 440                                  ;; leave some room for custom or multiple images.
 441                                  (size (* 40 (expt 2 20)))
 442                                  (label "GNU-ESP") ;cosmetic only
 443                                  ;; Use "vfat" here since this property is used
 444                                  ;; when mounting. The actual FAT-ness is based
 445                                  ;; on file system size (16 in this case).
 446                                  (file-system "vfat")
 447                                  (flags '(esp))))))))
 448                (initialize-hard-disk "/dev/vda"
 449                                      #:partitions partitions
 450                                      #:grub-efi #$grub-efi
 451                                      #:bootloader-package
 452                                      #$(bootloader-package bootloader)
 453                                      #:bootcfg #$bootcfg-drv
 454                                      #:bootcfg-location
 455                                      #$(bootloader-configuration-file bootloader)
 456                                      #:bootloader-installer
 457                                      #$(bootloader-installer bootloader)))))))
 458    #:system system
 459    #:make-disk-image? #t
 460    #:disk-image-size disk-image-size
 461    #:disk-image-format disk-image-format
 462    #:references-graphs inputs))
 463
 464 (define* (system-docker-image os
 465                               #:key
 466                               (name "guixsd-docker-image")
 467                               register-closures?)
 468   "Build a docker image.  OS is the desired <operating-system>.  NAME is the
 469 base name to use for the output file.  When REGISTER-CLOSURES? is not #f,
 470 register the closure of OS with Guix in the resulting Docker image.  This only
 471 makes sense when you want to build a Guix System Docker image that has Guix
 472 installed inside of it.  If you don't need Guix (e.g., your Docker
 473 image just contains a web server that is started by the Shepherd), then you
 474 should set REGISTER-CLOSURES? to #f."
 475   (define schema
 476     (and register-closures?
 477          (local-file (search-path %load-path
 478                                   "guix/store/schema.sql"))))
 479
 480   (let ((os    (containerized-operating-system os '()))
 481         (name  (string-append name ".tar.gz"))
 482         (graph "system-graph"))
 483     (define build
 484       (with-extensions (cons guile-json           ;for (guix docker)
 485                              gcrypt-sqlite3&co)   ;for (guix store database)
 486         (with-imported-modules `(,@(source-module-closure
 487                                     '((guix docker)
 488                                       (guix store database)
 489                                       (guix build utils)
 490                                       (guix build store-copy)
 491                                       (gnu build vm))
 492                                     #:select? not-config?)
 493                                  ((guix config) => ,(make-config.scm)))
 494           #~(begin
 495               (use-modules (guix docker)
 496                            (guix build utils)
 497                            (gnu build vm)
 498                            (srfi srfi-19)
 499                            (guix build store-copy)
 500                            (guix store database))
 501
 502               ;; Set the SQL schema location.
 503               (sql-schema #$schema)
 504
 505               (let* (;; This initializer requires elevated privileges that are
 506                      ;; not normally available in the build environment (e.g.,
 507                      ;; it needs to create device nodes).  In order to obtain
 508                      ;; such privileges, we run it as root in a VM.
 509                      (initialize (root-partition-initializer
 510                                   #:closures '(#$graph)
 511                                   #:register-closures? #$register-closures?
 512                                   #:system-directory #$os
 513                                   ;; De-duplication would fail due to
 514                                   ;; cross-device link errors, so don't do it.
 515                                   #:deduplicate? #f))
 516                      ;; Even as root in a VM, the initializer would fail due to
 517                      ;; lack of privileges if we use a root-directory that is on
 518                      ;; a file system that is shared with the host (e.g., /tmp).
 519                      (root-directory "/guixsd-system-root"))
 520                 (set-path-environment-variable "PATH" '("bin" "sbin") '(#+tar))
 521                 (mkdir root-directory)
 522                 (initialize root-directory)
 523                 (build-docker-image
 524                  (string-append "/xchg/" #$name) ;; The output file.
 525                  (cons* root-directory
 526                         (map store-info-item
 527                              (call-with-input-file
 528                                  (string-append "/xchg/" #$graph)
 529                                read-reference-graph)))
 530                  #$os
 531                  #:compressor '(#+(file-append gzip "/bin/gzip") "-9n")
 532                  #:creation-time (make-time time-utc 0 1)
 533                  #:transformations `((,root-directory -> "")))
 534
 535                 ;; Make sure the tarball is fully written before rebooting.
 536                 (sync))))))
 537     (expression->derivation-in-linux-vm
 538      name build
 539      #:make-disk-image? #f
 540      #:single-file-output? #t
 541      #:references-graphs `((,graph ,os)))))
 542
 543 \f
 544 ;;;
 545 ;;; VM and disk images.
 546 ;;;
 547
 548 (define* (operating-system-uuid os #:optional (type 'dce))
 549   "Compute UUID object with a deterministic \"UUID\" for OS, of the given
 550 TYPE (one of 'iso9660 or 'dce).  Return a UUID object."
 551   ;; Note: For this to be deterministic, we must not hash things that contains
 552   ;; (directly or indirectly) procedures, for example.  That rules out
 553   ;; anything that contains gexps, thunk or delayed record fields, etc.
 554
 555   (define service-name
 556     (compose service-type-name service-kind))
 557
 558   (define (file-system-digest fs)
 559     ;; Return a hashable digest that does not contain 'dependencies' since
 560     ;; this field can contain procedures.
 561     (let ((device (file-system-device fs)))
 562       (list (file-system-mount-point fs)
 563             (file-system-type fs)
 564             (cond ((file-system-label? device)
 565                    (file-system-label->string device))
 566                   ((uuid? device)
 567                    (uuid->string device))
 568                   ((string? device)
 569                    device)
 570                   (else #f))
 571             (file-system-options fs))))
 572
 573   (if (eq? type 'iso9660)
 574       (let ((pad (compose (cut string-pad <> 2 #\0)
 575                           number->string))
 576             (h   (hash (map service-name (operating-system-services os))
 577                        3600)))
 578         (bytevector->uuid
 579          (string->iso9660-uuid
 580           (string-append "1970-01-01-"
 581                          (pad (hash (operating-system-host-name os) 24)) "-"
 582                          (pad (quotient h 60)) "-"
 583                          (pad (modulo h 60)) "-"
 584                          (pad (hash (map file-system-digest
 585                                          (operating-system-file-systems os))
 586                                     100))))
 587          'iso9660))
 588       (bytevector->uuid
 589        (uint-list->bytevector
 590         (list (hash file-system-type
 591                     (- (expt 2 32) 1))
 592               (hash (operating-system-host-name os)
 593                     (- (expt 2 32) 1))
 594               (hash (map service-name (operating-system-services os))
 595                     (- (expt 2 32) 1))
 596               (hash (map file-system-digest (operating-system-file-systems os))
 597                     (- (expt 2 32) 1)))
 598         (endianness little)
 599         4)
 600        type)))
 601
 602 (define* (system-disk-image os
 603                             #:key
 604                             (name "disk-image")
 605                             (file-system-type "ext4")
 606                             (disk-image-size (* 900 (expt 2 20)))
 607                             (volatile? #t))
 608   "Return the derivation of a disk image of DISK-IMAGE-SIZE bytes of the
 609 system described by OS.  Said image can be copied on a USB stick as is.  When
 610 VOLATILE? is true, the root file system is made volatile; this is useful
 611 to USB sticks meant to be read-only."
 612   (define normalize-label
 613     ;; ISO labels are all-caps (case-insensitive), but since
 614     ;; 'find-partition-by-label' is case-sensitive, make it all-caps here.
 615     (if (string=? "iso9660" file-system-type)
 616         string-upcase
 617         identity))
 618
 619   (define root-label
 620     ;; Volume name of the root file system.
 621     (normalize-label "Guix_image"))
 622
 623   (define root-uuid
 624     ;; UUID of the root file system, computed in a deterministic fashion.
 625     ;; This is what we use to locate the root file system so it has to be
 626     ;; different from the user's own file system UUIDs.
 627     (operating-system-uuid os
 628                            (if (string=? file-system-type "iso9660")
 629                                'iso9660
 630                                'dce)))
 631
 632   (define file-systems-to-keep
 633     (remove (lambda (fs)
 634               (string=? (file-system-mount-point fs) "/"))
 635             (operating-system-file-systems os)))
 636
 637   (let* ((os (operating-system (inherit os)
 638                ;; Since this is meant to be used on real hardware, don't
 639                ;; install QEMU networking or anything like that.  Assume USB
 640                ;; mass storage devices (usb-storage.ko) are available.
 641                (initrd (lambda (file-systems . rest)
 642                          (apply (operating-system-initrd os)
 643                                 file-systems
 644                                 #:volatile-root? #t
 645                                 rest)))
 646
 647                (bootloader (if (string=? "iso9660" file-system-type)
 648                                (bootloader-configuration
 649                                  (inherit (operating-system-bootloader os))
 650                                  (bootloader grub-mkrescue-bootloader))
 651                                (operating-system-bootloader os)))
 652
 653                ;; Force our own root file system.
 654                (file-systems (cons (file-system
 655                                      (mount-point "/")
 656                                      (device root-uuid)
 657                                      (type file-system-type))
 658                                    file-systems-to-keep))))
 659         (bootcfg (operating-system-bootcfg os)))
 660     (if (string=? "iso9660" file-system-type)
 661         (iso9660-image #:name name
 662                        #:file-system-label root-label
 663                        #:file-system-uuid root-uuid
 664                        #:os os
 665                        #:register-closures? #t
 666                        #:bootcfg-drv bootcfg
 667                        #:bootloader (bootloader-configuration-bootloader
 668                                      (operating-system-bootloader os))
 669                        #:inputs `(("system" ,os)
 670                                   ("bootcfg" ,bootcfg)))
 671         (qemu-image #:name name
 672                     #:os os
 673                     #:bootcfg-drv bootcfg
 674                     #:bootloader (bootloader-configuration-bootloader
 675                                   (operating-system-bootloader os))
 676                     #:disk-image-size disk-image-size
 677                     #:disk-image-format "raw"
 678                     #:file-system-type file-system-type
 679                     #:file-system-label root-label
 680                     #:file-system-uuid root-uuid
 681                     #:copy-inputs? #t
 682                     #:register-closures? #t
 683                     #:inputs `(("system" ,os)
 684                                ("bootcfg" ,bootcfg))))))
 685
 686 (define* (system-qemu-image os
 687                             #:key
 688                             (file-system-type "ext4")
 689                             (disk-image-size (* 900 (expt 2 20))))
 690   "Return the derivation of a freestanding QEMU image of DISK-IMAGE-SIZE bytes
 691 of the GNU system as described by OS."
 692   (define file-systems-to-keep
 693     ;; Keep only file systems other than root and not normally bound to real
 694     ;; devices.
 695     (remove (lambda (fs)
 696               (let ((target (file-system-mount-point fs))
 697                     (source (file-system-device fs)))
 698                 (or (string=? target "/")
 699                     (string-prefix? "/dev/" source))))
 700             (operating-system-file-systems os)))
 701
 702   (define root-uuid
 703     ;; UUID of the root file system.
 704     (operating-system-uuid os
 705                            (if (string=? file-system-type "iso9660")
 706                                'iso9660
 707                                'dce)))
 708
 709
 710   (let* ((os (operating-system (inherit os)
 711                ;; Assume we have an initrd with the whole QEMU shebang.
 712
 713                ;; Force our own root file system.  Refer to it by UUID so that
 714                ;; it works regardless of how the image is used ("qemu -hda",
 715                ;; Xen, etc.).
 716                (file-systems (cons (file-system
 717                                      (mount-point "/")
 718                                      (device root-uuid)
 719                                      (type file-system-type))
 720                                    file-systems-to-keep))))
 721          (bootcfg (operating-system-bootcfg os)))
 722     (qemu-image  #:os os
 723                  #:bootcfg-drv bootcfg
 724                  #:bootloader (bootloader-configuration-bootloader
 725                                (operating-system-bootloader os))
 726                  #:disk-image-size disk-image-size
 727                  #:file-system-type file-system-type
 728                  #:file-system-uuid root-uuid
 729                  #:inputs `(("system" ,os)
 730                             ("bootcfg" ,bootcfg))
 731                  #:copy-inputs? #t)))
 732
 733 \f
 734 ;;;
 735 ;;; VMs that share file systems with the host.
 736 ;;;
 737
 738 (define (file-system->mount-tag fs)
 739   "Return a 9p mount tag for host file system FS."
 740   ;; QEMU mount tags must be ASCII, at most 31-byte long, cannot contain
 741   ;; slashes, and cannot start with '_'.  Compute an identifier that
 742   ;; corresponds to the rules.
 743   (string-append "TAG"
 744                  (string-drop (bytevector->base32-string
 745                                (sha1 (string->utf8 fs)))
 746                               4)))
 747
 748 (define (mapping->file-system mapping)
 749   "Return a 9p file system that realizes MAPPING."
 750   (match mapping
 751     (($ <file-system-mapping> source target writable?)
 752      (file-system
 753        (mount-point target)
 754        (device (file-system->mount-tag source))
 755        (type "9p")
 756        (flags (if writable? '() '(read-only)))
 757        (options "trans=virtio,cache=loose")
 758        (check? #f)
 759        (create-mount-point? #t)))))
 760
 761 (define* (virtualized-operating-system os mappings #:optional (full-boot? #f))
 762   "Return an operating system based on OS suitable for use in a virtualized
 763 environment with the store shared with the host.  MAPPINGS is a list of
 764 <file-system-mapping> to realize in the virtualized OS."
 765   (define user-file-systems
 766     ;; Remove file systems that conflict with those added below, or that are
 767     ;; normally bound to real devices.
 768     (remove (lambda (fs)
 769               (let ((target (file-system-mount-point fs))
 770                     (source (file-system-device fs)))
 771                 (or (string=? target (%store-prefix))
 772                     (string=? target "/")
 773                     (and (string? source)
 774                          (string-prefix? "/dev/" source))
 775
 776                     ;; Labels and UUIDs are necessarily invalid in the VM.
 777                     (and (file-system-mount? fs)
 778                          (or (file-system-label? source)
 779                              (uuid? source))))))
 780             (operating-system-file-systems os)))
 781
 782   (define virtual-file-systems
 783     (cons (file-system
 784             (mount-point "/")
 785             (device "/dev/vda1")
 786             (type "ext4"))
 787
 788           (append (map mapping->file-system mappings)
 789                   user-file-systems)))
 790
 791   (operating-system (inherit os)
 792
 793     ;; XXX: Until we run QEMU with UEFI support (with the OVMF firmware),
 794     ;; force the traditional i386/BIOS method.
 795     ;; See <https://bugs.gnu.org/28768>.
 796     (bootloader (bootloader-configuration
 797                   (inherit (operating-system-bootloader os))
 798                   (bootloader grub-bootloader)
 799                   (target "/dev/vda")))
 800
 801     (initrd (lambda (file-systems . rest)
 802               (apply (operating-system-initrd os)
 803                      file-systems
 804                      #:volatile-root? #t
 805                      rest)))
 806
 807     ;; Disable swap.
 808     (swap-devices '())
 809
 810     ;; XXX: When FULL-BOOT? is true, do not add a 9p mount for /gnu/store
 811     ;; since that would lead the bootloader config to look for the kernel and
 812     ;; initrd in it.
 813     (file-systems (if full-boot?
 814                       virtual-file-systems
 815                       (cons
 816                        (file-system
 817                          (inherit (mapping->file-system %store-mapping))
 818                          (needed-for-boot? #t))
 819                        virtual-file-systems)))))
 820
 821 (define* (system-qemu-image/shared-store
 822           os
 823           #:key
 824           full-boot?
 825           (disk-image-size (* (if full-boot? 500 30) (expt 2 20))))
 826   "Return a derivation that builds a QEMU image of OS that shares its store
 827 with the host.
 828
 829 When FULL-BOOT? is true, return an image that does a complete boot sequence,
 830 bootloaded included; thus, make a disk image that contains everything the
 831 bootloader refers to: OS kernel, initrd, bootloader data, etc."
 832   (define root-uuid
 833     ;; Use a fixed UUID to improve determinism.
 834     (operating-system-uuid os 'dce))
 835
 836   (define bootcfg
 837     (operating-system-bootcfg os))
 838
 839   ;; XXX: When FULL-BOOT? is true, we end up creating an image that contains
 840   ;; BOOTCFG and all its dependencies, including the output of OS.
 841   ;; This is more than needed (we only need the kernel, initrd, GRUB for its
 842   ;; font, and the background image), but it's hard to filter that.
 843   (qemu-image #:os os
 844               #:bootcfg-drv bootcfg
 845               #:bootloader (bootloader-configuration-bootloader
 846                             (operating-system-bootloader os))
 847               #:disk-image-size disk-image-size
 848               #:file-system-uuid root-uuid
 849               #:inputs (if full-boot?
 850                            `(("bootcfg" ,bootcfg))
 851                            '())
 852
 853               ;; XXX: Passing #t here is too slow, so let it off by default.
 854               #:register-closures? #f
 855               #:copy-inputs? full-boot?))
 856
 857 (define* (common-qemu-options image shared-fs)
 858   "Return the a string-value gexp with the common QEMU options to boot IMAGE,
 859 with '-virtfs' options for the host file systems listed in SHARED-FS."
 860
 861   (define (virtfs-option fs)
 862     #~(format #f "-virtfs local,path=~s,security_model=none,mount_tag=~s"
 863               #$fs #$(file-system->mount-tag fs)))
 864
 865   #~(;; Only enable kvm if we see /dev/kvm exists.
 866      ;; This allows users without hardware virtualization to still use these
 867      ;; commands.
 868      #$@(if (file-exists? "/dev/kvm")
 869             '("-enable-kvm")
 870             '())
 871
 872      "-no-reboot"
 873      "-net nic,model=virtio"
 874      "-object" "rng-random,filename=/dev/urandom,id=guixsd-vm-rng"
 875      "-device" "virtio-rng-pci,rng=guixsd-vm-rng"
 876
 877      #$@(map virtfs-option shared-fs)
 878      "-vga std"
 879      (format #f "-drive file=~a,if=virtio,cache=writeback,werror=report,readonly"
 880              #$image)))
 881
 882 (define* (system-qemu-image/shared-store-script os
 883                                                 #:key
 884                                                 (qemu qemu)
 885                                                 (graphic? #t)
 886                                                 (memory-size 256)
 887                                                 (mappings '())
 888                                                 full-boot?
 889                                                 (disk-image-size
 890                                                  (* (if full-boot? 500 70)
 891                                                     (expt 2 20)))
 892                                                 (options '()))
 893   "Return a derivation that builds a script to run a virtual machine image of
 894 OS that shares its store with the host.  The virtual machine runs with
 895 MEMORY-SIZE MiB of memory.
 896
 897 MAPPINGS is a list of <file-system-mapping> specifying mapping of host file
 898 systems into the guest.
 899
 900 When FULL-BOOT? is true, the returned script runs everything starting from the
 901 bootloader; otherwise it directly starts the operating system kernel.  The
 902 DISK-IMAGE-SIZE parameter specifies the size in bytes of the root disk image;
 903 it is mostly useful when FULL-BOOT?  is true."
 904   (mlet* %store-monad ((os ->  (virtualized-operating-system os mappings full-boot?))
 905                        (image  (system-qemu-image/shared-store
 906                                 os
 907                                 #:full-boot? full-boot?
 908                                 #:disk-image-size disk-image-size)))
 909     (define kernel-arguments
 910       #~(list #$@(if graphic? #~() #~("console=ttyS0"))
 911               #+@(operating-system-kernel-arguments os "/dev/vda1")))
 912
 913     (define qemu-exec
 914       #~(list (string-append #$qemu "/bin/" #$(qemu-command (%current-system)))
 915               #$@(if full-boot?
 916                      #~()
 917                      #~("-kernel" #$(operating-system-kernel-file os)
 918                         "-initrd" #$(file-append os "/initrd")
 919                         (format #f "-append ~s"
 920                                 (string-join #$kernel-arguments " "))))
 921               #$@(common-qemu-options image
 922                                       (map file-system-mapping-source
 923                                            (cons %store-mapping mappings)))
 924               "-m " (number->string #$memory-size)
 925               #$@options))
 926
 927     (define builder
 928       #~(call-with-output-file #$output
 929           (lambda (port)
 930             (format port "#!~a~% exec ~a \"$@\"~%"
 931                     #$(file-append bash "/bin/sh")
 932                     (string-join #$qemu-exec " "))
 933             (chmod port #o555))))
 934
 935     (gexp->derivation "run-vm.sh" builder)))
 936
 937 \f
 938 ;;;
 939 ;;; High-level abstraction.
 940 ;;;
 941
 942 (define-record-type* <virtual-machine> %virtual-machine
 943   make-virtual-machine
 944   virtual-machine?
 945   (operating-system virtual-machine-operating-system) ;<operating-system>
 946   (qemu             virtual-machine-qemu              ;<package>
 947                     (default qemu))
 948   (graphic?         virtual-machine-graphic?      ;Boolean
 949                     (default #f))
 950   (memory-size      virtual-machine-memory-size   ;integer (MiB)
 951                     (default 256))
 952   (disk-image-size  virtual-machine-disk-image-size   ;integer (bytes)
 953                     (default 'guess))
 954   (port-forwardings virtual-machine-port-forwardings ;list of integer pairs
 955                     (default '())))
 956
 957 (define-syntax virtual-machine
 958   (syntax-rules ()
 959     "Declare a virtual machine running the specified OS, with the given
 960 options."
 961     ((_ os)                                       ;shortcut
 962      (%virtual-machine (operating-system os)))
 963     ((_ fields ...)
 964      (%virtual-machine fields ...))))
 965
 966 (define (port-forwardings->qemu-options forwardings)
 967   "Return the QEMU option for the given port FORWARDINGS as a string, where
 968 FORWARDINGS is a list of host-port/guest-port pairs."
 969   (string-join
 970    (map (match-lambda
 971           ((host-port . guest-port)
 972            (string-append "hostfwd=tcp::"
 973                           (number->string host-port)
 974                           "-:" (number->string guest-port))))
 975         forwardings)
 976    ","))
 977
 978 (define-gexp-compiler (virtual-machine-compiler (vm <virtual-machine>)
 979                                                 system target)
 980   ;; XXX: SYSTEM and TARGET are ignored.
 981   (match vm
 982     (($ <virtual-machine> os qemu graphic? memory-size disk-image-size ())
 983      (system-qemu-image/shared-store-script os
 984                                             #:qemu qemu
 985                                             #:graphic? graphic?
 986                                             #:memory-size memory-size
 987                                             #:disk-image-size
 988                                             disk-image-size))
 989     (($ <virtual-machine> os qemu graphic? memory-size disk-image-size
 990                           forwardings)
 991      (let ((options
 992             `("-net" ,(string-append
 993                        "user,"
 994                        (port-forwardings->qemu-options forwardings)))))
 995        (system-qemu-image/shared-store-script os
 996                                               #:qemu qemu
 997                                               #:graphic? graphic?
 998                                               #:memory-size memory-size
 999                                               #:disk-image-size
1000                                               disk-image-size
1001                                               #:options options)))))
1002
1003 ;;; vm.scm ends here