gnu/system/vm.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2013, 2014, 2015 Ludovic Courtès <ludo@gnu.org>
   3 ;;;
   4 ;;; This file is part of GNU Guix.
   5 ;;;
   6 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   7 ;;; under the terms of the GNU General Public License as published by
   8 ;;; the Free Software Foundation; either version 3 of the License, or (at
   9 ;;; your option) any later version.
  10 ;;;
  11 ;;; GNU Guix is distributed in the hope that it will be useful, but
  12 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  13 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 ;;; GNU General Public License for more details.
  15 ;;;
  16 ;;; You should have received a copy of the GNU General Public License
  17 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  18
  19 (define-module (gnu system vm)
  20   #:use-module (guix config)
  21   #:use-module (guix store)
  22   #:use-module (guix gexp)
  23   #:use-module (guix derivations)
  24   #:use-module (guix packages)
  25   #:use-module (guix monads)
  26   #:use-module (guix records)
  27
  28   #:use-module ((gnu build vm)
  29                 #:select (qemu-command))
  30   #:use-module (gnu packages base)
  31   #:use-module (gnu packages guile)
  32   #:use-module (gnu packages gawk)
  33   #:use-module (gnu packages bash)
  34   #:use-module (gnu packages less)
  35   #:use-module (gnu packages qemu)
  36   #:use-module (gnu packages disk)
  37   #:use-module (gnu packages zile)
  38   #:use-module (gnu packages grub)
  39   #:use-module (gnu packages linux)
  40   #:use-module (gnu packages package-management)
  41   #:use-module ((gnu packages make-bootstrap)
  42                 #:select (%guile-static-stripped))
  43   #:use-module (gnu packages admin)
  44
  45   #:use-module (gnu system shadow)
  46   #:use-module (gnu system linux)
  47   #:use-module (gnu system linux-initrd)
  48   #:use-module (gnu system grub)
  49   #:use-module (gnu system file-systems)
  50   #:use-module (gnu system)
  51   #:use-module (gnu services)
  52
  53   #:use-module (srfi srfi-1)
  54   #:use-module (srfi srfi-26)
  55   #:use-module (ice-9 match)
  56
  57   #:export (expression->derivation-in-linux-vm
  58             qemu-image
  59             system-qemu-image
  60
  61             system-qemu-image/shared-store
  62             system-qemu-image/shared-store-script
  63             system-disk-image))
  64
  65 \f
  66 ;;; Commentary:
  67 ;;;
  68 ;;; Tools to evaluate build expressions within virtual machines.
  69 ;;;
  70 ;;; Code:
  71
  72 (define %linux-vm-file-systems
  73   ;; File systems mounted for 'derivation-in-linux-vm'.  The store and /xchg
  74   ;; directory are shared with the host over 9p.
  75   (list (file-system
  76           (mount-point (%store-prefix))
  77           (device "store")
  78           (type "9p")
  79           (needed-for-boot? #t)
  80           (options "trans=virtio")
  81           (check? #f))
  82         (file-system
  83           (mount-point "/xchg")
  84           (device "xchg")
  85           (type "9p")
  86           (needed-for-boot? #t)
  87           (options "trans=virtio")
  88           (check? #f))))
  89
  90 (define* (expression->derivation-in-linux-vm name exp
  91                                              #:key
  92                                              (system (%current-system))
  93                                              (linux linux-libre)
  94                                              initrd
  95                                              (qemu qemu-headless)
  96                                              (env-vars '())
  97                                              (modules
  98                                               '((gnu build vm)
  99                                                 (gnu build install)
 100                                                 (gnu build linux-boot)
 101                                                 (gnu build linux-modules)
 102                                                 (gnu build file-systems)
 103                                                 (guix elf)
 104                                                 (guix build utils)
 105                                                 (guix build store-copy)))
 106                                              (guile-for-build
 107                                               (%guile-for-build))
 108
 109                                              (make-disk-image? #f)
 110                                              (references-graphs #f)
 111                                              (memory-size 256)
 112                                              (disk-image-format "qcow2")
 113                                              (disk-image-size
 114                                               (* 100 (expt 2 20))))
 115   "Evaluate EXP in a QEMU virtual machine running LINUX with INITRD (a
 116 derivation).  In the virtual machine, EXP has access to all its inputs from the
 117 store; it should put its output files in the `/xchg' directory, which is
 118 copied to the derivation's output when the VM terminates.  The virtual machine
 119 runs with MEMORY-SIZE MiB of memory.
 120
 121 When MAKE-DISK-IMAGE? is true, then create a QEMU disk image of type
 122 DISK-IMAGE-FORMAT (e.g., 'qcow2' or 'raw'), of DISK-IMAGE-SIZE bytes and
 123 return it.
 124
 125 MODULES is the set of modules imported in the execution environment of EXP.
 126
 127 When REFERENCES-GRAPHS is true, it must be a list of file name/store path
 128 pairs, as for `derivation'.  The files containing the reference graphs are
 129 made available under the /xchg CIFS share."
 130   (mlet* %store-monad
 131       ((module-dir   (imported-modules modules))
 132        (compiled     (compiled-modules modules))
 133        (user-builder (gexp->file "builder-in-linux-vm" exp))
 134        (loader       (gexp->file "linux-vm-loader"
 135                                  #~(begin
 136                                      (set! %load-path
 137                                            (cons #$module-dir %load-path))
 138                                      (set! %load-compiled-path
 139                                            (cons #$compiled
 140                                                  %load-compiled-path))
 141                                      (primitive-load #$user-builder))))
 142        (coreutils -> (canonical-package coreutils))
 143        (initrd       (if initrd                   ; use the default initrd?
 144                          (return initrd)
 145                          (base-initrd %linux-vm-file-systems
 146                                       #:linux linux
 147                                       #:virtio? #t
 148                                       #:qemu-networking? #t))))
 149
 150     (define builder
 151       ;; Code that launches the VM that evaluates EXP.
 152       #~(begin
 153           (use-modules (guix build utils)
 154                        (gnu build vm))
 155
 156           (let ((inputs  '#$(list qemu coreutils))
 157                 (linux   (string-append #$linux "/bzImage"))
 158                 (initrd  (string-append #$initrd "/initrd"))
 159                 (loader  #$loader)
 160                 (graphs  '#$(match references-graphs
 161                               (((graph-files . _) ...) graph-files)
 162                               (_ #f))))
 163
 164             (set-path-environment-variable "PATH" '("bin") inputs)
 165
 166             (load-in-linux-vm loader
 167                               #:output #$output
 168                               #:linux linux #:initrd initrd
 169                               #:memory-size #$memory-size
 170                               #:make-disk-image? #$make-disk-image?
 171                               #:disk-image-format #$disk-image-format
 172                               #:disk-image-size #$disk-image-size
 173                               #:references-graphs graphs))))
 174
 175     (gexp->derivation name builder
 176                       ;; TODO: Require the "kvm" feature.
 177                       #:system system
 178                       #:env-vars env-vars
 179                       #:modules modules
 180                       #:guile-for-build guile-for-build
 181                       #:references-graphs references-graphs)))
 182
 183 (define* (qemu-image #:key
 184                      (name "qemu-image")
 185                      (system (%current-system))
 186                      (qemu qemu-headless)
 187                      (disk-image-size (* 100 (expt 2 20)))
 188                      (disk-image-format "qcow2")
 189                      (file-system-type "ext4")
 190                      file-system-label
 191                      os-derivation
 192                      grub-configuration
 193                      (register-closures? #t)
 194                      (inputs '())
 195                      copy-inputs?)
 196   "Return a bootable, stand-alone QEMU image of type DISK-IMAGE-FORMAT (e.g.,
 197 'qcow2' or 'raw'), with a root partition of type FILE-SYSTEM-TYPE.
 198 Optionally, FILE-SYSTEM-LABEL can be specified as the volume name for the root
 199 partition.  The returned image is a full disk image that runs OS-DERIVATION,
 200 with a GRUB installation that uses GRUB-CONFIGURATION as its configuration
 201 file (GRUB-CONFIGURATION must be the name of a file in the VM.)
 202
 203 INPUTS is a list of inputs (as for packages).  When COPY-INPUTS? is true, copy
 204 all of INPUTS into the image being built.  When REGISTER-CLOSURES? is true,
 205 register INPUTS in the store database of the image so that Guix can be used in
 206 the image."
 207   (expression->derivation-in-linux-vm
 208    name
 209    #~(begin
 210        (use-modules (gnu build vm)
 211                     (guix build utils))
 212
 213        (let ((inputs
 214               '#$(append (list qemu parted grub e2fsprogs util-linux)
 215                          (map canonical-package
 216                               (list sed grep coreutils findutils gawk))
 217                          (if register-closures? (list guix) '())))
 218
 219              ;; This variable is unused but allows us to add INPUTS-TO-COPY
 220              ;; as inputs.
 221              (to-register
 222               '#$(map (match-lambda
 223                        ((name thing) thing)
 224                        ((name thing output) `(,thing ,output)))
 225                       inputs)))
 226
 227          (set-path-environment-variable "PATH" '("bin" "sbin") inputs)
 228
 229          (let ((graphs '#$(match inputs
 230                             (((names . _) ...)
 231                              names))))
 232            (initialize-hard-disk "/dev/vda"
 233                                  #:system-directory #$os-derivation
 234                                  #:grub.cfg #$grub-configuration
 235                                  #:closures graphs
 236                                  #:copy-closures? #$copy-inputs?
 237                                  #:register-closures? #$register-closures?
 238                                  #:disk-image-size #$disk-image-size
 239                                  #:file-system-type #$file-system-type
 240                                  #:file-system-label #$file-system-label)
 241            (reboot))))
 242    #:system system
 243    #:make-disk-image? #t
 244    #:disk-image-size disk-image-size
 245    #:disk-image-format disk-image-format
 246    #:references-graphs inputs))
 247
 248 \f
 249 ;;;
 250 ;;; VM and disk images.
 251 ;;;
 252
 253 (define* (system-disk-image os
 254                             #:key
 255                             (name "disk-image")
 256                             (file-system-type "ext4")
 257                             (disk-image-size (* 900 (expt 2 20)))
 258                             (volatile? #t))
 259   "Return the derivation of a disk image of DISK-IMAGE-SIZE bytes of the
 260 system described by OS.  Said image can be copied on a USB stick as is.  When
 261 VOLATILE? is true, the root file system is made volatile; this is useful
 262 to USB sticks meant to be read-only."
 263   (define root-label
 264     ;; Volume name of the root file system.  Since we don't know which device
 265     ;; will hold it, we use the volume name to find it (using the UUID would
 266     ;; be even better, but somewhat less convenient.)
 267     "gnu-disk-image")
 268
 269   (define file-systems-to-keep
 270     (remove (lambda (fs)
 271               (string=? (file-system-mount-point fs) "/"))
 272             (operating-system-file-systems os)))
 273
 274   (let ((os (operating-system (inherit os)
 275               ;; Since this is meant to be used on real hardware, don't
 276               ;; install QEMU networking or anything like that.  Assume USB
 277               ;; mass storage devices (usb-storage.ko) are available.
 278               (initrd (lambda (file-systems . rest)
 279                         (apply base-initrd file-systems
 280                                #:volatile-root? #t
 281                                rest)))
 282
 283               ;; Force our own root file system.
 284               (file-systems (cons (file-system
 285                                     (mount-point "/")
 286                                     (device root-label)
 287                                     (title 'label)
 288                                     (type file-system-type))
 289                                   file-systems-to-keep)))))
 290
 291     (mlet* %store-monad ((os-drv   (operating-system-derivation os))
 292                          (grub.cfg (operating-system-grub.cfg os)))
 293       (qemu-image #:name name
 294                   #:os-derivation os-drv
 295                   #:grub-configuration grub.cfg
 296                   #:disk-image-size disk-image-size
 297                   #:disk-image-format "raw"
 298                   #:file-system-type file-system-type
 299                   #:file-system-label root-label
 300                   #:copy-inputs? #t
 301                   #:register-closures? #t
 302                   #:inputs `(("system" ,os-drv)
 303                              ("grub.cfg" ,grub.cfg))))))
 304
 305 (define* (system-qemu-image os
 306                             #:key
 307                             (file-system-type "ext4")
 308                             (disk-image-size (* 900 (expt 2 20))))
 309   "Return the derivation of a freestanding QEMU image of DISK-IMAGE-SIZE bytes
 310 of the GNU system as described by OS."
 311   (define file-systems-to-keep
 312     ;; Keep only file systems other than root and not normally bound to real
 313     ;; devices.
 314     (remove (lambda (fs)
 315               (let ((target (file-system-mount-point fs))
 316                     (source (file-system-device fs)))
 317                 (or (string=? target "/")
 318                     (string-prefix? "/dev/" source))))
 319             (operating-system-file-systems os)))
 320
 321   (let ((os (operating-system (inherit os)
 322               ;; Use an initrd with the whole QEMU shebang.
 323               (initrd (lambda (file-systems . rest)
 324                         (apply base-initrd file-systems
 325                                #:virtio? #t
 326                                #:qemu-networking? #t
 327                                rest)))
 328
 329               ;; Force our own root file system.
 330               (file-systems (cons (file-system
 331                                     (mount-point "/")
 332                                     (device "/dev/sda1")
 333                                     (type file-system-type))
 334                                   file-systems-to-keep)))))
 335     (mlet* %store-monad
 336         ((os-drv      (operating-system-derivation os))
 337          (grub.cfg    (operating-system-grub.cfg os)))
 338       (qemu-image  #:os-derivation os-drv
 339                    #:grub-configuration grub.cfg
 340                    #:disk-image-size disk-image-size
 341                    #:file-system-type file-system-type
 342                    #:inputs `(("system" ,os-drv)
 343                               ("grub.cfg" ,grub.cfg))
 344                    #:copy-inputs? #t))))
 345
 346 \f
 347 ;;;
 348 ;;; VMs that share file systems with the host.
 349 ;;;
 350
 351 (define (file-system->mount-tag fs)
 352   "Return a 9p mount tag for host file system FS."
 353   ;; QEMU mount tags cannot contain slashes and cannot start with '_'.
 354   ;; Compute an identifier that corresponds to the rules.
 355   (string-append "TAG"
 356                  (string-map (match-lambda
 357                               (#\/ #\_)
 358                               (chr chr))
 359                              fs)))
 360
 361 (define (mapping->file-system mapping)
 362   "Return a 9p file system that realizes MAPPING."
 363   (match mapping
 364     (($ <file-system-mapping> source target writable?)
 365      (file-system
 366        (mount-point target)
 367        (device (file-system->mount-tag source))
 368        (type "9p")
 369        (flags (if writable? '() '(read-only)))
 370        (options (string-append "trans=virtio"))
 371        (check? #f)
 372        (create-mount-point? #t)))))
 373
 374 (define (virtualized-operating-system os mappings)
 375   "Return an operating system based on OS suitable for use in a virtualized
 376 environment with the store shared with the host.  MAPPINGS is a list of
 377 <file-system-mapping> to realize in the virtualized OS."
 378   (define user-file-systems
 379     ;; Remove file systems that conflict with those added below, or that are
 380     ;; normally bound to real devices.
 381     (remove (lambda (fs)
 382               (let ((target (file-system-mount-point fs))
 383                     (source (file-system-device fs)))
 384                 (or (string=? target (%store-prefix))
 385                     (string=? target "/")
 386                     (string-prefix? "/dev/" source))))
 387             (operating-system-file-systems os)))
 388
 389   (operating-system (inherit os)
 390     (initrd (lambda (file-systems . rest)
 391               (apply base-initrd file-systems
 392                      #:volatile-root? #t
 393                      #:virtio? #t
 394                      #:qemu-networking? #t
 395                      rest)))
 396
 397     ;; Disable swap.
 398     (swap-devices '())
 399
 400     (file-systems (cons* (file-system
 401                            (mount-point "/")
 402                            (device "/dev/vda1")
 403                            (type "ext4"))
 404
 405                          (file-system (inherit
 406                                        (mapping->file-system %store-mapping))
 407                             (needed-for-boot? #t))
 408
 409                          (append (map mapping->file-system mappings)
 410                                  user-file-systems)))))
 411
 412 (define* (system-qemu-image/shared-store
 413           os
 414           #:key
 415           full-boot?
 416           (disk-image-size (* (if full-boot? 500 15) (expt 2 20))))
 417   "Return a derivation that builds a QEMU image of OS that shares its store
 418 with the host.
 419
 420 When FULL-BOOT? is true, return an image that does a complete boot sequence,
 421 bootloaded included; thus, make a disk image that contains everything the
 422 bootloader refers to: OS kernel, initrd, bootloader data, etc."
 423   (mlet* %store-monad ((os-drv   (operating-system-derivation os))
 424                        (grub.cfg (operating-system-grub.cfg os)))
 425     ;; XXX: When FULL-BOOT? is true, we end up creating an image that contains
 426     ;; GRUB.CFG and all its dependencies, including the output of OS-DRV.
 427     ;; This is more than needed (we only need the kernel, initrd, GRUB for its
 428     ;; font, and the background image), but it's hard to filter that.
 429     (qemu-image #:os-derivation os-drv
 430                 #:grub-configuration grub.cfg
 431                 #:disk-image-size disk-image-size
 432                 #:inputs (if full-boot?
 433                              `(("grub.cfg" ,grub.cfg))
 434                              '())
 435
 436                 ;; XXX: Passing #t here is too slow, so let it off by default.
 437                 #:register-closures? #f
 438                 #:copy-inputs? full-boot?)))
 439
 440 (define* (common-qemu-options image shared-fs)
 441   "Return the a string-value gexp with the common QEMU options to boot IMAGE,
 442 with '-virtfs' options for the host file systems listed in SHARED-FS."
 443   (define (virtfs-option fs)
 444     #~(string-append "-virtfs local,path=\"" #$fs
 445                      "\",security_model=none,mount_tag=\""
 446                      #$(file-system->mount-tag fs)
 447                      "\" "))
 448
 449   #~(string-append
 450      " -enable-kvm -no-reboot -net nic,model=virtio \
 451   " #$@(map virtfs-option shared-fs) " \
 452   -net user \
 453   -serial stdio -vga std \
 454   -drive file=" #$image
 455   ",if=virtio,cache=writeback,werror=report,readonly \
 456   -m 256"))
 457
 458 (define* (system-qemu-image/shared-store-script os
 459                                                 #:key
 460                                                 (qemu qemu)
 461                                                 (graphic? #t)
 462                                                 (mappings '())
 463                                                 full-boot?
 464                                                 (disk-image-size
 465                                                  (* (if full-boot? 500 15)
 466                                                     (expt 2 20))))
 467   "Return a derivation that builds a script to run a virtual machine image of
 468 OS that shares its store with the host.
 469
 470 MAPPINGS is a list of <file-system-mapping> specifying mapping of host file
 471 systems into the guest.
 472
 473 When FULL-BOOT? is true, the returned script runs everything starting from the
 474 bootloader; otherwise it directly starts the operating system kernel.  The
 475 DISK-IMAGE-SIZE parameter specifies the size in bytes of the root disk image;
 476 it is mostly useful when FULL-BOOT?  is true."
 477   (mlet* %store-monad ((os ->  (virtualized-operating-system os mappings))
 478                        (os-drv (operating-system-derivation os))
 479                        (image  (system-qemu-image/shared-store
 480                                 os
 481                                 #:full-boot? full-boot?
 482                                 #:disk-image-size disk-image-size)))
 483     (define builder
 484       #~(call-with-output-file #$output
 485           (lambda (port)
 486             (display
 487              (string-append "#!" #$bash "/bin/sh
 488 exec " #$qemu "/bin/" #$(qemu-command (%current-system))
 489
 490 #$@(if full-boot?
 491        #~()
 492        #~(" -kernel " #$(operating-system-kernel os) "/bzImage \
 493             -initrd " #$os-drv "/initrd \
 494             -append \"" #$(if graphic? "" "console=ttyS0 ")
 495             "--system=" #$os-drv " --load=" #$os-drv "/boot --root=/dev/vda1\" "))
 496 #$(common-qemu-options image
 497                        (map file-system-mapping-source
 498                             (cons %store-mapping mappings)))
 499 " \"$@\"\n")
 500              port)
 501             (chmod port #o555))))
 502
 503     (gexp->derivation "run-vm.sh" builder)))
 504
 505 ;;; vm.scm ends here