guix/build/go-build-system.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2016 Petter <petter@mykolab.ch>
   3 ;;; Copyright © 2017, 2019 Leo Famulari <leo@famulari.name>
   4 ;;; Copyright © 2019 Maxim Cournoyer <maxim.cournoyer@gmail.com>
   5 ;;; Copyright © 2020 Jack Hill <jackhill@jackhill.us>
   6 ;;; Copyright © 2020 Jakub Kądziołka <kuba@kadziolka.net>
   7 ;;;
   8 ;;; This file is part of GNU Guix.
   9 ;;;
  10 ;;; GNU Guix is free software; you can redistribute it and/or modify it
  11 ;;; under the terms of the GNU General Public License as published by
  12 ;;; the Free Software Foundation; either version 3 of the License, or (at
  13 ;;; your option) any later version.
  14 ;;;
  15 ;;; GNU Guix is distributed in the hope that it will be useful, but
  16 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  17 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 ;;; GNU General Public License for more details.
  19 ;;;
  20 ;;; You should have received a copy of the GNU General Public License
  21 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  22
  23 (define-module (guix build go-build-system)
  24   #:use-module ((guix build gnu-build-system) #:prefix gnu:)
  25   #:use-module (guix build union)
  26   #:use-module (guix build utils)
  27   #:use-module (ice-9 match)
  28   #:use-module (ice-9 ftw)
  29   #:use-module (srfi srfi-1)
  30   #:use-module (rnrs io ports)
  31   #:use-module (rnrs bytevectors)
  32   #:export (%standard-phases
  33             go-build))
  34
  35 ;; Commentary:
  36 ;;
  37 ;; Build procedures for Go packages.  This is the builder-side code.
  38 ;;
  39 ;; Software written in Go is either a 'package' (i.e. library) or 'command'
  40 ;; (i.e. executable).  Both types can be built with either the `go build` or `go
  41 ;; install` commands.  However, `go build` discards the result of the build
  42 ;; process for Go libraries, so we use `go install`, which preserves the
  43 ;; results. [0]
  44
  45 ;; Go software is developed and built within a particular file system hierarchy
  46 ;; structure called a 'workspace' [1].  This workspace can be found by Go via
  47 ;; the GOPATH environment variable.  Typically, all Go source code and compiled
  48 ;; objects are kept in a single workspace, but GOPATH may be a list of
  49 ;; directories [2].  In this go-build-system we create a file system union of
  50 ;; the Go-language dependencies. Previously, we made GOPATH a list of store
  51 ;; directories, but stopped because Go programs started keeping references to
  52 ;; these directories in Go 1.11:
  53 ;; <https://bugs.gnu.org/33620>.
  54 ;;
  55 ;; Go software, whether a package or a command, is uniquely named using an
  56 ;; 'import path'.  The import path is based on the URL of the software's source.
  57 ;; Because most source code is provided over the internet, the import path is
  58 ;; typically a combination of the remote URL and the source repository's file
  59 ;; system structure. For example, the Go port of the common `du` command is
  60 ;; hosted on github.com, at <https://github.com/calmh/du>.  Thus, the import
  61 ;; path is <github.com/calmh/du>. [3]
  62 ;;
  63 ;; It may be possible to automatically guess a package's import path based on
  64 ;; the source URL, but we don't try that in this revision of the
  65 ;; go-build-system.
  66 ;;
  67 ;; Modules of modular Go libraries are named uniquely with their
  68 ;; file system paths.  For example, the supplemental but "standardized"
  69 ;; libraries developed by the Go upstream developers are available at
  70 ;; <https://golang.org/x/{net,text,crypto, et cetera}>.  The Go IPv4
  71 ;; library's import path is <golang.org/x/net/ipv4>.  The source of
  72 ;; such modular libraries must be unpacked at the top-level of the
  73 ;; file system structure of the library.  So the IPv4 library should be
  74 ;; unpacked to <golang.org/x/net>.  This is handled in the
  75 ;; go-build-system with the optional #:unpack-path key.
  76 ;;
  77 ;; In general, Go software is built using a standardized build mechanism
  78 ;; that does not require any build scripts like Makefiles.  This means
  79 ;; that all modules of modular libraries cannot be built with a single
  80 ;; command.  Each module must be built individually.  This complicates
  81 ;; certain cases, and these issues are currently resolved by creating a
  82 ;; file system union of the required modules of such libraries.  I think
  83 ;; this could be improved in future revisions of the go-build-system.
  84 ;;
  85 ;; TODO:
  86 ;; * Avoid copying dependencies into the build environment and / or avoid using
  87 ;; a tmpdir when creating the inputs union.
  88 ;; * Use Go modules [4]
  89 ;; * Re-use compiled packages [5]
  90 ;; * Avoid the go-inputs hack
  91 ;; * Stop needing remove-go-references (-trimpath ? )
  92 ;; * Remove module packages, only offering the full Git repos? This is
  93 ;; more idiomatic, I think, because Go downloads Git repos, not modules.
  94 ;; What are the trade-offs?
  95 ;;
  96 ;; [0] `go build`:
  97 ;; https://golang.org/cmd/go/#hdr-Compile_packages_and_dependencies
  98 ;; `go install`:
  99 ;; https://golang.org/cmd/go/#hdr-Compile_and_install_packages_and_dependencies
 100 ;; [1] Go workspace example, from <https://golang.org/doc/code.html#Workspaces>:
 101 ;; bin/
 102 ;;     hello                          # command executable
 103 ;;     outyet                         # command executable
 104 ;; pkg/
 105 ;;     linux_amd64/
 106 ;;         github.com/golang/example/
 107 ;;             stringutil.a           # package object
 108 ;; src/
 109 ;;     github.com/golang/example/
 110 ;;         .git/                      # Git repository metadata
 111 ;;         hello/
 112 ;;             hello.go               # command source
 113 ;;         outyet/
 114 ;;              main.go               # command source
 115 ;;              main_test.go          # test source
 116 ;;         stringutil/
 117 ;;             reverse.go             # package source
 118 ;;             reverse_test.go        # test source
 119 ;;         golang.org/x/image/
 120 ;;             .git/                  # Git repository metadata
 121 ;;             bmp/
 122 ;;                 reader.go          # package source
 123 ;;                 writer.go          # package source
 124 ;;     ... (many more repositories and packages omitted) ...
 125 ;;
 126 ;; [2] https://golang.org/doc/code.html#GOPATH
 127 ;; [3] https://golang.org/doc/code.html#ImportPaths
 128 ;; [4] https://golang.org/cmd/go/#hdr-Modules__module_versions__and_more
 129 ;; [5] https://bugs.gnu.org/32919
 130 ;;
 131 ;; Code:
 132
 133 (define* (setup-go-environment #:key inputs outputs #:allow-other-keys)
 134   "Prepare a Go build environment for INPUTS and OUTPUTS.  Build a file system
 135 union of INPUTS.  Export GOPATH, which helps the compiler find the source code
 136 of the package being built and its dependencies, and GOBIN, which determines
 137 where executables (\"commands\") are installed to.  This phase is sometimes used
 138 by packages that use (guix build-system gnu) but have a handful of Go
 139 dependencies, so it should be self-contained."
 140   ;; The Go cache is required starting in Go 1.12.  We don't actually use it but
 141   ;; we need it to be a writable directory.
 142   (setenv "GOCACHE" "/tmp/go-cache")
 143   ;; Using the current working directory as GOPATH makes it easier for packagers
 144   ;; who need to manipulate the unpacked source code.
 145   (setenv "GOPATH" (getcwd))
 146   ;; Go 1.13 uses go modules by default. The go build system does not
 147   ;; currently support modules, so turn modules off to continue using the old
 148   ;; GOPATH behavior.
 149   (setenv "GO111MODULE" "off")
 150   (setenv "GOBIN" (string-append (assoc-ref outputs "out") "/bin"))
 151   (let ((tmpdir (tmpnam)))
 152     (match (go-inputs inputs)
 153       (((names . directories) ...)
 154        (union-build tmpdir (filter directory-exists? directories)
 155                     #:create-all-directories? #t
 156                     #:log-port (%make-void-port "w"))))
 157     ;; XXX A little dance because (guix build union) doesn't use mkdir-p.
 158     (copy-recursively tmpdir
 159                       (string-append (getenv "GOPATH"))
 160                       #:keep-mtime? #t)
 161     (delete-file-recursively tmpdir))
 162   #t)
 163
 164 (define* (unpack #:key source import-path unpack-path #:allow-other-keys)
 165   "Relative to $GOPATH, unpack SOURCE in UNPACK-PATH, or IMPORT-PATH when
 166 UNPACK-PATH is unset.  If the SOURCE archive has a single top level directory,
 167 it is stripped so that the sources appear directly under UNPACK-PATH.  When
 168 SOURCE is a directory, copy its content into UNPACK-PATH instead of
 169 unpacking."
 170   (define (unpack-maybe-strip source dest)
 171     (let* ((scratch-dir (string-append (or (getenv "TMPDIR") "/tmp")
 172                                        "/scratch-dir"))
 173            (out (mkdir-p scratch-dir)))
 174       (with-directory-excursion scratch-dir
 175         (if (string-suffix? ".zip" source)
 176             (invoke "unzip" source)
 177             (invoke "tar" "-xvf" source))
 178         (let ((top-level-files (remove (lambda (x)
 179                                          (member x '("." "..")))
 180                                        (scandir "."))))
 181           (match top-level-files
 182             ((top-level-file)
 183              (when (file-is-directory? top-level-file)
 184                (copy-recursively top-level-file dest #:keep-mtime? #t)))
 185             (_
 186              (copy-recursively "." dest #:keep-mtime? #t)))))
 187       (delete-file-recursively scratch-dir)))
 188
 189   (when (string-null? import-path)
 190     (display "WARNING: The Go import path is unset.\n"))
 191   (when (string-null? unpack-path)
 192     (set! unpack-path import-path))
 193   (let ((dest (string-append (getenv "GOPATH") "/src/" unpack-path)))
 194     (mkdir-p dest)
 195     (if (file-is-directory? source)
 196         (copy-recursively source dest #:keep-mtime? #t)
 197         (unpack-maybe-strip source dest)))
 198   #t)
 199
 200 (define (go-package? name)
 201   (string-prefix? "go-" name))
 202
 203 (define (go-inputs inputs)
 204   "Return the alist of INPUTS that are Go software."
 205   ;; XXX This should not check the file name of the store item. Instead we
 206   ;; should pass, from the host side, the list of inputs that are packages using
 207   ;; the go-build-system.
 208   (alist-delete "go" ; Exclude the Go compiler
 209     (alist-delete "source" ; Exclude the source code of the package being built
 210       (filter (match-lambda
 211                 ((label . directory)
 212                  (go-package? ((compose package-name->name+version
 213                                         strip-store-file-name)
 214                                directory)))
 215                 (_ #f))
 216               inputs))))
 217
 218 (define* (build #:key import-path build-flags #:allow-other-keys)
 219   "Build the package named by IMPORT-PATH."
 220   (with-throw-handler
 221     #t
 222     (lambda _
 223       (apply invoke "go" "install"
 224               "-v" ; print the name of packages as they are compiled
 225               "-x" ; print each command as it is invoked
 226               ;; Respectively, strip the symbol table and debug
 227               ;; information, and the DWARF symbol table.
 228               "-ldflags=-s -w"
 229               `(,@build-flags ,import-path)))
 230     (lambda (key . args)
 231       (display (string-append "Building '" import-path "' failed.\n"
 232                               "Here are the results of `go env`:\n"))
 233       (invoke "go" "env"))))
 234
 235 ;; Can this also install commands???
 236 (define* (check #:key tests? import-path #:allow-other-keys)
 237   "Run the tests for the package named by IMPORT-PATH."
 238   (when tests?
 239     (invoke "go" "test" import-path))
 240   #t)
 241
 242 (define* (install #:key install-source? outputs import-path unpack-path #:allow-other-keys)
 243   "Install the source code of IMPORT-PATH to the primary output directory.
 244 Compiled executable files (Go \"commands\") should have already been installed
 245 to the store based on $GOBIN in the build phase.
 246 XXX We can't make use of compiled libraries (Go \"packages\")."
 247   (when install-source?
 248     (if (string-null? import-path)
 249         ((display "WARNING: The Go import path is unset.\n")))
 250     (let* ((out (assoc-ref outputs "out"))
 251            (source (string-append (getenv "GOPATH") "/src/" import-path))
 252            (dest (string-append out "/src/" import-path)))
 253       (mkdir-p dest)
 254       (copy-recursively source dest #:keep-mtime? #t)))
 255   #t)
 256
 257 (define* (remove-store-reference file file-name
 258                                   #:optional (store (%store-directory)))
 259   "Remove from FILE occurrences of FILE-NAME in STORE; return #t when FILE-NAME
 260 is encountered in FILE, #f otherwise. This implementation reads FILE one byte at
 261 a time, which is slow. Instead, we should use the Boyer-Moore string search
 262 algorithm; there is an example in (guix build grafts)."
 263   (define pattern
 264     (string-take file-name
 265                  (+ 34 (string-length (%store-directory)))))
 266
 267   (with-fluids ((%default-port-encoding #f))
 268     (with-atomic-file-replacement file
 269       (lambda (in out)
 270         ;; We cannot use `regexp-exec' here because it cannot deal with
 271         ;; strings containing NUL characters.
 272         (format #t "removing references to `~a' from `~a'...~%" file-name file)
 273         (setvbuf in 'block 65536)
 274         (setvbuf out 'block 65536)
 275         (fold-port-matches (lambda (match result)
 276                              (put-bytevector out (string->utf8 store))
 277                              (put-u8 out (char->integer #\/))
 278                              (put-bytevector out
 279                                              (string->utf8
 280                                               "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee-"))
 281                              #t)
 282                            #f
 283                            pattern
 284                            in
 285                            (lambda (char result)
 286                              (put-u8 out (char->integer char))
 287                              result))))))
 288
 289 (define* (remove-go-references #:key allow-go-reference?
 290                                inputs outputs #:allow-other-keys)
 291   "Remove any references to the Go compiler from the compiled Go executable
 292 files in OUTPUTS."
 293 ;; We remove this spurious reference to save bandwidth when installing Go
 294 ;; executables. It would be better to not embed the reference in the first
 295 ;; place, but I'm not sure how to do that. The subject was discussed at:
 296 ;; <https://lists.gnu.org/archive/html/guix-devel/2017-10/msg00207.html>
 297   (if allow-go-reference?
 298     #t
 299     (let ((go (assoc-ref inputs "go"))
 300           (bin "/bin"))
 301       (for-each (lambda (output)
 302                   (when (file-exists? (string-append (cdr output)
 303                                                      bin))
 304                     (for-each (lambda (file)
 305                                 (remove-store-reference file go))
 306                               (find-files (string-append (cdr output) bin)))))
 307                 outputs)
 308       #t)))
 309
 310 (define %standard-phases
 311   (modify-phases gnu:%standard-phases
 312     (delete 'bootstrap)
 313     (delete 'configure)
 314     (delete 'patch-generated-file-shebangs)
 315     (add-before 'unpack 'setup-go-environment setup-go-environment)
 316     (replace 'unpack unpack)
 317     (replace 'build build)
 318     (replace 'check check)
 319     (replace 'install install)
 320     (add-after 'install 'remove-go-references remove-go-references)))
 321
 322 (define* (go-build #:key inputs (phases %standard-phases)
 323                       #:allow-other-keys #:rest args)
 324   "Build the given Go package, applying all of PHASES in order."
 325   (apply gnu:gnu-build #:inputs inputs #:phases phases args))