guix/build/pyproject-build-system.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2021 Lars-Dominik Braun <lars@6xq.net>
   3 ;;; Copyright © 2022 Marius Bakke <marius@gnu.org>
   4 ;;;
   5 ;;; This file is part of GNU Guix.
   6 ;;;
   7 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   8 ;;; under the terms of the GNU General Public License as published by
   9 ;;; the Free Software Foundation; either version 3 of the License, or (at
  10 ;;; your option) any later version.
  11 ;;;
  12 ;;; GNU Guix is distributed in the hope that it will be useful, but
  13 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 ;;; GNU General Public License for more details.
  16 ;;;
  17 ;;; You should have received a copy of the GNU General Public License
  18 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 (define-module (guix build pyproject-build-system)
  21   #:use-module ((guix build python-build-system) #:prefix python:)
  22   #:use-module (guix build utils)
  23   #:use-module (guix build json)
  24   #:use-module (ice-9 match)
  25   #:use-module (ice-9 ftw)
  26   #:use-module (ice-9 format)
  27   #:use-module (ice-9 rdelim)
  28   #:use-module (ice-9 regex)
  29   #:use-module (srfi srfi-1)
  30   #:use-module (srfi srfi-26)
  31   #:use-module (srfi srfi-34)
  32   #:use-module (srfi srfi-35)
  33   #:export (%standard-phases
  34             add-installed-pythonpath
  35             site-packages
  36             python-version
  37             pyproject-build))
  38
  39 ;;; Commentary:
  40 ;;;
  41 ;;; PEP 517-compatible build system for Python packages.
  42 ;;;
  43 ;;; PEP 517 mandates the use of a TOML file called pyproject.toml at the
  44 ;;; project root, describing build and runtime dependencies, as well as the
  45 ;;; build system, which can be different from setuptools. This module uses
  46 ;;; that file to extract the build system used and call its wheel-building
  47 ;;; entry point build_wheel (see 'build). setuptools’ wheel builder is
  48 ;;; used as a fallback if either no pyproject.toml exists or it does not
  49 ;;; declare a build-system. It supports config_settings through the
  50 ;;; standard #:configure-flags argument.
  51 ;;;
  52 ;;; This wheel, which is just a ZIP file with a file structure defined
  53 ;;; by PEP 427 (https://www.python.org/dev/peps/pep-0427/), is then unpacked
  54 ;;; and its contents are moved to the appropriate locations in 'install.
  55 ;;;
  56 ;;; Then entry points, as defined by the PyPa Entry Point Specification
  57 ;;; (https://packaging.python.org/specifications/entry-points/) are read
  58 ;;; from a file called entry_points.txt in the package’s site-packages
  59 ;;; subdirectory and scripts are written to bin/. These are not part of a
  60 ;;; wheel and expected to be created by the installing utility.
  61 ;;; TODO: Add support for PEP-621 entry points.
  62 ;;;
  63 ;;; Caveats:
  64 ;;; - There is no support for in-tree build backends.
  65 ;;;
  66 ;;; Code:
  67 ;;;
  68
  69 ;; Re-export these variables from python-build-system as many packages
  70 ;; rely on these.
  71 (define python-version python:python-version)
  72 (define site-packages python:site-packages)
  73 (define add-installed-pythonpath python:add-installed-pythonpath)
  74
  75 ;; Base error type.
  76 (define-condition-type &python-build-error &error python-build-error?)
  77
  78 ;; Raised when 'check cannot find a valid test system in the inputs.
  79 (define-condition-type &test-system-not-found &python-build-error
  80                        test-system-not-found?)
  81
  82 ;; Raised when multiple wheels are created by 'build.
  83 (define-condition-type &cannot-extract-multiple-wheels &python-build-error
  84                        cannot-extract-multiple-wheels?)
  85
  86 ;; Raised, when no wheel has been built by the build system.
  87 (define-condition-type &no-wheels-built &python-build-error no-wheels-built?)
  88
  89 (define* (build #:key outputs build-backend configure-flags #:allow-other-keys)
  90   "Build a given Python package."
  91
  92   (define (pyproject.toml->build-backend file)
  93     "Look up the build backend in a pyproject.toml file."
  94     (call-with-input-file file
  95       (lambda (in)
  96         (let loop
  97           ((line (read-line in 'concat)))
  98           (if (eof-object? line) #f
  99               (let ((m (string-match "build-backend = [\"'](.+)[\"']" line)))
 100                 (if m
 101                     (match:substring m 1)
 102                     (loop (read-line in 'concat)))))))))
 103
 104   (let* ((wheel-output (assoc-ref outputs "wheel"))
 105          (wheel-dir (if wheel-output wheel-output "dist"))
 106          ;; There is no easy way to get data from Guile into Python via
 107          ;; s-expressions, but we have JSON serialization already, which Python
 108          ;; also supports out-of-the-box.
 109          (config-settings (call-with-output-string
 110                             (cut write-json configure-flags <>)))
 111          ;; python-setuptools’ default backend supports setup.py *and*
 112          ;; pyproject.toml. Allow overriding this automatic detection via
 113          ;; build-backend.
 114          (auto-build-backend (if (file-exists? "pyproject.toml")
 115                                  (pyproject.toml->build-backend
 116                                   "pyproject.toml")
 117                                  #f))
 118          ;; Use build system detection here and not in importer, because a) we
 119          ;; have alot of legacy packages and b) the importer cannot update arbitrary
 120          ;; fields in case a package switches its build system.
 121          (use-build-backend (or build-backend
 122                                 auto-build-backend
 123                                 "setuptools.build_meta")))
 124     (format #t
 125      "Using '~a' to build wheels, auto-detected '~a', override '~a'.~%"
 126      use-build-backend auto-build-backend build-backend)
 127     (mkdir-p wheel-dir)
 128     ;; Call the PEP 517 build function, which drops a .whl into wheel-dir.
 129     (invoke "python" "-c"
 130      "import sys, importlib, json
 131 config_settings = json.loads (sys.argv[3])
 132 builder = importlib.import_module(sys.argv[1])
 133 builder.build_wheel(sys.argv[2], config_settings=config_settings)"
 134      use-build-backend
 135      wheel-dir
 136      config-settings)))
 137
 138 (define* (check #:key tests? test-backend test-flags #:allow-other-keys)
 139   "Run the test suite of a given Python package."
 140   (if tests?
 141       ;; Unfortunately with PEP 517 there is no common method to specify test
 142       ;; systems.  Guess test system based on inputs instead.
 143       (let* ((pytest (which "pytest"))
 144              (nosetests (which "nosetests"))
 145              (nose2 (which "nose2"))
 146              (have-setup-py (file-exists? "setup.py"))
 147              (use-test-backend
 148               (or test-backend
 149                   ;; Prefer pytest
 150                   (if pytest 'pytest #f)
 151                   (if nosetests 'nose #f)
 152                   (if nose2 'nose2 #f)
 153                   ;; But fall back to setup.py, which should work for most
 154                   ;; packages. XXX: would be nice not to depend on setup.py here?
 155                   ;; fails more often than not to find any tests at all.  Maybe
 156                   ;; we can run `python -m unittest`?
 157                   (if have-setup-py 'setup.py #f))))
 158         (format #t "Using ~a~%" use-test-backend)
 159         (match use-test-backend
 160           ('pytest
 161            (apply invoke (cons pytest (or test-flags '("-vv")))))
 162           ('nose
 163            (apply invoke (cons nosetests (or test-flags '("-v")))))
 164           ('nose2
 165            (apply invoke (cons nose2 (or test-flags '("-v" "--pretty-assert")))))
 166           ('setup.py
 167            (apply invoke (append '("python" "setup.py")
 168                                  (or test-flags '("test" "-v")))))
 169           ;; The developer should explicitly disable tests in this case.
 170           (else (raise (condition (&test-system-not-found))))))
 171       (format #t "test suite not run~%")))
 172
 173 (define* (install #:key inputs outputs #:allow-other-keys)
 174   "Install a wheel file according to PEP 427"
 175   ;; See https://www.python.org/dev/peps/pep-0427/#installing-a-wheel-distribution-1-0-py32-none-any-whl
 176   (let ((site-dir (site-packages inputs outputs))
 177         (python (assoc-ref inputs "python"))
 178         (out (assoc-ref outputs "out")))
 179     (define (extract file)
 180       "Extract wheel (ZIP file) into site-packages directory"
 181       ;; Use Python’s zipfile to avoid extra dependency
 182       (invoke "python" "-m" "zipfile" "-e" file site-dir))
 183
 184     (define python-hashbang
 185       (string-append "#!" python "/bin/python"))
 186
 187     (define* (merge-directories source destination
 188                                 #:optional (post-move #f))
 189       "Move all files in SOURCE into DESTINATION, merging the two directories."
 190       (format #t "Merging directory ~a into ~a~%" source destination)
 191       (for-each (lambda (file)
 192                   (format #t "~a/~a -> ~a/~a~%"
 193                           source file destination file)
 194                   (mkdir-p destination)
 195                   (rename-file (string-append source "/" file)
 196                                (string-append destination "/" file))
 197                   (when post-move
 198                     (post-move file)))
 199                 (scandir source
 200                          (negate (cut member <> '("." "..")))))
 201       (rmdir source))
 202
 203     (define (expand-data-directory directory)
 204       "Move files from all .data subdirectories to their respective\ndestinations."
 205       ;; Python’s distutils.command.install defines this mapping from source to
 206       ;; destination mapping.
 207       (let ((source (string-append directory "/scripts"))
 208             (destination (string-append out "/bin")))
 209         (when (file-exists? source)
 210           (merge-directories source destination
 211                              (lambda (f)
 212                                (let ((dest-path (string-append destination
 213                                                                "/" f)))
 214                                  (chmod dest-path #o755)
 215                                  ;; PEP 427 recommends that installers rewrite
 216                                  ;; this odd shebang.
 217                                  (substitute* dest-path
 218                                    (("#!python")
 219                                     python-hashbang)))))))
 220       ;; Data can be contained in arbitrary directory structures.  Most
 221       ;; commonly it is used for share/.
 222       (let ((source (string-append directory "/data"))
 223             (destination out))
 224         (when (file-exists? source)
 225           (merge-directories source destination)))
 226       (let* ((distribution (car (string-split (basename directory) #\-)))
 227              (source (string-append directory "/headers"))
 228              (destination (string-append out "/include/python"
 229                                          (python-version python)
 230                                          "/" distribution)))
 231         (when (file-exists? source)
 232           (merge-directories source destination))))
 233
 234     (define (list-directories base predicate)
 235       ;; Cannot use find-files here, because it’s recursive.
 236       (scandir base
 237                (lambda (name)
 238                  (let ((stat (lstat (string-append base "/" name))))
 239                    (and (not (member name '("." "..")))
 240                         (eq? (stat:type stat) 'directory)
 241                         (predicate name stat))))))
 242
 243     (let* ((wheel-output (assoc-ref outputs "wheel"))
 244            (wheel-dir (if wheel-output wheel-output "dist"))
 245            (wheels (map (cut string-append wheel-dir "/" <>)
 246                         (scandir wheel-dir
 247                                  (cut string-suffix? ".whl" <>)))))
 248       (cond
 249         ((> (length wheels) 1)
 250          ;; This code does not support multiple wheels yet, because their
 251          ;; outputs would have to be merged properly.
 252          (raise (condition (&cannot-extract-multiple-wheels))))
 253         ((= (length wheels) 0)
 254          (raise (condition (&no-wheels-built)))))
 255       (for-each extract wheels))
 256     (let ((datadirs (map (cut string-append site-dir "/" <>)
 257                          (list-directories site-dir
 258                                            (file-name-predicate "\\.data$")))))
 259       (for-each (lambda (directory)
 260                   (expand-data-directory directory)
 261                   (rmdir directory)) datadirs))))
 262
 263 (define* (compile-bytecode #:key inputs outputs #:allow-other-keys)
 264   "Compile installed byte-code in site-packages."
 265   (let* ((site-dir (site-packages inputs outputs))
 266          (python (assoc-ref inputs "python"))
 267          (major-minor (map string->number
 268                            (take (string-split (python-version python) #\.) 2)))
 269          (<3.7? (match major-minor
 270                   ((major minor)
 271                    (or (< major 3)
 272                        (and (= major 3)
 273                             (< minor 7)))))))
 274     (if <3.7?
 275         ;; These versions don’t have the hash invalidation modes and do
 276         ;; not produce reproducible bytecode files.
 277         (format #t "Skipping bytecode compilation for Python version ~a < 3.7~%"
 278                 (python-version python))
 279         (invoke "python" "-m" "compileall"
 280                 "--invalidation-mode=unchecked-hash" site-dir))))
 281
 282 (define* (create-entrypoints #:key inputs outputs #:allow-other-keys)
 283   "Implement Entry Points Specification
 284 (https://packaging.python.org/specifications/entry-points/) by PyPa,
 285 which creates runnable scripts in bin/ from entry point specification
 286 file entry_points.txt.  This is necessary, because wheels do not contain
 287 these binaries and installers are expected to create them."
 288
 289   (define (entry-points.txt->entry-points file)
 290     "Specialized parser for Python configfile-like files, in particular
 291 entry_points.txt.  Returns a list of console_script and gui_scripts
 292 entry points."
 293     (call-with-input-file file
 294       (lambda (in)
 295         (let loop ((line (read-line in))
 296                    (inside #f)
 297                    (result '()))
 298           (if (eof-object? line)
 299               result
 300               (let* ((group-match (string-match "^\\[(.+)\\]$" line))
 301                      (group-name (if group-match
 302                                      (match:substring group-match 1)
 303                                      #f))
 304                      (next-inside (if (not group-name)
 305                                       inside
 306                                       (or (string=? group-name
 307                                                     "console_scripts")
 308                                           (string=? group-name "gui_scripts"))))
 309                      (item-match (string-match
 310                                   "^([^ =]+)\\s*=\\s*([^:]+):(.+)$" line)))
 311                 (if (and inside item-match)
 312                     (loop (read-line in)
 313                           next-inside
 314                           (cons (list (match:substring item-match 1)
 315                                       (match:substring item-match 2)
 316                                       (match:substring item-match 3))
 317                                 result))
 318                     (loop (read-line in) next-inside result))))))))
 319
 320   (define (create-script path name module function)
 321     "Create a Python script from an entry point’s NAME, MODULE and FUNCTION
 322 and return write it to PATH/NAME."
 323     (let ((interpreter (which "python"))
 324           (file-path (string-append path "/" name)))
 325       (format #t "Creating entry point for '~a.~a' at '~a'.~%"
 326               module function file-path)
 327       (call-with-output-file file-path
 328         (lambda (port)
 329           ;; Technically the script could also include search-paths,
 330           ;; but having a generic 'wrap phases also handles manually
 331           ;; written entry point scripts.
 332           (format port "#!~a
 333 # Auto-generated entry point script.
 334 import sys
 335 import ~a as mod
 336 sys.exit (mod.~a ())~%" interpreter module function)))
 337         (chmod file-path #o755)))
 338
 339   (let* ((site-dir (site-packages inputs outputs))
 340          (out (assoc-ref outputs "out"))
 341          (bin-dir (string-append out "/bin"))
 342          (entry-point-files (find-files site-dir "^entry_points.txt$")))
 343     (mkdir-p bin-dir)
 344     (for-each (lambda (f)
 345                 (for-each (lambda (ep)
 346                             (apply create-script
 347                                    (cons bin-dir ep)))
 348                           (entry-points.txt->entry-points f)))
 349               entry-point-files)))
 350
 351 (define* (set-SOURCE-DATE-EPOCH* #:rest _)
 352   "Set the 'SOURCE_DATE_EPOCH' environment variable.  This is used by tools
 353 that incorporate timestamps as a way to tell them to use a fixed timestamp.
 354 See https://reproducible-builds.org/specs/source-date-epoch/."
 355   ;; Use a post-1980 timestamp because the Zip format used in wheels do
 356   ;; not support timestamps before 1980.
 357   (setenv "SOURCE_DATE_EPOCH" "315619200"))
 358
 359 (define %standard-phases
 360   ;; The build phase only builds C extensions and copies the Python sources,
 361   ;; while the install phase copies then byte-compiles the sources to the
 362   ;; prefix directory.  The check phase is moved after the installation phase
 363   ;; to ease testing the built package.
 364   (modify-phases python:%standard-phases
 365     (replace 'set-SOURCE-DATE-EPOCH set-SOURCE-DATE-EPOCH*)
 366     (replace 'build build)
 367     (replace 'install install)
 368     (delete 'check)
 369     ;; Must be before tests, so they can use installed packages’ entry points.
 370     (add-before 'wrap 'create-entrypoints create-entrypoints)
 371     (add-after 'wrap 'check check)
 372     (add-before 'check 'compile-bytecode compile-bytecode)))
 373
 374 (define* (pyproject-build #:key inputs (phases %standard-phases)
 375                           #:allow-other-keys #:rest args)
 376   "Build the given Python package, applying all of PHASES in order."
 377   (apply python:python-build #:inputs inputs #:phases phases args))
 378
 379 ;;; pyproject-build-system.scm ends here