guix/build/python-build-system.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2013, 2015, 2016, 2018 Ludovic Courtès <ludo@gnu.org>
   3 ;;; Copyright © 2013 Andreas Enge <andreas@enge.fr>
   4 ;;; Copyright © 2013 Nikita Karetnikov <nikita@karetnikov.org>
   5 ;;; Copyright © 2015, 2018 Mark H Weaver <mhw@netris.org>
   6 ;;; Copyright © 2016 Hartmut Goebel <h.goebel@crazy-compilers.com>
   7 ;;; Copyright © 2018 Ricardo Wurmus <rekado@elephly.net>
   8 ;;;
   9 ;;; This file is part of GNU Guix.
  10 ;;;
  11 ;;; GNU Guix is free software; you can redistribute it and/or modify it
  12 ;;; under the terms of the GNU General Public License as published by
  13 ;;; the Free Software Foundation; either version 3 of the License, or (at
  14 ;;; your option) any later version.
  15 ;;;
  16 ;;; GNU Guix is distributed in the hope that it will be useful, but
  17 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  18 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 ;;; GNU General Public License for more details.
  20 ;;;
  21 ;;; You should have received a copy of the GNU General Public License
  22 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  23
  24 (define-module (guix build python-build-system)
  25   #:use-module ((guix build gnu-build-system) #:prefix gnu:)
  26   #:use-module (guix build utils)
  27   #:use-module (ice-9 match)
  28   #:use-module (ice-9 ftw)
  29   #:use-module (srfi srfi-1)
  30   #:use-module (srfi srfi-26)
  31   #:export (%standard-phases
  32             add-installed-pythonpath
  33             site-packages
  34             python-build))
  35
  36 ;; Commentary:
  37 ;;
  38 ;; Builder-side code of the standard Python package build procedure.
  39 ;;
  40 ;;
  41 ;; Backgound about the Python installation methods
  42 ;;
  43 ;; In Python there are different ways to install packages: distutils,
  44 ;; setuptools, easy_install and pip.  All of these are sharing the file
  45 ;; setup.py, introduced with distutils in Python 2.0. The setup.py file can be
  46 ;; considered as a kind of Makefile accepting targets (or commands) like
  47 ;; "build" and "install".  As of autumn 2016 the recommended way to install
  48 ;; Python packages is using pip.
  49 ;;
  50 ;; For both distutils and setuptools, running "python setup.py install" is the
  51 ;; way to install Python packages.  With distutils the "install" command
  52 ;; basically copies all packages into <prefix>/lib/pythonX.Y/site-packages.
  53 ;;
  54 ;; Some time later "setuptools" was established to enhance distutils.  To use
  55 ;; setuptools, the developer imports setuptools in setup.py.  When importing
  56 ;; setuptools, the original "install" command gets overwritten by setuptools'
  57 ;; "install" command.
  58 ;;
  59 ;; The command-line tools easy_install and pip are both capable of finding and
  60 ;; downloading the package source from PyPI (the Python Package Index).  Both
  61 ;; of them import setuptools and execute the "setup.py" file under their
  62 ;; control.  Thus the "setup.py" behaves as if the developer had imported
  63 ;; setuptools within setup.py - even is still using only distutils.
  64 ;;
  65 ;; Setuptools' "install" command (to be more precise: the "easy_install"
  66 ;; command which is called by "install") will put the path of the currently
  67 ;; installed version of each package and it's dependencies (as declared in
  68 ;; setup.py) into an "easy-install.pth" file.  In Guix each packages gets its
  69 ;; own "site-packages" directory and thus an "easy-install.pth" of its own.
  70 ;; To avoid conflicts, the python build system renames the file to
  71 ;; <packagename>.pth in the phase rename-pth-file.  To ensure that Python will
  72 ;; process the .pth file, easy_install also creates a basic "site.py" in each
  73 ;; "site-packages" directory.  The file is the same for all packages, thus
  74 ;; there is no need to rename it.  For more information about .pth files and
  75 ;; the site module, please refere to
  76 ;; https://docs.python.org/3/library/site.html.
  77 ;;
  78 ;; The .pth files contain the file-system paths (pointing to the store) of all
  79 ;; dependencies.  So the dependency is hidden in the .pth file but is not
  80 ;; visible in the file-system.  Now if packages A and B both required packages
  81 ;; P, but in different versions, Guix will not detect this when installing
  82 ;; both A and B to a profile. (For details and example see
  83 ;; https://lists.gnu.org/archive/html/guix-devel/2016-10/msg01233.html.)
  84 ;;
  85 ;; Pip behaves a bit different then easy_install: it always executes
  86 ;; "setup.py" with the option "--single-version-externally-managed" set.  This
  87 ;; makes setuptools' "install" command run the original "install" command
  88 ;; instead of the "easy_install" command, so no .pth file (and no site.py)
  89 ;; will be created.  The "site-packages" directory only contains the package
  90 ;; and the related .egg-info directory.
  91 ;;
  92 ;; This is exactly what we need for Guix and this is what we mimic in the
  93 ;; install phase below.
  94 ;;
  95 ;; As a draw back, the magic of the .pth file of linking to the other required
  96 ;; packages is gone and these packages have now to be declared as
  97 ;; "propagated-inputs".
  98 ;;
  99 ;; Note: Importing setuptools also adds two sub-commands: "install_egg_info"
 100 ;; and "install_scripts".  These sub-commands are executed even if
 101 ;; "--single-version-externally-managed" is set, thus the .egg-info directory
 102 ;; and the scripts defined in entry-points will always be created.
 103
 104
 105 (define setuptools-shim
 106   ;; Run setup.py with "setuptools" being imported, which will patch
 107   ;; "distutils". This is needed for packages using "distutils" instead of
 108   ;; "setuptools" since the former does not understand the
 109   ;; "--single-version-externally-managed" flag.
 110   ;; Python code taken from pip 9.0.1 pip/utils/setuptools_build.py
 111   (string-append
 112    "import setuptools, tokenize;__file__='setup.py';"
 113    "f=getattr(tokenize, 'open', open)(__file__);"
 114    "code=f.read().replace('\\r\\n', '\\n');"
 115    "f.close();"
 116    "exec(compile(code, __file__, 'exec'))"))
 117
 118 (define (call-setuppy command params use-setuptools?)
 119   (if (file-exists? "setup.py")
 120       (begin
 121          (format #t "running \"python setup.py\" with command ~s and parameters ~s~%"
 122                 command params)
 123          (if use-setuptools?
 124              (apply invoke "python" "-c" setuptools-shim
 125                     command params)
 126              (apply invoke "python" "./setup.py" command params)))
 127       (error "no setup.py found")))
 128
 129 (define* (build #:key use-setuptools? #:allow-other-keys)
 130   "Build a given Python package."
 131   (call-setuppy "build" '() use-setuptools?)
 132   #t)
 133
 134 (define* (check #:key tests? test-target use-setuptools? #:allow-other-keys)
 135   "Run the test suite of a given Python package."
 136   (if tests?
 137       ;; Running `setup.py test` creates an additional .egg-info directory in
 138       ;; build/lib in some cases, e.g. if the source is in a sub-directory
 139       ;; (given with `package_dir`). This will by copied to the output, too,
 140       ;; so we need to remove.
 141       (let ((before (find-files "build" "\\.egg-info$" #:directories? #t)))
 142         (call-setuppy test-target '() use-setuptools?)
 143         (let* ((after (find-files "build" "\\.egg-info$" #:directories? #t))
 144                (inter (lset-difference string=? after before)))
 145           (for-each delete-file-recursively inter)))
 146       (format #t "test suite not run~%"))
 147   #t)
 148
 149 (define (get-python-version python)
 150   (let* ((version     (last (string-split python #\-)))
 151          (components  (string-split version #\.))
 152          (major+minor (take components 2)))
 153     (string-join major+minor ".")))
 154
 155 (define (site-packages inputs outputs)
 156   "Return the path of the current output's Python site-package."
 157   (let* ((out (assoc-ref outputs "out"))
 158          (python (assoc-ref inputs "python")))
 159     (string-append out "/lib/python"
 160                    (get-python-version python)
 161                    "/site-packages/")))
 162
 163 (define (add-installed-pythonpath inputs outputs)
 164   "Prepend the Python site-package of OUTPUT to PYTHONPATH.  This is useful
 165 when running checks after installing the package."
 166   (let ((old-path (getenv "PYTHONPATH"))
 167         (add-path (site-packages inputs outputs)))
 168     (setenv "PYTHONPATH"
 169             (string-append add-path
 170                            (if old-path (string-append ":" old-path) "")))
 171     #t))
 172
 173 (define* (install #:key outputs (configure-flags '()) use-setuptools?
 174                   #:allow-other-keys)
 175   "Install a given Python package."
 176   (let* ((out (assoc-ref outputs "out"))
 177          (params (append (list (string-append "--prefix=" out))
 178                          (if use-setuptools?
 179                              ;; distutils does not accept these flags
 180                              (list "--single-version-externally-managed"
 181                                     "--root=/")
 182                              '())
 183                          configure-flags)))
 184     (call-setuppy "install" params use-setuptools?)
 185     #t))
 186
 187 (define* (wrap #:key inputs outputs #:allow-other-keys)
 188   (define (list-of-files dir)
 189     (map (cut string-append dir "/" <>)
 190          (or (scandir dir (lambda (f)
 191                             (let ((s (stat (string-append dir "/" f))))
 192                               (eq? 'regular (stat:type s)))))
 193              '())))
 194
 195   (define bindirs
 196     (append-map (match-lambda
 197                  ((_ . dir)
 198                   (list (string-append dir "/bin")
 199                         (string-append dir "/sbin"))))
 200                 outputs))
 201
 202   (let* ((out  (assoc-ref outputs "out"))
 203          (python (assoc-ref inputs "python"))
 204          (var `("PYTHONPATH" prefix
 205                 ,(cons (string-append out "/lib/python"
 206                                       (get-python-version python)
 207                                       "/site-packages")
 208                        (search-path-as-string->list
 209                         (or (getenv "PYTHONPATH") ""))))))
 210     (for-each (lambda (dir)
 211                 (let ((files (list-of-files dir)))
 212                   (for-each (cut wrap-program <> var)
 213                             files)))
 214               bindirs)
 215     #t))
 216
 217 (define* (rename-pth-file #:key name inputs outputs #:allow-other-keys)
 218   "Rename easy-install.pth to NAME.pth to avoid conflicts between packages
 219 installed with setuptools."
 220   ;; Even if the "easy-install.pth" is not longer created, we kept this phase.
 221   ;; There still may be packages creating an "easy-install.pth" manually for
 222   ;; some good reason.
 223   (let* ((out (assoc-ref outputs "out"))
 224          (python (assoc-ref inputs "python"))
 225          (site-packages (string-append out "/lib/python"
 226                                        (get-python-version python)
 227                                        "/site-packages"))
 228          (easy-install-pth (string-append site-packages "/easy-install.pth"))
 229          (new-pth (string-append site-packages "/" name ".pth")))
 230     (when (file-exists? easy-install-pth)
 231       (rename-file easy-install-pth new-pth))
 232     #t))
 233
 234 (define* (ensure-no-mtimes-pre-1980 #:rest _)
 235   "Ensure that there are no mtimes before 1980-01-02 in the source tree."
 236   ;; Rationale: patch-and-repack creates tarballs with timestamps at the POSIX
 237   ;; epoch, 1970-01-01 UTC.  This causes problems with Python packages,
 238   ;; because Python eggs are ZIP files, and the ZIP format does not support
 239   ;; timestamps before 1980.
 240   (let ((early-1980 315619200))  ; 1980-01-02 UTC
 241     (ftw "." (lambda (file stat flag)
 242                (unless (<= early-1980 (stat:mtime stat))
 243                  (utime file early-1980 early-1980))
 244                #t))
 245     #t))
 246
 247 (define* (enable-bytecode-determinism #:rest _)
 248   "Improve determinism of pyc files."
 249   ;; Use deterministic hashes for strings, bytes, and datetime objects.
 250   (setenv "PYTHONHASHSEED" "0")
 251   #t)
 252
 253 (define %standard-phases
 254   ;; 'configure' phase is not needed.
 255   (modify-phases gnu:%standard-phases
 256     (add-after 'unpack 'ensure-no-mtimes-pre-1980 ensure-no-mtimes-pre-1980)
 257     (add-after 'ensure-no-mtimes-pre-1980 'enable-bytecode-determinism
 258       enable-bytecode-determinism)
 259     (delete 'bootstrap)
 260     (delete 'configure)
 261     (replace 'install install)
 262     (replace 'check check)
 263     (replace 'build build)
 264     (add-after 'install 'wrap wrap)
 265     (add-before 'strip 'rename-pth-file rename-pth-file)))
 266
 267 (define* (python-build #:key inputs (phases %standard-phases)
 268                        #:allow-other-keys #:rest args)
 269   "Build the given Python package, applying all of PHASES in order."
 270   (apply gnu:gnu-build #:inputs inputs #:phases phases args))
 271
 272 ;;; python-build-system.scm ends here