gnu/packages/python.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2013 Nikita Karetnikov <nikita@karetnikov.org>
   3 ;;; Copyright © 2013, 2014, 2015, 2016, 2017, 2018, 2021 Ludovic Courtès <ludo@gnu.org>
   4 ;;; Copyright © 2013, 2014, 2015, 2016 Andreas Enge <andreas@enge.fr>
   5 ;;; Copyright © 2014, 2015 Mark H Weaver <mhw@netris.org>
   6 ;;; Copyright © 2014, 2017, 2019 Eric Bavier <bavier@member.fsf.org>
   7 ;;; Copyright © 2014, 2015 Federico Beffa <beffa@fbengineering.ch>
   8 ;;; Copyright © 2015 Omar Radwan <toxemicsquire4@gmail.com>
   9 ;;; Copyright © 2015 Pierre-Antoine Rault <par@rigelk.eu>
  10 ;;; Copyright © 2015, 2016, 2017, 2018, 2019 Ricardo Wurmus <rekado@elephly.net>
  11 ;;; Copyright © 2015, 2016 Christine Lemmer-Webber <cwebber@dustycloud.org>
  12 ;;; Copyright © 2015 Eric Dvorsak <eric@dvorsak.fr>
  13 ;;; Copyright © 2015, 2016 David Thompson <davet@gnu.org>
  14 ;;; Copyright © 2015, 2016, 2017, 2021 Leo Famulari <leo@famulari.name>
  15 ;;; Copyright © 2015, 2017 Ben Woodcroft <donttrustben@gmail.com>
  16 ;;; Copyright © 2015, 2016 Erik Edrosa <erik.edrosa@gmail.com>
  17 ;;; Copyright © 2015, 2016, 2017, 2018, 2019, 2020 Efraim Flashner <efraim@flashner.co.il>
  18 ;;; Copyright © 2015, 2017 Kyle Meyer <kyle@kyleam.com>
  19 ;;; Copyright © 2015, 2016 Chris Marusich <cmmarusich@gmail.com>
  20 ;;; Copyright © 2016 Danny Milosavljevic <dannym+a@scratchpost.org>
  21 ;;; Copyright © 2016 Lukas Gradl <lgradl@openmailbox.org>
  22 ;;; Copyright © 2016, 2018 Hartmut Goebel <h.goebel@crazy-compilers.com>
  23 ;;; Copyright © 2016 Daniel Pimentel <d4n1@d4n1.org>
  24 ;;; Copyright © 2016 Sou Bunnbu <iyzsong@gmail.com>
  25 ;;; Copyright © 2016, 2017 Troy Sankey <sankeytms@gmail.com>
  26 ;;; Copyright © 2016, 2017 Nikita <nikita@n0.is>
  27 ;;; Copyright © 2016 Dylan Jeffers <sapientech@sapientech@openmailbox.org>
  28 ;;; Copyright © 2016 David Craven <david@craven.ch>
  29 ;;; Copyright © 2016, 2017, 2018, 2019, 2020, 2021 Marius Bakke <marius@gnu.org>
  30 ;;; Copyright © 2016, 2017 Stefan Reichör <stefan@xsteve.at>
  31 ;;; Copyright © 2016, 2017 Alex Vong <alexvong1995@gmail.com>
  32 ;;; Copyright © 2016, 2017, 2018 Arun Isaac <arunisaac@systemreboot.net>
  33 ;;; Copyright © 2016–2018, 2021 Tobias Geerinckx-Rice <me@tobias.gr>
  34 ;;; Copyright © 2016, 2017, 2018, 2021 Julien Lepiller <julien@lepiller.eu>
  35 ;;; Copyright © 2016, 2017 Thomas Danckaert <post@thomasdanckaert.be>
  36 ;;; Copyright © 2017 Carlo Zancanaro <carlo@zancanaro.id.au>
  37 ;;; Copyright © 2017 Frederick M. Muriithi <fredmanglis@gmail.com>
  38 ;;; Copyright © 2017, 2018 Adriano Peluso <catonano@gmail.com>
  39 ;;; Copyright © 2017 Ben Sturmfels <ben@sturm.com.au>
  40 ;;; Copyright © 2017, 2018, 2019 Mathieu Othacehe <m.othacehe@gmail.com>
  41 ;;; Copyright © 2017 José Miguel Sánchez García <jmi2k@openmailbox.org>
  42 ;;; Copyright © 2017 Roel Janssen <roel@gnu.org>
  43 ;;; Copyright © 2017, 2018 Kei Kebreau <kkebreau@posteo.net>
  44 ;;; Copyright © 2017 Rutger Helling <rhelling@mykolab.com>
  45 ;;; Copyright © 2017 Brendan Tildesley <mail@brendan.scot>
  46 ;;; Copyright © 2018 Ethan R. Jones <ethanrjones97@gmail.com
  47 ;;; Copyright © 2018 Fis Trivial <ybbs.daans@hotmail.com>
  48 ;;; Copyright © 2018 Vijayalakshmi Vedantham <vijimay12@gmail.com>
  49 ;;; Copyright © 2018 Mathieu Lirzin <mthl@gnu.org>
  50 ;;; Copyright © 2018 Adam Massmann <massmannak@gmail.com>
  51 ;;; Copyright © 2016, 2018 Tomáš Čech <sleep_walker@gnu.org>
  52 ;;; Copyright © 2018 Nicolas Goaziou <mail@nicolasgoaziou.fr>
  53 ;;; Copyright © 2018 Oleg Pykhalov <go.wigust@gmail.com>
  54 ;;; Copyright © 2018 Clément Lassieur <clement@lassieur.org>
  55 ;;; Copyright © 2018, 2019, 2020, 2021 Maxim Cournoyer <maxim.cournoyer@gmail.com>
  56 ;;; Copyright © 2018 Luther Thompson <lutheroto@gmail.com>
  57 ;;; Copyright © 2018 Vagrant Cascadian <vagrant@debian.org>
  58 ;;; Copyright © 2019 Tanguy Le Carrour <tanguy@bioneland.org>
  59 ;;; Copyright © 2020 Jan (janneke) Nieuwenhuizen <janneke@gnu.org>
  60 ;;; Copyright © 2020, 2021 Greg Hogan <code@greghogan.com>
  61 ;;;
  62 ;;; This file is part of GNU Guix.
  63 ;;;
  64 ;;; GNU Guix is free software; you can redistribute it and/or modify it
  65 ;;; under the terms of the GNU General Public License as published by
  66 ;;; the Free Software Foundation; either version 3 of the License, or (at
  67 ;;; your option) any later version.
  68 ;;;
  69 ;;; GNU Guix is distributed in the hope that it will be useful, but
  70 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  71 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  72 ;;; GNU General Public License for more details.
  73 ;;;
  74 ;;; You should have received a copy of the GNU General Public License
  75 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  76
  77 (define-module (gnu packages python)
  78   #:use-module ((guix licenses) #:prefix license:)
  79   #:use-module (gnu packages)
  80   #:use-module (gnu packages base)
  81   #:use-module (gnu packages bash)
  82   #:use-module (gnu packages certs)
  83   #:use-module (gnu packages check)
  84   #:use-module (gnu packages compression)
  85   #:use-module (gnu packages dbm)
  86   #:use-module (gnu packages hurd)
  87   #:use-module (gnu packages libffi)
  88   #:use-module (gnu packages ncurses)
  89   #:use-module (gnu packages pkg-config)
  90   #:use-module (gnu packages python-xyz)
  91   #:use-module (gnu packages readline)
  92   #:use-module (gnu packages shells)
  93   #:use-module (gnu packages sqlite)
  94   #:use-module (gnu packages tcl)
  95   #:use-module (gnu packages tls)
  96   #:use-module (gnu packages xml)
  97   #:use-module (guix gexp)
  98   #:use-module (guix packages)
  99   #:use-module (guix download)
 100   #:use-module (guix utils)
 101   #:use-module (guix build-system gnu)
 102   #:use-module (guix build-system trivial)
 103   #:use-module (srfi srfi-1)
 104   #:use-module (srfi srfi-26)
 105
 106   #:export (customize-site
 107             guix-pythonpath-search-path))
 108
 109 (define* (customize-site version)
 110   "Generate a install-sitecustomize.py phase, using VERSION."
 111   `(lambda* (#:key native-inputs inputs outputs #:allow-other-keys)
 112      (let* ((out (assoc-ref outputs "out"))
 113             (site-packages (string-append
 114                             out "/lib/python"
 115                             ,(version-major+minor version)
 116                             "/site-packages"))
 117             (sitecustomize.py (assoc-ref (or native-inputs inputs)
 118                                          "sitecustomize.py"))
 119             (dest (string-append site-packages "/sitecustomize.py")))
 120        (mkdir-p site-packages)
 121        (copy-file sitecustomize.py dest)
 122        ;; Set the correct permissions on the installed file, else the byte
 123        ;; compilation phase fails with a permission denied error.
 124        (chmod dest #o644))))
 125
 126 (define (guix-pythonpath-search-path version)
 127   "Generate a GUIX_PYTHONPATH search path specification, using VERSION."
 128   (search-path-specification (variable "GUIX_PYTHONPATH")
 129                              (files (list (string-append
 130                                            "lib/python"
 131                                            (version-major+minor version)
 132                                            "/site-packages")))))
 133
 134 (define-public python-2.7
 135   (package
 136     (name "python2")
 137     (version "2.7.18")
 138     (source
 139      (origin
 140        (method url-fetch)
 141        (uri (string-append "https://www.python.org/ftp/python/"
 142                            version "/Python-" version ".tar.xz"))
 143        (sha256
 144         (base32
 145          "0hzgxl94hnflis0d6m4szjx0b52gah7wpmcg5g00q7am6xwhwb5n"))
 146        (patches (search-patches "python-2.7-search-paths.patch"
 147                                 "python-2-deterministic-build-info.patch"
 148                                 "python-2.7-site-prefixes.patch"
 149                                 "python-2.7-source-date-epoch.patch"
 150                                 "python-2.7-adjust-tests.patch"
 151                                 "python-cross-compile.patch"
 152                                 "python-2.7-CVE-2021-3177.patch"
 153                                 "python-2.7-no-static-lib.patch"))
 154        (modules '((guix build utils)))
 155        (snippet
 156         '(begin
 157            ;; Ensure the bundled copies of these libraries are not used.
 158            (for-each delete-file-recursively
 159                      '("Modules/_ctypes/libffi" "Modules/expat" "Modules/zlib"))
 160
 161            (substitute* "Modules/Setup.dist"
 162              ;; Link Expat instead of embedding the bundled one.
 163              (("^#pyexpat.*") "pyexpat pyexpat.c -lexpat\n"))
 164
 165            ;; Suboptimal to delete failing tests here, but if we delete them in
 166            ;; the arguments then we need to make sure to strip out that phase
 167            ;; when it gets inherited by python and python-minimal.
 168            (for-each delete-file
 169                      '("Lib/test/test_compileall.py"
 170                        "Lib/test/test_ctypes.py" ; fails on mips64el
 171                        "Lib/test/test_distutils.py"
 172                        "Lib/test/test_import.py"
 173                        "Lib/test/test_shutil.py"
 174                        "Lib/test/test_socket.py"
 175                        "Lib/test/test_subprocess.py"))))))
 176     (outputs '("out"
 177                "tk"                     ;tkinter; adds 50 MiB to the closure
 178                "idle"))                 ;programming environment; weighs 5MB
 179     (build-system gnu-build-system)
 180     (arguments
 181      `(#:test-target "test"
 182        #:configure-flags
 183        (list "--enable-shared"          ;allow embedding
 184              "--with-system-expat"      ;for XML support
 185              "--with-system-ffi"        ;build ctypes
 186              "--with-ensurepip=install" ;install pip and setuptools
 187              "--with-computed-gotos"    ;main interpreter loop optimization
 188              "--enable-unicode=ucs4"
 189
 190              ;; FIXME: These flags makes Python significantly faster, but
 191              ;; leads to non-reproducible binaries.
 192              ;; "--with-lto"               ;increase size by 20MB, but 15% speedup
 193              ;; "--enable-optimizations"
 194
 195              ;; Prevent the installed _sysconfigdata.py from retaining a reference
 196              ;; to coreutils.
 197              "INSTALL=install -c"
 198              "MKDIR_P=mkdir -p"
 199
 200              ;; Disable runtime check failing if cross-compiling, see:
 201              ;; https://lists.yoctoproject.org/pipermail/poky/2013-June/008997.html
 202              ,@(if (%current-target-system)
 203                    '("ac_cv_buggy_getaddrinfo=no"
 204                      "ac_cv_file__dev_ptmx=no"
 205                      "ac_cv_file__dev_ptc=no")
 206                    '())
 207              ;; -fno-semantic-interposition reinstates some optimizations by gcc
 208              ;; leading to around 15% speedup. This is the default starting from
 209              ;; python 3.10.
 210              "CFLAGS=-fno-semantic-interposition"
 211              (string-append "LDFLAGS=-Wl,-rpath="
 212                             (assoc-ref %outputs "out") "/lib"
 213                             " -fno-semantic-interposition"))
 214        ;; With no -j argument tests use all available cpus, so provide one.
 215        #:make-flags
 216        (list (string-append
 217               (format #f "TESTOPTS=-j~d" (parallel-job-count))
 218               ;; Exclude the following tests as they fail
 219               ;; non-deterministically with "error: [Errno 104] Connection
 220               ;; reset by peer."  Python 3 seems unaffected.  A potential fix,
 221               ;; yet to be backported to Python 2, is available at:
 222               ;; https://github.com/python/cpython/commit/529525fb5a8fd9b96ab4021311a598c77588b918.
 223               " --exclude test_urllib2_localnet test_httplib"))
 224
 225        #:modules ((ice-9 ftw) (ice-9 match)
 226                   (guix build utils) (guix build gnu-build-system))
 227        #:phases
 228        (modify-phases %standard-phases
 229          (add-before
 230              'configure 'patch-lib-shells
 231            (lambda _
 232              ;; This variable is used in setup.py to enable cross compilation
 233              ;; specific switches. As it is not set properly by configure
 234              ;; script, set it manually.
 235              ,@(if (%current-target-system)
 236                    '((setenv "_PYTHON_HOST_PLATFORM" ""))
 237                    '())
 238              ;; Filter for existing files, since some may not exist in all
 239              ;; versions of python that are built with this recipe.
 240              (substitute* (filter file-exists?
 241                                   '("Lib/subprocess.py"
 242                                     "Lib/popen2.py"
 243                                     "Lib/distutils/tests/test_spawn.py"
 244                                     "Lib/test/support/__init__.py"
 245                                     "Lib/test/test_subprocess.py"))
 246                (("/bin/sh") (which "sh")))))
 247          ,@(if (hurd-system?)
 248                `((add-before 'build 'patch-regen-for-hurd
 249                    (lambda* (#:key inputs #:allow-other-keys)
 250                      (let ((libc (assoc-ref inputs "libc")))
 251                        (substitute* "Lib/plat-generic/regen"
 252                          (("/usr/include/") (string-append libc "/include/")))))))
 253                '())
 254          (add-before 'configure 'do-not-record-configure-flags
 255            (lambda* (#:key configure-flags #:allow-other-keys)
 256              ;; Remove configure flags from the installed '_sysconfigdata.py'
 257              ;; and 'Makefile' so we don't end up keeping references to the
 258              ;; build tools.
 259              ;;
 260              ;; Preserve at least '--with-system-ffi' since otherwise the
 261              ;; thing tries to build libffi, fails, and we end up with a
 262              ;; Python that lacks ctypes.
 263              (substitute* "configure"
 264                (("^CONFIG_ARGS=.*$")
 265                 (format #f "CONFIG_ARGS='~a'\n"
 266                         (if (member "--with-system-ffi" configure-flags)
 267                             "--with-system-ffi"
 268                             ""))))))
 269          (add-before 'check 'pre-check
 270            (lambda _
 271              ;; 'Lib/test/test_site.py' needs a valid $HOME
 272              (setenv "HOME" (getcwd))))
 273          (add-after 'unpack 'set-source-file-times-to-1980
 274            ;; XXX One of the tests uses a ZIP library to pack up some of the
 275            ;; source tree, and fails with "ZIP does not support timestamps
 276            ;; before 1980".  Work around this by setting the file times in the
 277            ;; source tree to sometime in early 1980.
 278            (lambda _
 279              (let ((circa-1980 (* 10 366 24 60 60)))
 280                (ftw "." (lambda (file stat flag)
 281                           (utime file circa-1980 circa-1980)
 282                           #t)))))
 283          (add-after 'install 'remove-tests
 284            ;; Remove 25 MiB of unneeded unit tests.  Keep test_support.*
 285            ;; because these files are used by some libraries out there.
 286            (lambda* (#:key outputs #:allow-other-keys)
 287              (let ((out (assoc-ref outputs "out")))
 288                (match (scandir (string-append out "/lib")
 289                                (lambda (name)
 290                                  (string-prefix? "python" name)))
 291                  ((pythonX.Y)
 292                   (let ((testdir (string-append out "/lib/" pythonX.Y
 293                                                 "/test")))
 294                     (with-directory-excursion testdir
 295                       (for-each delete-file-recursively
 296                                 (scandir testdir
 297                                          (match-lambda
 298                                            ((or "." "..") #f)
 299                                            ("support" #f)
 300                                            (file
 301                                             (not
 302                                              (string-prefix? "test_support."
 303                                                              file))))))
 304                       (call-with-output-file "__init__.py" (const #t))))
 305                   (let ((libdir (string-append out "/lib/" pythonX.Y)))
 306                     (for-each
 307                      (lambda (directory)
 308                        (let ((dir (string-append libdir "/" directory)))
 309                          (when (file-exists? dir)
 310                            (delete-file-recursively dir))))
 311                      '("email/test" "ctypes/test" "unittest/test" "tkinter/test"
 312                        "sqlite3/test" "bsddb/test" "lib-tk/test" "json/tests"
 313                        "distutils/tests"))))))))
 314          (add-after 'remove-tests 'move-tk-inter
 315            (lambda* (#:key outputs #:allow-other-keys)
 316              ;; When Tkinter support is built move it to a separate output so
 317              ;; that the main output doesn't contain a reference to Tcl/Tk.
 318              (let ((out (assoc-ref outputs "out"))
 319                    (tk  (assoc-ref outputs "tk")))
 320                (when tk
 321                  (match (find-files out "tkinter.*\\.so")
 322                    ((tkinter.so)
 323                     ;; The .so is in OUT/lib/pythonX.Y/lib-dynload, but we
 324                     ;; want it under TK/lib/pythonX.Y/site-packages.
 325                     (let* ((len    (string-length out))
 326                            (target (string-append
 327                                     tk "/"
 328                                     (string-drop
 329                                      (dirname (dirname tkinter.so))
 330                                      len)
 331                                     "/site-packages")))
 332                       (install-file tkinter.so target)
 333                       (delete-file tkinter.so))))))))
 334          (add-after 'move-tk-inter 'move-idle
 335            (lambda* (#:key outputs #:allow-other-keys)
 336              ;; when idle is built, move it to a separate output to save some
 337              ;; space (5MB)
 338              (let ((out (assoc-ref outputs "out"))
 339                    (idle (assoc-ref outputs "idle")))
 340                (when idle
 341                  (for-each
 342                   (lambda (file)
 343                     (let ((target (string-append idle "/bin/" (basename file))))
 344                       (install-file file (dirname target))
 345                       (delete-file file)))
 346                   (find-files (string-append out "/bin") "^idle"))
 347                  (match (find-files out "^idlelib$" #:directories? #t)
 348                    ((idlelib)
 349                     (let* ((len (string-length out))
 350                            (target (string-append idle "/"
 351                                                   (string-drop idlelib len)
 352                                                   "/site-packages")))
 353                       (mkdir-p (dirname target))
 354                       (rename-file idlelib target))))))))
 355          (add-after 'move-idle 'rebuild-bytecode
 356            (lambda* (#:key outputs #:allow-other-keys)
 357              (let ((out (assoc-ref outputs "out")))
 358                ;; Disable hash randomization to ensure the generated .pycs
 359                ;; are reproducible.
 360                (setenv "PYTHONHASHSEED" "0")
 361                (for-each
 362                 (lambda (output)
 363                   (for-each (lambda (opt)
 364                               (format #t "Compiling with optimization level: ~a\n"
 365                                       (if (null? opt) "none" (car opt)))
 366                               (apply invoke
 367                                      `(,,(if (%current-target-system)
 368                                              "python2"
 369                                              '(string-append out "/bin/python"))
 370                                        ,@opt
 371                                        "-m" "compileall"
 372                                        "-f" ; force rebuild
 373                                        ;; Don't build lib2to3, because it contains
 374                                        ;; Python 3 code.
 375                                        "-x" "lib2to3/.*"
 376                                        ,output)))
 377                             ;; Python 2 has a single file extension (.pyo) for the
 378                             ;; chosen level of optimization, so it doesn't make
 379                             ;; sense to byte compile with more than one level.
 380                             (list '() '("-OO"))))
 381                 (map cdr outputs)))))
 382          (add-after 'install 'install-sitecustomize.py
 383            ,(customize-site version)))))
 384     (inputs
 385      (list bzip2
 386            expat
 387            gdbm
 388            libffi ; for ctypes
 389            sqlite ; for sqlite extension
 390            openssl
 391            readline
 392            zlib
 393            tcl
 394            tk))                     ; for tkinter
 395     (native-inputs
 396      `(("pkg-config" ,pkg-config)
 397        ("sitecustomize.py" ,(local-file (search-auxiliary-file
 398                                          "python/sitecustomize.py")))
 399        ;; When cross-compiling, a native version of Python itself is needed.
 400        ,@(if (%current-target-system)
 401              `(("python2" ,this-package)
 402                ("which" ,which))
 403              '())))
 404     (native-search-paths
 405      (list (guix-pythonpath-search-path version)))
 406     (home-page "https://www.python.org")
 407     (synopsis "High-level, dynamically-typed programming language")
 408     (description
 409      "Python is a remarkably powerful dynamic programming language that
 410 is used in a wide variety of application domains.  Some of its key
 411 distinguishing features include: clear, readable syntax; strong
 412 introspection capabilities; intuitive object orientation; natural
 413 expression of procedural code; full modularity, supporting hierarchical
 414 packages; exception-based error handling; and very high level dynamic
 415 data types.")
 416     (properties '((cpe-name . "python")))
 417     (license license:psfl)))
 418
 419 ;; Current 2.x version.
 420 (define-public python-2 python-2.7)
 421
 422 (define-public python2-called-python
 423   ;; Both 2.x and 3.x used to be called "python".  In commit
 424   ;; a7714d42de2c3082f3609d1e63c83d703fb39cf9 (March 2018), we renamed the
 425   ;; Python 2.x package to "python2".
 426   (package/inherit python-2
 427     (name "python")
 428     (properties `((superseded . ,python-2)))))
 429
 430 (define-public python-3.9
 431   (package
 432     (inherit python-2)
 433     (name "python")
 434     (version "3.9.9")
 435     (source (origin
 436               (method url-fetch)
 437               (uri (string-append "https://www.python.org/ftp/python/"
 438                                   version "/Python-" version ".tar.xz"))
 439               (patches (search-patches
 440                         "python-3-arm-alignment.patch"
 441                         "python-3-deterministic-build-info.patch"
 442                         "python-3-fix-tests.patch"
 443                         "python-3-hurd-configure.patch"
 444                         "python-3-search-paths.patch"
 445                         "python-3-no-static-lib.patch"))
 446               (sha256
 447                (base32
 448                 "09vd7g71i11iz5ydqghwc8kaxr0vgji94hhwwnj77h3kll28r0h6"))
 449               (modules '((guix build utils)))
 450               (snippet
 451                '(begin
 452                   ;; Delete the bundled copy of libexpat.
 453                   (delete-file-recursively "Modules/expat")
 454                   (substitute* "Modules/Setup"
 455                     ;; Link Expat instead of embedding the bundled one.
 456                     (("^#pyexpat.*") "pyexpat pyexpat.c -lexpat\n"))
 457                   ;; Delete windows binaries
 458                   (for-each delete-file
 459                             (find-files "Lib/distutils/command" "\\.exe$"))))))
 460     (arguments
 461      (substitute-keyword-arguments (package-arguments python-2)
 462        ((#:make-flags _)
 463         `(list (string-append
 464                 (format #f "TESTOPTS=-j~d" (parallel-job-count))
 465                 ;; test_mmap fails on low-memory systems
 466                 " --exclude test_mmap test_socket"
 467                 ,@(if (hurd-target?)
 468                       '(" test_posix"      ;multiple errors
 469                         " test_time"
 470                         " test_pty"
 471                         " test_shutil"
 472                         " test_tempfile"   ;chflags: invalid argument:
 473                                            ;  tbv14c9t/dir0/dir0/dir0/test0.txt
 474                         " test_asyncio"    ;runs over 10min
 475                         " test_os"         ;stty: 'standard input':
 476                                            ;  Inappropriate ioctl for device
 477                         " test_openpty"    ;No such file or directory
 478                         " test_selectors"  ;assertEqual(NUM_FDS // 2, len(fds))
 479                                            ;  32752 != 4
 480                         " test_compileall" ;multiple errors
 481                         " test_poll"       ;list index out of range
 482                         " test_subprocess" ;runs over 10min
 483                         " test_asyncore"   ;multiple errors
 484                         " test_threadsignals"
 485                         " test_eintr"      ;Process return code is -14
 486                         " test_io"         ;multiple errors
 487                         " test_logging"
 488                         " test_signal"
 489                         " test_threading"  ;runs over 10min
 490                         " test_flags"      ;ERROR
 491                         " test_bidirectional_pty"
 492                         " test_create_unix_connection"
 493                         " test_unix_sock_client_ops"
 494                         " test_open_unix_connection"
 495                         " test_open_unix_connection_error"
 496                         " test_read_pty_output"
 497                         " test_write_pty")
 498                       '()))))
 499        ((#:phases phases)
 500         `(modify-phases ,phases
 501            ,@(if (hurd-system?)
 502                  `((delete 'patch-regen-for-hurd)) ;regen was removed after 3.5.9
 503                  '())
 504            (add-after 'unpack 'remove-windows-binaries
 505              (lambda _
 506                ;; Delete .exe from embedded .whl (zip) files
 507                (for-each
 508                 (lambda (whl)
 509                   (let ((dir "whl-content")
 510                         (circa-1980 (* 10 366 24 60 60)))
 511                     (mkdir-p dir)
 512                     (with-directory-excursion dir
 513                       (let ((whl (string-append "../" whl)))
 514                         (invoke "unzip" whl)
 515                         (for-each delete-file
 516                                   (find-files "." "\\.exe$"))
 517                         (delete-file whl)
 518                         ;; Reset timestamps to prevent them from ending
 519                         ;; up in the Zip archive.
 520                         (ftw "." (lambda (file stat flag)
 521                                    (utime file circa-1980 circa-1980)
 522                                    #t))
 523                         (apply invoke "zip" "-X" whl
 524                                (find-files "." #:directories? #t))))
 525                     (delete-file-recursively dir)))
 526                 (find-files "Lib/ensurepip" "\\.whl$"))))
 527            (add-before 'check 'set-TZDIR
 528              (lambda* (#:key inputs native-inputs #:allow-other-keys)
 529                ;; test_email requires the Olson time zone database.
 530                (setenv "TZDIR"
 531                        (string-append (assoc-ref
 532                                        (or native-inputs inputs) "tzdata")
 533                                       "/share/zoneinfo"))))
 534            (replace 'rebuild-bytecode
 535              (lambda* (#:key outputs #:allow-other-keys)
 536                (let ((out (assoc-ref outputs "out")))
 537                  ;; Disable hash randomization to ensure the generated .pycs
 538                  ;; are reproducible.
 539                  (setenv "PYTHONHASHSEED" "0")
 540
 541                  (for-each (lambda (output)
 542                              ;; XXX: Delete existing pycs generated by the build
 543                              ;; system beforehand because the -f argument does
 544                              ;; not necessarily overwrite all files, leading to
 545                              ;; indeterministic results.
 546                              (for-each (lambda (pyc)
 547                                          (delete-file pyc))
 548                                        (find-files output "\\.pyc$"))
 549
 550                              (apply invoke
 551                                     `(,,(if (%current-target-system)
 552                                             "python3"
 553                                             '(string-append out
 554                                                             "/bin/python3"))
 555                                       "-m" "compileall"
 556                                       "-o" "0" "-o" "1" "-o" "2"
 557                                       "-f" ; force rebuild
 558                                       "--invalidation-mode=unchecked-hash"
 559                                       ;; Don't build lib2to3, because it's
 560                                       ;; Python 2 code.
 561                                       "-x" "lib2to3/.*"
 562                                       ,output)))
 563                            (map cdr outputs)))))
 564            (replace 'install-sitecustomize.py
 565              ,(customize-site version))))))
 566     (native-inputs
 567      `(("tzdata" ,tzdata-for-tests)
 568        ("unzip" ,unzip)
 569        ("zip" ,(@ (gnu packages compression) zip))
 570        ,@(if (%current-target-system)
 571              `(("python3" ,this-package))
 572              '())
 573        ,@(package-native-inputs python-2)))
 574     (native-search-paths
 575      (list (guix-pythonpath-search-path version)
 576            ;; Used to locate tzdata by the zoneinfo module introduced in
 577            ;; Python 3.9.
 578            (search-path-specification
 579             (variable "PYTHONTZPATH")
 580             (files (list "share/zoneinfo")))))))
 581
 582 ;; Current 3.x version.
 583 (define-public python-3 python-3.9)
 584
 585 ;; Current major version.
 586 (define-public python python-3)
 587
 588 ;; Minimal variants of Python, mostly used to break the cycle between Tk and
 589 ;; Python (Tk -> libxcb -> Python.)
 590
 591 (define-public python2-minimal
 592   (package/inherit python-2
 593     (name "python2-minimal")
 594     (outputs '("out"))
 595
 596     ;; Keep zlib, which is used by 'pip' (via the 'zipimport' module), which
 597     ;; is invoked upon 'make install'.  'pip' also expects 'ctypes' and thus
 598     ;; libffi.  Expat is needed for XML support which is expected by a lot
 599     ;; of libraries out there.
 600     (inputs `(("expat" ,expat)
 601               ("libffi" ,libffi)
 602               ("zlib" ,zlib)))))
 603
 604 (define-public python-minimal
 605   (package/inherit python
 606     (name "python-minimal")
 607     (outputs '("out"))
 608
 609     ;; Build fails due to missing ctypes without libffi.
 610     ;; OpenSSL is a mandatory dependency of Python 3.x, for urllib;
 611     ;; zlib is required by 'zipimport', used by pip.  Expat is needed
 612     ;; for XML support, which is generally expected to be available.
 613     (inputs `(("expat" ,expat)
 614               ("libffi" ,libffi)
 615               ("openssl" ,openssl)
 616               ("zlib" ,zlib)))))
 617
 618 (define-public python-debug
 619   (package/inherit python
 620     (name "python-debug")
 621     (outputs '("out" "debug"))
 622     (build-system gnu-build-system)
 623     (arguments
 624      (substitute-keyword-arguments (package-arguments python)
 625        ((#:configure-flags flags '())
 626         `(cons "--with-pydebug" ,flags))))
 627     (synopsis
 628      "High-level, dynamically-typed programming language (for debugging)")
 629     (description
 630      "This variant of Python provides an interpreter built with
 631 @code{--with-pydebug} to help develop and debug extensions.  See
 632 @url{https://pythonextensionpatterns.readthedocs.io/en/latest/debugging/debug.html},
 633 for more information.")))
 634
 635 (define* (wrap-python3 python
 636                        #:optional
 637                        (name (string-append (package-name python) "-wrapper")))
 638   (package/inherit python
 639     (name name)
 640     (source #f)
 641     (build-system trivial-build-system)
 642     (outputs '("out"))
 643     (inputs `(("bash" ,bash)))
 644     (propagated-inputs `(("python" ,python)))
 645     (arguments
 646      `(#:modules ((guix build utils))
 647        #:builder
 648          (begin
 649            (use-modules (guix build utils))
 650            (let ((bin (string-append (assoc-ref %outputs "out") "/bin"))
 651                  (python (string-append (assoc-ref %build-inputs "python") "/bin/")))
 652                 (mkdir-p bin)
 653                 (for-each
 654                   (lambda (old new)
 655                     (symlink (string-append python old)
 656                              (string-append bin "/" new)))
 657                   `("python3" ,"pydoc3" ,"pip3")
 658                   `("python"  ,"pydoc"  ,"pip"))
 659                 ;; python-config outputs search paths based upon its location,
 660                 ;; use a bash wrapper to avoid changing its outputs.
 661                 (let ((bash (string-append (assoc-ref %build-inputs "bash")
 662                                            "/bin/bash"))
 663                       (old  (string-append python "python3-config"))
 664                       (new  (string-append bin "/python-config")))
 665                   (with-output-to-file new
 666                     (lambda ()
 667                       (format #t "#!~a~%" bash)
 668                       (format #t "exec \"~a\" \"$@\"~%" old)
 669                       (chmod new #o755))))))))
 670     (synopsis "Wrapper for the Python 3 commands")
 671     (description
 672      "This package provides wrappers for the commands of Python@tie{}3.x such
 673 that they can be invoked under their usual name---e.g., @command{python}
 674 instead of @command{python3} or @command{pip} instead of @command{pip3}.
 675 To function properly, this package should not be installed together with the
 676 @command{python} package.")))
 677
 678 (define-public python-wrapper (wrap-python3 python))
 679 (define-public python-minimal-wrapper (wrap-python3 python-minimal))
 680
 681 (define-public micropython
 682   (package
 683     (name "micropython")
 684     (version "1.15")
 685     (source
 686       (origin
 687         (method url-fetch)
 688         (uri (string-append "https://github.com/micropython/micropython/"
 689                             "releases/download/v" version
 690                             "/micropython-" version ".tar.xz"))
 691         (sha256
 692          (base32 "04sfrfcljhfps340l4wh5ffwkhw1ydraday8nv92nv7gmnrj1l2j"))
 693       (modules '((guix build utils)))
 694       (snippet
 695        '(begin
 696           (delete-file-recursively "ports/cc3200/FreeRTOS")
 697           (with-directory-excursion "lib"
 698             ;; TODO: Unbundle axtls and berkley-db-1.xx
 699             (for-each delete-file-recursively
 700                       '("libffi" "lwip" "stm32lib" "nrfx")))
 701           #t))))
 702     (build-system gnu-build-system)
 703     (arguments
 704      `(#:phases
 705        (modify-phases %standard-phases
 706          (add-before 'build 'build-mpy-cross
 707            (lambda* (#:key make-flags #:allow-other-keys)
 708              (with-directory-excursion "mpy-cross"
 709                (apply invoke "make" make-flags))))
 710          (add-after 'build-mpy-cross 'prepare-build
 711            (lambda _
 712              (chdir "ports/unix")
 713              ;; see: https://github.com/micropython/micropython/pull/4246
 714              (substitute* "Makefile"
 715                (("-Os") "-Os -ffp-contract=off"))
 716              #t))
 717          (replace 'install-license-files
 718            ;; We don't build in the root directory so the file isn't found.
 719            (lambda* (#:key outputs #:allow-other-keys)
 720              (let* ((out  (assoc-ref outputs "out"))
 721                     (dest (string-append out "/share/doc/" ,name "-" ,version "/")))
 722                (install-file "../../LICENSE" dest))
 723              #t))
 724          (delete 'configure)) ; no configure
 725        #:make-flags (list (string-append "PREFIX=" (assoc-ref %outputs "out"))
 726                           "V=1")
 727        #:test-target "test"))
 728     (native-inputs
 729      `(("pkg-config" ,pkg-config)
 730        ("python" ,python-wrapper)))
 731     (inputs
 732      (list libffi))
 733     (home-page "https://micropython.org/")
 734     (synopsis "Python implementation for microcontrollers and constrained systems")
 735     (description "MicroPython is a lean and efficient implementation of the
 736 Python 3 programming language that includes a small subset of the Python
 737 standard library and is optimised to run on microcontrollers and in constrained
 738 environments.  MicroPython is packed full of advanced features such as an
 739 interactive prompt, arbitrary precision integers, closures, list comprehension,
 740 generators, exception handling and more.  Still it is compact enough to fit and
 741 run within just 256k of code space and 16k of RAM.  MicroPython aims to be as
 742 compatible with normal Python as possible to allow you to transfer code with
 743 ease from the desktop to a microcontroller or embedded system.")
 744     (license license:expat)))
 745
 746 (define-public pypy3
 747   (package
 748     (name "pypy3")
 749     (version "7.3.5")
 750     (source (origin
 751               (method url-fetch)
 752               (uri (string-append "https://downloads.python.org/pypy/"
 753                                   "pypy3.7-v" version "-src.tar.bz2"))
 754               (sha256
 755                (base32
 756                 "18lrdmpcczlbk3cfarkgwqdmilrybz56i1dafk8dkjlyk90gw86r"))))
 757     (build-system gnu-build-system)
 758     (native-inputs
 759      `(("python-2" ,python-2)
 760        ("pkg-config" ,pkg-config)
 761        ("tar" ,tar)                     ; Required for package.py
 762        ("python2-pycparser" ,python2-pycparser)
 763        ("python2-hypothesis" ,python2-hypothesis)
 764        ("nss-certs" ,nss-certs)         ; For ssl tests
 765        ("gzip" ,gzip)))
 766     (inputs
 767      (list libffi
 768            zlib
 769            ncurses
 770            openssl
 771            expat
 772            bzip2
 773            sqlite
 774            gdbm
 775            tcl
 776            tk
 777            glibc
 778            xz))                     ; liblzma
 779     (arguments
 780      `(#:tests? #f                     ;FIXME: 43 out of 364 tests are failing
 781        #:modules ((ice-9 ftw) (ice-9 match)
 782                   (guix build utils) (guix build gnu-build-system))
 783        #:disallowed-references (,nss-certs)
 784        #:phases (modify-phases %standard-phases
 785                   (delete 'configure)
 786                   (add-after 'unpack 'patch-source
 787                     (lambda* (#:key inputs outputs #:allow-other-keys)
 788                       (substitute* '("rpython/rlib/clibffi.py")
 789                         ;; find_library does not work for libc
 790                         (("ctypes\\.util\\.find_library\\('c'\\)") "'libc.so'"))
 791                       (substitute* '("lib_pypy/cffi/_pycparser/ply/cpp.py")
 792                         ;; Make reproducible (XXX: unused?)
 793                         (("time\\.localtime\\(\\)") "time.gmtime(0)"))
 794                       (substitute* '("pypy/module/sys/version.py")
 795                         ;; Make reproducible
 796                         (("t\\.gmtime\\(\\)") "t.gmtime(0)"))
 797                       (substitute* '("lib_pypy/_tkinter/tklib_build.py")
 798                         ;; Link to versioned libtcl and libtk
 799                         (("linklibs = \\['tcl', 'tk'\\]")
 800                          "linklibs = ['tcl8.6', 'tk8.6']")
 801                         (("incdirs = \\[\\]")
 802                          (string-append "incdirs = ['"
 803                                         (assoc-ref inputs "tcl")
 804                                         "/include', '"
 805                                         (assoc-ref inputs "tk")
 806                                         "/include']")))
 807                       (substitute* '("lib_pypy/_curses_build.py")
 808                         ;; Find curses
 809                         (("/usr/local") (assoc-ref inputs "ncurses")))
 810                       (substitute* '("lib_pypy/_dbm.py")
 811                         ;; Use gdbm compat library, so we don’t need to pull
 812                         ;; in bdb.
 813                         (("ctypes.util.find_library\\('db'\\)")
 814                          (format #f "'~a/lib/libgdbm_compat.so'"
 815                                  (assoc-ref inputs "gdbm"))))
 816                       (substitute* '("lib_pypy/_sqlite3_build.py")
 817                         ;; Always use search paths
 818                         (("sys\\.platform\\.startswith\\('freebsd'\\)") "True")
 819                         ;; Find sqlite3
 820                         (("/usr/local") (assoc-ref inputs "sqlite"))
 821                         (("libname = 'sqlite3'")
 822                          (string-append "libname = '"
 823                                         (assoc-ref inputs "sqlite")
 824                                         "/lib/libsqlite3.so.0'")))
 825                       (substitute* '("lib-python/3/subprocess.py")
 826                         ;; Fix shell path
 827                         (("/bin/sh")
 828                          (search-input-file inputs "/bin/sh")))
 829                       (substitute* '("lib-python/3/distutils/unixccompiler.py")
 830                         ;; gcc-toolchain does not provide symlink cc -> gcc
 831                         (("\"cc\"") "\"gcc\""))))
 832                   (add-after
 833                       'unpack 'set-source-file-times-to-1980
 834                     ;; copied from python package, required by zip testcase
 835                     (lambda _
 836                       (let ((circa-1980 (* 10 366 24 60 60)))
 837                         (ftw "." (lambda (file stat flag)
 838                                    (utime file circa-1980 circa-1980)
 839                                    #t)))))
 840                   (replace 'build
 841                     (lambda* (#:key inputs #:allow-other-keys)
 842                       (with-directory-excursion "pypy/goal"
 843                         ;; Build with jit optimization.
 844                         (invoke "python2"
 845                                 "../../rpython/bin/rpython"
 846                                 (string-append "--make-jobs="
 847                                                (number->string (parallel-job-count)))
 848                                 "-Ojit"
 849                                 "targetpypystandalone"
 850                                 "--allworkingmodules"))
 851                       ;; Build c modules and package everything, so tests work.
 852                       (with-directory-excursion "pypy/tool/release"
 853                         (invoke "python2" "package.py"
 854                                 "--archive-name" "pypy-dist"
 855                                 "--builddir" (getcwd)))))
 856                   (replace 'check
 857                     (lambda* (#:key tests? #:allow-other-keys)
 858                       (if tests?
 859                           (begin
 860                             (setenv "HOME" "/tmp") ; test_with_pip tries to
 861                                         ; access ~/.cache/pip
 862                             ;; Run library tests only (no interpreter unit
 863                             ;; tests). This is what Gentoo does.
 864                             (invoke
 865                              "python2"
 866                              "pypy/test_all.py"
 867                              "--pypy=pypy/tool/release/pypy-dist/bin/pypy3"
 868                              "lib-python"))
 869                           (format #t "test suite not run~%"))))
 870                   (replace 'install
 871                     (lambda* (#:key inputs outputs #:allow-other-keys)
 872                       (let* ((out (assoc-ref outputs "out"))
 873                              (bin-pypy3 (string-append out "/bin/pypy3"))
 874                              (shebang-match-python "#!.+/bin/python")
 875                              (shebang-pypy3 (string-append "#!" bin-pypy3))
 876                              (dist-dir "pypy/tool/release/pypy-dist"))
 877                         (with-directory-excursion dist-dir
 878                           ;; Delete test data.
 879                           (for-each
 880                            (lambda (x)
 881                              (delete-file-recursively (string-append
 882                                                        "lib-python/3/" x)))
 883                            '("tkinter/test"
 884                              "test"
 885                              "sqlite3/test"
 886                              "lib2to3/tests"
 887                              "idlelib/idle_test"
 888                              "distutils/tests"
 889                              "ctypes/test"
 890                              "unittest/test"))
 891                           ;; Patch shebang referencing python2
 892                           (substitute* '("lib-python/3/cgi.py"
 893                                          "lib-python/3/encodings/rot_13.py")
 894                             ((shebang-match-python) shebang-pypy3))
 895                           (with-fluids ((%default-port-encoding "ISO-8859-1"))
 896                             (substitute* '("lib_pypy/_md5.py"
 897                                            "lib_pypy/_sha1.py")
 898                               ((shebang-match-python) shebang-pypy3))))
 899                         (copy-recursively dist-dir out)))))))
 900     (home-page "https://www.pypy.org/")
 901     (synopsis "Python implementation with just-in-time compilation")
 902     (description "PyPy is a faster, alternative implementation of the Python
 903 programming language employing a just-in-time compiler.  It supports most
 904 Python code natively, including C extensions.")
 905     (license (list license:expat     ; pypy itself; _pytest/
 906                    license:psfl      ; python standard library in lib-python/
 907                    license:asl2.0    ; dotviewer/font/ and some of lib-python/
 908                    license:gpl3+ ; ./rpython/rlib/rvmprof/src/shared/libbacktrace/dwarf2.*
 909                    license:bsd-3 ; lib_pypy/cffi/_pycparser/ply/
 910                    (license:non-copyleft
 911                     "http://www.unicode.org/copyright.html")))))
 912