tests/pypi.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2014 David Thompson <davet@gnu.org>
   3 ;;; Copyright © 2016 Ricardo Wurmus <rekado@elephly.net>
   4 ;;; Copyright © 2019 Maxim Cournoyer <maxim.cournoyer@gmail.com>
   5 ;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz>
   6 ;;; Copyright © 2022 Vivien Kraus <vivien@planete-kraus.eu>
   7 ;;;
   8 ;;; This file is part of GNU Guix.
   9 ;;;
  10 ;;; GNU Guix is free software; you can redistribute it and/or modify it
  11 ;;; under the terms of the GNU General Public License as published by
  12 ;;; the Free Software Foundation; either version 3 of the License, or (at
  13 ;;; your option) any later version.
  14 ;;;
  15 ;;; GNU Guix is distributed in the hope that it will be useful, but
  16 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  17 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  18 ;;; GNU General Public License for more details.
  19 ;;;
  20 ;;; You should have received a copy of the GNU General Public License
  21 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  22
  23 (define-module (test-pypi)
  24   #:use-module (guix import pypi)
  25   #:use-module (guix base32)
  26   #:use-module (guix memoization)
  27   #:use-module (guix utils)
  28   #:use-module (gcrypt hash)
  29   #:use-module (guix tests)
  30   #:use-module (guix build-system python)
  31   #:use-module ((guix build utils) #:select (delete-file-recursively which mkdir-p))
  32   #:use-module ((guix diagnostics) #:select (guix-warning-port))
  33   #:use-module (json)
  34   #:use-module (srfi srfi-26)
  35   #:use-module (srfi srfi-34)
  36   #:use-module (srfi srfi-35)
  37   #:use-module (srfi srfi-64)
  38   #:use-module (ice-9 match)
  39   #:use-module (ice-9 optargs))
  40
  41 (define* (foo-json #:key (name "foo") (name-in-url #f))
  42   "Create a JSON description of an example pypi package, named @var{name},
  43 optionally using a different @var{name in its URL}."
  44   (scm->json-string
  45    `((info
  46       . ((version . "1.0.0")
  47          (name . ,name)
  48          (license . "GNU LGPL")
  49          (summary . "summary")
  50          (home_page . "http://example.com")
  51          (classifiers . #())
  52          (download_url . "")))
  53      (urls . #())
  54      (releases
  55       . ((1.0.0
  56           . #(((url . ,(format #f "https://example.com/~a-1.0.0.egg"
  57                                (or name-in-url name)))
  58                (packagetype . "bdist_egg"))
  59               ((url . ,(format #f "https://example.com/~a-1.0.0.tar.gz"
  60                                (or name-in-url name)))
  61                (packagetype . "sdist"))
  62               ((url . ,(format #f "https://example.com/~a-1.0.0-py2.py3-none-any.whl"
  63                                (or name-in-url name)))
  64                (packagetype . "bdist_wheel")))))))))
  65
  66 (define test-json-1
  67   (foo-json))
  68
  69 (define test-json-2
  70   (foo-json #:name "foo-99"))
  71
  72 (define test-source-hash
  73   "")
  74
  75 (define test-specifications
  76   '("Fizzy [foo, bar]"
  77     "PickyThing<1.6,>1.9,!=1.9.6,<2.0a0,==2.4c1"
  78     "SomethingWithMarker[foo]>1.0;python_version<\"2.7\""
  79     "requests [security,tests] >= 2.8.1, == 2.8.* ; python_version < \"2.7\""
  80     "pip @ https://github.com/pypa/pip/archive/1.3.1.zip#\
  81 sha1=da9234ee9982d4bbb3c72346a6de940a148ea686"))
  82
  83 (define test-requires.txt "\
  84 # A comment
  85  # A comment after a space
  86 foo ~= 3
  87 bar != 2
  88
  89 [test]
  90 pytest (>=2.5.0)
  91 ")
  92
  93 ;; Beaker contains only optional dependencies.
  94 (define test-requires.txt-beaker "\
  95 [crypto]
  96 pycryptopp>=0.5.12
  97
  98 [cryptography]
  99 cryptography
 100
 101 [testsuite]
 102 Mock
 103 coverage
 104 ")
 105
 106 (define test-metadata "\
 107 Classifier: Programming Language :: Python :: 3.7
 108 Requires-Dist: baz ~= 3
 109 Requires-Dist: bar != 2
 110 Provides-Extra: test
 111 Requires-Dist: pytest (>=2.5.0) ; extra == 'test'
 112 ")
 113
 114 (define test-metadata-with-extras "
 115 Classifier: Programming Language :: Python :: 3.7
 116 Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
 117 Requires-Dist: wrapt (<2,>=1)
 118 Requires-Dist: bar
 119
 120 Provides-Extra: dev
 121 Requires-Dist: tox ; extra == 'dev'
 122 Requires-Dist: bumpversion (<1) ; extra == 'dev'
 123 ")
 124
 125 ;;; Provides-Extra can appear before Requires-Dist.
 126 (define test-metadata-with-extras-jedi "\
 127 Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*
 128 Provides-Extra: testing
 129 Requires-Dist: parso (>=0.3.0)
 130 Provides-Extra: testing
 131 Requires-Dist: pytest (>=3.1.0); extra == 'testing'
 132 ")
 133
 134 \f
 135 (test-begin "pypi")
 136
 137 (test-equal "guix-package->pypi-name, old URL style"
 138   "psutil"
 139   (guix-package->pypi-name
 140    (dummy-package "foo"
 141                   (source (dummy-origin
 142                            (uri
 143                             "https://pypi.org/packages/source/p/psutil/psutil-4.3.0.tar.gz"))))))
 144
 145 (test-equal "guix-package->pypi-name, new URL style"
 146   "certbot"
 147   (guix-package->pypi-name
 148    (dummy-package "foo"
 149                   (source (dummy-origin
 150                            (uri
 151                             "https://pypi.org/packages/a2/3b/4756e6a0ceb14e084042a2a65c615d68d25621c6fd446d0fc10d14c4ce7d/certbot-0.8.1.tar.gz"))))))
 152
 153 (test-equal "guix-package->pypi-name, several URLs"
 154   "cram"
 155   (guix-package->pypi-name
 156    (dummy-package "foo"
 157                   (source
 158                    (dummy-origin
 159                     (uri (list "https://bitheap.org/cram/cram-0.7.tar.gz"
 160                                (pypi-uri "cram" "0.7"))))))))
 161
 162 (test-equal "guix-package->pypi-name, honor 'upstream-name'"
 163   "bar-3"
 164   (guix-package->pypi-name
 165    (dummy-package "foo"
 166                   (properties
 167                    '((upstream-name . "bar-3"))))))
 168
 169 (test-equal "specification->requirement-name"
 170   '("Fizzy" "PickyThing" "SomethingWithMarker" "requests" "pip")
 171   (map specification->requirement-name test-specifications))
 172
 173 (test-equal "parse-requires.txt"
 174   (list '("foo" "bar") '("pytest"))
 175   (mock ((ice-9 ports) call-with-input-file
 176          call-with-input-string)
 177         (parse-requires.txt test-requires.txt)))
 178
 179 (test-equal "parse-requires.txt - Beaker"
 180   (list '() '("Mock" "coverage"))
 181   (mock ((ice-9 ports) call-with-input-file
 182          call-with-input-string)
 183         (parse-requires.txt test-requires.txt-beaker)))
 184
 185 (test-equal "parse-wheel-metadata, with extras"
 186   (list '("wrapt" "bar") '("tox" "bumpversion"))
 187   (mock ((ice-9 ports) call-with-input-file
 188          call-with-input-string)
 189         (parse-wheel-metadata test-metadata-with-extras)))
 190
 191 (test-equal "parse-wheel-metadata, with extras - Jedi"
 192   (list '("parso") '("pytest"))
 193   (mock ((ice-9 ports) call-with-input-file
 194          call-with-input-string)
 195         (parse-wheel-metadata test-metadata-with-extras-jedi)))
 196
 197 (test-equal "find-project-url, with numpy"
 198   "numpy"
 199   (find-project-url
 200    "numpy"
 201    "https://files.pythonhosted.org/packages/0a/c8/a62767a6b374a0dfb02d2a0456e5f56a372cdd1689dbc6ffb6bf1ddedbc0/numpy-1.22.1.zip"))
 202
 203 (test-equal "find-project-url, uWSGI"
 204   "uwsgi"
 205   (find-project-url
 206    "uWSGI"
 207    "https://files.pythonhosted.org/packages/24/fd/93851e4a076719199868d4c918cc93a52742e68370188c1c570a6e42a54f/uwsgi-2.0.20.tar.gz"))
 208
 209 (test-equal "find-project-url, flake8-array-spacing"
 210   "flake8_array_spacing"
 211   (find-project-url
 212    "flake8-array-spacing"
 213    "https://files.pythonhosted.org/packages/a4/21/ff29b901128b681b7de7a2787b3aeb3e1f3cba4a8c0cffa9712cbff016bc/flake8_array_spacing-0.2.0.tar.gz"))
 214
 215 (test-equal "find-project-url, foo/goo"
 216   "foo"
 217   (find-project-url
 218    "foo"
 219    "https://files.pythonhosted.org/packages/f0/f00/goo-0.0.0.tar.gz"))
 220
 221 (test-assert "pypi->guix-package, no wheel"
 222   ;; Replace network resources with sample data.
 223     (mock ((guix import utils) url-fetch
 224            (lambda (url file-name)
 225              (match url
 226                ("https://example.com/foo-1.0.0.tar.gz"
 227                 (begin
 228                   ;; Unusual requires.txt location should still be found.
 229                   (mkdir-p "foo-1.0.0/src/bizarre.egg-info")
 230                   (with-output-to-file "foo-1.0.0/src/bizarre.egg-info/requires.txt"
 231                     (lambda ()
 232                       (display test-requires.txt)))
 233                   (parameterize ((current-output-port (%make-void-port "rw+")))
 234                     (system* "tar" "czvf" file-name "foo-1.0.0/"))
 235                   (delete-file-recursively "foo-1.0.0")
 236                   (set! test-source-hash
 237                     (call-with-input-file file-name port-sha256))))
 238                ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
 239                (_ (error "Unexpected URL: " url)))))
 240           (mock ((guix http-client) http-fetch
 241                  (lambda (url . rest)
 242                    (match url
 243                      ("https://pypi.org/pypi/foo/json"
 244                       (values (open-input-string test-json-1)
 245                               (string-length test-json-1)))
 246                      ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
 247                      (_ (error "Unexpected URL: " url)))))
 248                 (match (pypi->guix-package "foo")
 249                   (('package
 250                      ('name "python-foo")
 251                      ('version "1.0.0")
 252                      ('source ('origin
 253                                 ('method 'url-fetch)
 254                                 ('uri ('pypi-uri "foo" 'version))
 255                                 ('sha256
 256                                  ('base32
 257                                   (? string? hash)))))
 258                      ('build-system 'python-build-system)
 259                      ('propagated-inputs ('list 'python-bar 'python-foo))
 260                      ('native-inputs ('list 'python-pytest))
 261                      ('home-page "http://example.com")
 262                      ('synopsis "summary")
 263                      ('description "summary")
 264                      ('license 'license:lgpl2.0))
 265                    (and (string=? (bytevector->nix-base32-string
 266                                    test-source-hash)
 267                                   hash)
 268                         (equal? (pypi->guix-package "foo" #:version "1.0.0")
 269                                 (pypi->guix-package "foo"))
 270                         (guard (c ((error? c) #t))
 271                           (pypi->guix-package "foo" #:version "42"))))
 272                   (x
 273                    (pk 'fail x #f))))))
 274
 275 (test-skip (if (which "zip") 0 1))
 276 (test-assert "pypi->guix-package, wheels"
 277   ;; Replace network resources with sample data.
 278   (mock ((guix import utils) url-fetch
 279          (lambda (url file-name)
 280            (match url
 281              ("https://example.com/foo-1.0.0.tar.gz"
 282               (begin
 283                 (mkdir-p "foo-1.0.0/foo.egg-info/")
 284                 (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt"
 285                   (lambda ()
 286                     (display "wrong data to make sure we're testing wheels ")))
 287                 (parameterize ((current-output-port (%make-void-port "rw+")))
 288                   (system* "tar" "czvf" file-name "foo-1.0.0/"))
 289                 (delete-file-recursively "foo-1.0.0")
 290                 (set! test-source-hash
 291                   (call-with-input-file file-name port-sha256))))
 292              ("https://example.com/foo-1.0.0-py2.py3-none-any.whl"
 293               (begin
 294                 (mkdir "foo-1.0.0.dist-info")
 295                 (with-output-to-file "foo-1.0.0.dist-info/METADATA"
 296                   (lambda ()
 297                     (display test-metadata)))
 298                 (let ((zip-file (string-append file-name ".zip")))
 299                   ;; zip always adds a "zip" extension to the file it creates,
 300                   ;; so we need to rename it.
 301                   (system* "zip" "-q" zip-file "foo-1.0.0.dist-info/METADATA")
 302                   (rename-file zip-file file-name))
 303                 (delete-file-recursively "foo-1.0.0.dist-info")))
 304              (_ (error "Unexpected URL: " url)))))
 305         (mock ((guix http-client) http-fetch
 306                (lambda (url . rest)
 307                  (match url
 308                    ("https://pypi.org/pypi/foo/json"
 309                     (values (open-input-string test-json-1)
 310                             (string-length test-json-1)))
 311                    ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
 312                    (_ (error "Unexpected URL: " url)))))
 313               ;; Not clearing the memoization cache here would mean returning the value
 314               ;; computed in the previous test.
 315               (invalidate-memoization! pypi->guix-package)
 316               (match (pypi->guix-package "foo")
 317                 (('package
 318                    ('name "python-foo")
 319                    ('version "1.0.0")
 320                    ('source ('origin
 321                               ('method 'url-fetch)
 322                               ('uri ('pypi-uri "foo" 'version))
 323                               ('sha256
 324                                ('base32
 325                                 (? string? hash)))))
 326                    ('build-system 'python-build-system)
 327                    ('propagated-inputs ('list 'python-bar 'python-baz))
 328                    ('native-inputs ('list 'python-pytest))
 329                    ('home-page "http://example.com")
 330                    ('synopsis "summary")
 331                    ('description "summary")
 332                    ('license 'license:lgpl2.0))
 333                  (string=? (bytevector->nix-base32-string
 334                             test-source-hash)
 335                            hash))
 336                 (x
 337                  (pk 'fail x #f))))))
 338
 339 (test-assert "pypi->guix-package, no usable requirement file."
 340   ;; Replace network resources with sample data.
 341   (mock ((guix import utils) url-fetch
 342          (lambda (url file-name)
 343            (match url
 344              ("https://example.com/foo-1.0.0.tar.gz"
 345               (mkdir-p "foo-1.0.0/foo.egg-info/")
 346               (parameterize ((current-output-port (%make-void-port "rw+")))
 347                 (system* "tar" "czvf" file-name "foo-1.0.0/"))
 348               (delete-file-recursively "foo-1.0.0")
 349               (set! test-source-hash
 350                 (call-with-input-file file-name port-sha256)))
 351              ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
 352              (_ (error "Unexpected URL: " url)))))
 353         (mock ((guix http-client) http-fetch
 354                (lambda (url . rest)
 355                  (match url
 356                    ("https://pypi.org/pypi/foo/json"
 357                     (values (open-input-string test-json-1)
 358                             (string-length test-json-1)))
 359                    ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f)
 360                    (_ (error "Unexpected URL: " url)))))
 361               ;; Not clearing the memoization cache here would mean returning the value
 362               ;; computed in the previous test.
 363               (invalidate-memoization! pypi->guix-package)
 364               (match (pypi->guix-package "foo")
 365                 (('package
 366                    ('name "python-foo")
 367                    ('version "1.0.0")
 368                    ('source ('origin
 369                               ('method 'url-fetch)
 370                               ('uri ('pypi-uri "foo" 'version))
 371                               ('sha256
 372                                ('base32
 373                                 (? string? hash)))))
 374                    ('build-system 'python-build-system)
 375                    ('home-page "http://example.com")
 376                    ('synopsis "summary")
 377                    ('description "summary")
 378                    ('license 'license:lgpl2.0))
 379                  (string=? (bytevector->nix-base32-string
 380                             test-source-hash)
 381                            hash))
 382                 (x
 383                  (pk 'fail x #f))))))
 384
 385 (test-assert "pypi->guix-package, package name contains \"-\" followed by digits"
 386   ;; Replace network resources with sample data.
 387   (mock ((guix import utils) url-fetch
 388          (lambda (url file-name)
 389            (match url
 390              ("https://example.com/foo-99-1.0.0.tar.gz"
 391               (begin
 392                 ;; Unusual requires.txt location should still be found.
 393                 (mkdir-p "foo-99-1.0.0/src/bizarre.egg-info")
 394                 (with-output-to-file "foo-99-1.0.0/src/bizarre.egg-info/requires.txt"
 395                   (lambda ()
 396                     (display test-requires.txt)))
 397                 (parameterize ((current-output-port (%make-void-port "rw+")))
 398                   (system* "tar" "czvf" file-name "foo-99-1.0.0/"))
 399                 (delete-file-recursively "foo-99-1.0.0")
 400                 (set! test-source-hash
 401                   (call-with-input-file file-name port-sha256))))
 402              ("https://example.com/foo-99-1.0.0-py2.py3-none-any.whl" #f)
 403              (_ (error "Unexpected URL: " url)))))
 404         (mock ((guix http-client) http-fetch
 405                (lambda (url . rest)
 406                  (match url
 407                    ("https://pypi.org/pypi/foo-99/json"
 408                     (values (open-input-string test-json-2)
 409                             (string-length test-json-2)))
 410                    ("https://example.com/foo-99-1.0.0-py2.py3-none-any.whl" #f)
 411                    (_ (error "Unexpected URL: " url)))))
 412               (match (pypi->guix-package "foo-99")
 413                 (('package
 414                    ('name "python-foo-99")
 415                    ('version "1.0.0")
 416                    ('source ('origin
 417                               ('method 'url-fetch)
 418                               ('uri ('pypi-uri "foo-99" 'version))
 419                               ('sha256
 420                                ('base32
 421                                 (? string? hash)))))
 422                    ('properties ('quote (("upstream-name" . "foo-99"))))
 423                    ('build-system 'python-build-system)
 424                    ('propagated-inputs ('list 'python-bar 'python-foo))
 425                    ('native-inputs ('list 'python-pytest))
 426                    ('home-page "http://example.com")
 427                    ('synopsis "summary")
 428                    ('description "summary")
 429                    ('license 'license:lgpl2.0))
 430                  (string=? (bytevector->nix-base32-string
 431                             test-source-hash)
 432                            hash))
 433                 (x
 434                  (pk 'fail x #f))))))
 435
 436 (test-end "pypi")