| 1 | ;;; GNU Guix --- Functional package management for GNU |
| 2 | ;;; Copyright © 2014 David Thompson <davet@gnu.org> |
| 3 | ;;; Copyright © 2016 Ricardo Wurmus <rekado@elephly.net> |
| 4 | ;;; Copyright © 2019 Maxim Cournoyer <maxim.cournoyer@gmail.com> |
| 5 | ;;; Copyright © 2021 Xinglu Chen <public@yoctocell.xyz> |
| 6 | ;;; Copyright © 2022 Vivien Kraus <vivien@planete-kraus.eu> |
| 7 | ;;; |
| 8 | ;;; This file is part of GNU Guix. |
| 9 | ;;; |
| 10 | ;;; GNU Guix is free software; you can redistribute it and/or modify it |
| 11 | ;;; under the terms of the GNU General Public License as published by |
| 12 | ;;; the Free Software Foundation; either version 3 of the License, or (at |
| 13 | ;;; your option) any later version. |
| 14 | ;;; |
| 15 | ;;; GNU Guix is distributed in the hope that it will be useful, but |
| 16 | ;;; WITHOUT ANY WARRANTY; without even the implied warranty of |
| 17 | ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 18 | ;;; GNU General Public License for more details. |
| 19 | ;;; |
| 20 | ;;; You should have received a copy of the GNU General Public License |
| 21 | ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. |
| 22 | |
| 23 | (define-module (test-pypi) |
| 24 | #:use-module (guix import pypi) |
| 25 | #:use-module (guix base32) |
| 26 | #:use-module (guix memoization) |
| 27 | #:use-module (guix utils) |
| 28 | #:use-module (gcrypt hash) |
| 29 | #:use-module (guix tests) |
| 30 | #:use-module (guix build-system python) |
| 31 | #:use-module ((guix build utils) #:select (delete-file-recursively which mkdir-p)) |
| 32 | #:use-module ((guix diagnostics) #:select (guix-warning-port)) |
| 33 | #:use-module (json) |
| 34 | #:use-module (srfi srfi-26) |
| 35 | #:use-module (srfi srfi-34) |
| 36 | #:use-module (srfi srfi-35) |
| 37 | #:use-module (srfi srfi-64) |
| 38 | #:use-module (ice-9 match) |
| 39 | #:use-module (ice-9 optargs)) |
| 40 | |
| 41 | (define* (foo-json #:key (name "foo") (name-in-url #f)) |
| 42 | "Create a JSON description of an example pypi package, named @var{name}, |
| 43 | optionally using a different @var{name in its URL}." |
| 44 | (scm->json-string |
| 45 | `((info |
| 46 | . ((version . "1.0.0") |
| 47 | (name . ,name) |
| 48 | (license . "GNU LGPL") |
| 49 | (summary . "summary") |
| 50 | (home_page . "http://example.com") |
| 51 | (classifiers . #()) |
| 52 | (download_url . ""))) |
| 53 | (urls . #()) |
| 54 | (releases |
| 55 | . ((1.0.0 |
| 56 | . #(((url . ,(format #f "https://example.com/~a-1.0.0.egg" |
| 57 | (or name-in-url name))) |
| 58 | (packagetype . "bdist_egg")) |
| 59 | ((url . ,(format #f "https://example.com/~a-1.0.0.tar.gz" |
| 60 | (or name-in-url name))) |
| 61 | (packagetype . "sdist")) |
| 62 | ((url . ,(format #f "https://example.com/~a-1.0.0-py2.py3-none-any.whl" |
| 63 | (or name-in-url name))) |
| 64 | (packagetype . "bdist_wheel"))))))))) |
| 65 | |
| 66 | (define test-json-1 |
| 67 | (foo-json)) |
| 68 | |
| 69 | (define test-json-2 |
| 70 | (foo-json #:name "foo-99")) |
| 71 | |
| 72 | (define test-source-hash |
| 73 | "") |
| 74 | |
| 75 | (define test-specifications |
| 76 | '("Fizzy [foo, bar]" |
| 77 | "PickyThing<1.6,>1.9,!=1.9.6,<2.0a0,==2.4c1" |
| 78 | "SomethingWithMarker[foo]>1.0;python_version<\"2.7\"" |
| 79 | "requests [security,tests] >= 2.8.1, == 2.8.* ; python_version < \"2.7\"" |
| 80 | "pip @ https://github.com/pypa/pip/archive/1.3.1.zip#\ |
| 81 | sha1=da9234ee9982d4bbb3c72346a6de940a148ea686")) |
| 82 | |
| 83 | (define test-requires.txt "\ |
| 84 | # A comment |
| 85 | # A comment after a space |
| 86 | foo ~= 3 |
| 87 | bar != 2 |
| 88 | |
| 89 | [test] |
| 90 | pytest (>=2.5.0) |
| 91 | ") |
| 92 | |
| 93 | ;; Beaker contains only optional dependencies. |
| 94 | (define test-requires.txt-beaker "\ |
| 95 | [crypto] |
| 96 | pycryptopp>=0.5.12 |
| 97 | |
| 98 | [cryptography] |
| 99 | cryptography |
| 100 | |
| 101 | [testsuite] |
| 102 | Mock |
| 103 | coverage |
| 104 | ") |
| 105 | |
| 106 | (define test-metadata "\ |
| 107 | Classifier: Programming Language :: Python :: 3.7 |
| 108 | Requires-Dist: baz ~= 3 |
| 109 | Requires-Dist: bar != 2 |
| 110 | Provides-Extra: test |
| 111 | Requires-Dist: pytest (>=2.5.0) ; extra == 'test' |
| 112 | ") |
| 113 | |
| 114 | (define test-metadata-with-extras " |
| 115 | Classifier: Programming Language :: Python :: 3.7 |
| 116 | Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* |
| 117 | Requires-Dist: wrapt (<2,>=1) |
| 118 | Requires-Dist: bar |
| 119 | |
| 120 | Provides-Extra: dev |
| 121 | Requires-Dist: tox ; extra == 'dev' |
| 122 | Requires-Dist: bumpversion (<1) ; extra == 'dev' |
| 123 | ") |
| 124 | |
| 125 | ;;; Provides-Extra can appear before Requires-Dist. |
| 126 | (define test-metadata-with-extras-jedi "\ |
| 127 | Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.* |
| 128 | Provides-Extra: testing |
| 129 | Requires-Dist: parso (>=0.3.0) |
| 130 | Provides-Extra: testing |
| 131 | Requires-Dist: pytest (>=3.1.0); extra == 'testing' |
| 132 | ") |
| 133 | |
| 134 | \f |
| 135 | (test-begin "pypi") |
| 136 | |
| 137 | (test-equal "guix-package->pypi-name, old URL style" |
| 138 | "psutil" |
| 139 | (guix-package->pypi-name |
| 140 | (dummy-package "foo" |
| 141 | (source (dummy-origin |
| 142 | (uri |
| 143 | "https://pypi.org/packages/source/p/psutil/psutil-4.3.0.tar.gz")))))) |
| 144 | |
| 145 | (test-equal "guix-package->pypi-name, new URL style" |
| 146 | "certbot" |
| 147 | (guix-package->pypi-name |
| 148 | (dummy-package "foo" |
| 149 | (source (dummy-origin |
| 150 | (uri |
| 151 | "https://pypi.org/packages/a2/3b/4756e6a0ceb14e084042a2a65c615d68d25621c6fd446d0fc10d14c4ce7d/certbot-0.8.1.tar.gz")))))) |
| 152 | |
| 153 | (test-equal "guix-package->pypi-name, several URLs" |
| 154 | "cram" |
| 155 | (guix-package->pypi-name |
| 156 | (dummy-package "foo" |
| 157 | (source |
| 158 | (dummy-origin |
| 159 | (uri (list "https://bitheap.org/cram/cram-0.7.tar.gz" |
| 160 | (pypi-uri "cram" "0.7")))))))) |
| 161 | |
| 162 | (test-equal "guix-package->pypi-name, honor 'upstream-name'" |
| 163 | "bar-3" |
| 164 | (guix-package->pypi-name |
| 165 | (dummy-package "foo" |
| 166 | (properties |
| 167 | '((upstream-name . "bar-3")))))) |
| 168 | |
| 169 | (test-equal "specification->requirement-name" |
| 170 | '("Fizzy" "PickyThing" "SomethingWithMarker" "requests" "pip") |
| 171 | (map specification->requirement-name test-specifications)) |
| 172 | |
| 173 | (test-equal "parse-requires.txt" |
| 174 | (list '("foo" "bar") '("pytest")) |
| 175 | (mock ((ice-9 ports) call-with-input-file |
| 176 | call-with-input-string) |
| 177 | (parse-requires.txt test-requires.txt))) |
| 178 | |
| 179 | (test-equal "parse-requires.txt - Beaker" |
| 180 | (list '() '("Mock" "coverage")) |
| 181 | (mock ((ice-9 ports) call-with-input-file |
| 182 | call-with-input-string) |
| 183 | (parse-requires.txt test-requires.txt-beaker))) |
| 184 | |
| 185 | (test-equal "parse-wheel-metadata, with extras" |
| 186 | (list '("wrapt" "bar") '("tox" "bumpversion")) |
| 187 | (mock ((ice-9 ports) call-with-input-file |
| 188 | call-with-input-string) |
| 189 | (parse-wheel-metadata test-metadata-with-extras))) |
| 190 | |
| 191 | (test-equal "parse-wheel-metadata, with extras - Jedi" |
| 192 | (list '("parso") '("pytest")) |
| 193 | (mock ((ice-9 ports) call-with-input-file |
| 194 | call-with-input-string) |
| 195 | (parse-wheel-metadata test-metadata-with-extras-jedi))) |
| 196 | |
| 197 | (test-equal "find-project-url, with numpy" |
| 198 | "numpy" |
| 199 | (find-project-url |
| 200 | "numpy" |
| 201 | "https://files.pythonhosted.org/packages/0a/c8/a62767a6b374a0dfb02d2a0456e5f56a372cdd1689dbc6ffb6bf1ddedbc0/numpy-1.22.1.zip")) |
| 202 | |
| 203 | (test-equal "find-project-url, uWSGI" |
| 204 | "uwsgi" |
| 205 | (find-project-url |
| 206 | "uWSGI" |
| 207 | "https://files.pythonhosted.org/packages/24/fd/93851e4a076719199868d4c918cc93a52742e68370188c1c570a6e42a54f/uwsgi-2.0.20.tar.gz")) |
| 208 | |
| 209 | (test-equal "find-project-url, flake8-array-spacing" |
| 210 | "flake8_array_spacing" |
| 211 | (find-project-url |
| 212 | "flake8-array-spacing" |
| 213 | "https://files.pythonhosted.org/packages/a4/21/ff29b901128b681b7de7a2787b3aeb3e1f3cba4a8c0cffa9712cbff016bc/flake8_array_spacing-0.2.0.tar.gz")) |
| 214 | |
| 215 | (test-equal "find-project-url, foo/goo" |
| 216 | "foo" |
| 217 | (find-project-url |
| 218 | "foo" |
| 219 | "https://files.pythonhosted.org/packages/f0/f00/goo-0.0.0.tar.gz")) |
| 220 | |
| 221 | (test-assert "pypi->guix-package, no wheel" |
| 222 | ;; Replace network resources with sample data. |
| 223 | (mock ((guix import utils) url-fetch |
| 224 | (lambda (url file-name) |
| 225 | (match url |
| 226 | ("https://example.com/foo-1.0.0.tar.gz" |
| 227 | (begin |
| 228 | ;; Unusual requires.txt location should still be found. |
| 229 | (mkdir-p "foo-1.0.0/src/bizarre.egg-info") |
| 230 | (with-output-to-file "foo-1.0.0/src/bizarre.egg-info/requires.txt" |
| 231 | (lambda () |
| 232 | (display test-requires.txt))) |
| 233 | (parameterize ((current-output-port (%make-void-port "rw+"))) |
| 234 | (system* "tar" "czvf" file-name "foo-1.0.0/")) |
| 235 | (delete-file-recursively "foo-1.0.0") |
| 236 | (set! test-source-hash |
| 237 | (call-with-input-file file-name port-sha256)))) |
| 238 | ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f) |
| 239 | (_ (error "Unexpected URL: " url))))) |
| 240 | (mock ((guix http-client) http-fetch |
| 241 | (lambda (url . rest) |
| 242 | (match url |
| 243 | ("https://pypi.org/pypi/foo/json" |
| 244 | (values (open-input-string test-json-1) |
| 245 | (string-length test-json-1))) |
| 246 | ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f) |
| 247 | (_ (error "Unexpected URL: " url))))) |
| 248 | (match (pypi->guix-package "foo") |
| 249 | (('package |
| 250 | ('name "python-foo") |
| 251 | ('version "1.0.0") |
| 252 | ('source ('origin |
| 253 | ('method 'url-fetch) |
| 254 | ('uri ('pypi-uri "foo" 'version)) |
| 255 | ('sha256 |
| 256 | ('base32 |
| 257 | (? string? hash))))) |
| 258 | ('build-system 'python-build-system) |
| 259 | ('propagated-inputs ('list 'python-bar 'python-foo)) |
| 260 | ('native-inputs ('list 'python-pytest)) |
| 261 | ('home-page "http://example.com") |
| 262 | ('synopsis "summary") |
| 263 | ('description "summary") |
| 264 | ('license 'license:lgpl2.0)) |
| 265 | (and (string=? (bytevector->nix-base32-string |
| 266 | test-source-hash) |
| 267 | hash) |
| 268 | (equal? (pypi->guix-package "foo" #:version "1.0.0") |
| 269 | (pypi->guix-package "foo")) |
| 270 | (guard (c ((error? c) #t)) |
| 271 | (pypi->guix-package "foo" #:version "42")))) |
| 272 | (x |
| 273 | (pk 'fail x #f)))))) |
| 274 | |
| 275 | (test-skip (if (which "zip") 0 1)) |
| 276 | (test-assert "pypi->guix-package, wheels" |
| 277 | ;; Replace network resources with sample data. |
| 278 | (mock ((guix import utils) url-fetch |
| 279 | (lambda (url file-name) |
| 280 | (match url |
| 281 | ("https://example.com/foo-1.0.0.tar.gz" |
| 282 | (begin |
| 283 | (mkdir-p "foo-1.0.0/foo.egg-info/") |
| 284 | (with-output-to-file "foo-1.0.0/foo.egg-info/requires.txt" |
| 285 | (lambda () |
| 286 | (display "wrong data to make sure we're testing wheels "))) |
| 287 | (parameterize ((current-output-port (%make-void-port "rw+"))) |
| 288 | (system* "tar" "czvf" file-name "foo-1.0.0/")) |
| 289 | (delete-file-recursively "foo-1.0.0") |
| 290 | (set! test-source-hash |
| 291 | (call-with-input-file file-name port-sha256)))) |
| 292 | ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" |
| 293 | (begin |
| 294 | (mkdir "foo-1.0.0.dist-info") |
| 295 | (with-output-to-file "foo-1.0.0.dist-info/METADATA" |
| 296 | (lambda () |
| 297 | (display test-metadata))) |
| 298 | (let ((zip-file (string-append file-name ".zip"))) |
| 299 | ;; zip always adds a "zip" extension to the file it creates, |
| 300 | ;; so we need to rename it. |
| 301 | (system* "zip" "-q" zip-file "foo-1.0.0.dist-info/METADATA") |
| 302 | (rename-file zip-file file-name)) |
| 303 | (delete-file-recursively "foo-1.0.0.dist-info"))) |
| 304 | (_ (error "Unexpected URL: " url))))) |
| 305 | (mock ((guix http-client) http-fetch |
| 306 | (lambda (url . rest) |
| 307 | (match url |
| 308 | ("https://pypi.org/pypi/foo/json" |
| 309 | (values (open-input-string test-json-1) |
| 310 | (string-length test-json-1))) |
| 311 | ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f) |
| 312 | (_ (error "Unexpected URL: " url))))) |
| 313 | ;; Not clearing the memoization cache here would mean returning the value |
| 314 | ;; computed in the previous test. |
| 315 | (invalidate-memoization! pypi->guix-package) |
| 316 | (match (pypi->guix-package "foo") |
| 317 | (('package |
| 318 | ('name "python-foo") |
| 319 | ('version "1.0.0") |
| 320 | ('source ('origin |
| 321 | ('method 'url-fetch) |
| 322 | ('uri ('pypi-uri "foo" 'version)) |
| 323 | ('sha256 |
| 324 | ('base32 |
| 325 | (? string? hash))))) |
| 326 | ('build-system 'python-build-system) |
| 327 | ('propagated-inputs ('list 'python-bar 'python-baz)) |
| 328 | ('native-inputs ('list 'python-pytest)) |
| 329 | ('home-page "http://example.com") |
| 330 | ('synopsis "summary") |
| 331 | ('description "summary") |
| 332 | ('license 'license:lgpl2.0)) |
| 333 | (string=? (bytevector->nix-base32-string |
| 334 | test-source-hash) |
| 335 | hash)) |
| 336 | (x |
| 337 | (pk 'fail x #f)))))) |
| 338 | |
| 339 | (test-assert "pypi->guix-package, no usable requirement file." |
| 340 | ;; Replace network resources with sample data. |
| 341 | (mock ((guix import utils) url-fetch |
| 342 | (lambda (url file-name) |
| 343 | (match url |
| 344 | ("https://example.com/foo-1.0.0.tar.gz" |
| 345 | (mkdir-p "foo-1.0.0/foo.egg-info/") |
| 346 | (parameterize ((current-output-port (%make-void-port "rw+"))) |
| 347 | (system* "tar" "czvf" file-name "foo-1.0.0/")) |
| 348 | (delete-file-recursively "foo-1.0.0") |
| 349 | (set! test-source-hash |
| 350 | (call-with-input-file file-name port-sha256))) |
| 351 | ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f) |
| 352 | (_ (error "Unexpected URL: " url))))) |
| 353 | (mock ((guix http-client) http-fetch |
| 354 | (lambda (url . rest) |
| 355 | (match url |
| 356 | ("https://pypi.org/pypi/foo/json" |
| 357 | (values (open-input-string test-json-1) |
| 358 | (string-length test-json-1))) |
| 359 | ("https://example.com/foo-1.0.0-py2.py3-none-any.whl" #f) |
| 360 | (_ (error "Unexpected URL: " url))))) |
| 361 | ;; Not clearing the memoization cache here would mean returning the value |
| 362 | ;; computed in the previous test. |
| 363 | (invalidate-memoization! pypi->guix-package) |
| 364 | (match (pypi->guix-package "foo") |
| 365 | (('package |
| 366 | ('name "python-foo") |
| 367 | ('version "1.0.0") |
| 368 | ('source ('origin |
| 369 | ('method 'url-fetch) |
| 370 | ('uri ('pypi-uri "foo" 'version)) |
| 371 | ('sha256 |
| 372 | ('base32 |
| 373 | (? string? hash))))) |
| 374 | ('build-system 'python-build-system) |
| 375 | ('home-page "http://example.com") |
| 376 | ('synopsis "summary") |
| 377 | ('description "summary") |
| 378 | ('license 'license:lgpl2.0)) |
| 379 | (string=? (bytevector->nix-base32-string |
| 380 | test-source-hash) |
| 381 | hash)) |
| 382 | (x |
| 383 | (pk 'fail x #f)))))) |
| 384 | |
| 385 | (test-assert "pypi->guix-package, package name contains \"-\" followed by digits" |
| 386 | ;; Replace network resources with sample data. |
| 387 | (mock ((guix import utils) url-fetch |
| 388 | (lambda (url file-name) |
| 389 | (match url |
| 390 | ("https://example.com/foo-99-1.0.0.tar.gz" |
| 391 | (begin |
| 392 | ;; Unusual requires.txt location should still be found. |
| 393 | (mkdir-p "foo-99-1.0.0/src/bizarre.egg-info") |
| 394 | (with-output-to-file "foo-99-1.0.0/src/bizarre.egg-info/requires.txt" |
| 395 | (lambda () |
| 396 | (display test-requires.txt))) |
| 397 | (parameterize ((current-output-port (%make-void-port "rw+"))) |
| 398 | (system* "tar" "czvf" file-name "foo-99-1.0.0/")) |
| 399 | (delete-file-recursively "foo-99-1.0.0") |
| 400 | (set! test-source-hash |
| 401 | (call-with-input-file file-name port-sha256)))) |
| 402 | ("https://example.com/foo-99-1.0.0-py2.py3-none-any.whl" #f) |
| 403 | (_ (error "Unexpected URL: " url))))) |
| 404 | (mock ((guix http-client) http-fetch |
| 405 | (lambda (url . rest) |
| 406 | (match url |
| 407 | ("https://pypi.org/pypi/foo-99/json" |
| 408 | (values (open-input-string test-json-2) |
| 409 | (string-length test-json-2))) |
| 410 | ("https://example.com/foo-99-1.0.0-py2.py3-none-any.whl" #f) |
| 411 | (_ (error "Unexpected URL: " url))))) |
| 412 | (match (pypi->guix-package "foo-99") |
| 413 | (('package |
| 414 | ('name "python-foo-99") |
| 415 | ('version "1.0.0") |
| 416 | ('source ('origin |
| 417 | ('method 'url-fetch) |
| 418 | ('uri ('pypi-uri "foo-99" 'version)) |
| 419 | ('sha256 |
| 420 | ('base32 |
| 421 | (? string? hash))))) |
| 422 | ('properties ('quote (("upstream-name" . "foo-99")))) |
| 423 | ('build-system 'python-build-system) |
| 424 | ('propagated-inputs ('list 'python-bar 'python-foo)) |
| 425 | ('native-inputs ('list 'python-pytest)) |
| 426 | ('home-page "http://example.com") |
| 427 | ('synopsis "summary") |
| 428 | ('description "summary") |
| 429 | ('license 'license:lgpl2.0)) |
| 430 | (string=? (bytevector->nix-base32-string |
| 431 | test-source-hash) |
| 432 | hash)) |
| 433 | (x |
| 434 | (pk 'fail x #f)))))) |
| 435 | |
| 436 | (test-end "pypi") |