Commit | Line | Data |
---|---|---|
1b3e9685 DT |
1 | ;;; GNU Guix --- Functional package management for GNU |
2 | ;;; Copyright © 2014 David Thompson <davet@gnu.org> | |
d1cb7e95 | 3 | ;;; Copyright © 2015 Cyril Roelandt <tipecaml@gmail.com> |
522773b7 | 4 | ;;; Copyright © 2015, 2016 Ludovic Courtès <ludo@gnu.org> |
1b3e9685 DT |
5 | ;;; |
6 | ;;; This file is part of GNU Guix. | |
7 | ;;; | |
8 | ;;; GNU Guix is free software; you can redistribute it and/or modify it | |
9 | ;;; under the terms of the GNU General Public License as published by | |
10 | ;;; the Free Software Foundation; either version 3 of the License, or (at | |
11 | ;;; your option) any later version. | |
12 | ;;; | |
13 | ;;; GNU Guix is distributed in the hope that it will be useful, but | |
14 | ;;; WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
16 | ;;; GNU General Public License for more details. | |
17 | ;;; | |
18 | ;;; You should have received a copy of the GNU General Public License | |
19 | ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. | |
20 | ||
21 | (define-module (guix import pypi) | |
22 | #:use-module (ice-9 binary-ports) | |
23 | #:use-module (ice-9 match) | |
24 | #:use-module (ice-9 pretty-print) | |
25 | #:use-module (ice-9 regex) | |
ff986890 | 26 | #:use-module ((ice-9 rdelim) #:select (read-line)) |
1b3e9685 | 27 | #:use-module (srfi srfi-1) |
ff986890 | 28 | #:use-module (srfi srfi-26) |
85dce718 LC |
29 | #:use-module (srfi srfi-34) |
30 | #:use-module (srfi srfi-35) | |
1b3e9685 DT |
31 | #:use-module (rnrs bytevectors) |
32 | #:use-module (json) | |
33 | #:use-module (web uri) | |
ff986890 | 34 | #:use-module (guix ui) |
1b3e9685 | 35 | #:use-module (guix utils) |
8173ceee LC |
36 | #:use-module ((guix build utils) |
37 | #:select ((package-name->name+version | |
38 | . hyphen-package-name->name+version))) | |
1b3e9685 | 39 | #:use-module (guix import utils) |
bab020d7 | 40 | #:use-module ((guix download) #:prefix download:) |
1ff2619b | 41 | #:use-module (guix import json) |
1b3e9685 | 42 | #:use-module (guix packages) |
bab020d7 | 43 | #:use-module (guix upstream) |
1b3e9685 DT |
44 | #:use-module (guix licenses) |
45 | #:use-module (guix build-system python) | |
1b3e9685 | 46 | #:use-module (gnu packages python) |
8173ceee LC |
47 | #:export (guix-package->pypi-name |
48 | pypi->guix-package | |
bab020d7 | 49 | %pypi-updater)) |
1b3e9685 | 50 | |
1b3e9685 | 51 | (define (pypi-fetch name) |
467a3c93 LC |
52 | "Return an alist representation of the PyPI metadata for the package NAME, |
53 | or #f on failure." | |
32728adb LC |
54 | ;; XXX: We want to silence the download progress report, which is especially |
55 | ;; annoying for 'guix refresh', but we have to use a file port. | |
56 | (call-with-output-file "/dev/null" | |
57 | (lambda (null) | |
58 | (with-error-to-port null | |
59 | (lambda () | |
60 | (json-fetch (string-append "https://pypi.python.org/pypi/" | |
61 | name "/json"))))))) | |
1b3e9685 | 62 | |
85dce718 LC |
63 | ;; For packages found on PyPI that lack a source distribution. |
64 | (define-condition-type &missing-source-error &error | |
65 | missing-source-error? | |
66 | (package missing-source-error-package)) | |
67 | ||
1b3e9685 DT |
68 | (define (latest-source-release pypi-package) |
69 | "Return the latest source release for PYPI-PACKAGE." | |
70 | (let ((releases (assoc-ref* pypi-package "releases" | |
71 | (assoc-ref* pypi-package "info" "version")))) | |
72 | (or (find (lambda (release) | |
73 | (string=? "sdist" (assoc-ref release "packagetype"))) | |
74 | releases) | |
85dce718 LC |
75 | (raise (condition (&missing-source-error |
76 | (package pypi-package))))))) | |
1b3e9685 | 77 | |
266785d2 CR |
78 | (define (latest-wheel-release pypi-package) |
79 | "Return the url of the wheel for the latest release of pypi-package, | |
80 | or #f if there isn't any." | |
81 | (let ((releases (assoc-ref* pypi-package "releases" | |
82 | (assoc-ref* pypi-package "info" "version")))) | |
83 | (or (find (lambda (release) | |
84 | (string=? "bdist_wheel" (assoc-ref release "packagetype"))) | |
85 | releases) | |
86 | #f))) | |
87 | ||
ff986890 CR |
88 | (define (python->package-name name) |
89 | "Given the NAME of a package on PyPI, return a Guix-compliant name for the | |
90 | package." | |
91 | (if (string-prefix? "python-" name) | |
92 | (snake-case name) | |
93 | (string-append "python-" (snake-case name)))) | |
94 | ||
bab020d7 CR |
95 | (define (guix-package->pypi-name package) |
96 | "Given a Python PACKAGE built from pypi.python.org, return the name of the | |
97 | package on PyPI." | |
98 | (let ((source-url (and=> (package-source package) origin-uri))) | |
8173ceee LC |
99 | (hyphen-package-name->name+version |
100 | (basename (file-sans-extension source-url))))) | |
bab020d7 | 101 | |
266785d2 CR |
102 | (define (wheel-url->extracted-directory wheel-url) |
103 | (match (string-split (basename wheel-url) #\-) | |
104 | ((name version _ ...) | |
105 | (string-append name "-" version ".dist-info")))) | |
106 | ||
ff986890 CR |
107 | (define (maybe-inputs package-inputs) |
108 | "Given a list of PACKAGE-INPUTS, tries to generate the 'inputs' field of a | |
109 | package definition." | |
110 | (match package-inputs | |
111 | (() | |
112 | '()) | |
113 | ((package-inputs ...) | |
114 | `((inputs (,'quasiquote ,package-inputs)))))) | |
115 | ||
266785d2 CR |
116 | (define (guess-requirements source-url wheel-url tarball) |
117 | "Given SOURCE-URL, WHEEL-URL and a TARBALL of the package, return a list of | |
118 | the required packages specified in the requirements.txt file. TARBALL will be | |
119 | extracted in the current directory, and will be deleted." | |
ff986890 CR |
120 | |
121 | (define (tarball-directory url) | |
122 | ;; Given the URL of the package's tarball, return the name of the directory | |
123 | ;; that will be created upon decompressing it. If the filetype is not | |
124 | ;; supported, return #f. | |
125 | ;; TODO: Support more archive formats. | |
126 | (let ((basename (substring url (+ 1 (string-rindex url #\/))))) | |
127 | (cond | |
128 | ((string-suffix? ".tar.gz" basename) | |
129 | (string-drop-right basename 7)) | |
130 | ((string-suffix? ".tar.bz2" basename) | |
131 | (string-drop-right basename 8)) | |
132 | (else | |
133 | (begin | |
134 | (warning (_ "Unsupported archive format: \ | |
135 | cannot determine package dependencies")) | |
136 | #f))))) | |
137 | ||
138 | (define (clean-requirement s) | |
139 | ;; Given a requirement LINE, as can be found in a Python requirements.txt | |
140 | ;; file, remove everything other than the actual name of the required | |
141 | ;; package, and return it. | |
142 | (string-take s | |
143 | (or (string-index s #\space) | |
144 | (string-length s)))) | |
145 | ||
146 | (define (comment? line) | |
147 | ;; Return #t if the given LINE is a comment, #f otherwise. | |
148 | (eq? (string-ref (string-trim line) 0) #\#)) | |
149 | ||
150 | (define (read-requirements requirements-file) | |
151 | ;; Given REQUIREMENTS-FILE, a Python requirements.txt file, return a list | |
152 | ;; of name/variable pairs describing the requirements. | |
153 | (call-with-input-file requirements-file | |
154 | (lambda (port) | |
155 | (let loop ((result '())) | |
156 | (let ((line (read-line port))) | |
157 | (if (eof-object? line) | |
158 | result | |
159 | (cond | |
160 | ((or (string-null? line) (comment? line)) | |
161 | (loop result)) | |
162 | (else | |
163 | (loop (cons (python->package-name (clean-requirement line)) | |
164 | result)))))))))) | |
165 | ||
266785d2 CR |
166 | (define (read-wheel-metadata wheel-archive) |
167 | ;; Given WHEEL-ARCHIVE, a ZIP Python wheel archive, return the package's | |
168 | ;; requirements. | |
169 | (let* ((dirname (wheel-url->extracted-directory wheel-url)) | |
170 | (json-file (string-append dirname "/metadata.json"))) | |
171 | (and (zero? (system* "unzip" "-q" wheel-archive json-file)) | |
172 | (dynamic-wind | |
173 | (const #t) | |
174 | (lambda () | |
175 | (call-with-input-file json-file | |
176 | (lambda (port) | |
177 | (let* ((metadata (json->scm port)) | |
178 | (run_requires (hash-ref metadata "run_requires")) | |
aebd383d CR |
179 | (requirements (if run_requires |
180 | (hash-ref (list-ref run_requires 0) | |
181 | "requires") | |
182 | '()))) | |
266785d2 CR |
183 | (map (lambda (r) |
184 | (python->package-name (clean-requirement r))) | |
185 | requirements))))) | |
186 | (lambda () | |
187 | (delete-file json-file) | |
188 | (rmdir dirname)))))) | |
189 | ||
190 | (define (guess-requirements-from-wheel) | |
191 | ;; Return the package's requirements using the wheel, or #f if an error | |
192 | ;; occurs. | |
193 | (call-with-temporary-output-file | |
194 | (lambda (temp port) | |
195 | (if wheel-url | |
196 | (and (url-fetch wheel-url temp) | |
197 | (read-wheel-metadata temp)) | |
198 | #f)))) | |
199 | ||
200 | ||
201 | (define (guess-requirements-from-source) | |
202 | ;; Return the package's requirements by guessing them from the source. | |
203 | (let ((dirname (tarball-directory source-url))) | |
204 | (if (string? dirname) | |
205 | (let* ((req-file (string-append dirname "/requirements.txt")) | |
206 | (exit-code (system* "tar" "xf" tarball req-file))) | |
207 | ;; TODO: support more formats. | |
208 | (if (zero? exit-code) | |
209 | (dynamic-wind | |
210 | (const #t) | |
211 | (lambda () | |
212 | (read-requirements req-file)) | |
213 | (lambda () | |
214 | (delete-file req-file) | |
215 | (rmdir dirname))) | |
216 | (begin | |
217 | (warning (_ "'tar xf' failed with exit code ~a\n") | |
218 | exit-code) | |
219 | '()))) | |
220 | '()))) | |
221 | ||
222 | ;; First, try to compute the requirements using the wheel, since that is the | |
223 | ;; most reliable option. If a wheel is not provided for this package, try | |
224 | ;; getting them by reading the "requirements.txt" file from the source. Note | |
225 | ;; that "requirements.txt" is not mandatory, so this is likely to fail. | |
226 | (or (guess-requirements-from-wheel) | |
227 | (guess-requirements-from-source))) | |
228 | ||
229 | ||
230 | (define (compute-inputs source-url wheel-url tarball) | |
ff986890 CR |
231 | "Given the SOURCE-URL of an already downloaded TARBALL, return a list of |
232 | name/variable pairs describing the required inputs of this package." | |
233 | (sort | |
234 | (map (lambda (input) | |
235 | (list input (list 'unquote (string->symbol input)))) | |
236 | (append '("python-setuptools") | |
237 | ;; Argparse has been part of Python since 2.7. | |
238 | (remove (cut string=? "python-argparse" <>) | |
266785d2 | 239 | (guess-requirements source-url wheel-url tarball)))) |
ff986890 CR |
240 | (lambda args |
241 | (match args | |
242 | (((a _ ...) (b _ ...)) | |
243 | (string-ci<? a b)))))) | |
1b3e9685 | 244 | |
266785d2 | 245 | (define (make-pypi-sexp name version source-url wheel-url home-page synopsis |
1b3e9685 DT |
246 | description license) |
247 | "Return the `package' s-expression for a python package with the given NAME, | |
248 | VERSION, SOURCE-URL, HOME-PAGE, SYNOPSIS, DESCRIPTION, and LICENSE." | |
ff986890 CR |
249 | (call-with-temporary-output-file |
250 | (lambda (temp port) | |
251 | (and (url-fetch source-url temp) | |
252 | `(package | |
253 | (name ,(python->package-name name)) | |
254 | (version ,version) | |
255 | (source (origin | |
256 | (method url-fetch) | |
522773b7 LC |
257 | |
258 | ;; Sometimes 'pypi-uri' doesn't quite work due to mixed | |
259 | ;; cases in NAME, for instance, as is the case with | |
260 | ;; "uwsgi". In that case, fall back to a full URL. | |
261 | (uri ,(if (equal? (pypi-uri name version) source-url) | |
262 | `(pypi-uri ,name version) | |
263 | `(string-append | |
264 | ,@(factorize-uri source-url version)))) | |
265 | ||
ff986890 CR |
266 | (sha256 |
267 | (base32 | |
268 | ,(guix-hash-url temp))))) | |
269 | (build-system python-build-system) | |
266785d2 | 270 | ,@(maybe-inputs (compute-inputs source-url wheel-url temp)) |
ff986890 CR |
271 | (home-page ,home-page) |
272 | (synopsis ,synopsis) | |
273 | (description ,description) | |
140b3048 | 274 | (license ,(license->symbol license))))))) |
1b3e9685 DT |
275 | |
276 | (define (pypi->guix-package package-name) | |
277 | "Fetch the metadata for PACKAGE-NAME from pypi.python.org, and return the | |
467a3c93 | 278 | `package' s-expression corresponding to that package, or #f on failure." |
1b3e9685 | 279 | (let ((package (pypi-fetch package-name))) |
467a3c93 | 280 | (and package |
85dce718 LC |
281 | (guard (c ((missing-source-error? c) |
282 | (let ((package (missing-source-error-package c))) | |
283 | (leave (_ "no source release for pypi package ~a ~a~%") | |
284 | (assoc-ref* package "info" "name") | |
285 | (assoc-ref* package "info" "version"))))) | |
286 | (let ((name (assoc-ref* package "info" "name")) | |
287 | (version (assoc-ref* package "info" "version")) | |
288 | (release (assoc-ref (latest-source-release package) "url")) | |
266785d2 | 289 | (wheel (assoc-ref (latest-wheel-release package) "url")) |
85dce718 LC |
290 | (synopsis (assoc-ref* package "info" "summary")) |
291 | (description (assoc-ref* package "info" "summary")) | |
292 | (home-page (assoc-ref* package "info" "home_page")) | |
293 | (license (string->license (assoc-ref* package "info" "license")))) | |
266785d2 | 294 | (make-pypi-sexp name version release wheel home-page synopsis |
85dce718 | 295 | description license)))))) |
bab020d7 CR |
296 | |
297 | (define (pypi-package? package) | |
298 | "Return true if PACKAGE is a Python package from PyPI." | |
299 | ||
300 | (define (pypi-url? url) | |
301 | (string-prefix? "https://pypi.python.org/" url)) | |
302 | ||
303 | (let ((source-url (and=> (package-source package) origin-uri)) | |
304 | (fetch-method (and=> (package-source package) origin-method))) | |
305 | (and (eq? fetch-method download:url-fetch) | |
306 | (match source-url | |
307 | ((? string?) | |
308 | (pypi-url? source-url)) | |
309 | ((source-url ...) | |
310 | (any pypi-url? source-url)))))) | |
311 | ||
7d27a025 LC |
312 | (define (latest-release package) |
313 | "Return an <upstream-source> for the latest release of PACKAGE." | |
85dce718 | 314 | (guard (c ((missing-source-error? c) #f)) |
7d27a025 | 315 | (let* ((pypi-name (guix-package->pypi-name package)) |
85dce718 LC |
316 | (metadata (pypi-fetch pypi-name)) |
317 | (version (assoc-ref* metadata "info" "version")) | |
318 | (url (assoc-ref (latest-source-release metadata) "url"))) | |
319 | (upstream-source | |
7d27a025 | 320 | (package (package-name package)) |
85dce718 LC |
321 | (version version) |
322 | (urls (list url)))))) | |
bab020d7 CR |
323 | |
324 | (define %pypi-updater | |
325 | (upstream-updater | |
326 | (name 'pypi) | |
327 | (description "Updater for PyPI packages") | |
328 | (pred pypi-package?) | |
329 | (latest latest-release))) |