Merge remote-tracking branch 'origin/master' into qt-updates
[jackhill/guix/guix.git] / guix / gnu-maintenance.scm
1 ;;; GNU Guix --- Functional package management for GNU
2 ;;; Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
3 ;;; Copyright © 2012, 2013 Nikita Karetnikov <nikita@karetnikov.org>
4 ;;;
5 ;;; This file is part of GNU Guix.
6 ;;;
7 ;;; GNU Guix is free software; you can redistribute it and/or modify it
8 ;;; under the terms of the GNU General Public License as published by
9 ;;; the Free Software Foundation; either version 3 of the License, or (at
10 ;;; your option) any later version.
11 ;;;
12 ;;; GNU Guix is distributed in the hope that it will be useful, but
13 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
14 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 ;;; GNU General Public License for more details.
16 ;;;
17 ;;; You should have received a copy of the GNU General Public License
18 ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
19
20 (define-module (guix gnu-maintenance)
21 #:use-module (web uri)
22 #:use-module (web client)
23 #:use-module (web response)
24 #:use-module (ice-9 regex)
25 #:use-module (ice-9 match)
26 #:use-module (srfi srfi-1)
27 #:use-module (srfi srfi-11)
28 #:use-module (srfi srfi-26)
29 #:use-module (rnrs io ports)
30 #:use-module (system foreign)
31 #:use-module (guix http-client)
32 #:use-module (guix ftp-client)
33 #:use-module (guix utils)
34 #:use-module (guix memoization)
35 #:use-module (guix records)
36 #:use-module (guix upstream)
37 #:use-module (guix packages)
38 #:use-module (guix zlib)
39 #:export (gnu-package-name
40 gnu-package-mundane-name
41 gnu-package-copyright-holder
42 gnu-package-savannah
43 gnu-package-fsd
44 gnu-package-language
45 gnu-package-logo
46 gnu-package-doc-category
47 gnu-package-doc-summary
48 gnu-package-doc-description
49 gnu-package-doc-urls
50 gnu-package-download-url
51
52 official-gnu-packages
53 find-package
54 gnu-package?
55
56 release-file?
57 releases
58 latest-release
59 gnu-release-archive-types
60 gnu-package-name->name+version
61
62 %gnu-updater
63 %gnu-ftp-updater
64 %kde-updater
65 %xorg-updater
66 %kernel.org-updater))
67
68 ;;; Commentary:
69 ;;;
70 ;;; Code for dealing with the maintenance of GNU packages, such as
71 ;;; auto-updates.
72 ;;;
73 ;;; Code:
74
75 \f
76 ;;;
77 ;;; List of GNU packages.
78 ;;;
79
80 (define %gnumaint-base-url
81 "http://cvs.savannah.gnu.org/viewvc/*checkout*/womb/gnumaint/")
82
83 (define %package-list-url
84 (string->uri
85 (string-append %gnumaint-base-url "rec/gnupackages.rec")))
86
87 (define %package-description-url
88 ;; This file contains package descriptions in recutils format.
89 ;; See <https://lists.gnu.org/archive/html/guix-devel/2013-10/msg00071.html>
90 ;; and <https://lists.gnu.org/archive/html/guix-devel/2018-06/msg00362.html>.
91 (string->uri
92 (string-append %gnumaint-base-url "rec/pkgblurbs.rec")))
93
94 (define-record-type* <gnu-package-descriptor>
95 gnu-package-descriptor
96 make-gnu-package-descriptor
97
98 gnu-package-descriptor?
99
100 (name gnu-package-name)
101 (mundane-name gnu-package-mundane-name)
102 (copyright-holder gnu-package-copyright-holder)
103 (savannah gnu-package-savannah)
104 (fsd gnu-package-fsd)
105 (language gnu-package-language) ; list of strings
106 (logo gnu-package-logo)
107 (doc-category gnu-package-doc-category)
108 (doc-summary gnu-package-doc-summary)
109 (doc-description gnu-package-doc-description) ; taken from 'pkgdescr.txt'
110 (doc-urls gnu-package-doc-urls) ; list of strings
111 (download-url gnu-package-download-url))
112
113 (define* (official-gnu-packages
114 #:optional (fetch http-fetch/cached))
115 "Return a list of records, which are GNU packages. Use FETCH,
116 to fetch the list of GNU packages over HTTP."
117 (define (read-records port)
118 ;; Return a list of alists. Each alist contains fields of a GNU
119 ;; package.
120 (let loop ((alist (recutils->alist port))
121 (result '()))
122 (if (null? alist)
123 (reverse result)
124 (loop (recutils->alist port)
125
126 ;; Ignore things like "%rec" (info "(recutils) Record
127 ;; Descriptors").
128 (if (assoc-ref alist "package")
129 (cons alist result)
130 result)))))
131
132 (define official-description
133 (let ((db (read-records (fetch %package-description-url #:text? #t))))
134 (lambda (name)
135 ;; Return the description found upstream for package NAME, or #f.
136 (and=> (find (lambda (alist)
137 (equal? name (assoc-ref alist "package")))
138 db)
139 (lambda (record)
140 (let ((field (assoc-ref record "blurb")))
141 ;; The upstream description file uses "redirect PACKAGE" as
142 ;; a blurb in cases where the description of the two
143 ;; packages should be considered the same (e.g., GTK+ has
144 ;; "redirect gnome".) This is usually not acceptable for
145 ;; us because we prefer to have distinct descriptions in
146 ;; such cases. Thus, ignore the 'blurb' field when that
147 ;; happens.
148 (and field
149 (not (string-prefix? "redirect " field))
150 field)))))))
151
152 (map (lambda (alist)
153 (let ((name (assoc-ref alist "package")))
154 (alist->record `(("description" . ,(official-description name))
155 ,@alist)
156 make-gnu-package-descriptor
157 (list "package" "mundane_name" "copyright_holder"
158 "savannah" "fsd" "language" "logo"
159 "doc_category" "doc_summary" "description"
160 "doc_url"
161 "download_url")
162 '("doc_url" "language"))))
163 (let* ((port (fetch %package-list-url #:text? #t))
164 (lst (read-records port)))
165 (close-port port)
166 lst)))
167
168 (define (find-package name)
169 "Find GNU package called NAME and return it. Return #f if it was not
170 found."
171 (find (lambda (package)
172 (string=? name (gnu-package-name package)))
173 (official-gnu-packages)))
174
175 (define gnu-package?
176 (let ((official-gnu-packages (memoize official-gnu-packages)))
177 (mlambdaq (package)
178 "Return true if PACKAGE is a GNU package. This procedure may access the
179 network to check in GNU's database."
180 (define (mirror-type url)
181 (let ((uri (string->uri url)))
182 (and (eq? (uri-scheme uri) 'mirror)
183 (cond
184 ((member (uri-host uri)
185 '("gnu" "gnupg" "gcc" "gnome"))
186 ;; Definitely GNU.
187 'gnu)
188 ((equal? (uri-host uri) "cran")
189 ;; Possibly GNU: mirror://cran could be either GNU R itself
190 ;; or a non-GNU package.
191 #f)
192 (else
193 ;; Definitely non-GNU.
194 'non-gnu)))))
195
196 (define (gnu-home-page? package)
197 (letrec-syntax ((>> (syntax-rules ()
198 ((_ value proc)
199 (and=> value proc))
200 ((_ value proc rest ...)
201 (and=> value
202 (lambda (next)
203 (>> (proc next) rest ...)))))))
204 (>> package package-home-page
205 string->uri uri-host
206 (lambda (host)
207 (member host '("www.gnu.org" "gnu.org"))))))
208
209 (or (gnu-home-page? package)
210 (let ((url (and=> (package-source package) origin-uri))
211 (name (package-upstream-name package)))
212 (case (and (string? url) (mirror-type url))
213 ((gnu) #t)
214 ((non-gnu) #f)
215 (else
216 (and (member name (map gnu-package-name (official-gnu-packages)))
217 #t))))))))
218
219 \f
220 ;;;
221 ;;; Latest release.
222 ;;;
223
224 (define (ftp-server/directory package)
225 "Return the FTP server and directory where PACKAGE's tarball are stored."
226 (let ((name (package-upstream-name package)))
227 (values (or (assoc-ref (package-properties package) 'ftp-server)
228 "ftp.gnu.org")
229 (or (assoc-ref (package-properties package) 'ftp-directory)
230 (string-append "/gnu/" name)))))
231
232 (define (sans-extension tarball)
233 "Return TARBALL without its .tar.* or .zip extension."
234 (let ((end (or (string-contains tarball ".tar")
235 (string-contains tarball ".zip"))))
236 (substring tarball 0 end)))
237
238 (define %tarball-rx
239 ;; The .zip extensions is notably used for freefont-ttf.
240 ;; The "-src" pattern is for "TeXmacs-1.0.7.9-src.tar.gz".
241 ;; The "-gnu[0-9]" pattern is for "icecat-38.4.0-gnu1.tar.bz2".
242 (make-regexp "^([^.]+)-([0-9]|[^-])+(-(src|gnu[0-9]))?\\.(tar\\.|zip$)"))
243
244 (define %alpha-tarball-rx
245 (make-regexp "^.*-.*[0-9](-|~)?(alpha|beta|rc|cvs|svn|git)-?[0-9\\.]*\\.tar\\."))
246
247 (define (release-file? project file)
248 "Return #f if FILE is not a release tarball of PROJECT, otherwise return
249 true."
250 (and (not (string-suffix? ".sig" file))
251 (and=> (regexp-exec %tarball-rx file)
252 (lambda (match)
253 ;; Filter out unrelated files, like `guile-www-1.1.1'.
254 ;; Case-insensitive for things like "TeXmacs" vs. "texmacs".
255 ;; The "-src" suffix is for "freefont-src-20120503.tar.gz".
256 (and=> (match:substring match 1)
257 (lambda (name)
258 (or (string-ci=? name project)
259 (string-ci=? name
260 (string-append project
261 "-src")))))))
262 (not (regexp-exec %alpha-tarball-rx file))
263 (let ((s (sans-extension file)))
264 (regexp-exec %package-name-rx s))))
265
266 (define (tarball->version tarball)
267 "Return the version TARBALL corresponds to. TARBALL is a file name like
268 \"coreutils-8.23.tar.xz\"."
269 (let-values (((name version)
270 (gnu-package-name->name+version (sans-extension tarball))))
271 version))
272
273 (define* (releases project
274 #:key
275 (server "ftp.gnu.org")
276 (directory (string-append "/gnu/" project)))
277 "Return the list of <upstream-release> of PROJECT as a list of release
278 name/directory pairs."
279 ;; TODO: Parse something like fencepost.gnu.org:/gd/gnuorg/packages-ftp.
280 (define conn (ftp-open server))
281
282 (let loop ((directories (list directory))
283 (result '()))
284 (match directories
285 (()
286 (ftp-close conn)
287 (coalesce-sources result))
288 ((directory rest ...)
289 (let* ((files (ftp-list conn directory))
290 (subdirs (filter-map (match-lambda
291 ((name 'directory . _) name)
292 (_ #f))
293 files)))
294 (define (file->url file)
295 (string-append "ftp://" server directory "/" file))
296
297 (define (file->source file)
298 (let ((url (file->url file)))
299 (upstream-source
300 (package project)
301 (version (tarball->version file))
302 (urls (list url))
303 (signature-urls (list (string-append url ".sig"))))))
304
305 (loop (append (map (cut string-append directory "/" <>)
306 subdirs)
307 rest)
308 (append
309 ;; Filter out signatures, deltas, and files which
310 ;; are potentially not releases of PROJECT--e.g.,
311 ;; in /gnu/guile, filter out guile-oops and
312 ;; guile-www; in mit-scheme, filter out binaries.
313 (filter-map (match-lambda
314 ((file 'file . _)
315 (and (release-file? project file)
316 (file->source file)))
317 (_ #f))
318 files)
319 result)))))))
320
321 (define* (latest-ftp-release project
322 #:key
323 (server "ftp.gnu.org")
324 (directory (string-append "/gnu/" project))
325 (keep-file? (const #t))
326 (file->signature (cut string-append <> ".sig"))
327 (ftp-open ftp-open) (ftp-close ftp-close))
328 "Return an <upstream-source> for the latest release of PROJECT on SERVER
329 under DIRECTORY, or #f. Use FTP-OPEN and FTP-CLOSE to open (resp. close) FTP
330 connections; this can be useful to reuse connections.
331
332 KEEP-FILE? is a predicate to decide whether to enter a directory and to
333 consider a given file (source tarball) as a valid candidate based on its name.
334
335 FILE->SIGNATURE must be a procedure; it is passed a source file URL and must
336 return the corresponding signature URL, or #f it signatures are unavailable."
337 (define (latest a b)
338 (if (version>? a b) a b))
339
340 (define (latest-release a b)
341 (if (version>? (upstream-source-version a) (upstream-source-version b))
342 a b))
343
344 (define patch-directory-name?
345 ;; Return #t for patch directory names such as 'bash-4.2-patches'.
346 (cut string-suffix? "patches" <>))
347
348 (define conn (ftp-open server))
349
350 (define (file->url directory file)
351 (string-append "ftp://" server directory "/" file))
352
353 (define (file->source directory file)
354 (let ((url (file->url directory file)))
355 (upstream-source
356 (package project)
357 (version (tarball->version file))
358 (urls (list url))
359 (signature-urls (match (file->signature url)
360 (#f #f)
361 (sig (list sig)))))))
362
363 (let loop ((directory directory)
364 (result #f))
365 (let* ((entries (ftp-list conn directory))
366
367 ;; Filter out things like /gnupg/patches. Filter out "w32"
368 ;; directories as found on ftp.gnutls.org.
369 (subdirs (filter-map (match-lambda
370 (((? patch-directory-name? dir)
371 'directory . _)
372 #f)
373 (("w32" 'directory . _)
374 #f)
375 (("unstable" 'directory . _)
376 ;; As seen at ftp.gnupg.org/gcrypt/pinentry.
377 #f)
378 ((directory 'directory . _)
379 directory)
380 (_ #f))
381 entries))
382
383 ;; Whether or not SUBDIRS is empty, compute the latest releases
384 ;; for the current directory. This is necessary for packages
385 ;; such as 'sharutils' that have a sub-directory that contains
386 ;; only an older release.
387 (releases (filter-map (match-lambda
388 ((file 'file . _)
389 (and (release-file? project file)
390 (keep-file? file)
391 (file->source directory file)))
392 (_ #f))
393 entries)))
394
395 ;; Assume that SUBDIRS correspond to versions, and jump into the
396 ;; one with the highest version number.
397 (let* ((release (reduce latest-release #f
398 (coalesce-sources releases)))
399 (result (if (and result release)
400 (latest-release release result)
401 (or release result)))
402 (target (reduce latest #f subdirs)))
403 (if target
404 (loop (string-append directory "/" target)
405 result)
406 (begin
407 (ftp-close conn)
408 result))))))
409
410 (define* (latest-release package
411 #:key
412 (server "ftp.gnu.org")
413 (directory (string-append "/gnu/" package)))
414 "Return the <upstream-source> for the latest version of PACKAGE or #f.
415 PACKAGE must be the canonical name of a GNU package."
416 (latest-ftp-release package
417 #:server server
418 #:directory directory))
419
420 (define-syntax-rule (false-if-ftp-error exp)
421 "Return #f if an FTP error is raise while evaluating EXP; return the result
422 of EXP otherwise."
423 (catch 'ftp-error
424 (lambda ()
425 exp)
426 (lambda (key port . rest)
427 (if (ftp-connection? port)
428 (ftp-close port)
429 (close-port port))
430 #f)))
431
432 (define (latest-release* package)
433 "Like 'latest-release', but (1) take a <package> object, and (2) ignore FTP
434 errors that might occur when PACKAGE is not actually a GNU package, or not
435 hosted on ftp.gnu.org, or not under that name (this is the case for
436 \"emacs-auctex\", for instance.)"
437 (let-values (((server directory)
438 (ftp-server/directory package)))
439 (false-if-ftp-error (latest-release (package-upstream-name package)
440 #:server server
441 #:directory directory))))
442
443 (define %gnu-file-list-uri
444 ;; URI of the file list for ftp.gnu.org.
445 (string->uri "https://ftp.gnu.org/find.txt.gz"))
446
447 (define ftp.gnu.org-files
448 (mlambda ()
449 "Return the list of files available at ftp.gnu.org."
450
451 ;; XXX: Memoize the whole procedure to work around the fact that
452 ;; 'http-fetch/cached' caches the gzipped version.
453
454 (define (trim-leading-components str)
455 ;; Trim the leading ".", if any, in "./gnu/foo".
456 (string-trim str (char-set #\.)))
457
458 (define (string->lines str)
459 (string-tokenize str (char-set-complement (char-set #\newline))))
460
461 ;; Since https://ftp.gnu.org honors 'If-Modified-Since', the hard-coded
462 ;; TTL can be relatively short.
463 (let ((port (http-fetch/cached %gnu-file-list-uri #:ttl (* 15 60))))
464 (map trim-leading-components
465 (call-with-gzip-input-port port
466 (compose string->lines get-string-all))))))
467
468 (define (latest-gnu-release package)
469 "Return the latest release of PACKAGE, a GNU package available via
470 ftp.gnu.org.
471
472 This method does not rely on FTP access at all; instead, it browses the file
473 list available from %GNU-FILE-LIST-URI over HTTP(S)."
474 (let-values (((server directory)
475 (ftp-server/directory package))
476 ((name)
477 (package-upstream-name package)))
478 (let* ((files (ftp.gnu.org-files))
479 (relevant (filter (lambda (file)
480 (and (string-prefix? "/gnu" file)
481 (string-contains file directory)
482 (release-file? name (basename file))))
483 files)))
484 (match (sort relevant (lambda (file1 file2)
485 (version>? (sans-extension (basename file1))
486 (sans-extension (basename file2)))))
487 ((and tarballs (reference _ ...))
488 (let* ((version (tarball->version reference))
489 (tarballs (filter (lambda (file)
490 (string=? (sans-extension
491 (basename file))
492 (sans-extension
493 (basename reference))))
494 tarballs)))
495 (upstream-source
496 (package name)
497 (version version)
498 (urls (map (lambda (file)
499 (string-append "mirror://gnu/"
500 (string-drop file
501 (string-length "/gnu/"))))
502 tarballs))
503 (signature-urls (map (cut string-append <> ".sig") urls)))))
504 (()
505 #f)))))
506
507 (define %package-name-rx
508 ;; Regexp for a package name, e.g., "foo-X.Y". Since TeXmacs uses
509 ;; "TeXmacs-X.Y-src", the `-src' suffix is allowed.
510 (make-regexp "^(.*)-(([0-9]|\\.)+)(-src)?"))
511
512 (define (gnu-package-name->name+version name+version)
513 "Return the package name and version number extracted from NAME+VERSION."
514 (let ((match (regexp-exec %package-name-rx name+version)))
515 (if (not match)
516 (values name+version #f)
517 (values (match:substring match 1) (match:substring match 2)))))
518
519 (define gnome-package?
520 (url-prefix-predicate "mirror://gnome/"))
521
522 (define (pure-gnu-package? package)
523 "Return true if PACKAGE is a non-Emacs and non-GNOME GNU package. This
524 excludes AucTeX, for instance, whose releases are now uploaded to
525 elpa.gnu.org, and all the GNOME packages; EMMS is included though, because its
526 releases are on gnu.org."
527 (and (or (not (string-prefix? "emacs-" (package-name package)))
528 (gnu-hosted? package))
529 (not (gnome-package? package))
530 (gnu-package? package)))
531
532 (define gnu-hosted?
533 (url-prefix-predicate "mirror://gnu/"))
534
535 (define (latest-kde-release package)
536 "Return the latest release of PACKAGE, the name of an KDE.org package."
537 (let ((uri (string->uri (origin-uri (package-source package)))))
538 (false-if-ftp-error
539 (latest-ftp-release
540 (package-upstream-name package)
541 #:server "mirrors.mit.edu"
542 #:directory
543 (string-append "/kde" (dirname (dirname (uri-path uri))))
544 #:file->signature (const #f)))))
545
546 (define (latest-xorg-release package)
547 "Return the latest release of PACKAGE, the name of an X.org package."
548 (let ((uri (string->uri (origin-uri (package-source package)))))
549 (false-if-ftp-error
550 (latest-ftp-release
551 (package-name package)
552 #:server "ftp.freedesktop.org"
553 #:directory
554 (string-append "/pub/xorg/" (dirname (uri-path uri)))))))
555
556 (define (latest-kernel.org-release package)
557 "Return the latest release of PACKAGE, the name of a kernel.org package."
558 (let ((uri (string->uri (origin-uri (package-source package)))))
559 (false-if-ftp-error
560 (latest-ftp-release
561 (package-name package)
562 #:server "ftp.free.fr" ;a mirror reachable over FTP
563 #:directory (string-append "/mirrors/ftp.kernel.org"
564 (dirname (uri-path uri)))
565
566 ;; kernel.org provides "foo-x.y.tar.sign" files, which are signatures of
567 ;; the uncompressed tarball.
568 #:file->signature (lambda (tarball)
569 (string-append (file-sans-extension tarball)
570 ".sign"))))))
571
572 (define %gnu-updater
573 ;; This is for everything at ftp.gnu.org.
574 (upstream-updater
575 (name 'gnu)
576 (description "Updater for GNU packages")
577 (pred gnu-hosted?)
578 (latest latest-gnu-release)))
579
580 (define %gnu-ftp-updater
581 ;; This is for GNU packages taken from alternate locations, such as
582 ;; alpha.gnu.org, ftp.gnupg.org, etc. It is obsolescent.
583 (upstream-updater
584 (name 'gnu-ftp)
585 (description "Updater for GNU packages only available via FTP")
586 (pred (lambda (package)
587 (and (not (gnu-hosted? package))
588 (pure-gnu-package? package))))
589 (latest latest-release*)))
590
591 (define %kde-updater
592 (upstream-updater
593 (name 'kde)
594 (description "Updater for KDE packages")
595 (pred (url-prefix-predicate "mirror://kde/"))
596 (latest latest-kde-release)))
597
598 (define %xorg-updater
599 (upstream-updater
600 (name 'xorg)
601 (description "Updater for X.org packages")
602 (pred (url-prefix-predicate "mirror://xorg/"))
603 (latest latest-xorg-release)))
604
605 (define %kernel.org-updater
606 (upstream-updater
607 (name 'kernel.org)
608 (description "Updater for packages hosted on kernel.org")
609 (pred (url-prefix-predicate "mirror://kernel.org/"))
610 (latest latest-kernel.org-release)))
611
612 ;;; gnu-maintenance.scm ends here