guix/gnu-maintenance.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org>
   3 ;;; Copyright © 2012, 2013 Nikita Karetnikov <nikita@karetnikov.org>
   4 ;;;
   5 ;;; This file is part of GNU Guix.
   6 ;;;
   7 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   8 ;;; under the terms of the GNU General Public License as published by
   9 ;;; the Free Software Foundation; either version 3 of the License, or (at
  10 ;;; your option) any later version.
  11 ;;;
  12 ;;; GNU Guix is distributed in the hope that it will be useful, but
  13 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 ;;; GNU General Public License for more details.
  16 ;;;
  17 ;;; You should have received a copy of the GNU General Public License
  18 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 (define-module (guix gnu-maintenance)
  21   #:use-module (web uri)
  22   #:use-module (web client)
  23   #:use-module (web response)
  24   #:use-module (ice-9 regex)
  25   #:use-module (ice-9 match)
  26   #:use-module (srfi srfi-1)
  27   #:use-module (srfi srfi-11)
  28   #:use-module (srfi srfi-26)
  29   #:use-module (rnrs io ports)
  30   #:use-module (system foreign)
  31   #:use-module (guix http-client)
  32   #:use-module (guix ftp-client)
  33   #:use-module (guix utils)
  34   #:use-module (guix memoization)
  35   #:use-module (guix records)
  36   #:use-module (guix upstream)
  37   #:use-module (guix packages)
  38   #:use-module (guix zlib)
  39   #:export (gnu-package-name
  40             gnu-package-mundane-name
  41             gnu-package-copyright-holder
  42             gnu-package-savannah
  43             gnu-package-fsd
  44             gnu-package-language
  45             gnu-package-logo
  46             gnu-package-doc-category
  47             gnu-package-doc-summary
  48             gnu-package-doc-description
  49             gnu-package-doc-urls
  50             gnu-package-download-url
  51
  52             official-gnu-packages
  53             find-package
  54             gnu-package?
  55
  56             release-file?
  57             releases
  58             latest-release
  59             gnu-release-archive-types
  60             gnu-package-name->name+version
  61
  62             %gnu-updater
  63             %gnu-ftp-updater
  64             %kde-updater
  65             %xorg-updater
  66             %kernel.org-updater))
  67
  68 ;;; Commentary:
  69 ;;;
  70 ;;; Code for dealing with the maintenance of GNU packages, such as
  71 ;;; auto-updates.
  72 ;;;
  73 ;;; Code:
  74
  75 \f
  76 ;;;
  77 ;;; List of GNU packages.
  78 ;;;
  79
  80 (define %gnumaint-base-url
  81   "http://cvs.savannah.gnu.org/viewvc/*checkout*/womb/gnumaint/")
  82
  83 (define %package-list-url
  84   (string->uri
  85    (string-append %gnumaint-base-url "gnupackages.txt")))
  86
  87 (define %package-description-url
  88   ;; This file contains package descriptions in recutils format.
  89   ;; See <https://lists.gnu.org/archive/html/guix-devel/2013-10/msg00071.html>.
  90   (string->uri
  91    (string-append %gnumaint-base-url "pkgblurbs.txt")))
  92
  93 (define-record-type* <gnu-package-descriptor>
  94   gnu-package-descriptor
  95   make-gnu-package-descriptor
  96
  97   gnu-package-descriptor?
  98
  99   (name             gnu-package-name)
 100   (mundane-name     gnu-package-mundane-name)
 101   (copyright-holder gnu-package-copyright-holder)
 102   (savannah         gnu-package-savannah)
 103   (fsd              gnu-package-fsd)
 104   (language         gnu-package-language)         ; list of strings
 105   (logo             gnu-package-logo)
 106   (doc-category     gnu-package-doc-category)
 107   (doc-summary      gnu-package-doc-summary)
 108   (doc-description  gnu-package-doc-description)  ; taken from 'pkgdescr.txt'
 109   (doc-urls         gnu-package-doc-urls)         ; list of strings
 110   (download-url     gnu-package-download-url))
 111
 112 (define* (official-gnu-packages
 113           #:optional (fetch http-fetch/cached))
 114   "Return a list of records, which are GNU packages.  Use FETCH,
 115 to fetch the list of GNU packages over HTTP."
 116   (define (read-records port)
 117     ;; Return a list of alists.  Each alist contains fields of a GNU
 118     ;; package.
 119     (let loop ((alist  (recutils->alist port))
 120                (result '()))
 121       (if (null? alist)
 122           (reverse result)
 123           (loop (recutils->alist port)
 124                 (cons alist result)))))
 125
 126   (define official-description
 127     (let ((db (read-records (fetch %package-description-url #:text? #t))))
 128       (lambda (name)
 129         ;; Return the description found upstream for package NAME, or #f.
 130         (and=> (find (lambda (alist)
 131                        (equal? name (assoc-ref alist "package")))
 132                      db)
 133                (lambda (record)
 134                  (let ((field (assoc-ref record "blurb")))
 135                    ;; The upstream description file uses "redirect PACKAGE" as
 136                    ;; a blurb in cases where the description of the two
 137                    ;; packages should be considered the same (e.g., GTK+ has
 138                    ;; "redirect gnome".)  This is usually not acceptable for
 139                    ;; us because we prefer to have distinct descriptions in
 140                    ;; such cases.  Thus, ignore the 'blurb' field when that
 141                    ;; happens.
 142                    (and field
 143                         (not (string-prefix? "redirect " field))
 144                         field)))))))
 145
 146   (map (lambda (alist)
 147          (let ((name (assoc-ref alist "package")))
 148            (alist->record `(("description" . ,(official-description name))
 149                             ,@alist)
 150                           make-gnu-package-descriptor
 151                           (list "package" "mundane-name" "copyright-holder"
 152                                 "savannah" "fsd" "language" "logo"
 153                                 "doc-category" "doc-summary" "description"
 154                                 "doc-url"
 155                                 "download-url")
 156                           '("doc-url" "language"))))
 157        (let* ((port (fetch %package-list-url #:text? #t))
 158               (lst  (read-records port)))
 159          (close-port port)
 160          lst)))
 161
 162 (define (find-package name)
 163   "Find GNU package called NAME and return it.  Return #f if it was not
 164 found."
 165   (find (lambda (package)
 166           (string=? name (gnu-package-name package)))
 167         (official-gnu-packages)))
 168
 169 (define gnu-package?
 170   (let ((official-gnu-packages (memoize official-gnu-packages)))
 171     (mlambdaq (package)
 172       "Return true if PACKAGE is a GNU package.  This procedure may access the
 173 network to check in GNU's database."
 174       (define (mirror-type url)
 175         (let ((uri (string->uri url)))
 176           (and (eq? (uri-scheme uri) 'mirror)
 177                (cond
 178                 ((member (uri-host uri)
 179                          '("gnu" "gnupg" "gcc" "gnome"))
 180                  ;; Definitely GNU.
 181                  'gnu)
 182                 ((equal? (uri-host uri) "cran")
 183                  ;; Possibly GNU: mirror://cran could be either GNU R itself
 184                  ;; or a non-GNU package.
 185                  #f)
 186                 (else
 187                  ;; Definitely non-GNU.
 188                  'non-gnu)))))
 189
 190       (define (gnu-home-page? package)
 191         (letrec-syntax ((>> (syntax-rules ()
 192                               ((_ value proc)
 193                                (and=> value proc))
 194                               ((_ value proc rest ...)
 195                                (and=> value
 196                                       (lambda (next)
 197                                         (>> (proc next) rest ...)))))))
 198           (>> package package-home-page
 199               string->uri uri-host
 200               (lambda (host)
 201                 (member host '("www.gnu.org" "gnu.org"))))))
 202
 203       (or (gnu-home-page? package)
 204           (let ((url  (and=> (package-source package) origin-uri))
 205                 (name (package-upstream-name package)))
 206             (case (and (string? url) (mirror-type url))
 207               ((gnu) #t)
 208               ((non-gnu) #f)
 209               (else
 210                (and (member name (map gnu-package-name (official-gnu-packages)))
 211                     #t))))))))
 212
 213 \f
 214 ;;;
 215 ;;; Latest release.
 216 ;;;
 217
 218 (define (ftp-server/directory package)
 219   "Return the FTP server and directory where PACKAGE's tarball are stored."
 220   (let ((name (package-upstream-name package)))
 221     (values (or (assoc-ref (package-properties package) 'ftp-server)
 222                 "ftp.gnu.org")
 223             (or (assoc-ref (package-properties package) 'ftp-directory)
 224                 (string-append "/gnu/" name)))))
 225
 226 (define (sans-extension tarball)
 227   "Return TARBALL without its .tar.* or .zip extension."
 228   (let ((end (or (string-contains tarball ".tar")
 229                  (string-contains tarball ".zip"))))
 230     (substring tarball 0 end)))
 231
 232 (define %tarball-rx
 233   ;; The .zip extensions is notably used for freefont-ttf.
 234   ;; The "-src" pattern is for "TeXmacs-1.0.7.9-src.tar.gz".
 235   ;; The "-gnu[0-9]" pattern is for "icecat-38.4.0-gnu1.tar.bz2".
 236   (make-regexp "^([^.]+)-([0-9]|[^-])+(-(src|gnu[0-9]))?\\.(tar\\.|zip$)"))
 237
 238 (define %alpha-tarball-rx
 239   (make-regexp "^.*-.*[0-9](-|~)?(alpha|beta|rc|cvs|svn|git)-?[0-9\\.]*\\.tar\\."))
 240
 241 (define (release-file? project file)
 242   "Return #f if FILE is not a release tarball of PROJECT, otherwise return
 243 true."
 244   (and (not (string-suffix? ".sig" file))
 245        (and=> (regexp-exec %tarball-rx file)
 246               (lambda (match)
 247                 ;; Filter out unrelated files, like `guile-www-1.1.1'.
 248                 ;; Case-insensitive for things like "TeXmacs" vs. "texmacs".
 249                 ;; The "-src" suffix is for "freefont-src-20120503.tar.gz".
 250                 (and=> (match:substring match 1)
 251                        (lambda (name)
 252                          (or (string-ci=? name project)
 253                              (string-ci=? name
 254                                           (string-append project
 255                                                          "-src")))))))
 256        (not (regexp-exec %alpha-tarball-rx file))
 257        (let ((s (sans-extension file)))
 258          (regexp-exec %package-name-rx s))))
 259
 260 (define (tarball->version tarball)
 261   "Return the version TARBALL corresponds to.  TARBALL is a file name like
 262 \"coreutils-8.23.tar.xz\"."
 263   (let-values (((name version)
 264                 (gnu-package-name->name+version (sans-extension tarball))))
 265     version))
 266
 267 (define* (releases project
 268                    #:key
 269                    (server "ftp.gnu.org")
 270                    (directory (string-append "/gnu/" project)))
 271   "Return the list of <upstream-release> of PROJECT as a list of release
 272 name/directory pairs."
 273   ;; TODO: Parse something like fencepost.gnu.org:/gd/gnuorg/packages-ftp.
 274   (define conn (ftp-open server))
 275
 276   (let loop ((directories (list directory))
 277              (result      '()))
 278     (match directories
 279       (()
 280        (ftp-close conn)
 281        (coalesce-sources result))
 282       ((directory rest ...)
 283        (let* ((files   (ftp-list conn directory))
 284               (subdirs (filter-map (match-lambda
 285                                      ((name 'directory . _) name)
 286                                      (_ #f))
 287                                    files)))
 288          (define (file->url file)
 289            (string-append "ftp://" server directory "/" file))
 290
 291          (define (file->source file)
 292            (let ((url (file->url file)))
 293              (upstream-source
 294               (package project)
 295               (version (tarball->version file))
 296               (urls (list url))
 297               (signature-urls (list (string-append url ".sig"))))))
 298
 299          (loop (append (map (cut string-append directory "/" <>)
 300                             subdirs)
 301                        rest)
 302                (append
 303                 ;; Filter out signatures, deltas, and files which
 304                 ;; are potentially not releases of PROJECT--e.g.,
 305                 ;; in /gnu/guile, filter out guile-oops and
 306                 ;; guile-www; in mit-scheme, filter out binaries.
 307                 (filter-map (match-lambda
 308                               ((file 'file . _)
 309                                (and (release-file? project file)
 310                                     (file->source file)))
 311                               (_ #f))
 312                             files)
 313                 result)))))))
 314
 315 (define* (latest-ftp-release project
 316                              #:key
 317                              (server "ftp.gnu.org")
 318                              (directory (string-append "/gnu/" project))
 319                              (keep-file? (const #t))
 320                              (file->signature (cut string-append <> ".sig"))
 321                              (ftp-open ftp-open) (ftp-close ftp-close))
 322   "Return an <upstream-source> for the latest release of PROJECT on SERVER
 323 under DIRECTORY, or #f.  Use FTP-OPEN and FTP-CLOSE to open (resp. close) FTP
 324 connections; this can be useful to reuse connections.
 325
 326 KEEP-FILE? is a predicate to decide whether to enter a directory and to
 327 consider a given file (source tarball) as a valid candidate based on its name.
 328
 329 FILE->SIGNATURE must be a procedure; it is passed a source file URL and must
 330 return the corresponding signature URL, or #f it signatures are unavailable."
 331   (define (latest a b)
 332     (if (version>? a b) a b))
 333
 334   (define (latest-release a b)
 335     (if (version>? (upstream-source-version a) (upstream-source-version b))
 336         a b))
 337
 338   (define contains-digit?
 339     (cut string-any char-set:digit <>))
 340
 341   (define patch-directory-name?
 342     ;; Return #t for patch directory names such as 'bash-4.2-patches'.
 343     (cut string-suffix? "patches" <>))
 344
 345   (define conn (ftp-open server))
 346
 347   (define (file->url directory file)
 348     (string-append "ftp://" server directory "/" file))
 349
 350   (define (file->source directory file)
 351     (let ((url (file->url directory file)))
 352       (upstream-source
 353        (package project)
 354        (version (tarball->version file))
 355        (urls (list url))
 356        (signature-urls (match (file->signature url)
 357                          (#f #f)
 358                          (sig (list sig)))))))
 359
 360   (let loop ((directory directory)
 361              (result    #f))
 362     (let* ((entries (ftp-list conn directory))
 363
 364            ;; Filter out sub-directories that do not contain digits---e.g.,
 365            ;; /gnuzilla/lang and /gnupg/patches.  Filter out "w32"
 366            ;; directories as found on ftp.gnutls.org.
 367            (subdirs (filter-map (match-lambda
 368                                   (((? patch-directory-name? dir)
 369                                     'directory . _)
 370                                    #f)
 371                                   (("w32" 'directory . _)
 372                                    #f)
 373                                   (((? contains-digit? dir) 'directory . _)
 374                                    (and (keep-file? dir) dir))
 375                                   (_ #f))
 376                                 entries))
 377
 378            ;; Whether or not SUBDIRS is empty, compute the latest releases
 379            ;; for the current directory.  This is necessary for packages
 380            ;; such as 'sharutils' that have a sub-directory that contains
 381            ;; only an older release.
 382            (releases (filter-map (match-lambda
 383                                    ((file 'file . _)
 384                                     (and (release-file? project file)
 385                                          (keep-file? file)
 386                                          (file->source directory file)))
 387                                    (_ #f))
 388                                  entries)))
 389
 390       ;; Assume that SUBDIRS correspond to versions, and jump into the
 391       ;; one with the highest version number.
 392       (let* ((release  (reduce latest-release #f
 393                                (coalesce-sources releases)))
 394              (result   (if (and result release)
 395                            (latest-release release result)
 396                            (or release result)))
 397              (target   (reduce latest #f subdirs)))
 398         (if target
 399             (loop (string-append directory "/" target)
 400                   result)
 401             (begin
 402               (ftp-close conn)
 403               result))))))
 404
 405 (define* (latest-release package
 406                          #:key
 407                          (server "ftp.gnu.org")
 408                          (directory (string-append "/gnu/" package)))
 409   "Return the <upstream-source> for the latest version of PACKAGE or #f.
 410 PACKAGE must be the canonical name of a GNU package."
 411   (latest-ftp-release package
 412                       #:server server
 413                       #:directory directory))
 414
 415 (define-syntax-rule (false-if-ftp-error exp)
 416   "Return #f if an FTP error is raise while evaluating EXP; return the result
 417 of EXP otherwise."
 418   (catch 'ftp-error
 419     (lambda ()
 420       exp)
 421     (lambda (key port . rest)
 422       (if (ftp-connection? port)
 423           (ftp-close port)
 424           (close-port port))
 425       #f)))
 426
 427 (define (latest-release* package)
 428   "Like 'latest-release', but (1) take a <package> object, and (2) ignore FTP
 429 errors that might occur when PACKAGE is not actually a GNU package, or not
 430 hosted on ftp.gnu.org, or not under that name (this is the case for
 431 \"emacs-auctex\", for instance.)"
 432   (let-values (((server directory)
 433                 (ftp-server/directory package)))
 434     (false-if-ftp-error (latest-release (package-upstream-name package)
 435                                         #:server server
 436                                         #:directory directory))))
 437
 438 (define %gnu-file-list-uri
 439   ;; URI of the file list for ftp.gnu.org.
 440   (string->uri "https://ftp.gnu.org/find.txt.gz"))
 441
 442 (define ftp.gnu.org-files
 443   (mlambda ()
 444     "Return the list of files available at ftp.gnu.org."
 445
 446     ;; XXX: Memoize the whole procedure to work around the fact that
 447     ;; 'http-fetch/cached' caches the gzipped version.
 448
 449     (define (trim-leading-components str)
 450       ;; Trim the leading ".", if any, in "./gnu/foo".
 451       (string-trim str (char-set #\.)))
 452
 453     (define (string->lines str)
 454       (string-tokenize str (char-set-complement (char-set #\newline))))
 455
 456     ;; Since https://ftp.gnu.org honors 'If-Modified-Since', the hard-coded
 457     ;; TTL can be relatively short.
 458     (let ((port (http-fetch/cached %gnu-file-list-uri #:ttl (* 15 60))))
 459       (map trim-leading-components
 460            (call-with-gzip-input-port port
 461              (compose string->lines get-string-all))))))
 462
 463 (define (latest-gnu-release package)
 464   "Return the latest release of PACKAGE, a GNU package available via
 465 ftp.gnu.org.
 466
 467 This method does not rely on FTP access at all; instead, it browses the file
 468 list available from %GNU-FILE-LIST-URI over HTTP(S)."
 469   (let-values (((server directory)
 470                 (ftp-server/directory package))
 471                ((name)
 472                 (package-upstream-name package)))
 473     (let* ((files    (ftp.gnu.org-files))
 474            (relevant (filter (lambda (file)
 475                                (and (string-prefix? "/gnu" file)
 476                                     (string-contains file directory)
 477                                     (release-file? name (basename file))))
 478                              files)))
 479       (match (sort relevant (lambda (file1 file2)
 480                               (version>? (sans-extension (basename file1))
 481                                          (sans-extension (basename file2)))))
 482         ((and tarballs (reference _ ...))
 483          (let* ((version  (tarball->version reference))
 484                 (tarballs (filter (lambda (file)
 485                                     (string=? (sans-extension
 486                                                (basename file))
 487                                               (sans-extension
 488                                                (basename reference))))
 489                                   tarballs)))
 490            (upstream-source
 491             (package name)
 492             (version version)
 493             (urls (map (lambda (file)
 494                          (string-append "mirror://gnu/"
 495                                         (string-drop file
 496                                                      (string-length "/gnu/"))))
 497                        tarballs))
 498             (signature-urls (map (cut string-append <> ".sig") urls)))))
 499         (()
 500          #f)))))
 501
 502 (define %package-name-rx
 503   ;; Regexp for a package name, e.g., "foo-X.Y".  Since TeXmacs uses
 504   ;; "TeXmacs-X.Y-src", the `-src' suffix is allowed.
 505   (make-regexp "^(.*)-(([0-9]|\\.)+)(-src)?"))
 506
 507 (define (gnu-package-name->name+version name+version)
 508   "Return the package name and version number extracted from NAME+VERSION."
 509   (let ((match (regexp-exec %package-name-rx name+version)))
 510     (if (not match)
 511         (values name+version #f)
 512         (values (match:substring match 1) (match:substring match 2)))))
 513
 514 (define gnome-package?
 515   (url-prefix-predicate "mirror://gnome/"))
 516
 517 (define (pure-gnu-package? package)
 518   "Return true if PACKAGE is a non-Emacs and non-GNOME GNU package.  This
 519 excludes AucTeX, for instance, whose releases are now uploaded to
 520 elpa.gnu.org, and all the GNOME packages; EMMS is included though, because its
 521 releases are on gnu.org."
 522   (and (or (not (string-prefix? "emacs-" (package-name package)))
 523            (gnu-hosted? package))
 524        (not (gnome-package? package))
 525        (gnu-package? package)))
 526
 527 (define gnu-hosted?
 528   (url-prefix-predicate "mirror://gnu/"))
 529
 530 (define (latest-kde-release package)
 531   "Return the latest release of PACKAGE, the name of an KDE.org package."
 532   (let ((uri (string->uri (origin-uri (package-source package)))))
 533     (false-if-ftp-error
 534      (latest-ftp-release
 535       (package-upstream-name package)
 536       #:server "mirrors.mit.edu"
 537       #:directory
 538       (string-append "/kde" (dirname (dirname (uri-path uri))))
 539       #:file->signature (const #f)))))
 540
 541 (define (latest-xorg-release package)
 542   "Return the latest release of PACKAGE, the name of an X.org package."
 543   (let ((uri (string->uri (origin-uri (package-source package)))))
 544     (false-if-ftp-error
 545      (latest-ftp-release
 546       (package-name package)
 547       #:server "ftp.freedesktop.org"
 548       #:directory
 549       (string-append "/pub/xorg/" (dirname (uri-path uri)))))))
 550
 551 (define (latest-kernel.org-release package)
 552   "Return the latest release of PACKAGE, the name of a kernel.org package."
 553   (let ((uri (string->uri (origin-uri (package-source package)))))
 554     (false-if-ftp-error
 555      (latest-ftp-release
 556       (package-name package)
 557       #:server "ftp.free.fr"                      ;a mirror reachable over FTP
 558       #:directory (string-append "/mirrors/ftp.kernel.org"
 559                                  (dirname (uri-path uri)))
 560
 561       ;; kernel.org provides "foo-x.y.tar.sign" files, which are signatures of
 562       ;; the uncompressed tarball.
 563       #:file->signature (lambda (tarball)
 564                           (string-append (file-sans-extension tarball)
 565                                          ".sign"))))))
 566
 567 (define %gnu-updater
 568   ;; This is for everything at ftp.gnu.org.
 569   (upstream-updater
 570    (name 'gnu)
 571    (description "Updater for GNU packages")
 572    (pred gnu-hosted?)
 573    (latest latest-gnu-release)))
 574
 575 (define %gnu-ftp-updater
 576   ;; This is for GNU packages taken from alternate locations, such as
 577   ;; alpha.gnu.org, ftp.gnupg.org, etc.  It is obsolescent.
 578   (upstream-updater
 579    (name 'gnu-ftp)
 580    (description "Updater for GNU packages only available via FTP")
 581    (pred (lambda (package)
 582            (and (not (gnu-hosted? package))
 583                 (pure-gnu-package? package))))
 584    (latest latest-release*)))
 585
 586 (define %kde-updater
 587   (upstream-updater
 588     (name 'kde)
 589     (description "Updater for KDE packages")
 590     (pred (url-prefix-predicate "mirror://kde/"))
 591     (latest latest-kde-release)))
 592
 593 (define %xorg-updater
 594   (upstream-updater
 595    (name 'xorg)
 596    (description "Updater for X.org packages")
 597    (pred (url-prefix-predicate "mirror://xorg/"))
 598    (latest latest-xorg-release)))
 599
 600 (define %kernel.org-updater
 601   (upstream-updater
 602    (name 'kernel.org)
 603    (description "Updater for packages hosted on kernel.org")
 604    (pred (url-prefix-predicate "mirror://kernel.org/"))
 605    (latest latest-kernel.org-release)))
 606
 607 ;;; gnu-maintenance.scm ends here