guix/gnu-maintenance.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org>
   3 ;;; Copyright © 2012, 2013 Nikita Karetnikov <nikita@karetnikov.org>
   4 ;;;
   5 ;;; This file is part of GNU Guix.
   6 ;;;
   7 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   8 ;;; under the terms of the GNU General Public License as published by
   9 ;;; the Free Software Foundation; either version 3 of the License, or (at
  10 ;;; your option) any later version.
  11 ;;;
  12 ;;; GNU Guix is distributed in the hope that it will be useful, but
  13 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 ;;; GNU General Public License for more details.
  16 ;;;
  17 ;;; You should have received a copy of the GNU General Public License
  18 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 (define-module (guix gnu-maintenance)
  21   #:use-module (web uri)
  22   #:use-module (web client)
  23   #:use-module (web response)
  24   #:use-module (ice-9 regex)
  25   #:use-module (ice-9 match)
  26   #:use-module (srfi srfi-1)
  27   #:use-module (srfi srfi-11)
  28   #:use-module (srfi srfi-26)
  29   #:use-module (system foreign)
  30   #:use-module (guix http-client)
  31   #:use-module (guix ftp-client)
  32   #:use-module (guix utils)
  33   #:use-module (guix combinators)
  34   #:use-module (guix records)
  35   #:use-module (guix upstream)
  36   #:use-module (guix packages)
  37   #:export (gnu-package-name
  38             gnu-package-mundane-name
  39             gnu-package-copyright-holder
  40             gnu-package-savannah
  41             gnu-package-fsd
  42             gnu-package-language
  43             gnu-package-logo
  44             gnu-package-doc-category
  45             gnu-package-doc-summary
  46             gnu-package-doc-description
  47             gnu-package-doc-urls
  48             gnu-package-download-url
  49
  50             official-gnu-packages
  51             find-package
  52             gnu-package?
  53
  54             release-file?
  55             releases
  56             latest-release
  57             gnu-release-archive-types
  58             gnu-package-name->name+version
  59
  60             %gnu-updater
  61             %gnome-updater
  62             %kde-updater
  63             %xorg-updater
  64             %kernel.org-updater))
  65
  66 ;;; Commentary:
  67 ;;;
  68 ;;; Code for dealing with the maintenance of GNU packages, such as
  69 ;;; auto-updates.
  70 ;;;
  71 ;;; Code:
  72
  73 \f
  74 ;;;
  75 ;;; List of GNU packages.
  76 ;;;
  77
  78 (define %gnumaint-base-url
  79   "http://cvs.savannah.gnu.org/viewvc/*checkout*/womb/gnumaint/")
  80
  81 (define %package-list-url
  82   (string->uri
  83    (string-append %gnumaint-base-url "gnupackages.txt")))
  84
  85 (define %package-description-url
  86   ;; This file contains package descriptions in recutils format.
  87   ;; See <https://lists.gnu.org/archive/html/guix-devel/2013-10/msg00071.html>.
  88   (string->uri
  89    (string-append %gnumaint-base-url "pkgblurbs.txt")))
  90
  91 (define-record-type* <gnu-package-descriptor>
  92   gnu-package-descriptor
  93   make-gnu-package-descriptor
  94
  95   gnu-package-descriptor?
  96
  97   (name             gnu-package-name)
  98   (mundane-name     gnu-package-mundane-name)
  99   (copyright-holder gnu-package-copyright-holder)
 100   (savannah         gnu-package-savannah)
 101   (fsd              gnu-package-fsd)
 102   (language         gnu-package-language)         ; list of strings
 103   (logo             gnu-package-logo)
 104   (doc-category     gnu-package-doc-category)
 105   (doc-summary      gnu-package-doc-summary)
 106   (doc-description  gnu-package-doc-description)  ; taken from 'pkgdescr.txt'
 107   (doc-urls         gnu-package-doc-urls)         ; list of strings
 108   (download-url     gnu-package-download-url))
 109
 110 (define* (official-gnu-packages
 111           #:optional (fetch http-fetch/cached))
 112   "Return a list of records, which are GNU packages.  Use FETCH,
 113 to fetch the list of GNU packages over HTTP."
 114   (define (read-records port)
 115     ;; Return a list of alists.  Each alist contains fields of a GNU
 116     ;; package.
 117     (let loop ((alist  (recutils->alist port))
 118                (result '()))
 119       (if (null? alist)
 120           (reverse result)
 121           (loop (recutils->alist port)
 122                 (cons alist result)))))
 123
 124   (define official-description
 125     (let ((db (read-records (fetch %package-description-url #:text? #t))))
 126       (lambda (name)
 127         ;; Return the description found upstream for package NAME, or #f.
 128         (and=> (find (lambda (alist)
 129                        (equal? name (assoc-ref alist "package")))
 130                      db)
 131                (lambda (record)
 132                  (let ((field (assoc-ref record "blurb")))
 133                    ;; The upstream description file uses "redirect PACKAGE" as
 134                    ;; a blurb in cases where the description of the two
 135                    ;; packages should be considered the same (e.g., GTK+ has
 136                    ;; "redirect gnome".)  This is usually not acceptable for
 137                    ;; us because we prefer to have distinct descriptions in
 138                    ;; such cases.  Thus, ignore the 'blurb' field when that
 139                    ;; happens.
 140                    (and field
 141                         (not (string-prefix? "redirect " field))
 142                         field)))))))
 143
 144   (map (lambda (alist)
 145          (let ((name (assoc-ref alist "package")))
 146            (alist->record `(("description" . ,(official-description name))
 147                             ,@alist)
 148                           make-gnu-package-descriptor
 149                           (list "package" "mundane-name" "copyright-holder"
 150                                 "savannah" "fsd" "language" "logo"
 151                                 "doc-category" "doc-summary" "description"
 152                                 "doc-url"
 153                                 "download-url")
 154                           '("doc-url" "language"))))
 155        (let* ((port (fetch %package-list-url #:text? #t))
 156               (lst  (read-records port)))
 157          (close-port port)
 158          lst)))
 159
 160 (define (find-package name)
 161   "Find GNU package called NAME and return it.  Return #f if it was not
 162 found."
 163   (find (lambda (package)
 164           (string=? name (gnu-package-name package)))
 165         (official-gnu-packages)))
 166
 167 (define gnu-package?
 168   (memoize
 169    (let ((official-gnu-packages (memoize official-gnu-packages)))
 170      (lambda (package)
 171        "Return true if PACKAGE is a GNU package.  This procedure may access the
 172 network to check in GNU's database."
 173        (define (mirror-type url)
 174          (let ((uri (string->uri url)))
 175            (and (eq? (uri-scheme uri) 'mirror)
 176                 (cond
 177                  ((member (uri-host uri)
 178                           '("gnu" "gnupg" "gcc" "gnome"))
 179                   ;; Definitely GNU.
 180                   'gnu)
 181                  ((equal? (uri-host uri) "cran")
 182                   ;; Possibly GNU: mirror://cran could be either GNU R itself
 183                   ;; or a non-GNU package.
 184                   #f)
 185                  (else
 186                   ;; Definitely non-GNU.
 187                   'non-gnu)))))
 188
 189        (define (gnu-home-page? package)
 190          (letrec-syntax ((>> (syntax-rules ()
 191                                ((_ value proc)
 192                                 (and=> value proc))
 193                                ((_ value proc rest ...)
 194                                 (and=> value
 195                                        (lambda (next)
 196                                          (>> (proc next) rest ...)))))))
 197            (>> package package-home-page
 198                string->uri uri-host
 199                (lambda (host)
 200                  (member host '("www.gnu.org" "gnu.org"))))))
 201
 202        (or (gnu-home-page? package)
 203            (let ((url  (and=> (package-source package) origin-uri))
 204                  (name (or (assq-ref (package-properties package)
 205                                      'upstream-name)
 206                            (package-name package))))
 207              (case (and (string? url) (mirror-type url))
 208                ((gnu) #t)
 209                ((non-gnu) #f)
 210                (else
 211                 (and (member name (map gnu-package-name (official-gnu-packages)))
 212                      #t)))))))))
 213
 214 \f
 215 ;;;
 216 ;;; Latest release.
 217 ;;;
 218
 219 (define (ftp-server/directory package)
 220   "Return the FTP server and directory where PACKAGE's tarball are stored."
 221   (let ((name (or (assq-ref (package-properties package) 'upstream-name)
 222                   (package-name package))))
 223     (values (or (assoc-ref (package-properties package) 'ftp-server)
 224                 "ftp.gnu.org")
 225             (or (assoc-ref (package-properties package) 'ftp-directory)
 226                 (string-append "/gnu/" name)))))
 227
 228 (define (sans-extension tarball)
 229   "Return TARBALL without its .tar.* or .zip extension."
 230   (let ((end (or (string-contains tarball ".tar")
 231                  (string-contains tarball ".zip"))))
 232     (substring tarball 0 end)))
 233
 234 (define %tarball-rx
 235   ;; The .zip extensions is notably used for freefont-ttf.
 236   ;; The "-src" pattern is for "TeXmacs-1.0.7.9-src.tar.gz".
 237   ;; The "-gnu[0-9]" pattern is for "icecat-38.4.0-gnu1.tar.bz2".
 238   (make-regexp "^([^.]+)-([0-9]|[^-])+(-(src|gnu[0-9]))?\\.(tar\\.|zip$)"))
 239
 240 (define %alpha-tarball-rx
 241   (make-regexp "^.*-.*[0-9](-|~)?(alpha|beta|rc|cvs|svn|git)-?[0-9\\.]*\\.tar\\."))
 242
 243 (define (release-file? project file)
 244   "Return #f if FILE is not a release tarball of PROJECT, otherwise return
 245 true."
 246   (and (not (string-suffix? ".sig" file))
 247        (and=> (regexp-exec %tarball-rx file)
 248               (lambda (match)
 249                 ;; Filter out unrelated files, like `guile-www-1.1.1'.
 250                 ;; Case-insensitive for things like "TeXmacs" vs. "texmacs".
 251                 ;; The "-src" suffix is for "freefont-src-20120503.tar.gz".
 252                 (and=> (match:substring match 1)
 253                        (lambda (name)
 254                          (or (string-ci=? name project)
 255                              (string-ci=? name
 256                                           (string-append project
 257                                                          "-src")))))))
 258        (not (regexp-exec %alpha-tarball-rx file))
 259        (let ((s (sans-extension file)))
 260          (regexp-exec %package-name-rx s))))
 261
 262 (define (tarball->version tarball)
 263   "Return the version TARBALL corresponds to.  TARBALL is a file name like
 264 \"coreutils-8.23.tar.xz\"."
 265   (let-values (((name version)
 266                 (gnu-package-name->name+version (sans-extension tarball))))
 267     version))
 268
 269 (define* (releases project
 270                    #:key
 271                    (server "ftp.gnu.org")
 272                    (directory (string-append "/gnu/" project)))
 273   "Return the list of <upstream-release> of PROJECT as a list of release
 274 name/directory pairs."
 275   ;; TODO: Parse something like fencepost.gnu.org:/gd/gnuorg/packages-ftp.
 276   (define conn (ftp-open server))
 277
 278   (let loop ((directories (list directory))
 279              (result      '()))
 280     (match directories
 281       (()
 282        (ftp-close conn)
 283        (coalesce-sources result))
 284       ((directory rest ...)
 285        (let* ((files   (ftp-list conn directory))
 286               (subdirs (filter-map (match-lambda
 287                                      ((name 'directory . _) name)
 288                                      (_ #f))
 289                                    files)))
 290          (define (file->url file)
 291            (string-append "ftp://" server directory "/" file))
 292
 293          (define (file->source file)
 294            (let ((url (file->url file)))
 295              (upstream-source
 296               (package project)
 297               (version (tarball->version file))
 298               (urls (list url))
 299               (signature-urls (list (string-append url ".sig"))))))
 300
 301          (loop (append (map (cut string-append directory "/" <>)
 302                             subdirs)
 303                        rest)
 304                (append
 305                 ;; Filter out signatures, deltas, and files which
 306                 ;; are potentially not releases of PROJECT--e.g.,
 307                 ;; in /gnu/guile, filter out guile-oops and
 308                 ;; guile-www; in mit-scheme, filter out binaries.
 309                 (filter-map (match-lambda
 310                               ((file 'file . _)
 311                                (and (release-file? project file)
 312                                     (file->source file)))
 313                               (_ #f))
 314                             files)
 315                 result)))))))
 316
 317 (define* (latest-ftp-release project
 318                              #:key
 319                              (server "ftp.gnu.org")
 320                              (directory (string-append "/gnu/" project))
 321                              (keep-file? (const #t))
 322                              (file->signature (cut string-append <> ".sig"))
 323                              (ftp-open ftp-open) (ftp-close ftp-close))
 324   "Return an <upstream-source> for the latest release of PROJECT on SERVER
 325 under DIRECTORY, or #f.  Use FTP-OPEN and FTP-CLOSE to open (resp. close) FTP
 326 connections; this can be useful to reuse connections.
 327
 328 KEEP-FILE? is a predicate to decide whether to enter a directory and to
 329 consider a given file (source tarball) as a valid candidate based on its name.
 330
 331 FILE->SIGNATURE must be a procedure; it is passed a source file URL and must
 332 return the corresponding signature URL, or #f it signatures are unavailable."
 333   (define (latest a b)
 334     (if (version>? a b) a b))
 335
 336   (define (latest-release a b)
 337     (if (version>? (upstream-source-version a) (upstream-source-version b))
 338         a b))
 339
 340   (define contains-digit?
 341     (cut string-any char-set:digit <>))
 342
 343   (define patch-directory-name?
 344     ;; Return #t for patch directory names such as 'bash-4.2-patches'.
 345     (cut string-suffix? "patches" <>))
 346
 347   (define conn (ftp-open server))
 348
 349   (define (file->url directory file)
 350     (string-append "ftp://" server directory "/" file))
 351
 352   (define (file->source directory file)
 353     (let ((url (file->url directory file)))
 354       (upstream-source
 355        (package project)
 356        (version (tarball->version file))
 357        (urls (list url))
 358        (signature-urls (match (file->signature url)
 359                          (#f #f)
 360                          (sig (list sig)))))))
 361
 362   (let loop ((directory directory)
 363              (result    #f))
 364     (let* ((entries (ftp-list conn directory))
 365
 366            ;; Filter out sub-directories that do not contain digits---e.g.,
 367            ;; /gnuzilla/lang and /gnupg/patches.  Filter out "w32"
 368            ;; directories as found on ftp.gnutls.org.
 369            (subdirs (filter-map (match-lambda
 370                                   (((? patch-directory-name? dir)
 371                                     'directory . _)
 372                                    #f)
 373                                   (("w32" 'directory . _)
 374                                    #f)
 375                                   (((? contains-digit? dir) 'directory . _)
 376                                    (and (keep-file? dir) dir))
 377                                   (_ #f))
 378                                 entries))
 379
 380            ;; Whether or not SUBDIRS is empty, compute the latest releases
 381            ;; for the current directory.  This is necessary for packages
 382            ;; such as 'sharutils' that have a sub-directory that contains
 383            ;; only an older release.
 384            (releases (filter-map (match-lambda
 385                                    ((file 'file . _)
 386                                     (and (release-file? project file)
 387                                          (keep-file? file)
 388                                          (file->source directory file)))
 389                                    (_ #f))
 390                                  entries)))
 391
 392       ;; Assume that SUBDIRS correspond to versions, and jump into the
 393       ;; one with the highest version number.
 394       (let* ((release  (reduce latest-release #f
 395                                (coalesce-sources releases)))
 396              (result   (if (and result release)
 397                            (latest-release release result)
 398                            (or release result)))
 399              (target   (reduce latest #f subdirs)))
 400         (if target
 401             (loop (string-append directory "/" target)
 402                   result)
 403             (begin
 404               (ftp-close conn)
 405               result))))))
 406
 407 (define* (latest-release package
 408                          #:key
 409                          (server "ftp.gnu.org")
 410                          (directory (string-append "/gnu/" package)))
 411   "Return the <upstream-source> for the latest version of PACKAGE or #f.
 412 PACKAGE must be the canonical name of a GNU package."
 413   (latest-ftp-release package
 414                       #:server server
 415                       #:directory directory))
 416
 417 (define-syntax-rule (false-if-ftp-error exp)
 418   "Return #f if an FTP error is raise while evaluating EXP; return the result
 419 of EXP otherwise."
 420   (catch 'ftp-error
 421     (lambda ()
 422       exp)
 423     (lambda (key port . rest)
 424       (if (ftp-connection? port)
 425           (ftp-close port)
 426           (close-port port))
 427       #f)))
 428
 429 (define (latest-release* package)
 430   "Like 'latest-release', but (1) take a <package> object, and (2) ignore FTP
 431 errors that might occur when PACKAGE is not actually a GNU package, or not
 432 hosted on ftp.gnu.org, or not under that name (this is the case for
 433 \"emacs-auctex\", for instance.)"
 434   (let-values (((server directory)
 435                 (ftp-server/directory package)))
 436     (let ((name (or (assoc-ref (package-properties package) 'upstream-name)
 437                     (package-name package))))
 438       (false-if-ftp-error (latest-release name
 439                                           #:server server
 440                                           #:directory directory)))))
 441
 442 (define %package-name-rx
 443   ;; Regexp for a package name, e.g., "foo-X.Y".  Since TeXmacs uses
 444   ;; "TeXmacs-X.Y-src", the `-src' suffix is allowed.
 445   (make-regexp "^(.*)-(([0-9]|\\.)+)(-src)?"))
 446
 447 (define (gnu-package-name->name+version name+version)
 448   "Return the package name and version number extracted from NAME+VERSION."
 449   (let ((match (regexp-exec %package-name-rx name+version)))
 450     (if (not match)
 451         (values name+version #f)
 452         (values (match:substring match 1) (match:substring match 2)))))
 453
 454 (define (pure-gnu-package? package)
 455   "Return true if PACKAGE is a non-Emacs and non-GNOME GNU package.  This
 456 excludes AucTeX, for instance, whose releases are now uploaded to
 457 elpa.gnu.org, and all the GNOME packages; EMMS is included though, because its
 458 releases are on gnu.org."
 459   (and (or (not (string-prefix? "emacs-" (package-name package)))
 460            (gnu-hosted? package))
 461        (not (gnome-package? package))
 462        (gnu-package? package)))
 463
 464 (define (url-prefix-predicate prefix)
 465   "Return a predicate that returns true when passed a package where one of its
 466 source URLs starts with PREFIX."
 467   (lambda (package)
 468     (define matching-uri?
 469       (match-lambda
 470         ((? string? uri)
 471          (string-prefix? prefix uri))
 472         (_
 473          #f)))
 474
 475     (match (package-source package)
 476       ((? origin? origin)
 477        (match (origin-uri origin)
 478          ((? matching-uri?) #t)
 479          (_                 #f)))
 480       (_ #f))))
 481
 482 (define gnu-hosted?
 483   (url-prefix-predicate "mirror://gnu/"))
 484
 485 (define gnome-package?
 486   (url-prefix-predicate "mirror://gnome/"))
 487
 488 (define (latest-gnome-release package)
 489   "Return the latest release of PACKAGE, the name of a GNOME package."
 490   (define %not-dot
 491     (char-set-complement (char-set #\.)))
 492
 493   (define (even-minor-version? version)
 494     (match (string-tokenize version %not-dot)
 495       (((= string->number major) (= string->number minor) . rest)
 496        (and minor (even? minor)))
 497       (_
 498        #t)))                                      ;cross fingers
 499
 500   (define (even-numbered? file)
 501     ;; Return true if FILE somehow denotes an even-numbered file name.  The
 502     ;; trick here is that we want this to match both directories such as
 503     ;; "3.18.6" and actual file names such as "gtk+-3.18.6.tar.bz2".
 504     (let-values (((name version) (package-name->name+version file)))
 505       (even-minor-version? (or version name))))
 506
 507   (define upstream-name
 508     ;; Some packages like "NetworkManager" have camel-case names.
 509     (or (assoc-ref (package-properties package) 'upstream-name)
 510         (package-name package)))
 511
 512   (false-if-ftp-error
 513    (latest-ftp-release upstream-name
 514                        #:server "ftp.gnome.org"
 515                        #:directory (string-append "/pub/gnome/sources/"
 516                                                   upstream-name)
 517
 518
 519                        ;; <https://www.gnome.org/gnome-3/source/> explains
 520                        ;; that odd minor version numbers represent development
 521                        ;; releases, which we are usually not interested in.
 522                        #:keep-file? even-numbered?
 523
 524                        ;; ftp.gnome.org provides no signatures, only
 525                        ;; checksums.
 526                        #:file->signature (const #f))))
 527
 528
 529 (define (latest-kde-release package)
 530   "Return the latest release of PACKAGE, the name of an KDE.org package."
 531   (let ((uri (string->uri (origin-uri (package-source package)))))
 532     (false-if-ftp-error
 533      (latest-ftp-release
 534       (or (assoc-ref (package-properties package) 'upstream-name)
 535           (package-name package))
 536       #:server "mirrors.mit.edu"
 537       #:directory
 538       (string-append "/kde" (dirname (dirname (uri-path uri))))
 539       #:file->signature (const #f)))))
 540
 541 (define (latest-xorg-release package)
 542   "Return the latest release of PACKAGE, the name of an X.org package."
 543   (let ((uri (string->uri (origin-uri (package-source package)))))
 544     (false-if-ftp-error
 545      (latest-ftp-release
 546       (package-name package)
 547       #:server "ftp.freedesktop.org"
 548       #:directory
 549       (string-append "/pub/xorg/" (dirname (uri-path uri)))))))
 550
 551 (define (latest-kernel.org-release package)
 552   "Return the latest release of PACKAGE, the name of a kernel.org package."
 553   (let ((uri (string->uri (origin-uri (package-source package)))))
 554     (false-if-ftp-error
 555      (latest-ftp-release
 556       (package-name package)
 557       #:server "ftp.free.fr"                      ;a mirror reachable over FTP
 558       #:directory (string-append "/mirrors/ftp.kernel.org"
 559                                  (dirname (uri-path uri)))
 560
 561       ;; kernel.org provides "foo-x.y.tar.sign" files, which are signatures of
 562       ;; the uncompressed tarball.
 563       #:file->signature (lambda (tarball)
 564                           (string-append (file-sans-extension tarball)
 565                                          ".sign"))))))
 566
 567 (define %gnu-updater
 568   (upstream-updater
 569    (name 'gnu)
 570    (description "Updater for GNU packages")
 571    (pred pure-gnu-package?)
 572    (latest latest-release*)))
 573
 574 (define %gnome-updater
 575   (upstream-updater
 576    (name 'gnome)
 577    (description "Updater for GNOME packages")
 578    (pred gnome-package?)
 579    (latest latest-gnome-release)))
 580
 581 (define %kde-updater
 582   (upstream-updater
 583     (name 'kde)
 584     (description "Updater for KDE packages")
 585     (pred (url-prefix-predicate "mirror://kde/"))
 586     (latest latest-kde-release)))
 587
 588 (define %xorg-updater
 589   (upstream-updater
 590    (name 'xorg)
 591    (description "Updater for X.org packages")
 592    (pred (url-prefix-predicate "mirror://xorg/"))
 593    (latest latest-xorg-release)))
 594
 595 (define %kernel.org-updater
 596   (upstream-updater
 597    (name 'kernel.org)
 598    (description "Updater for packages hosted on kernel.org")
 599    (pred (url-prefix-predicate "mirror://kernel.org/"))
 600    (latest latest-kernel.org-release)))
 601
 602 ;;; gnu-maintenance.scm ends here