guix/download.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org>
   3 ;;; Copyright © 2013, 2014, 2015 Andreas Enge <andreas@enge.fr>
   4 ;;; Copyright © 2015 Federico Beffa <beffa@fbengineering.ch>
   5 ;;; Copyright © 2016 Alex Griffin <a@ajgrf.com>
   6 ;;; Copyright © 2016 David Craven <david@craven.ch>
   7 ;;; Copyright © 2017 Tobias Geerinckx-Rice <me@tobias.gr>
   8 ;;; Copyright © 2019 Guy Fleury Iteriteka <hoonandon@gmail.com>
   9 ;;;
  10 ;;; This file is part of GNU Guix.
  11 ;;;
  12 ;;; GNU Guix is free software; you can redistribute it and/or modify it
  13 ;;; under the terms of the GNU General Public License as published by
  14 ;;; the Free Software Foundation; either version 3 of the License, or (at
  15 ;;; your option) any later version.
  16 ;;;
  17 ;;; GNU Guix is distributed in the hope that it will be useful, but
  18 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  19 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 ;;; GNU General Public License for more details.
  21 ;;;
  22 ;;; You should have received a copy of the GNU General Public License
  23 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  24
  25 (define-module (guix download)
  26   #:use-module (ice-9 match)
  27   #:use-module (guix derivations)
  28   #:use-module (guix packages)
  29   #:use-module (guix store)
  30   #:autoload   (guix build download) (url-fetch)
  31   #:use-module (guix monads)
  32   #:use-module (guix gexp)
  33   #:use-module (guix utils)
  34   #:use-module (web uri)
  35   #:use-module (srfi srfi-1)
  36   #:use-module (srfi srfi-26)
  37   #:export (%mirrors
  38             %disarchive-mirrors
  39             (url-fetch* . url-fetch)
  40             url-fetch/executable
  41             url-fetch/tarbomb
  42             url-fetch/zipbomb
  43             download-to-store))
  44
  45 ;;; Commentary:
  46 ;;;
  47 ;;; Produce fixed-output derivations with data fetched over HTTP or FTP.
  48 ;;;
  49 ;;; Code:
  50
  51 (define %mirrors
  52   ;; Mirror lists used when `mirror://' URLs are passed.
  53   (let* ((gnu-mirrors
  54           '(;; This one redirects to a (supposedly) nearby and (supposedly)
  55             ;; up-to-date mirror.
  56             "https://ftpmirror.gnu.org/gnu/"
  57
  58             "ftp://ftp.cs.tu-berlin.de/pub/gnu/"
  59             "ftp://ftp.funet.fi/pub/mirrors/ftp.gnu.org/gnu/"
  60
  61             ;; This one is the master repository, and thus it's always
  62             ;; up-to-date.
  63             "http://ftp.gnu.org/pub/gnu/")))
  64     `((gnu ,@gnu-mirrors)
  65       (gcc
  66        "ftp://ftp.nluug.nl/mirror/languages/gcc/"
  67        "ftp://ftp.fu-berlin.de/unix/languages/gcc/"
  68        "ftp://ftp.irisa.fr/pub/mirrors/gcc.gnu.org/gcc/"
  69        "ftp://gcc.gnu.org/pub/gcc/"
  70        ,@(map (cut string-append <> "/gcc") gnu-mirrors))
  71       (gnupg
  72        "http://artfiles.org/gnupg.org"
  73        "http://www.crysys.hu/"
  74        "https://gnupg.org/ftp/gcrypt/"
  75        "ftp://mirrors.dotsrc.org/gcrypt/"
  76        "ftp://mirror.cict.fr/gnupg/"
  77        "ftp://ftp.franken.de/pub/crypt/mirror/ftp.gnupg.org/gcrypt/"
  78        "ftp://ftp.freenet.de/pub/ftp.gnupg.org/gcrypt/"
  79        "ftp://ftp.hi.is/pub/mirrors/gnupg/"
  80        "ftp://ftp.heanet.ie/mirrors/ftp.gnupg.org/gcrypt/"
  81        "ftp://ftp.bit.nl/mirror/gnupg/"
  82        "ftp://ftp.surfnet.nl/pub/security/gnupg/"
  83        "ftp://ftp.iasi.roedu.net/pub/mirrors/ftp.gnupg.org/"
  84        "ftp://ftp.sunet.se/pub/security/gnupg/"
  85        "ftp://mirror.switch.ch/mirror/gnupg/"
  86        "ftp://mirror.tje.me.uk/pub/mirrors/ftp.gnupg.org/"
  87        "ftp://ftp.mirrorservice.org/sites/ftp.gnupg.org/gcrypt/"
  88        "ftp://ftp.ring.gr.jp/pub/net/gnupg/"
  89        "ftp://ftp.gnupg.org/gcrypt/")
  90       (gnome
  91        "http://ftp.belnet.be/ftp.gnome.org/"
  92        "http://ftp.linux.org.uk/mirrors/ftp.gnome.org/"
  93        "http://ftp.gnome.org/pub/GNOME/"
  94        "https://download.gnome.org/"
  95        "http://mirror.yandex.ru/mirrors/ftp.gnome.org/")
  96       (hackage
  97        "http://hackage.haskell.org/")
  98       (savannah           ; http://download0.savannah.gnu.org/mirmon/savannah/
  99        "http://download.savannah.gnu.org/releases/"
 100        "http://nongnu.freemirror.org/nongnu/"
 101        "http://ftp.cc.uoc.gr/mirrors/nongnu.org/"
 102        "http://ftp.twaren.net/Unix/NonGNU/"
 103        "http://mirror.csclub.uwaterloo.ca/nongnu/"
 104        "http://nongnu.askapache.com/"
 105        "http://savannah.c3sl.ufpr.br/"
 106        "http://download.savannah.gnu.org/releases-noredirect/"
 107        "http://download-mirror.savannah.gnu.org/releases/"
 108        "ftp://ftp.twaren.net/Unix/NonGNU/"
 109        "ftp://mirror.csclub.uwaterloo.ca/nongnu/"
 110        "ftp://mirror.publicns.net/pub/nongnu/"
 111        "ftp://savannah.c3sl.ufpr.br/")
 112       (sourceforge ; https://sourceforge.net/p/forge/documentation/Mirrors/
 113        "http://downloads.sourceforge.net/project/"
 114        "http://ufpr.dl.sourceforge.net/project/"
 115        "http://heanet.dl.sourceforge.net/project/"
 116        "http://freefr.dl.sourceforge.net/project/"
 117        "http://internode.dl.sourceforge.net/project/"
 118        "http://jaist.dl.sourceforge.net/project/"
 119        "http://kent.dl.sourceforge.net/project/"
 120        "http://liquidtelecom.dl.sourceforge.net/project/"
 121        ;; "http://nbtelecom.dl.sourceforge.net/project/"  ;never returns 404s
 122        "http://nchc.dl.sourceforge.net/project/"
 123        "http://ncu.dl.sourceforge.net/project/"
 124        "http://netcologne.dl.sourceforge.net/project/"
 125        "http://netix.dl.sourceforge.net/project/"
 126        "http://pilotfiber.dl.sourceforge.net/project/"
 127        "http://superb-sea2.dl.sourceforge.net/project/"
 128        "http://tenet.dl.sourceforge.net/project/"
 129        "http://vorboss.dl.sourceforge.net/project/"
 130        "http://netassist.dl.sourceforge.net/project/")
 131       (netfilter.org ; https://www.netfilter.org/mirrors.html
 132        "http://ftp.netfilter.org/pub/"
 133        "ftp://ftp.es.netfilter.org/mirrors/netfilter/"
 134        "ftp://ftp.hu.netfilter.org/"
 135        "ftp://www.lt.netfilter.org/pub/")
 136       (kernel.org
 137        "http://linux-kernel.uio.no/pub/"
 138        "http://kernel.osuosl.org/pub/"
 139        "http://ftp.be.debian.org/pub/"
 140        "http://mirror.linux.org.au/"
 141        "ftp://ftp.funet.fi/pub/mirrors/ftp.kernel.org/pub/")
 142       (apache             ; from http://www.apache.org/mirrors/dist.html
 143        "http://www.eu.apache.org/dist/"
 144        "http://www.us.apache.org/dist/"
 145        "https://ftp.nluug.nl/internet/apache/"
 146        "http://apache.mirror.iweb.ca/"
 147        "http://mirrors.ircam.fr/pub/apache/"
 148        "http://apache.mirrors.ovh.net/ftp.apache.org/dist/"
 149        "http://apache-mirror.rbc.ru/pub/apache/"
 150        "ftp://ftp.osuosl.org/pub/apache/"
 151        "http://mirrors.ibiblio.org/apache/"
 152
 153        ;; As a last resort, try the archive.
 154        "http://archive.apache.org/dist/")
 155       (xorg               ; from http://www.x.org/wiki/Releases/Download
 156        "http://www.x.org/releases/" ; main mirrors
 157        "http://mirror.csclub.uwaterloo.ca/x.org/" ; North America
 158        "http://xorg.mirrors.pair.com/"
 159        "http://mirror.us.leaseweb.net/xorg/"
 160        "ftp://mirror.csclub.uwaterloo.ca/x.org/"
 161        "ftp://xorg.mirrors.pair.com/"
 162        "ftp://artfiles.org/x.org/" ; Europe
 163        "ftp://ftp.chg.ru/pub/X11/x.org/"
 164        "ftp://ftp.fu-berlin.de/unix/X11/FTP.X.ORG/"
 165        "ftp://ftp.gwdg.de/pub/x11/x.org/"
 166        "ftp://ftp.mirrorservice.org/sites/ftp.x.org/"
 167        "ftp://ftp.ntua.gr/pub/X11/"
 168        "ftp://ftp.piotrkosoft.net/pub/mirrors/ftp.x.org/"
 169        "ftp://ftp.portal-to-web.de/pub/mirrors/x.org/"
 170        "ftp://ftp.solnet.ch/mirror/x.org/"
 171        "ftp://mi.mirror.garr.it/mirrors/x.org/"
 172        "ftp://mirror.cict.fr/x.org/"
 173        "ftp://mirror.switch.ch/mirror/X11/"
 174        "ftp://mirrors.ircam.fr/pub/x.org/"
 175        "ftp://x.mirrors.skynet.be/pub/ftp.x.org/"
 176        "http://x.cs.pu.edu.tw/" ; East Asia
 177        "ftp://ftp.cs.cuhk.edu.hk/pub/X11"
 178        "ftp://ftp.u-aizu.ac.jp/pub/x11/x.org/"
 179        "ftp://ftp.yz.yamagata-u.ac.jp/pub/X11/x.org/"
 180        "ftp://ftp.kaist.ac.kr/x.org/"
 181        "ftp://mirrors.go-part.com/xorg/"
 182        "ftp://ftp.is.co.za/pub/x.org")            ; South Africa
 183       (cpan
 184        "http://www.cpan.org/"
 185        "http://cpan.metacpan.org/"
 186        ;; A selection of HTTP mirrors from http://www.cpan.org/SITES.html.
 187        ;; Europe.
 188        "http://ftp.belnet.be/mirror/ftp.cpan.org/"
 189        "http://mirrors.nic.cz/CPAN/"
 190        "http://mirror.ibcp.fr/pub/CPAN/"
 191        "http://ftp.ntua.gr/pub/lang/perl/"
 192        "http://mirror.as43289.net/pub/CPAN/"
 193        "http://cpan.cs.uu.nl/"
 194        "http://cpan.uib.no/"
 195        "http://cpan-mirror.rbc.ru/pub/CPAN/"
 196        "http://mirror.sbb.rs/CPAN/"
 197        "http://cpan.lnx.sk/"
 198        "http://ftp.rediris.es/mirror/CPAN/"
 199        "http://mirror.ox.ac.uk/sites/www.cpan.org/"
 200        ;; Africa.
 201        "http://mirror.liquidtelecom.com/CPAN/"
 202        "http://cpan.mirror.ac.za/"
 203        "http://mirror.is.co.za/pub/cpan/"
 204        "http://cpan.saix.net/"
 205        "http://mirror.ucu.ac.ug/cpan/"
 206        ;; North America.
 207        "http://mirrors.gossamer-threads.com/CPAN/"
 208        "http://mirror.csclub.uwaterloo.ca/CPAN/"
 209        "http://mirrors.ucr.ac.cr/CPAN/"
 210        "http://www.msg.com.mx/CPAN/"
 211        "http://mirrors.namecheap.com/CPAN/"
 212        "http://mirror.uic.edu/CPAN/"
 213        "http://mirror.datapipe.net/CPAN/"
 214        "http://mirror.cc.columbia.edu/pub/software/cpan/"
 215        "http://mirror.uta.edu/CPAN/"
 216        ;; South America.
 217        "http://cpan.mmgdesigns.com.ar/"
 218        "http://mirror.nbtelecom.com.br/CPAN/"
 219        "http://linorg.usp.br/CPAN/"
 220        "http://cpan.dcc.uchile.cl/"
 221        "http://mirror.cedia.org.ec/CPAN/"
 222        ;; Oceania.
 223        "http://cpan.mirror.serversaustralia.com.au/"
 224        "http://mirror.waia.asn.au/pub/cpan/"
 225        "http://mirror.as24220.net/pub/cpan/"
 226        "http://cpan.lagoon.nc/pub/CPAN/"
 227        "http://cpan.inspire.net.nz/"
 228        ;; Asia.
 229        "http://mirror.dhakacom.com/CPAN/"
 230        "http://mirrors.ustc.edu.cn/CPAN/"
 231        "http://ftp.cuhk.edu.hk/pub/packages/perl/CPAN/"
 232        "http://kambing.ui.ac.id/cpan/"
 233        "http://cpan.hostiran.ir/"
 234        "http://ftp.nara.wide.ad.jp/pub/CPAN/"
 235        "http://mirror.neolabs.kz/CPAN/"
 236        "http://cpan.nctu.edu.tw/"
 237        "http://cpan.ulak.net.tr/"
 238        "http://mirrors.vinahost.vn/CPAN/")
 239       (cran
 240        ;; Arbitrary mirrors from http://cran.r-project.org/mirrors.html
 241        ;; This one automatically redirects to servers worldwide
 242        "http://cran.r-project.org/"
 243        "http://cran.rstudio.com/"
 244        "http://cran.univ-lyon1.fr/"
 245        "http://cran.ism.ac.jp/"
 246        "http://cran.stat.auckland.ac.nz/"
 247        "http://cran.mirror.ac.za/"
 248        "http://cran.csie.ntu.edu.tw/")
 249       (imagemagick
 250        ;; from http://www.imagemagick.org/script/download.php
 251        ;; (without mirrors that are unavailable or not up to date)
 252        "https://sunsite.icm.edu.pl/packages/ImageMagick/releases"
 253        "http://mirror.checkdomain.de/imagemagick/releases"
 254        "ftp://ftp.u-aizu.ac.jp/pub/graphics/image/ImageMagick/imagemagick.org/releases"
 255        "ftp://ftp.nluug.nl/pub/ImageMagick/"
 256        "http://www.imagemagick.org/download/releases/"
 257        ;; Try this if all else fails (normally contains just the latest version).
 258        "http://www.imagemagick.org/download/")
 259       (debian
 260        "http://ftp.de.debian.org/debian/"
 261        "http://ftp.fr.debian.org/debian/"
 262        "http://ftp.debian.org/debian/"
 263        "http://archive.debian.org/debian/")
 264       (kde
 265        "http://download.kde.org"
 266        "http://download.kde.org/Attic" ; for when it gets archived.
 267        ;; Mirrors from http://files.kde.org/extra/mirrors.html
 268        ;; Europe
 269        "http://mirror.easyname.at/kde"
 270        "http://mirror.karneval.cz/pub/kde"
 271        "http://ftp.fi.muni.cz/pub/kde/"
 272        "http://mirror.oss.maxcdn.com/kde/"
 273        "http://ftp5.gwdg.de/pub/linux/kde/"
 274        "http://ftp-stud.fht-esslingen.de/Mirrors/ftp.kde.org/pub/kde/"
 275        "http://mirror.klaus-uwe.me/kde/ftp/"
 276        "http://kde.beta.mirror.ga/"
 277        "http://kde.alpha.mirror.ga/"
 278        "http://mirror.netcologne.de/kde"
 279        "http://vesta.informatik.rwth-aachen.de/ftp/pub/mirror/kde/"
 280        "http://ftp.rz.uni-wuerzburg.de/pub/unix/kde/"
 281        "http://mirrors.dotsrc.org/kde/"
 282        "http://ftp.funet.fi/pub/mirrors/ftp.kde.org/pub/kde/"
 283        "http://kde-mirror.freenux.org/"
 284        "http://mirrors.ircam.fr/pub/KDE/"
 285        "http://www-ftp.lip6.fr/pub/X11/kde/"
 286        "http://fr2.rpmfind.net/linux/KDE/"
 287        "http://kde.mirror.anlx.net/"
 288        "http://www.mirrorservice.org/sites/ftp.kde.org/pub/kde/"
 289        "http://ftp.heanet.ie/mirrors/ftp.kde.org/"
 290        "http://ftp.nluug.nl/pub/windowing/kde/"
 291        "http://ftp.surfnet.nl/windowing/kde/"
 292        "http://ftp.icm.edu.pl/pub/unix/kde/"
 293        "http://ftp.pbone.net/pub/kde/"
 294        "http://piotrkosoft.net/pub/mirrors/ftp.kde.org/"
 295        "http://mirrors.fe.up.pt/pub/kde/"
 296        "http://ftp.iasi.roedu.net/pub/mirrors/ftp.kde.org/"
 297        "http://ftp.acc.umu.se/mirror/kde.org/ftp/"
 298        "http://kde.ip-connect.vn.ua/"
 299        ;; North America
 300        "http://mirror.its.dal.ca/kde/"
 301        "http://mirror.csclub.uwaterloo.ca/kde/"
 302        "http://mirror.cc.columbia.edu/pub/software/kde/"
 303        "http://kde.mirrors.hoobly.com/"
 304        "http://ftp.ussg.iu.edu/kde/"
 305        "http://mirrors.mit.edu/kde/"
 306        "http://kde.mirrors.tds.net/pub/kde/"
 307        ;; Oceania
 308        "http://ftp.kddlabs.co.jp/pub/X11/kde/"
 309        "http://kde.mirror.uber.com.au/")
 310       (openbsd
 311        "https://ftp.openbsd.org/pub/OpenBSD/"
 312        ;; Anycast CDN redirecting to your friendly local mirror.
 313        "https://mirrors.evowise.com/pub/OpenBSD/"
 314        ;; Other HTTPS mirrors from https://www.openbsd.org/ftp.html
 315        "https://mirror.aarnet.edu.au/pub/OpenBSD/"
 316        "https://ftp2.eu.openbsd.org/pub/OpenBSD/"
 317        "https://openbsd.c3sl.ufpr.br/pub/OpenBSD/"
 318        "https://openbsd.ipacct.com/pub/OpenBSD/"
 319        "https://ftp.OpenBSD.org/pub/OpenBSD/"
 320        "https://openbsd.cs.toronto.edu/pub/OpenBSD/"
 321        "https://openbsd.delfic.org/pub/OpenBSD/"
 322        "https://openbsd.mirror.netelligent.ca/pub/OpenBSD/"
 323        "https://mirrors.ucr.ac.cr/pub/OpenBSD/"
 324        "https://mirrors.dotsrc.org/pub/OpenBSD/"
 325        "https://mirror.one.com/pub/OpenBSD/"
 326        "https://ftp.fr.openbsd.org/pub/OpenBSD/"
 327        "https://ftp2.fr.openbsd.org/pub/OpenBSD/"
 328        "https://mirrors.ircam.fr/pub/OpenBSD/"
 329        "https://ftp.spline.de/pub/OpenBSD/"
 330        "https://mirror.hs-esslingen.de/pub/OpenBSD/"
 331        "https://ftp.halifax.rwth-aachen.de/openbsd/"
 332        "https://ftp.hostserver.de/pub/OpenBSD/"
 333        "https://ftp.fau.de/pub/OpenBSD/"
 334        "https://ftp.cc.uoc.gr/pub/OpenBSD/"
 335        "https://openbsd.hk/pub/OpenBSD/"
 336        "https://ftp.heanet.ie/pub/OpenBSD/"
 337        "https://openbsd.mirror.garr.it/pub/OpenBSD/"
 338        "https://mirror.litnet.lt/pub/OpenBSD/"
 339        "https://mirror.meerval.net/pub/OpenBSD/"
 340        "https://ftp.nluug.nl/pub/OpenBSD/"
 341        "https://ftp.bit.nl/pub/OpenBSD/"
 342        "https://mirrors.dalenys.com/pub/OpenBSD/"
 343        "https://ftp.icm.edu.pl/pub/OpenBSD/"
 344        "https://ftp.rnl.tecnico.ulisboa.pt/pub/OpenBSD/"
 345        "https://mirrors.pidginhost.com/pub/OpenBSD/"
 346        "https://mirror.yandex.ru/pub/OpenBSD/"
 347        "https://ftp.eu.openbsd.org/pub/OpenBSD/"
 348        "https://ftp.yzu.edu.tw/pub/OpenBSD/"
 349        "https://www.mirrorservice.org/pub/OpenBSD/"
 350        "https://anorien.csc.warwick.ac.uk/pub/OpenBSD/"
 351        "https://mirror.bytemark.co.uk/pub/OpenBSD/"
 352        "https://mirrors.sonic.net/pub/OpenBSD/"
 353        "https://ftp3.usa.openbsd.org/pub/OpenBSD/"
 354        "https://mirrors.syringanetworks.net/pub/OpenBSD/"
 355        "https://openbsd.mirror.constant.com/pub/OpenBSD/"
 356        "https://ftp4.usa.openbsd.org/pub/OpenBSD/"
 357        "https://ftp5.usa.openbsd.org/pub/OpenBSD/"
 358        "https://mirror.esc7.net/pub/OpenBSD/")
 359       (mate
 360        "https://pub.mate-desktop.org/releases/"
 361        "http://pub.mate-desktop.org/releases/"))))
 362
 363 (define %mirror-file
 364   ;; Copy of the list of mirrors to a file.  This allows us to keep a single
 365   ;; copy in the store, and computing it here avoids repeated calls to
 366   ;; 'object->string'.
 367   (plain-file "mirrors" (object->string %mirrors)))
 368
 369 (define %content-addressed-mirrors
 370   ;; List of content-addressed mirrors.  Each mirror is represented as a
 371   ;; procedure that takes a file name, an algorithm (symbol) and a hash
 372   ;; (bytevector), and returns a URL or #f.
 373   '(begin
 374      (use-modules (guix base32))
 375
 376      (define (guix-publish host)
 377        (lambda (file algo hash)
 378          ;; Files served by 'guix publish' are accessible under a single
 379          ;; hash algorithm.
 380          (string-append "https://" host "/file/"
 381                         file "/" (symbol->string algo) "/"
 382                         (bytevector->nix-base32-string hash))))
 383
 384      ;; XXX: (guix base16) appeared in March 2017 (and thus 0.13.0) so old
 385      ;; installations of the daemon might lack it.  Thus, load it lazily to
 386      ;; avoid gratuitous errors.  See <https://bugs.gnu.org/33542>.
 387      (module-autoload! (current-module)
 388                        '(guix base16) '(bytevector->base16-string))
 389
 390      (list (guix-publish "ci.guix.gnu.org")
 391            (lambda (file algo hash)
 392              ;; 'tarballs.nixos.org' supports several algorithms.
 393              (string-append "https://tarballs.nixos.org/"
 394                             (symbol->string algo) "/"
 395                             (bytevector->nix-base32-string hash)))
 396            (lambda (file algo hash)
 397              ;; Software Heritage usually archives VCS history rather than
 398              ;; tarballs, but tarballs are sometimes available (and can be
 399              ;; explicitly stored there.)  For example, see
 400              ;; <https://archive.softwareheritage.org/api/1/content/sha256:92d0fa1c311cacefa89853bdb53c62f4110cdfda3820346b59cbd098f40f955e/>.
 401              (string-append "https://archive.softwareheritage.org/api/1/content/"
 402                             (symbol->string algo) ":"
 403                             (bytevector->base16-string hash) "/raw/")))))
 404
 405 (define %content-addressed-mirror-file
 406   ;; Content-addressed mirrors stored in a file.
 407   (plain-file "content-addressed-mirrors"
 408               (object->string %content-addressed-mirrors)))
 409
 410 (define %disarchive-mirrors
 411   '("https://disarchive.ngyro.com/"))
 412
 413 (define %disarchive-mirror-file
 414   (plain-file "disarchive-mirrors" (object->string %disarchive-mirrors)))
 415
 416 (define built-in-builders*
 417   (store-lift built-in-builders))
 418
 419 (define* (built-in-download file-name url
 420                             #:key system hash-algo hash
 421                             mirrors content-addressed-mirrors
 422                             disarchive-mirrors
 423                             executable?
 424                             (guile 'unused))
 425   "Download FILE-NAME from URL using the built-in 'download' builder.  When
 426 EXECUTABLE? is true, make the downloaded file executable.
 427
 428 This is an \"out-of-band\" download in that the returned derivation does not
 429 explicitly depend on Guile, GnuTLS, etc.  Instead, the daemon performs the
 430 download by itself using its own dependencies."
 431   (mlet %store-monad ((mirrors (lower-object mirrors))
 432                       (content-addressed-mirrors
 433                        (lower-object content-addressed-mirrors))
 434                       (disarchive-mirrors (lower-object disarchive-mirrors)))
 435     (raw-derivation file-name "builtin:download" '()
 436                     #:system system
 437                     #:hash-algo hash-algo
 438                     #:hash hash
 439                     #:recursive? executable?
 440                     #:sources (list mirrors
 441                                     content-addressed-mirrors
 442                                     disarchive-mirrors)
 443
 444                     ;; Honor the user's proxy and locale settings.
 445                     #:leaked-env-vars '("http_proxy" "https_proxy"
 446                                         "LC_ALL" "LC_MESSAGES" "LANG"
 447                                         "COLUMNS")
 448
 449                     #:env-vars `(("url" . ,(object->string url))
 450                                  ("mirrors" . ,mirrors)
 451                                  ("content-addressed-mirrors"
 452                                   . ,content-addressed-mirrors)
 453                                  ("disarchive-mirrors" . ,disarchive-mirrors)
 454                                  ,@(if executable?
 455                                        '(("executable" . "1"))
 456                                        '()))
 457
 458                     ;; Do not offload this derivation because we cannot be
 459                     ;; sure that the remote daemon supports the 'download'
 460                     ;; built-in.  We may remove this limitation when support
 461                     ;; for that built-in is widespread.
 462                     #:local-build? #t)))
 463
 464 (define* (url-fetch* url hash-algo hash
 465                      #:optional name
 466                      #:key (system (%current-system))
 467                      (guile (default-guile))
 468                      executable?)
 469   "Return a fixed-output derivation that fetches data from URL (a string, or a
 470 list of strings denoting alternate URLs), which is expected to have hash HASH
 471 of type HASH-ALGO (a symbol).  By default, the file name is the base name of
 472 URL; optionally, NAME can specify a different file name.  When EXECUTABLE? is
 473 true, make the downloaded file executable.
 474
 475 When one of the URL starts with mirror://, then its host part is
 476 interpreted as the name of a mirror scheme, taken from %MIRROR-FILE.
 477
 478 Alternatively, when URL starts with file://, return the corresponding file
 479 name in the store."
 480   (define file-name
 481     (match url
 482       ((head _ ...)
 483        (basename head))
 484       (_
 485        (basename url))))
 486
 487   (let ((uri (and (string? url) (string->uri url))))
 488     (if (or (and (string? url) (not uri))
 489             (and uri (memq (uri-scheme uri) '(#f file))))
 490         (interned-file (if uri (uri-path uri) url)
 491                        (or name file-name))
 492         (mlet %store-monad ((builtins (built-in-builders*)))
 493           ;; The "download" built-in builder was added in guix-daemon in
 494           ;; Nov. 2016 and made it in the 0.12.0 release of Dec. 2016.  We now
 495           ;; require it.
 496           (unless (member "download" builtins)
 497             (error "'guix-daemon' is too old, please upgrade" builtins))
 498
 499           (built-in-download (or name file-name) url
 500                              #:guile guile
 501                              #:system system
 502                              #:hash-algo hash-algo
 503                              #:hash hash
 504                              #:executable? executable?
 505                              #:mirrors %mirror-file
 506                              #:content-addressed-mirrors
 507                              %content-addressed-mirror-file
 508                              #:disarchive-mirrors
 509                              %disarchive-mirror-file)))))
 510
 511 (define* (url-fetch/executable url hash-algo hash
 512                                #:optional name
 513                                #:key (system (%current-system))
 514                                (guile (default-guile)))
 515   "Like 'url-fetch', but make the downloaded file executable."
 516   (url-fetch* url hash-algo hash name
 517               #:system system
 518               #:guile guile
 519               #:executable? #t))
 520
 521 (define* (url-fetch/tarbomb url hash-algo hash
 522                             #:optional name
 523                             #:key (system (%current-system))
 524                             (guile (default-guile)))
 525   "Similar to 'url-fetch' but unpack the file from URL in a directory of its
 526 own.  This helper makes it easier to deal with \"tar bombs\"."
 527   (define file-name
 528     (match url
 529       ((head _ ...)
 530        (basename head))
 531       (_
 532        (basename url))))
 533   (define gzip
 534     (module-ref (resolve-interface '(gnu packages compression)) 'gzip))
 535   (define tar
 536     (module-ref (resolve-interface '(gnu packages base)) 'tar))
 537
 538   (mlet %store-monad ((drv (url-fetch* url hash-algo hash
 539                                        (string-append "tarbomb-"
 540                                                       (or name file-name))
 541                                        #:system system
 542                                        #:guile guile))
 543                       (guile (package->derivation guile system)))
 544     ;; Take the tar bomb, and simply unpack it as a directory.
 545     ;; Use ungrafted tar/gzip so that the resulting tarball doesn't depend on
 546     ;; whether grafts are enabled.
 547     (gexp->derivation (or name file-name)
 548                       (with-imported-modules '((guix build utils))
 549                         #~(begin
 550                             (use-modules (guix build utils))
 551                             (mkdir #$output)
 552                             (setenv "PATH" (string-append #+gzip "/bin"))
 553                             (chdir #$output)
 554                             (invoke (string-append #+tar "/bin/tar")
 555                                     "xf" #$drv)))
 556                       #:system system
 557                       #:guile-for-build guile
 558                       #:graft? #f
 559                       #:local-build? #t)))
 560
 561 (define* (url-fetch/zipbomb url hash-algo hash
 562                             #:optional name
 563                             #:key (system (%current-system))
 564                             (guile (default-guile)))
 565   "Similar to 'url-fetch' but unpack the zip file at URL in a directory of its
 566 own.  This helper makes it easier to deal with \"zip bombs\"."
 567   (define file-name
 568     (match url
 569       ((head _ ...)
 570        (basename head))
 571       (_
 572        (basename url))))
 573   (define unzip
 574     (module-ref (resolve-interface '(gnu packages compression)) 'unzip))
 575
 576   (mlet %store-monad ((drv (url-fetch* url hash-algo hash
 577                                        (string-append "zipbomb-"
 578                                                       (or name file-name))
 579                                        #:system system
 580                                        #:guile guile))
 581                       (guile (package->derivation guile system)))
 582     ;; Take the zip bomb, and simply unpack it as a directory.
 583     ;; Use ungrafted unzip so that the resulting tarball doesn't depend on
 584     ;; whether grafts are enabled.
 585     (gexp->derivation (or name file-name)
 586                       (with-imported-modules '((guix build utils))
 587                         #~(begin
 588                             (use-modules (guix build utils))
 589                             (mkdir #$output)
 590                             (chdir #$output)
 591                             (invoke (string-append #+unzip "/bin/unzip")
 592                                     #$drv)))
 593                       #:system system
 594                       #:guile-for-build guile
 595                       #:graft? #f
 596                       #:local-build? #t)))
 597
 598 (define* (download-to-store store url #:optional (name (basename url))
 599                             #:key (log (current-error-port)) recursive?
 600                             (verify-certificate? #t))
 601   "Download from URL to STORE, either under NAME or URL's basename if
 602 omitted.  Write progress reports to LOG.  RECURSIVE? has the same effect as
 603 the same-named parameter of 'add-to-store'.  VERIFY-CERTIFICATE? determines
 604 whether or not to validate HTTPS server certificates."
 605   (define uri
 606     (string->uri url))
 607
 608   (if (or (not uri) (memq (uri-scheme uri) '(file #f)))
 609       (add-to-store store name recursive? "sha256"
 610                     (if uri (uri-path uri) url))
 611       (call-with-temporary-output-file
 612        (lambda (temp port)
 613          (let ((result
 614                 (parameterize ((current-output-port log))
 615                   (url-fetch url temp
 616                              #:mirrors %mirrors
 617                              #:verify-certificate? verify-certificate?))))
 618            (close port)
 619            (and result
 620                 (add-to-store store name recursive? "sha256" temp)))))))
 621
 622 ;;; download.scm ends here