guix/download.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2012, 2013, 2014, 2015, 2016 Ludovic Courtès <ludo@gnu.org>
   3 ;;; Copyright © 2013, 2014, 2015 Andreas Enge <andreas@enge.fr>
   4 ;;; Copyright © 2015 Federico Beffa <beffa@fbengineering.ch>
   5 ;;; Copyright © 2016 Alex Griffin <a@ajgrf.com>
   6 ;;; Copyright © 2016 David Craven <david@craven.ch>
   7 ;;; Copyright © 2017 Tobias Geerinckx-Rice <me@tobias.gr>
   8 ;;;
   9 ;;; This file is part of GNU Guix.
  10 ;;;
  11 ;;; GNU Guix is free software; you can redistribute it and/or modify it
  12 ;;; under the terms of the GNU General Public License as published by
  13 ;;; the Free Software Foundation; either version 3 of the License, or (at
  14 ;;; your option) any later version.
  15 ;;;
  16 ;;; GNU Guix is distributed in the hope that it will be useful, but
  17 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  18 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 ;;; GNU General Public License for more details.
  20 ;;;
  21 ;;; You should have received a copy of the GNU General Public License
  22 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  23
  24 (define-module (guix download)
  25   #:use-module (ice-9 match)
  26   #:use-module (guix derivations)
  27   #:use-module (guix packages)
  28   #:use-module (guix store)
  29   #:use-module ((guix build download) #:prefix build:)
  30   #:use-module (guix monads)
  31   #:use-module (guix gexp)
  32   #:use-module (guix utils)
  33   #:use-module (web uri)
  34   #:use-module (srfi srfi-1)
  35   #:use-module (srfi srfi-26)
  36   #:export (%mirrors
  37             url-fetch
  38             url-fetch/tarbomb
  39             url-fetch/zipbomb
  40             download-to-store))
  41
  42 ;;; Commentary:
  43 ;;;
  44 ;;; Produce fixed-output derivations with data fetched over HTTP or FTP.
  45 ;;;
  46 ;;; Code:
  47
  48 (define %mirrors
  49   ;; Mirror lists used when `mirror://' URLs are passed.
  50   (let* ((gnu-mirrors
  51           '(;; This one redirects to a (supposedly) nearby and (supposedly)
  52             ;; up-to-date mirror.
  53             "http://ftpmirror.gnu.org/"
  54
  55             "ftp://ftp.cs.tu-berlin.de/pub/gnu/"
  56             "ftp://ftp.funet.fi/pub/mirrors/ftp.gnu.org/gnu/"
  57
  58             ;; This one is the master repository, and thus it's always
  59             ;; up-to-date.
  60             "http://ftp.gnu.org/pub/gnu/")))
  61     `((gnu ,@gnu-mirrors)
  62       (gcc
  63        "ftp://ftp.nluug.nl/mirror/languages/gcc/"
  64        "ftp://ftp.fu-berlin.de/unix/languages/gcc/"
  65        "ftp://ftp.irisa.fr/pub/mirrors/gcc.gnu.org/gcc/"
  66        "ftp://gcc.gnu.org/pub/gcc/"
  67        ,@(map (cut string-append <> "/gcc") gnu-mirrors))
  68       (gnupg
  69        "http://gd.tuwien.ac.at/privacy/gnupg/"
  70        "http://artfiles.org/gnupg.org"
  71        "http://www.crysys.hu/"
  72        "https://gnupg.org/ftp/gcrypt/"
  73        "ftp://mirrors.dotsrc.org/gcrypt/"
  74        "ftp://mirror.cict.fr/gnupg/"
  75        "ftp://ftp.franken.de/pub/crypt/mirror/ftp.gnupg.org/gcrypt/"
  76        "ftp://ftp.freenet.de/pub/ftp.gnupg.org/gcrypt/"
  77        "ftp://ftp.hi.is/pub/mirrors/gnupg/"
  78        "ftp://ftp.heanet.ie/mirrors/ftp.gnupg.org/gcrypt/"
  79        "ftp://ftp.bit.nl/mirror/gnupg/"
  80        "ftp://ftp.surfnet.nl/pub/security/gnupg/"
  81        "ftp://ftp.iasi.roedu.net/pub/mirrors/ftp.gnupg.org/"
  82        "ftp://ftp.sunet.se/pub/security/gnupg/"
  83        "ftp://mirror.switch.ch/mirror/gnupg/"
  84        "ftp://mirror.tje.me.uk/pub/mirrors/ftp.gnupg.org/"
  85        "ftp://ftp.mirrorservice.org/sites/ftp.gnupg.org/gcrypt/"
  86        "ftp://ftp.ring.gr.jp/pub/net/gnupg/"
  87        "ftp://ftp.gnupg.org/gcrypt/")
  88       (gnome
  89        "http://ftp.belnet.be/ftp.gnome.org/"
  90        "http://ftp.linux.org.uk/mirrors/ftp.gnome.org/"
  91        "http://ftp.gnome.org/pub/GNOME/"
  92        "https://download.gnome.org/"
  93        "http://mirror.yandex.ru/mirrors/ftp.gnome.org/")
  94       (hackage
  95        "http://hackage.haskell.org/")
  96       (savannah
  97        "http://download.savannah.gnu.org/releases/"
  98        "ftp://ftp.twaren.net/Unix/NonGNU/"
  99        "ftp://mirror.csclub.uwaterloo.ca/nongnu/"
 100        "ftp://mirror.publicns.net/pub/nongnu/"
 101        "ftp://savannah.c3sl.ufpr.br/"
 102        "http://ftp.cc.uoc.gr/mirrors/nongnu.org/"
 103        "http://ftp.twaren.net/Unix/NonGNU/"
 104        "http://mirror.csclub.uwaterloo.ca/nongnu/"
 105        "http://nongnu.askapache.com/"
 106        "http://savannah.c3sl.ufpr.br/"
 107        "http://download.savannah.gnu.org/releases-noredirect/"
 108        "http://download-mirror.savannah.gnu.org/releases/")
 109       (sourceforge ; https://sourceforge.net/p/forge/documentation/Mirrors/
 110        "http://downloads.sourceforge.net/project/"
 111        "http://ufpr.dl.sourceforge.net/project/"
 112        "http://heanet.dl.sourceforge.net/project/"
 113        "http://freefr.dl.sourceforge.net/project/"
 114        "http://internode.dl.sourceforge.net/project/"
 115        "http://jaist.dl.sourceforge.net/project/"
 116        "http://kent.dl.sourceforge.net/project/"
 117        "http://liquidtelecom.dl.sourceforge.net/project/"
 118        "http://nbtelecom.dl.sourceforge.net/project/"
 119        "http://nchc.dl.sourceforge.net/project/"
 120        "http://ncu.dl.sourceforge.net/project/"
 121        "http://netcologne.dl.sourceforge.net/project/"
 122        "http://netix.dl.sourceforge.net/project/"
 123        "http://pilotfiber.dl.sourceforge.net/project/"
 124        "http://superb-sea2.dl.sourceforge.net/project/"
 125        "http://tenet.dl.sourceforge.net/project/"
 126        "http://vorboss.dl.sourceforge.net/project/"
 127        "http://netassist.dl.sourceforge.net/project/")
 128       (netfilter.org ; https://www.netfilter.org/mirrors.html
 129        "http://ftp.netfilter.org/pub/"
 130        "ftp://ftp.es.netfilter.org/mirrors/netfilter/"
 131        "ftp://ftp.hu.netfilter.org/"
 132        "ftp://www.lt.netfilter.org/pub/")
 133       (kernel.org
 134        "http://ramses.wh2.tu-dresden.de/pub/mirrors/kernel.org/"
 135        "http://linux-kernel.uio.no/pub/"
 136        "http://kernel.osuosl.org/pub/"
 137        "ftp://ftp.funet.fi/pub/mirrors/ftp.kernel.org/pub/"
 138        "http://ftp.be.debian.org/pub/"
 139        "http://mirror.linux.org.au/")
 140       (apache             ; from http://www.apache.org/mirrors/dist.html
 141        "http://www.eu.apache.org/dist/"
 142        "http://www.us.apache.org/dist/"
 143        "ftp://gd.tuwien.ac.at/pub/infosys/servers/http/apache/dist/"
 144        "http://apache.belnet.be/"
 145        "http://mirrors.ircam.fr/pub/apache/"
 146        "http://apache-mirror.rbc.ru/pub/apache/"
 147
 148        ;; As a last resort, try the archive.
 149        "http://archive.apache.org/dist/")
 150       (xorg               ; from http://www.x.org/wiki/Releases/Download
 151        "http://www.x.org/releases/" ; main mirrors
 152        "ftp://mirror.csclub.uwaterloo.ca/x.org/" ; North America
 153        "ftp://xorg.mirrors.pair.com/"
 154        "http://mirror.csclub.uwaterloo.ca/x.org/"
 155        "http://xorg.mirrors.pair.com/"
 156        "http://mirror.us.leaseweb.net/xorg/"
 157        "ftp://artfiles.org/x.org/" ; Europe
 158        "ftp://ftp.chg.ru/pub/X11/x.org/"
 159        "ftp://ftp.fu-berlin.de/unix/X11/FTP.X.ORG/"
 160        "ftp://ftp.gwdg.de/pub/x11/x.org/"
 161        "ftp://ftp.mirrorservice.org/sites/ftp.x.org/"
 162        "ftp://ftp.ntua.gr/pub/X11/"
 163        "ftp://ftp.piotrkosoft.net/pub/mirrors/ftp.x.org/"
 164        "ftp://ftp.portal-to-web.de/pub/mirrors/x.org/"
 165        "ftp://ftp.solnet.ch/mirror/x.org/"
 166        "ftp://gd.tuwien.ac.at/X11/"
 167        "ftp://mi.mirror.garr.it/mirrors/x.org/"
 168        "ftp://mirror.cict.fr/x.org/"
 169        "ftp://mirror.switch.ch/mirror/X11/"
 170        "ftp://mirrors.ircam.fr/pub/x.org/"
 171        "ftp://x.mirrors.skynet.be/pub/ftp.x.org/"
 172        "ftp://ftp.cs.cuhk.edu.hk/pub/X11" ; East Asia
 173        "ftp://ftp.u-aizu.ac.jp/pub/x11/x.org/"
 174        "ftp://ftp.yz.yamagata-u.ac.jp/pub/X11/x.org/"
 175        "ftp://ftp.kaist.ac.kr/x.org/"
 176        "ftp://mirrors.go-part.com/xorg/"
 177        "http://x.cs.pu.edu.tw/"
 178        "ftp://ftp.is.co.za/pub/x.org")            ; South Africa
 179       (cpan                              ; from http://www.cpan.org/SITES.html
 180        "http://mirror.ibcp.fr/pub/CPAN/"
 181        "ftp://ftp.ciril.fr/pub/cpan/"
 182        "ftp://artfiles.org/cpan.org/"
 183        "http://www.cpan.org/"
 184        "ftp://cpan.rinet.ru/pub/mirror/CPAN/"
 185        "ftp://cpan.inode.at/"
 186        "ftp://cpan.iht.co.il/"
 187        "ftp://ftp.osuosl.org/pub/CPAN/"
 188        "ftp://ftp.nara.wide.ad.jp/pub/CPAN/"
 189        "http://mirrors.163.com/cpan/"
 190        "ftp://cpan.mirror.ac.za/"
 191        "http://cpan.mirrors.ionfish.org/"
 192        "http://cpan.mirror.dkm.cz/pub/CPAN/"
 193        "http://cpan.mirror.iphh.net/"
 194        "http://mirrors.teentelecom.net/CPAN/"
 195        "http://mirror.teklinks.com/CPAN/"
 196        "http://cpan.weepeetelecom.be/"
 197        "http://mirrors.xservers.ro/CPAN/"
 198        "http://cpan.yimg.com/"
 199        "http://mirror.yazd.ac.ir/cpan/"
 200        "http://ftp.belnet.be/ftp.cpan.org/")
 201       (cran
 202        ;; Arbitrary mirrors from http://cran.r-project.org/mirrors.html
 203        ;; This one automatically redirects to servers worldwide
 204        "http://cran.r-project.org/"
 205        "http://cran.rstudio.com/"
 206        "http://cran.univ-lyon1.fr/"
 207        "http://cran.ism.ac.jp/"
 208        "http://cran.stat.auckland.ac.nz/"
 209        "http://cran.mirror.ac.za/"
 210        "http://cran.csie.ntu.edu.tw/")
 211       (imagemagick
 212        ;; from http://www.imagemagick.org/script/download.php
 213        ;; (without mirrors that are unavailable or not up to date)
 214        ;; mirrors keeping old versions at the top level
 215        "ftp://sunsite.icm.edu.pl/packages/ImageMagick/"
 216        ;; mirrors moving old versions to "legacy"
 217        "ftp://mirror.aarnet.edu.au/pub/imagemagick/"
 218        "http://mirror.checkdomain.de/imagemagick/"
 219        "ftp://ftp.kddlabs.co.jp/graphics/ImageMagick/"
 220        "ftp://ftp.u-aizu.ac.jp/pub/graphics/image/ImageMagick/imagemagick.org/"
 221        "ftp://ftp.nluug.nl/pub/ImageMagick/"
 222        "http://ftp.surfnet.nl/pub/ImageMagick/"
 223        "http://mirror.searchdaimon.com/ImageMagick"
 224        "ftp://ftp.tpnet.pl/pub/graphics/ImageMagick/"
 225        "http://mirror.is.co.za/pub/imagemagick/"
 226        "http://mirrors-usa.go-parts.com/mirrors/ImageMagick/"
 227        "ftp://ftp.fifi.org/pub/ImageMagick/"
 228        "http://www.imagemagick.org/download/"
 229        ;; one legacy location as a last resort
 230        "http://www.imagemagick.org/download/legacy/")
 231       (debian
 232        "http://ftp.de.debian.org/debian/"
 233        "http://ftp.fr.debian.org/debian/"
 234        "http://ftp.debian.org/debian/"
 235        "http://archive.debian.org/debian/")
 236       (kde
 237        "http://download.kde.org"
 238        ;; Mirrors from http://files.kde.org/extra/mirrors.html
 239        ;; Europe
 240        "http://mirror.easyname.at/kde"
 241        "http://mirror.karneval.cz/pub/kde"
 242        "http://ftp.fi.muni.cz/pub/kde/"
 243        "http://mirror.oss.maxcdn.com/kde/"
 244        "http://ftp5.gwdg.de/pub/linux/kde/"
 245        "http://ftp-stud.fht-esslingen.de/Mirrors/ftp.kde.org/pub/kde/"
 246        "http://mirror.klaus-uwe.me/kde/ftp/"
 247        "http://kde.beta.mirror.ga/"
 248        "http://kde.alpha.mirror.ga/"
 249        "http://mirror.netcologne.de/kde"
 250        "http://vesta.informatik.rwth-aachen.de/ftp/pub/mirror/kde/"
 251        "http://ftp.rz.uni-wuerzburg.de/pub/unix/kde/"
 252        "http://mirrors.dotsrc.org/kde/"
 253        "http://ftp.funet.fi/pub/mirrors/ftp.kde.org/pub/kde/"
 254        "http://kde-mirror.freenux.org/"
 255        "http://mirrors.ircam.fr/pub/KDE/"
 256        "http://www-ftp.lip6.fr/pub/X11/kde/"
 257        "http://fr2.rpmfind.net/linux/KDE/"
 258        "http://kde.mirror.anlx.net/"
 259        "http://www.mirrorservice.org/sites/ftp.kde.org/pub/kde/"
 260        "http://ftp.heanet.ie/mirrors/ftp.kde.org/"
 261        "http://ftp.nluug.nl/pub/windowing/kde/"
 262        "http://ftp.surfnet.nl/windowing/kde/"
 263        "http://ftp.icm.edu.pl/pub/unix/kde/"
 264        "http://ftp.pbone.net/pub/kde/"
 265        "http://piotrkosoft.net/pub/mirrors/ftp.kde.org/"
 266        "http://mirrors.fe.up.pt/pub/kde/"
 267        "http://ftp.iasi.roedu.net/pub/mirrors/ftp.kde.org/"
 268        "http://ftp.acc.umu.se/mirror/kde.org/ftp/"
 269        "http://kde.ip-connect.vn.ua/"
 270        ;; North America
 271        "http://mirror.its.dal.ca/kde/"
 272        "http://mirror.csclub.uwaterloo.ca/kde/"
 273        "http://mirror.cc.columbia.edu/pub/software/kde/"
 274        "http://mirrors-usa.go-parts.com/kde"
 275        "http://kde.mirrors.hoobly.com/"
 276        "http://ftp.ussg.iu.edu/kde/"
 277        "http://mirrors.mit.edu/kde/"
 278        "http://kde.mirrors.tds.net/pub/kde/"
 279        ;; Oceania
 280        "http://ftp.kddlabs.co.jp/pub/X11/kde/"
 281        "http://kde.mirror.uber.com.au/")
 282       (openbsd
 283        "https://ftp.openbsd.org/pub/OpenBSD/"
 284        ;; Anycast CDN redirecting to your friendly local mirror.
 285        "https://mirrors.evowise.com/pub/OpenBSD/"
 286        ;; Other HTTPS mirrors from https://www.openbsd.org/ftp.html
 287        "https://mirror.aarnet.edu.au/pub/OpenBSD/"
 288        "https://ftp2.eu.openbsd.org/pub/OpenBSD/"
 289        "https://openbsd.c3sl.ufpr.br/pub/OpenBSD/"
 290        "https://openbsd.ipacct.com/pub/OpenBSD/"
 291        "https://ftp.OpenBSD.org/pub/OpenBSD/"
 292        "https://openbsd.cs.toronto.edu/pub/OpenBSD/"
 293        "https://openbsd.delfic.org/pub/OpenBSD/"
 294        "https://openbsd.mirror.netelligent.ca/pub/OpenBSD/"
 295        "https://mirrors.ucr.ac.cr/pub/OpenBSD/"
 296        "https://mirrors.dotsrc.org/pub/OpenBSD/"
 297        "https://mirror.one.com/pub/OpenBSD/"
 298        "https://ftp.fr.openbsd.org/pub/OpenBSD/"
 299        "https://ftp2.fr.openbsd.org/pub/OpenBSD/"
 300        "https://mirrors.ircam.fr/pub/OpenBSD/"
 301        "https://ftp.spline.de/pub/OpenBSD/"
 302        "https://mirror.hs-esslingen.de/pub/OpenBSD/"
 303        "https://ftp.halifax.rwth-aachen.de/openbsd/"
 304        "https://ftp.hostserver.de/pub/OpenBSD/"
 305        "https://ftp.fau.de/pub/OpenBSD/"
 306        "https://ftp.cc.uoc.gr/pub/OpenBSD/"
 307        "https://openbsd.hk/pub/OpenBSD/"
 308        "https://ftp.heanet.ie/pub/OpenBSD/"
 309        "https://openbsd.mirror.garr.it/pub/OpenBSD/"
 310        "https://mirror.litnet.lt/pub/OpenBSD/"
 311        "https://mirror.meerval.net/pub/OpenBSD/"
 312        "https://ftp.nluug.nl/pub/OpenBSD/"
 313        "https://ftp.bit.nl/pub/OpenBSD/"
 314        "https://mirrors.dalenys.com/pub/OpenBSD/"
 315        "https://ftp.icm.edu.pl/pub/OpenBSD/"
 316        "https://ftp.rnl.tecnico.ulisboa.pt/pub/OpenBSD/"
 317        "https://mirrors.pidginhost.com/pub/OpenBSD/"
 318        "https://mirror.yandex.ru/pub/OpenBSD/"
 319        "https://ftp.eu.openbsd.org/pub/OpenBSD/"
 320        "https://ftp.yzu.edu.tw/pub/OpenBSD/"
 321        "https://www.mirrorservice.org/pub/OpenBSD/"
 322        "https://anorien.csc.warwick.ac.uk/pub/OpenBSD/"
 323        "https://mirror.bytemark.co.uk/pub/OpenBSD/"
 324        "https://mirrors.sonic.net/pub/OpenBSD/"
 325        "https://ftp3.usa.openbsd.org/pub/OpenBSD/"
 326        "https://mirrors.syringanetworks.net/pub/OpenBSD/"
 327        "https://openbsd.mirror.constant.com/pub/OpenBSD/"
 328        "https://ftp4.usa.openbsd.org/pub/OpenBSD/"
 329        "https://ftp5.usa.openbsd.org/pub/OpenBSD/"
 330        "https://mirror.esc7.net/pub/OpenBSD/"))))
 331
 332 (define %mirror-file
 333   ;; Copy of the list of mirrors to a file.  This allows us to keep a single
 334   ;; copy in the store, and computing it here avoids repeated calls to
 335   ;; 'object->string'.
 336   (plain-file "mirrors" (object->string %mirrors)))
 337
 338 (define %content-addressed-mirrors
 339   ;; List of content-addressed mirrors.  Each mirror is represented as a
 340   ;; procedure that takes a file name, an algorithm (symbol) and a hash
 341   ;; (bytevector), and returns a URL or #f.
 342   ;; Note: Avoid 'https' to mitigate <http://bugs.gnu.org/22774>.
 343   ;; TODO: Add more.
 344   '(list (lambda (file algo hash)
 345            ;; Files served by 'guix publish' are accessible under a single
 346            ;; hash algorithm.
 347            (string-append "http://mirror.hydra.gnu.org/file/"
 348                           file "/" (symbol->string algo) "/"
 349                           (bytevector->nix-base32-string hash)))
 350          (lambda (file algo hash)
 351            ;; 'tarballs.nixos.org' supports several algorithms.
 352            (string-append "http://tarballs.nixos.org/"
 353                           (symbol->string algo) "/"
 354                           (bytevector->nix-base32-string hash)))))
 355
 356 (define %content-addressed-mirror-file
 357   ;; Content-addressed mirrors stored in a file.
 358   (plain-file "content-addressed-mirrors"
 359               (object->string %content-addressed-mirrors)))
 360
 361 (define (gnutls-package)
 362   "Return the default GnuTLS package."
 363   (let ((module (resolve-interface '(gnu packages tls))))
 364     (module-ref module 'gnutls)))
 365
 366 (define built-in-builders*
 367   (let ((cache (make-weak-key-hash-table)))
 368     (lambda ()
 369       "Return, as a monadic value, the list of built-in builders supported by
 370 the daemon."
 371       (lambda (store)
 372         ;; Memoize the result to avoid repeated RPCs.
 373         (values (or (hashq-ref cache store)
 374                     (let ((result (built-in-builders store)))
 375                       (hashq-set! cache store result)
 376                       result))
 377                 store)))))
 378
 379 (define* (built-in-download file-name url
 380                             #:key system hash-algo hash
 381                             mirrors content-addressed-mirrors
 382                             (guile 'unused))
 383   "Download FILE-NAME from URL using the built-in 'download' builder.
 384
 385 This is an \"out-of-band\" download in that the returned derivation does not
 386 explicitly depend on Guile, GnuTLS, etc.  Instead, the daemon performs the
 387 download by itself using its own dependencies."
 388   (mlet %store-monad ((mirrors (lower-object mirrors))
 389                       (content-addressed-mirrors
 390                        (lower-object content-addressed-mirrors)))
 391     (raw-derivation file-name "builtin:download" '()
 392                     #:system system
 393                     #:hash-algo hash-algo
 394                     #:hash hash
 395                     #:inputs `((,mirrors)
 396                                (,content-addressed-mirrors))
 397
 398                     ;; Honor the user's proxy and locale settings.
 399                     #:leaked-env-vars '("http_proxy" "https_proxy"
 400                                         "LC_ALL" "LC_MESSAGES" "LANG"
 401                                         "COLUMNS")
 402
 403                     #:env-vars `(("url" . ,(object->string url))
 404                                  ("mirrors" . ,mirrors)
 405                                  ("content-addressed-mirrors"
 406                                   . ,content-addressed-mirrors))
 407
 408                     ;; Do not offload this derivation because we cannot be
 409                     ;; sure that the remote daemon supports the 'download'
 410                     ;; built-in.  We may remove this limitation when support
 411                     ;; for that built-in is widespread.
 412                     #:local-build? #t)))
 413
 414 (define* (in-band-download file-name url
 415                            #:key system hash-algo hash
 416                            mirrors content-addressed-mirrors
 417                            guile)
 418   "Download FILE-NAME from URL using a normal, \"in-band\" fixed-output
 419 derivation.
 420
 421 This is now deprecated since it has the drawback of causing bootstrapping
 422 issues: we may need to build GnuTLS just to be able to download the source of
 423 GnuTLS itself and its dependencies.  See <http://bugs.gnu.org/22774>."
 424   (define need-gnutls?
 425     ;; True if any of the URLs need TLS support.
 426     (let ((https? (cut string-prefix? "https://" <>)))
 427       (match url
 428         ((? string?)
 429          (https? url))
 430         ((url ...)
 431          (any https? url)))))
 432
 433   (define builder
 434     (with-imported-modules '((guix build download)
 435                              (guix build utils)
 436                              (guix ftp-client)
 437                              (guix base32)
 438                              (guix base64))
 439       #~(begin
 440           #+(if need-gnutls?
 441
 442                 ;; Add GnuTLS to the inputs and to the load path.
 443                 #~(eval-when (load expand eval)
 444                     (set! %load-path
 445                       (cons (string-append #+(gnutls-package)
 446                                            "/share/guile/site/"
 447                                            (effective-version))
 448                             %load-path)))
 449                 #~#t)
 450
 451           (use-modules (guix build download)
 452                        (guix base32))
 453
 454           (let ((value-from-environment (lambda (variable)
 455                                           (call-with-input-string
 456                                               (getenv variable)
 457                                             read))))
 458             (url-fetch (value-from-environment "guix download url")
 459                        #$output
 460                        #:mirrors (call-with-input-file #$mirrors read)
 461
 462                        ;; Content-addressed mirrors.
 463                        #:hashes
 464                        (value-from-environment "guix download hashes")
 465                        #:content-addressed-mirrors
 466                        (primitive-load #$content-addressed-mirrors)
 467
 468                        ;; No need to validate certificates since we know the
 469                        ;; hash of the expected result.
 470                        #:verify-certificate? #f)))))
 471
 472   (mlet %store-monad ((guile (package->derivation guile system)))
 473     (gexp->derivation file-name builder
 474                       #:guile-for-build guile
 475                       #:system system
 476                       #:hash-algo hash-algo
 477                       #:hash hash
 478
 479                       ;; Use environment variables and a fixed script
 480                       ;; name so there's only one script in store for
 481                       ;; all the downloads.
 482                       #:script-name "download"
 483                       #:env-vars
 484                       `(("guix download url" . ,(object->string url))
 485                         ("guix download hashes"
 486                          . ,(object->string `((,hash-algo . ,hash)))))
 487
 488                       ;; Honor the user's proxy settings.
 489                       #:leaked-env-vars '("http_proxy" "https_proxy")
 490
 491                       ;; In general, offloading downloads is not a good
 492                       ;; idea.  Daemons before 0.8.3 would also
 493                       ;; interpret this as "do not substitute" (see
 494                       ;; <https://bugs.gnu.org/18747>.)
 495                       #:local-build? #t)))
 496
 497 (define* (url-fetch url hash-algo hash
 498                     #:optional name
 499                     #:key (system (%current-system))
 500                     (guile (default-guile)))
 501   "Return a fixed-output derivation that fetches URL (a string, or a list of
 502 strings denoting alternate URLs), which is expected to have hash HASH of type
 503 HASH-ALGO (a symbol).  By default, the file name is the base name of URL;
 504 optionally, NAME can specify a different file name.
 505
 506 When one of the URL starts with mirror://, then its host part is
 507 interpreted as the name of a mirror scheme, taken from %MIRROR-FILE.
 508
 509 Alternately, when URL starts with file://, return the corresponding file name
 510 in the store."
 511   (define file-name
 512     (match url
 513       ((head _ ...)
 514        (basename head))
 515       (_
 516        (basename url))))
 517
 518   (let ((uri (and (string? url) (string->uri url))))
 519     (if (or (and (string? url) (not uri))
 520             (and uri (memq (uri-scheme uri) '(#f file))))
 521         (interned-file (if uri (uri-path uri) url)
 522                        (or name file-name))
 523         (mlet* %store-monad ((builtins (built-in-builders*))
 524                              (download -> (if (member "download" builtins)
 525                                               built-in-download
 526                                               in-band-download)))
 527           (download (or name file-name) url
 528                     #:guile guile
 529                     #:system system
 530                     #:hash-algo hash-algo
 531                     #:hash hash
 532                     #:mirrors %mirror-file
 533                     #:content-addressed-mirrors
 534                     %content-addressed-mirror-file)))))
 535
 536 (define* (url-fetch/tarbomb url hash-algo hash
 537                             #:optional name
 538                             #:key (system (%current-system))
 539                             (guile (default-guile)))
 540   "Similar to 'url-fetch' but unpack the file from URL in a directory of its
 541 own.  This helper makes it easier to deal with \"tar bombs\"."
 542   (define file-name
 543     (match url
 544       ((head _ ...)
 545        (basename head))
 546       (_
 547        (basename url))))
 548   (define gzip
 549     (module-ref (resolve-interface '(gnu packages compression)) 'gzip))
 550   (define tar
 551     (module-ref (resolve-interface '(gnu packages base)) 'tar))
 552
 553   (mlet %store-monad ((drv (url-fetch url hash-algo hash
 554                                       (string-append "tarbomb-"
 555                                                      (or name file-name))
 556                                       #:system system
 557                                       #:guile guile)))
 558     ;; Take the tar bomb, and simply unpack it as a directory.
 559     (gexp->derivation (or name file-name)
 560                       #~(begin
 561                           (mkdir #$output)
 562                           (setenv "PATH" (string-append #$gzip "/bin"))
 563                           (chdir #$output)
 564                           (zero? (system* (string-append #$tar "/bin/tar")
 565                                           "xf" #$drv)))
 566                       #:local-build? #t)))
 567
 568 (define* (url-fetch/zipbomb url hash-algo hash
 569                             #:optional name
 570                             #:key (system (%current-system))
 571                             (guile (default-guile)))
 572   "Similar to 'url-fetch' but unpack the zip file at URL in a directory of its
 573 own.  This helper makes it easier to deal with \"zip bombs\"."
 574   (define file-name
 575     (match url
 576       ((head _ ...)
 577        (basename head))
 578       (_
 579        (basename url))))
 580   (define unzip
 581     (module-ref (resolve-interface '(gnu packages compression)) 'unzip))
 582
 583   (mlet %store-monad ((drv (url-fetch url hash-algo hash
 584                                       (string-append "zipbomb-"
 585                                                      (or name file-name))
 586                                       #:system system
 587                                       #:guile guile)))
 588     ;; Take the zip bomb, and simply unpack it as a directory.
 589     (gexp->derivation (or name file-name)
 590                       #~(begin
 591                           (mkdir #$output)
 592                           (chdir #$output)
 593                           (zero? (system* (string-append #$unzip "/bin/unzip")
 594                                           #$drv)))
 595                       #:local-build? #t)))
 596
 597 (define* (download-to-store store url #:optional (name (basename url))
 598                             #:key (log (current-error-port)) recursive?
 599                             (verify-certificate? #t))
 600   "Download from URL to STORE, either under NAME or URL's basename if
 601 omitted.  Write progress reports to LOG.  RECURSIVE? has the same effect as
 602 the same-named parameter of 'add-to-store'.  VERIFY-CERTIFICATE? determines
 603 whether or not to validate HTTPS server certificates."
 604   (define uri
 605     (string->uri url))
 606
 607   (if (or (not uri) (memq (uri-scheme uri) '(file #f)))
 608       (add-to-store store name recursive? "sha256"
 609                     (if uri (uri-path uri) url))
 610       (call-with-temporary-output-file
 611        (lambda (temp port)
 612          (let ((result
 613                 (parameterize ((current-output-port log))
 614                   (build:url-fetch url temp
 615                                    #:mirrors %mirrors
 616                                    #:verify-certificate?
 617                                    verify-certificate?))))
 618            (close port)
 619            (and result
 620                 (add-to-store store name recursive? "sha256" temp)))))))
 621
 622 ;;; download.scm ends here