guix/scripts/publish.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2015 David Thompson <davet@gnu.org>
   3 ;;; Copyright © 2015, 2016, 2017 Ludovic Courtès <ludo@gnu.org>
   4 ;;;
   5 ;;; This file is part of GNU Guix.
   6 ;;;
   7 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   8 ;;; under the terms of the GNU General Public License as published by
   9 ;;; the Free Software Foundation; either version 3 of the License, or (at
  10 ;;; your option) any later version.
  11 ;;;
  12 ;;; GNU Guix is distributed in the hope that it will be useful, but
  13 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 ;;; GNU General Public License for more details.
  16 ;;;
  17 ;;; You should have received a copy of the GNU General Public License
  18 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 (define-module (guix scripts publish)
  21   #:use-module ((system repl server) #:prefix repl:)
  22   #:use-module (ice-9 binary-ports)
  23   #:use-module (ice-9 format)
  24   #:use-module (ice-9 match)
  25   #:use-module (ice-9 regex)
  26   #:use-module (ice-9 rdelim)
  27   #:use-module (ice-9 threads)
  28   #:use-module (rnrs bytevectors)
  29   #:use-module (srfi srfi-1)
  30   #:use-module (srfi srfi-2)
  31   #:use-module (srfi srfi-9)
  32   #:use-module (srfi srfi-9 gnu)
  33   #:use-module (srfi srfi-19)
  34   #:use-module (srfi srfi-26)
  35   #:use-module (srfi srfi-34)
  36   #:use-module (srfi srfi-37)
  37   #:use-module (web http)
  38   #:use-module (web request)
  39   #:use-module (web response)
  40   #:use-module (web server)
  41   #:use-module (web uri)
  42   #:autoload   (sxml simple) (sxml->xml)
  43   #:use-module (guix base32)
  44   #:use-module (guix base64)
  45   #:use-module (guix config)
  46   #:use-module (guix derivations)
  47   #:use-module (guix hash)
  48   #:use-module (guix pki)
  49   #:use-module (guix pk-crypto)
  50   #:use-module (guix workers)
  51   #:use-module (guix store)
  52   #:use-module ((guix serialization) #:select (write-file))
  53   #:use-module (guix zlib)
  54   #:use-module (guix cache)
  55   #:use-module (guix ui)
  56   #:use-module (guix scripts)
  57   #:use-module ((guix utils)
  58                 #:select (with-atomic-file-output compressed-file?))
  59   #:use-module ((guix build utils)
  60                 #:select (dump-port mkdir-p find-files))
  61   #:use-module ((guix build syscalls) #:select (set-thread-name))
  62   #:export (%public-key
  63             %private-key
  64
  65             guix-publish))
  66
  67 (define (show-help)
  68   (format #t (G_ "Usage: guix publish [OPTION]...
  69 Publish ~a over HTTP.\n") %store-directory)
  70   (display (G_ "
  71   -p, --port=PORT        listen on PORT"))
  72   (display (G_ "
  73       --listen=HOST      listen on the network interface for HOST"))
  74   (display (G_ "
  75   -u, --user=USER        change privileges to USER as soon as possible"))
  76   (display (G_ "
  77   -C, --compression[=LEVEL]
  78                          compress archives at LEVEL"))
  79   (display (G_ "
  80   -c, --cache=DIRECTORY  cache published items to DIRECTORY"))
  81   (display (G_ "
  82       --workers=N        use N workers to bake items"))
  83   (display (G_ "
  84       --ttl=TTL          announce narinfos can be cached for TTL seconds"))
  85   (display (G_ "
  86       --nar-path=PATH    use PATH as the prefix for nar URLs"))
  87   (display (G_ "
  88       --public-key=FILE  use FILE as the public key for signatures"))
  89   (display (G_ "
  90       --private-key=FILE use FILE as the private key for signatures"))
  91   (display (G_ "
  92   -r, --repl[=PORT]      spawn REPL server on PORT"))
  93   (newline)
  94   (display (G_ "
  95   -h, --help             display this help and exit"))
  96   (display (G_ "
  97   -V, --version          display version information and exit"))
  98   (newline)
  99   (show-bug-report-information))
 100
 101 (define (getaddrinfo* host)
 102   "Like 'getaddrinfo', but properly report errors."
 103   (catch 'getaddrinfo-error
 104     (lambda ()
 105       (getaddrinfo host))
 106     (lambda (key error)
 107       (leave (G_ "lookup of host '~a' failed: ~a~%")
 108              host (gai-strerror error)))))
 109
 110 ;; Nar compression parameters.
 111 (define-record-type <compression>
 112   (compression type level)
 113   compression?
 114   (type   compression-type)
 115   (level  compression-level))
 116
 117 (define %no-compression
 118   (compression 'none 0))
 119
 120 (define %default-gzip-compression
 121   ;; Since we compress on the fly, default to fast compression.
 122   (compression 'gzip 3))
 123
 124 (define (actual-compression item requested)
 125   "Return the actual compression used for ITEM, which may be %NO-COMPRESSION
 126 if ITEM is already compressed."
 127   (if (compressed-file? item)
 128       %no-compression
 129       requested))
 130
 131 (define %options
 132   (list (option '(#\h "help") #f #f
 133                 (lambda _
 134                   (show-help)
 135                   (exit 0)))
 136         (option '(#\V "version") #f #f
 137                 (lambda _
 138                   (show-version-and-exit "guix publish")))
 139         (option '(#\u "user") #t #f
 140                 (lambda (opt name arg result)
 141                   (alist-cons 'user arg result)))
 142         (option '(#\p "port") #t #f
 143                 (lambda (opt name arg result)
 144                   (alist-cons 'port (string->number* arg) result)))
 145         (option '("listen") #t #f
 146                 (lambda (opt name arg result)
 147                   (match (getaddrinfo* arg)
 148                     ((info _ ...)
 149                      (alist-cons 'address (addrinfo:addr info)
 150                                  result))
 151                     (()
 152                      (leave (G_ "lookup of host '~a' returned nothing")
 153                             name)))))
 154         (option '(#\C "compression") #f #t
 155                 (lambda (opt name arg result)
 156                   (match (if arg (string->number* arg) 3)
 157                     (0
 158                      (alist-cons 'compression %no-compression result))
 159                     (level
 160                      (if (zlib-available?)
 161                          (alist-cons 'compression
 162                                      (compression 'gzip level)
 163                                      result)
 164                          (begin
 165                            (warning (G_ "zlib support is missing; \
 166 compression disabled~%"))
 167                            result))))))
 168         (option '(#\c "cache") #t #f
 169                 (lambda (opt name arg result)
 170                   (alist-cons 'cache arg result)))
 171         (option '("workers") #t #f
 172                 (lambda (opt name arg result)
 173                   (alist-cons 'workers (string->number* arg)
 174                               result)))
 175         (option '("ttl") #t #f
 176                 (lambda (opt name arg result)
 177                   (let ((duration (string->duration arg)))
 178                     (unless duration
 179                       (leave (G_ "~a: invalid duration~%") arg))
 180                     (alist-cons 'narinfo-ttl (time-second duration)
 181                                 result))))
 182         (option '("nar-path") #t #f
 183                 (lambda (opt name arg result)
 184                   (alist-cons 'nar-path arg result)))
 185         (option '("public-key") #t #f
 186                 (lambda (opt name arg result)
 187                   (alist-cons 'public-key-file arg result)))
 188         (option '("private-key" "secret-key") #t #f
 189                 (lambda (opt name arg result)
 190                   (alist-cons 'private-key-file arg result)))
 191         (option '(#\r "repl") #f #t
 192                 (lambda (opt name arg result)
 193                   ;; If port unspecified, use default Guile REPL port.
 194                   (let ((port (and arg (string->number* arg))))
 195                     (alist-cons 'repl (or port 37146) result))))))
 196
 197 (define %default-options
 198   `((port . 8080)
 199
 200     ;; By default, serve nars under "/nar".
 201     (nar-path . "nar")
 202
 203     (public-key-file . ,%public-key-file)
 204     (private-key-file . ,%private-key-file)
 205
 206     ;; Default to fast & low compression.
 207     (compression . ,(if (zlib-available?)
 208                         %default-gzip-compression
 209                         %no-compression))
 210
 211     ;; Default number of workers when caching is enabled.
 212     (workers . ,(current-processor-count))
 213
 214     (address . ,(make-socket-address AF_INET INADDR_ANY 0))
 215     (repl . #f)))
 216
 217 ;; The key pair used to sign narinfos.
 218 (define %private-key
 219   (make-parameter #f))
 220 (define %public-key
 221   (make-parameter #f))
 222
 223 (define %nix-cache-info
 224   `(("StoreDir" . ,%store-directory)
 225     ("WantMassQuery" . 0)
 226     ("Priority" . 100)))
 227
 228 (define (signed-string s)
 229   "Sign the hash of the string S with the daemon's key."
 230   (let* ((public-key (%public-key))
 231          (hash (bytevector->hash-data (sha256 (string->utf8 s))
 232                                       #:key-type (key-type public-key))))
 233     (signature-sexp hash (%private-key) public-key)))
 234
 235 (define base64-encode-string
 236   (compose base64-encode string->utf8))
 237
 238 (define* (narinfo-string store store-path key
 239                          #:key (compression %no-compression)
 240                          (nar-path "nar") file-size)
 241   "Generate a narinfo key/value string for STORE-PATH; an exception is raised
 242 if STORE-PATH is invalid.  Produce a URL that corresponds to COMPRESSION.  The
 243 narinfo is signed with KEY.  NAR-PATH specifies the prefix for nar URLs.
 244 Optionally, FILE-SIZE can specify the size in bytes of the compressed NAR; it
 245 informs the client of how much needs to be downloaded."
 246   (let* ((path-info  (query-path-info store store-path))
 247          (compression (actual-compression store-path compression))
 248          (url        (encode-and-join-uri-path
 249                       `(,@(split-and-decode-uri-path nar-path)
 250                         ,@(match compression
 251                             (($ <compression> 'none)
 252                              '())
 253                             (($ <compression> type)
 254                              (list (symbol->string type))))
 255                         ,(basename store-path))))
 256          (hash       (bytevector->nix-base32-string
 257                       (path-info-hash path-info)))
 258          (size       (path-info-nar-size path-info))
 259          (file-size  (or file-size
 260                          (and (eq? compression %no-compression) size)))
 261          (references (string-join
 262                       (map basename (path-info-references path-info))
 263                       " "))
 264          (deriver    (path-info-deriver path-info))
 265          (base-info  (format #f
 266                              "\
 267 StorePath: ~a
 268 URL: ~a
 269 Compression: ~a
 270 NarHash: sha256:~a
 271 NarSize: ~d
 272 References: ~a~%~a"
 273                              store-path url
 274                              (compression-type compression)
 275                              hash size references
 276                              (if file-size
 277                                  (format #f "FileSize: ~a~%" file-size)
 278                                  "")))
 279          ;; Do not render a "Deriver" or "System" line if we are rendering
 280          ;; info for a derivation.
 281          (info       (if (not deriver)
 282                          base-info
 283                          (catch 'system-error
 284                            (lambda ()
 285                              (let ((drv (read-derivation-from-file deriver)))
 286                                (format #f "~aSystem: ~a~%Deriver: ~a~%"
 287                                        base-info (derivation-system drv)
 288                                        (basename deriver))))
 289                            (lambda args
 290                              ;; DERIVER might be missing, but that's fine:
 291                              ;; it's only used for <substitutable> where it's
 292                              ;; optional.  'System' is currently unused.
 293                              (if (= ENOENT (system-error-errno args))
 294                                  base-info
 295                                  (apply throw args))))))
 296          (signature  (base64-encode-string
 297                       (canonical-sexp->string (signed-string info)))))
 298     (format #f "~aSignature: 1;~a;~a~%" info (gethostname) signature)))
 299
 300 (define* (not-found request
 301                     #:key (phrase "Resource not found")
 302                     ttl)
 303   "Render 404 response for REQUEST."
 304   (values (build-response #:code 404
 305                           #:headers (if ttl
 306                                         `((cache-control (max-age . ,ttl)))
 307                                         '()))
 308           (string-append phrase ": "
 309                          (uri-path (request-uri request)))))
 310
 311 (define (render-nix-cache-info)
 312   "Render server information."
 313   (values '((content-type . (text/plain)))
 314           (lambda (port)
 315             (for-each (match-lambda
 316                        ((key . value)
 317                         (format port "~a: ~a~%" key value)))
 318                       %nix-cache-info))))
 319
 320 (define* (render-narinfo store request hash
 321                          #:key ttl (compression %no-compression)
 322                          (nar-path "nar"))
 323   "Render metadata for the store path corresponding to HASH.  If TTL is true,
 324 advertise it as the maximum validity period (in seconds) via the
 325 'Cache-Control' header.  This allows 'guix substitute' to cache it for an
 326 appropriate duration.  NAR-PATH specifies the prefix for nar URLs."
 327   (let ((store-path (hash-part->path store hash)))
 328     (if (string-null? store-path)
 329         (not-found request)
 330         (values `((content-type . (application/x-nix-narinfo))
 331                   ,@(if ttl
 332                         `((cache-control (max-age . ,ttl)))
 333                         '()))
 334                 (cut display
 335                   (narinfo-string store store-path (%private-key)
 336                                   #:nar-path nar-path
 337                                   #:compression compression)
 338                   <>)))))
 339
 340 (define* (nar-cache-file directory item
 341                              #:key (compression %no-compression))
 342   (string-append directory "/"
 343                  (symbol->string (compression-type compression))
 344                  "/" (basename item) ".nar"))
 345
 346 (define* (narinfo-cache-file directory item
 347                              #:key (compression %no-compression))
 348   (string-append directory "/"
 349                  (symbol->string (compression-type compression))
 350                  "/" (basename item)
 351                  ".narinfo"))
 352
 353 (define run-single-baker
 354   (let ((baking (make-weak-value-hash-table))
 355         (mutex  (make-mutex)))
 356     (lambda (item thunk)
 357       "Run THUNK, which is supposed to bake ITEM, but make sure only one
 358 thread is baking ITEM at a given time."
 359       (define selected?
 360         (with-mutex mutex
 361           (and (not (hash-ref baking item))
 362                (begin
 363                  (hash-set! baking item (current-thread))
 364                  #t))))
 365
 366       (when selected?
 367         (dynamic-wind
 368           (const #t)
 369           thunk
 370           (lambda ()
 371             (with-mutex mutex
 372               (hash-remove! baking item))))))))
 373
 374 (define-syntax-rule (single-baker item exp ...)
 375   "Bake ITEM by evaluating EXP, but make sure there's only one baker for ITEM
 376 at a time."
 377   (run-single-baker item (lambda () exp ...)))
 378
 379
 380 (define (narinfo-files cache)
 381   "Return the list of .narinfo files under CACHE."
 382   (if (file-is-directory? cache)
 383       (find-files cache
 384                   (lambda (file stat)
 385                     (string-suffix? ".narinfo" file)))
 386       '()))
 387
 388 (define (nar-expiration-time ttl)
 389   "Return the narinfo expiration time (in seconds since the Epoch).  The
 390 expiration time is +inf.0 when passed an item that is still in the store; in
 391 other cases, it is the last-access time of the item plus TTL.
 392
 393 This policy allows us to keep cached nars that correspond to valid store
 394 items.  Failing that, we could eventually have to recompute them and return
 395 404 in the meantime."
 396   (let ((expiration-time (file-expiration-time ttl)))
 397     (lambda (file)
 398       (let ((item (string-append (%store-prefix) "/"
 399                                  (basename file ".narinfo"))))
 400         ;; Note: We don't need to use 'valid-path?' here because FILE would
 401         ;; not exist if ITEM were not valid in the first place.
 402         (if (file-exists? item)
 403             +inf.0
 404             (expiration-time file))))))
 405
 406 (define* (render-narinfo/cached store request hash
 407                                 #:key ttl (compression %no-compression)
 408                                 (nar-path "nar")
 409                                 cache pool)
 410   "Respond to the narinfo request for REQUEST.  If the narinfo is available in
 411 CACHE, then send it; otherwise, return 404 and \"bake\" that nar and narinfo
 412 requested using POOL."
 413   (define (delete-entry narinfo)
 414     ;; Delete NARINFO and the corresponding nar from CACHE.
 415     (let ((nar (string-append (string-drop-right narinfo
 416                                                  (string-length ".narinfo"))
 417                               ".nar")))
 418       (delete-file* narinfo)
 419       (delete-file* nar)))
 420
 421   (let* ((item        (hash-part->path store hash))
 422          (compression (actual-compression item compression))
 423          (cached      (and (not (string-null? item))
 424                            (narinfo-cache-file cache item
 425                                                #:compression compression))))
 426     (cond ((string-null? item)
 427            (not-found request))
 428           ((file-exists? cached)
 429            ;; Narinfo is in cache, send it.
 430            (values `((content-type . (application/x-nix-narinfo))
 431                      ,@(if ttl
 432                            `((cache-control (max-age . ,ttl)))
 433                            '()))
 434                    (lambda (port)
 435                      (display (call-with-input-file cached
 436                                 read-string)
 437                               port))))
 438           ((and (file-exists? item)        ;cheaper than the 'valid-path?' RPC
 439                 (valid-path? store item))
 440            ;; Nothing in cache: bake the narinfo and nar in the background and
 441            ;; return 404.
 442            (eventually pool
 443              (single-baker item
 444                ;; Check whether CACHED has been produced in the meantime.
 445                (unless (file-exists? cached)
 446                  ;; (format #t "baking ~s~%" item)
 447                  (bake-narinfo+nar cache item
 448                                    #:ttl ttl
 449                                    #:compression compression
 450                                    #:nar-path nar-path)))
 451
 452              (when ttl
 453                (single-baker 'cache-cleanup
 454                  (maybe-remove-expired-cache-entries cache
 455                                                      narinfo-files
 456                                                      #:entry-expiration
 457                                                      (nar-expiration-time ttl)
 458                                                      #:delete-entry delete-entry
 459                                                      #:cleanup-period ttl))))
 460            (not-found request
 461                       #:phrase "We're baking it"
 462                       #:ttl 300))              ;should be available within 5m
 463           (else
 464            (not-found request)))))
 465
 466 (define* (bake-narinfo+nar cache item
 467                            #:key ttl (compression %no-compression)
 468                            (nar-path "/nar"))
 469   "Write the narinfo and nar for ITEM to CACHE."
 470   (let* ((compression (actual-compression item compression))
 471          (nar         (nar-cache-file cache item
 472                                       #:compression compression))
 473          (narinfo     (narinfo-cache-file cache item
 474                                           #:compression compression)))
 475
 476     (mkdir-p (dirname nar))
 477     (match (compression-type compression)
 478       ('gzip
 479        ;; Note: the file port gets closed along with the gzip port.
 480        (call-with-gzip-output-port (open-output-file (string-append nar ".tmp"))
 481          (lambda (port)
 482            (write-file item port))
 483          #:level (compression-level compression)
 484          #:buffer-size (* 128 1024))
 485        (rename-file (string-append nar ".tmp") nar))
 486       ('none
 487        ;; Cache nars even when compression is disabled so that we can
 488        ;; guarantee the TTL (see <https://bugs.gnu.org/28664>.)
 489        (with-atomic-file-output nar
 490          (lambda (port)
 491            (write-file item port)))))
 492
 493     (mkdir-p (dirname narinfo))
 494     (with-atomic-file-output narinfo
 495       (lambda (port)
 496         ;; Open a new connection to the store.  We cannot reuse the main
 497         ;; thread's connection to the store since we would end up sending
 498         ;; stuff concurrently on the same channel.
 499         (with-store store
 500           (display (narinfo-string store item
 501                                    (%private-key)
 502                                    #:nar-path nar-path
 503                                    #:compression compression
 504                                    #:file-size (and=> (stat nar #f)
 505                                                       stat:size))
 506                    port))))))
 507
 508 ;; XXX: Declare the 'Guix-Compression' HTTP header, which is in fact for
 509 ;; internal consumption: it allows us to pass the compression info to
 510 ;; 'http-write', as part of the workaround to <http://bugs.gnu.org/21093>.
 511 (declare-header! "Guix-Nar-Compression"
 512                  (lambda (str)
 513                    (match (call-with-input-string str read)
 514                      (('compression type level)
 515                       (compression type level))))
 516                  compression?
 517                  (lambda (compression port)
 518                    (match compression
 519                      (($ <compression> type level)
 520                       (write `(compression ,type ,level) port)))))
 521
 522 (define* (render-nar store request store-item
 523                      #:key (compression %no-compression))
 524   "Render archive of the store path corresponding to STORE-ITEM."
 525   (let ((store-path (string-append %store-directory "/" store-item)))
 526     ;; The ISO-8859-1 charset *must* be used otherwise HTTP clients will
 527     ;; interpret the byte stream as UTF-8 and arbitrarily change invalid byte
 528     ;; sequences.
 529     (if (valid-path? store store-path)
 530         (values `((content-type . (application/x-nix-archive
 531                                    (charset . "ISO-8859-1")))
 532                   (guix-nar-compression . ,compression))
 533                 ;; XXX: We're not returning the actual contents, deferring
 534                 ;; instead to 'http-write'.  This is a hack to work around
 535                 ;; <http://bugs.gnu.org/21093>.
 536                 store-path)
 537         (not-found request))))
 538
 539 (define* (render-nar/cached store cache request store-item
 540                             #:key (compression %no-compression))
 541   "Respond to REQUEST with a nar for STORE-ITEM.  If the nar is in CACHE,
 542 return it; otherwise, return 404."
 543   (let ((cached (nar-cache-file cache store-item
 544                                 #:compression compression)))
 545     (if (file-exists? cached)
 546         (values `((content-type . (application/octet-stream
 547                                    (charset . "ISO-8859-1"))))
 548                 ;; XXX: We're not returning the actual contents, deferring
 549                 ;; instead to 'http-write'.  This is a hack to work around
 550                 ;; <http://bugs.gnu.org/21093>.
 551                 cached)
 552         (not-found request))))
 553
 554 (define (render-content-addressed-file store request
 555                                        name algo hash)
 556   "Return the content of the result of the fixed-output derivation NAME that
 557 has the given HASH of type ALGO."
 558   ;; TODO: Support other hash algorithms.
 559   (if (and (eq? algo 'sha256) (= 32 (bytevector-length hash)))
 560       (let ((item (fixed-output-path name hash
 561                                      #:hash-algo algo
 562                                      #:recursive? #f)))
 563         (if (valid-path? store item)
 564             (values `((content-type . (application/octet-stream
 565                                        (charset . "ISO-8859-1"))))
 566                     ;; XXX: We're not returning the actual contents, deferring
 567                     ;; instead to 'http-write'.  This is a hack to work around
 568                     ;; <http://bugs.gnu.org/21093>.
 569                     item)
 570             (not-found request)))
 571       (not-found request)))
 572
 573 (define (render-home-page request)
 574   "Render the home page."
 575   (values `((content-type . (text/html (charset . "UTF-8"))))
 576           (call-with-output-string
 577             (lambda (port)
 578               (sxml->xml '(html
 579                            (head (title "GNU Guix Substitute Server"))
 580                            (body
 581                             (h1 "GNU Guix Substitute Server")
 582                             (p "Hi, "
 583                                (a (@ (href
 584                                       "https://gnu.org/s/guix/manual/html_node/Invoking-guix-publish.html"))
 585                                   (tt "guix publish"))
 586                                " speaking.  Welcome!")))
 587                          port)))))
 588
 589 (define (extract-narinfo-hash str)
 590   "Return the hash within the narinfo resource string STR, or false if STR
 591 is invalid."
 592   (and (string-suffix? ".narinfo" str)
 593        (let ((base (string-drop-right str 8)))
 594          (and (string-every %nix-base32-charset base)
 595               base))))
 596
 597 (define (get-request? request)
 598   "Return #t if REQUEST uses the GET method."
 599   (eq? (request-method request) 'GET))
 600
 601 (define (request-path-components request)
 602   "Split the URI path of REQUEST into a list of component strings.  For
 603 example: \"/foo/bar\" yields '(\"foo\" \"bar\")."
 604   (split-and-decode-uri-path (uri-path (request-uri request))))
 605
 606 \f
 607 ;;;
 608 ;;; Server.
 609 ;;;
 610
 611 (define %http-write
 612   (@@ (web server http) http-write))
 613
 614 (define (sans-content-length response)
 615   "Return RESPONSE without its 'content-length' header."
 616   (set-field response (response-headers)
 617              (alist-delete 'content-length
 618                            (response-headers response)
 619                            eq?)))
 620
 621 (define (with-content-length response length)
 622   "Return RESPONSE with a 'content-length' header set to LENGTH."
 623   (set-field response (response-headers)
 624              (alist-cons 'content-length length
 625                          (alist-delete 'content-length
 626                                        (response-headers response)
 627                                        eq?))))
 628
 629 (define-syntax-rule (swallow-EPIPE exp ...)
 630   "Swallow EPIPE errors raised by EXP..."
 631   (catch 'system-error
 632     (lambda ()
 633       exp ...)
 634     (lambda args
 635       (if (= EPIPE (system-error-errno args))
 636           (values)
 637           (apply throw args)))))
 638
 639 (define-syntax-rule (swallow-zlib-error exp ...)
 640   "Swallow 'zlib-error' exceptions raised by EXP..."
 641   (catch 'zlib-error
 642     (lambda ()
 643       exp ...)
 644     (const #f)))
 645
 646 (define (nar-response-port response)
 647   "Return a port on which to write the body of RESPONSE, the response of a
 648 /nar request, according to COMPRESSION."
 649   (match (assoc-ref (response-headers response) 'guix-nar-compression)
 650     (($ <compression> 'gzip level)
 651      ;; Note: We cannot used chunked encoding here because
 652      ;; 'make-gzip-output-port' wants a file port.
 653      (make-gzip-output-port (response-port response)
 654                             #:level level
 655                             #:buffer-size (* 64 1024)))
 656     (($ <compression> 'none)
 657      (response-port response))
 658     (#f
 659      (response-port response))))
 660
 661 (define (http-write server client response body)
 662   "Write RESPONSE and BODY to CLIENT, possibly in a separate thread to avoid
 663 blocking."
 664   (match (response-content-type response)
 665     (('application/x-nix-archive . _)
 666      ;; Sending the the whole archive can take time so do it in a separate
 667      ;; thread so that the main thread can keep working in the meantime.
 668      (call-with-new-thread
 669       (lambda ()
 670         (set-thread-name "publish nar")
 671         (let* ((response (write-response (sans-content-length response)
 672                                          client))
 673                (port     (begin
 674                            (force-output client)
 675                            (nar-response-port response))))
 676           ;; XXX: Given our ugly workaround for <http://bugs.gnu.org/21093> in
 677           ;; 'render-nar', BODY here is just the file name of the store item.
 678           ;; We call 'write-file' from here because we know that's the only
 679           ;; way to avoid building the whole nar in memory, which could
 680           ;; quickly become a real problem.  As a bonus, we even do
 681           ;; sendfile(2) directly from the store files to the socket.
 682           (swallow-zlib-error
 683            (swallow-EPIPE
 684             (write-file (utf8->string body) port)))
 685           (swallow-zlib-error
 686            (close-port port))
 687           (values)))))
 688     (('application/octet-stream . _)
 689      ;; Send a raw file in a separate thread.
 690      (call-with-new-thread
 691       (lambda ()
 692         (set-thread-name "publish file")
 693         (catch 'system-error
 694           (lambda ()
 695             (call-with-input-file (utf8->string body)
 696               (lambda (input)
 697                 (let* ((size     (stat:size (stat input)))
 698                        (response (write-response (with-content-length response
 699                                                                       size)
 700                                                  client))
 701                        (output   (response-port response)))
 702                   (if (file-port? output)
 703                       (sendfile output input size)
 704                       (dump-port input output))
 705                   (close-port output)
 706                   (values)))))
 707           (lambda args
 708             ;; If the file was GC'd behind our back, that's fine.  Likewise if
 709             ;; the client closes the connection.
 710             (unless (memv (system-error-errno args)
 711                           (list ENOENT EPIPE ECONNRESET))
 712               (apply throw args))
 713             (values))))))
 714     (_
 715      ;; Handle other responses sequentially.
 716      (%http-write server client response body))))
 717
 718 (define-server-impl concurrent-http-server
 719   ;; A variant of Guile's built-in HTTP server that offloads possibly long
 720   ;; responses to a different thread.
 721   (@@ (web server http) http-open)
 722   (@@ (web server http) http-read)
 723   http-write
 724   (@@ (web server http) http-close))
 725
 726 (define* (make-request-handler store
 727                                #:key
 728                                cache pool
 729                                narinfo-ttl
 730                                (nar-path "nar")
 731                                (compression %no-compression))
 732   (define nar-path?
 733     (let ((expected (split-and-decode-uri-path nar-path)))
 734       (cut equal? expected <>)))
 735
 736   (lambda (request body)
 737     (format #t "~a ~a~%"
 738             (request-method request)
 739             (uri-path (request-uri request)))
 740     (if (get-request? request)                    ;reject POST, PUT, etc.
 741         (match (request-path-components request)
 742           ;; /nix-cache-info
 743           (("nix-cache-info")
 744            (render-nix-cache-info))
 745           ;; /
 746           ((or () ("index.html"))
 747            (render-home-page request))
 748           ;; /<hash>.narinfo
 749           (((= extract-narinfo-hash (? string? hash)))
 750            ;; TODO: Register roots for HASH that will somehow remain for
 751            ;; NARINFO-TTL.
 752            (if cache
 753                (render-narinfo/cached store request hash
 754                                       #:cache cache
 755                                       #:pool pool
 756                                       #:ttl narinfo-ttl
 757                                       #:nar-path nar-path
 758                                       #:compression compression)
 759                (render-narinfo store request hash
 760                                #:ttl narinfo-ttl
 761                                #:nar-path nar-path
 762                                #:compression compression)))
 763           ;; /nar/file/NAME/sha256/HASH
 764           (("file" name "sha256" hash)
 765            (guard (c ((invalid-base32-character? c)
 766                       (not-found request)))
 767              (let ((hash (nix-base32-string->bytevector hash)))
 768                (render-content-addressed-file store request
 769                                               name 'sha256 hash))))
 770
 771           ;; Use different URLs depending on the compression type.  This
 772           ;; guarantees that /nar URLs remain valid even when 'guix publish'
 773           ;; is restarted with different compression parameters.
 774
 775           ;; /nar/gzip/<store-item>
 776           ((components ... "gzip" store-item)
 777            (if (and (nar-path? components) (zlib-available?))
 778                (let ((compression (match compression
 779                                     (($ <compression> 'gzip)
 780                                      compression)
 781                                     (_
 782                                      %default-gzip-compression))))
 783                  (if cache
 784                      (render-nar/cached store cache request store-item
 785                                         #:compression compression)
 786                      (render-nar store request store-item
 787                                  #:compression compression)))
 788                (not-found request)))
 789
 790           ;; /nar/<store-item>
 791           ((components ... store-item)
 792            (if (nar-path? components)
 793                (if cache
 794                    (render-nar/cached store cache request store-item
 795                                       #:compression %no-compression)
 796                    (render-nar store request store-item
 797                                #:compression %no-compression))
 798                (not-found request)))
 799
 800           (x (not-found request)))
 801         (not-found request))))
 802
 803 (define* (run-publish-server socket store
 804                              #:key (compression %no-compression)
 805                              (nar-path "nar") narinfo-ttl
 806                              cache pool)
 807   (run-server (make-request-handler store
 808                                     #:cache cache
 809                                     #:pool pool
 810                                     #:nar-path nar-path
 811                                     #:narinfo-ttl narinfo-ttl
 812                                     #:compression compression)
 813               concurrent-http-server
 814               `(#:socket ,socket)))
 815
 816 (define (open-server-socket address)
 817   "Return a TCP socket bound to ADDRESS, a socket address."
 818   (let ((sock (socket (sockaddr:fam address) SOCK_STREAM 0)))
 819     (setsockopt sock SOL_SOCKET SO_REUSEADDR 1)
 820     (bind sock address)
 821     sock))
 822
 823 (define (gather-user-privileges user)
 824   "Switch to the identity of USER, a user name."
 825   (catch 'misc-error
 826     (lambda ()
 827       (let ((user (getpw user)))
 828         (setgroups #())
 829         (setgid (passwd:gid user))
 830         (setuid (passwd:uid user))))
 831     (lambda (key proc message args . rest)
 832       (leave (G_ "user '~a' not found: ~a~%")
 833              user (apply format #f message args)))))
 834
 835 \f
 836 ;;;
 837 ;;; Entry point.
 838 ;;;
 839
 840 (define (guix-publish . args)
 841   (with-error-handling
 842     (let* ((opts    (args-fold* args %options
 843                                 (lambda (opt name arg result)
 844                                   (leave (G_ "~A: unrecognized option~%") name))
 845                                 (lambda (arg result)
 846                                   (leave (G_ "~A: extraneous argument~%") arg))
 847                                 %default-options))
 848            (user    (assoc-ref opts 'user))
 849            (port    (assoc-ref opts 'port))
 850            (ttl     (assoc-ref opts 'narinfo-ttl))
 851            (compression (assoc-ref opts 'compression))
 852            (address (let ((addr (assoc-ref opts 'address)))
 853                       (make-socket-address (sockaddr:fam addr)
 854                                            (sockaddr:addr addr)
 855                                            port)))
 856            (socket  (open-server-socket address))
 857            (nar-path  (assoc-ref opts 'nar-path))
 858            (repl-port (assoc-ref opts 'repl))
 859            (cache     (assoc-ref opts 'cache))
 860            (workers   (assoc-ref opts 'workers))
 861
 862            ;; Read the key right away so that (1) we fail early on if we can't
 863            ;; access them, and (2) we can then drop privileges.
 864            (public-key  (read-file-sexp (assoc-ref opts 'public-key-file)))
 865            (private-key (read-file-sexp (assoc-ref opts 'private-key-file))))
 866
 867       (when user
 868         ;; Now that we've read the key material and opened the socket, we can
 869         ;; drop privileges.
 870         (gather-user-privileges user))
 871
 872       (when (zero? (getuid))
 873         (warning (G_ "server running as root; \
 874 consider using the '--user' option!~%")))
 875
 876       (parameterize ((%public-key public-key)
 877                      (%private-key private-key))
 878         (format #t (G_ "publishing ~a on ~a, port ~d~%")
 879                 %store-directory
 880                 (inet-ntop (sockaddr:fam address) (sockaddr:addr address))
 881                 (sockaddr:port address))
 882         (when repl-port
 883           (repl:spawn-server (repl:make-tcp-server-socket #:port repl-port)))
 884
 885         ;; Set the name of the main thread.
 886         (set-thread-name "guix publish")
 887
 888         (with-store store
 889           (run-publish-server socket store
 890                               #:cache cache
 891                               #:pool (and cache (make-pool workers
 892                                                            #:thread-name
 893                                                            "publish worker"))
 894                               #:nar-path nar-path
 895                               #:compression compression
 896                               #:narinfo-ttl ttl))))))
 897
 898 ;;; Local Variables:
 899 ;;; eval: (put 'single-baker 'scheme-indent-function 1)
 900 ;;; End: