tests/publish.scm

   1 ;;; GNU Guix --- Functional package management for GNU
   2 ;;; Copyright © 2015 David Thompson <davet@gnu.org>
   3 ;;; Copyright © 2016, 2017, 2018 Ludovic Courtès <ludo@gnu.org>
   4 ;;;
   5 ;;; This file is part of GNU Guix.
   6 ;;;
   7 ;;; GNU Guix is free software; you can redistribute it and/or modify it
   8 ;;; under the terms of the GNU General Public License as published by
   9 ;;; the Free Software Foundation; either version 3 of the License, or (at
  10 ;;; your option) any later version.
  11 ;;;
  12 ;;; GNU Guix is distributed in the hope that it will be useful, but
  13 ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 ;;; GNU General Public License for more details.
  16 ;;;
  17 ;;; You should have received a copy of the GNU General Public License
  18 ;;; along with GNU Guix.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 ;; Avoid interference.
  21 (unsetenv "http_proxy")
  22
  23 (define-module (test-publish)
  24   #:use-module (guix scripts publish)
  25   #:use-module (guix tests)
  26   #:use-module (guix config)
  27   #:use-module (guix utils)
  28   #:use-module (guix hash)
  29   #:use-module (guix store)
  30   #:use-module (guix derivations)
  31   #:use-module (guix gexp)
  32   #:use-module (guix base32)
  33   #:use-module (guix base64)
  34   #:use-module ((guix records) #:select (recutils->alist))
  35   #:use-module ((guix serialization) #:select (restore-file))
  36   #:use-module (guix pk-crypto)
  37   #:use-module ((guix pki) #:select (%public-key-file %private-key-file))
  38   #:use-module (guix zlib)
  39   #:use-module (web uri)
  40   #:use-module (web client)
  41   #:use-module (web response)
  42   #:use-module (rnrs bytevectors)
  43   #:use-module (ice-9 binary-ports)
  44   #:use-module (srfi srfi-1)
  45   #:use-module (srfi srfi-26)
  46   #:use-module (srfi srfi-64)
  47   #:use-module (ice-9 format)
  48   #:use-module (ice-9 match)
  49   #:use-module (ice-9 rdelim))
  50
  51 (define %store
  52   (open-connection-for-tests))
  53
  54 (define %reference (add-text-to-store %store "ref" "foo"))
  55
  56 (define %item (add-text-to-store %store "item" "bar" (list %reference)))
  57
  58 (define (http-get-body uri)
  59   (call-with-values (lambda () (http-get uri))
  60     (lambda (response body) body)))
  61
  62 (define (http-get-port uri)
  63   (let ((socket (open-socket-for-uri uri)))
  64     ;; Make sure to use an unbuffered port so that we can then peek at the
  65     ;; underlying file descriptor via 'call-with-gzip-input-port'.
  66     (setvbuf socket _IONBF)
  67     (call-with-values
  68         (lambda ()
  69           (http-get uri #:port socket #:streaming? #t))
  70       (lambda (response port)
  71         ;; Don't (setvbuf port _IONBF) because of <http://bugs.gnu.org/19610>
  72         ;; (PORT might be a custom binary input port).
  73         port))))
  74
  75 (define (publish-uri route)
  76   (string-append "http://localhost:6789" route))
  77
  78 (define-syntax-rule (with-separate-output-ports exp ...)
  79   ;; Since ports aren't thread-safe in Guile 2.0, duplicate the output and
  80   ;; error ports to make sure the two threads don't end up stepping on each
  81   ;; other's toes.
  82   (with-output-to-port (duplicate-port (current-output-port) "w")
  83     (lambda ()
  84       (with-error-to-port (duplicate-port (current-error-port) "w")
  85         (lambda ()
  86           exp ...)))))
  87
  88 ;; Run a local publishing server in a separate thread.
  89 (with-separate-output-ports
  90  (call-with-new-thread
  91   (lambda ()
  92     (guix-publish "--port=6789" "-C0"))))     ;attempt to avoid port collision
  93
  94 (define (wait-until-ready port)
  95   ;; Wait until the server is accepting connections.
  96   (let ((conn (socket PF_INET SOCK_STREAM 0)))
  97     (let loop ()
  98       (unless (false-if-exception
  99                (connect conn AF_INET (inet-pton AF_INET "127.0.0.1") port))
 100         (loop)))))
 101
 102 (define (wait-for-file file)
 103   ;; Wait until FILE shows up.
 104   (let loop ((i 20))
 105     (cond ((file-exists? file)
 106            #t)
 107           ((zero? i)
 108            (error "file didn't show up" file))
 109           (else
 110            (pk 'wait-for-file file)
 111            (sleep 1)
 112            (loop (- i 1))))))
 113
 114 (define %gzip-magic-bytes
 115   ;; Magic bytes of gzip file.
 116   #vu8(#x1f #x8b))
 117
 118 ;; Wait until the two servers are ready.
 119 (wait-until-ready 6789)
 120
 121 ;; Initialize the public/private key SRFI-39 parameters.
 122 (%public-key (read-file-sexp %public-key-file))
 123 (%private-key (read-file-sexp %private-key-file))
 124
 125 \f
 126 (test-begin "publish")
 127
 128 (test-equal "/nix-cache-info"
 129   (format #f "StoreDir: ~a\nWantMassQuery: 0\nPriority: 100\n"
 130           %store-directory)
 131   (http-get-body (publish-uri "/nix-cache-info")))
 132
 133 (test-equal "/*.narinfo"
 134   (let* ((info (query-path-info %store %item))
 135          (unsigned-info
 136           (format #f
 137                   "StorePath: ~a
 138 URL: nar/~a
 139 Compression: none
 140 NarHash: sha256:~a
 141 NarSize: ~d
 142 References: ~a
 143 FileSize: ~a~%"
 144                   %item
 145                   (basename %item)
 146                   (bytevector->nix-base32-string
 147                    (path-info-hash info))
 148                   (path-info-nar-size info)
 149                   (basename (first (path-info-references info)))
 150                   (path-info-nar-size info)))
 151          (signature (base64-encode
 152                      (string->utf8
 153                       (canonical-sexp->string
 154                        ((@@ (guix scripts publish) signed-string)
 155                         unsigned-info))))))
 156     (format #f "~aSignature: 1;~a;~a~%"
 157             unsigned-info (gethostname) signature))
 158   (utf8->string
 159    (http-get-body
 160     (publish-uri
 161      (string-append "/" (store-path-hash-part %item) ".narinfo")))))
 162
 163 (test-equal "/*.narinfo with properly encoded '+' sign"
 164   ;; See <http://bugs.gnu.org/21888>.
 165   (let* ((item (add-text-to-store %store "fake-gtk+" "Congrats!"))
 166          (info (query-path-info %store item))
 167          (unsigned-info
 168           (format #f
 169                   "StorePath: ~a
 170 URL: nar/~a
 171 Compression: none
 172 NarHash: sha256:~a
 173 NarSize: ~d
 174 References: ~%\
 175 FileSize: ~a~%"
 176                   item
 177                   (uri-encode (basename item))
 178                   (bytevector->nix-base32-string
 179                    (path-info-hash info))
 180                   (path-info-nar-size info)
 181                   (path-info-nar-size info)))
 182          (signature (base64-encode
 183                      (string->utf8
 184                       (canonical-sexp->string
 185                        ((@@ (guix scripts publish) signed-string)
 186                         unsigned-info))))))
 187     (format #f "~aSignature: 1;~a;~a~%"
 188             unsigned-info (gethostname) signature))
 189
 190   (let ((item (add-text-to-store %store "fake-gtk+" "Congrats!")))
 191     (utf8->string
 192      (http-get-body
 193       (publish-uri
 194        (string-append "/" (store-path-hash-part item) ".narinfo"))))))
 195
 196 (test-equal "/nar/*"
 197   "bar"
 198   (call-with-temporary-output-file
 199    (lambda (temp port)
 200      (let ((nar (utf8->string
 201                  (http-get-body
 202                   (publish-uri
 203                    (string-append "/nar/" (basename %item)))))))
 204        (call-with-input-string nar (cut restore-file <> temp)))
 205      (call-with-input-file temp read-string))))
 206
 207 (unless (zlib-available?)
 208   (test-skip 1))
 209 (test-equal "/nar/gzip/*"
 210   "bar"
 211   (call-with-temporary-output-file
 212    (lambda (temp port)
 213      (let ((nar (http-get-port
 214                  (publish-uri
 215                   (string-append "/nar/gzip/" (basename %item))))))
 216        (call-with-gzip-input-port nar
 217          (cut restore-file <> temp)))
 218      (call-with-input-file temp read-string))))
 219
 220 (unless (zlib-available?)
 221   (test-skip 1))
 222 (test-equal "/nar/gzip/* is really gzip"
 223   %gzip-magic-bytes
 224   ;; Since 'gzdopen' (aka. 'call-with-gzip-input-port') transparently reads
 225   ;; uncompressed gzip, the test above doesn't check whether it's actually
 226   ;; gzip.  This is what this test does.  See <https://bugs.gnu.org/30184>.
 227   (let ((nar (http-get-port
 228               (publish-uri
 229                (string-append "/nar/gzip/" (basename %item))))))
 230     (get-bytevector-n nar (bytevector-length %gzip-magic-bytes))))
 231
 232 (unless (zlib-available?)
 233   (test-skip 1))
 234 (test-equal "/*.narinfo with compression"
 235   `(("StorePath" . ,%item)
 236     ("URL" . ,(string-append "nar/gzip/" (basename %item)))
 237     ("Compression" . "gzip"))
 238   (let ((thread (with-separate-output-ports
 239                  (call-with-new-thread
 240                   (lambda ()
 241                     (guix-publish "--port=6799" "-C5"))))))
 242     (wait-until-ready 6799)
 243     (let* ((url  (string-append "http://localhost:6799/"
 244                                 (store-path-hash-part %item) ".narinfo"))
 245            (body (http-get-port url)))
 246       (filter (lambda (item)
 247                 (match item
 248                   (("Compression" . _) #t)
 249                   (("StorePath" . _)  #t)
 250                   (("URL" . _) #t)
 251                   (_ #f)))
 252               (recutils->alist body)))))
 253
 254 (unless (zlib-available?)
 255   (test-skip 1))
 256 (test-equal "/*.narinfo for a compressed file"
 257   '("none" "nar")          ;compression-less nar
 258   ;; Assume 'guix publish -C' is already running on port 6799.
 259   (let* ((item (add-text-to-store %store "fake.tar.gz"
 260                                   "This is a fake compressed file."))
 261          (url  (string-append "http://localhost:6799/"
 262                               (store-path-hash-part item) ".narinfo"))
 263          (body (http-get-port url))
 264          (info (recutils->alist body)))
 265     (list (assoc-ref info "Compression")
 266           (dirname (assoc-ref info "URL")))))
 267
 268 (test-equal "custom nar path"
 269   ;; Serve nars at /foo/bar/chbouib instead of /nar.
 270   (list `(("StorePath" . ,%item)
 271           ("URL" . ,(string-append "foo/bar/chbouib/" (basename %item)))
 272           ("Compression" . "none"))
 273         200
 274         404)
 275   (let ((thread (with-separate-output-ports
 276                  (call-with-new-thread
 277                   (lambda ()
 278                     (guix-publish "--port=6798" "-C0"
 279                                   "--nar-path=///foo/bar//chbouib/"))))))
 280     (wait-until-ready 6798)
 281     (let* ((base    "http://localhost:6798/")
 282            (part    (store-path-hash-part %item))
 283            (url     (string-append base part ".narinfo"))
 284            (nar-url (string-append base "foo/bar/chbouib/"
 285                                    (basename %item)))
 286            (body    (http-get-port url)))
 287       (list (filter (lambda (item)
 288                       (match item
 289                         (("Compression" . _) #t)
 290                         (("StorePath" . _)  #t)
 291                         (("URL" . _) #t)
 292                         (_ #f)))
 293                     (recutils->alist body))
 294             (response-code (http-get nar-url))
 295             (response-code
 296              (http-get (string-append base "nar/" (basename %item))))))))
 297
 298 (test-equal "/nar/ with properly encoded '+' sign"
 299   "Congrats!"
 300   (let ((item (add-text-to-store %store "fake-gtk+" "Congrats!")))
 301     (call-with-temporary-output-file
 302      (lambda (temp port)
 303        (let ((nar (utf8->string
 304                    (http-get-body
 305                     (publish-uri
 306                      (string-append "/nar/" (uri-encode (basename item))))))))
 307          (call-with-input-string nar (cut restore-file <> temp)))
 308        (call-with-input-file temp read-string)))))
 309
 310 (test-equal "/nar/invalid"
 311   404
 312   (begin
 313     (call-with-output-file (string-append (%store-prefix) "/invalid")
 314       (lambda (port)
 315         (display "This file is not a valid store item." port)))
 316     (response-code (http-get (publish-uri (string-append "/nar/invalid"))))))
 317
 318 (test-equal "/file/NAME/sha256/HASH"
 319   "Hello, Guix world!"
 320   (let* ((data "Hello, Guix world!")
 321          (hash (call-with-input-string data port-sha256))
 322          (drv  (run-with-store %store
 323                  (gexp->derivation "the-file.txt"
 324                                    #~(call-with-output-file #$output
 325                                        (lambda (port)
 326                                          (display #$data port)))
 327                                    #:hash-algo 'sha256
 328                                    #:hash hash)))
 329          (out  (build-derivations %store (list drv))))
 330     (utf8->string
 331      (http-get-body
 332       (publish-uri
 333        (string-append "/file/the-file.txt/sha256/"
 334                       (bytevector->nix-base32-string hash)))))))
 335
 336 (test-equal "/file/NAME/sha256/INVALID-NIX-BASE32-STRING"
 337   404
 338   (let ((uri (publish-uri
 339               "/file/the-file.txt/sha256/not-a-nix-base32-string")))
 340     (response-code (http-get uri))))
 341
 342 (test-equal "/file/NAME/sha256/INVALID-HASH"
 343   404
 344   (let ((uri (publish-uri
 345               (string-append "/file/the-file.txt/sha256/"
 346                              (bytevector->nix-base32-string
 347                               (call-with-input-string "" port-sha256))))))
 348     (response-code (http-get uri))))
 349
 350 (unless (zlib-available?)
 351   (test-skip 1))
 352 (test-equal "with cache"
 353   (list #t
 354         `(("StorePath" . ,%item)
 355           ("URL" . ,(string-append "nar/gzip/" (basename %item)))
 356           ("Compression" . "gzip"))
 357         200                                       ;nar/gzip/…
 358         #t                                        ;Content-Length
 359         #t                                        ;FileSize
 360         404)                                      ;nar/…
 361   (call-with-temporary-directory
 362    (lambda (cache)
 363      (let ((thread (with-separate-output-ports
 364                     (call-with-new-thread
 365                      (lambda ()
 366                        (guix-publish "--port=6797" "-C2"
 367                                      (string-append "--cache=" cache)))))))
 368        (wait-until-ready 6797)
 369        (let* ((base     "http://localhost:6797/")
 370               (part     (store-path-hash-part %item))
 371               (url      (string-append base part ".narinfo"))
 372               (nar-url  (string-append base "nar/gzip/" (basename %item)))
 373               (cached   (string-append cache "/gzip/" (basename %item)
 374                                        ".narinfo"))
 375               (nar      (string-append cache "/gzip/"
 376                                        (basename %item) ".nar"))
 377               (response (http-get url)))
 378          (and (= 404 (response-code response))
 379
 380               ;; We should get an explicitly short TTL for 404 in this case
 381               ;; because it's going to become 200 shortly.
 382               (match (assq-ref (response-headers response) 'cache-control)
 383                 ((('max-age . ttl))
 384                  (< ttl 3600)))
 385
 386               (wait-for-file cached)
 387               (let* ((body         (http-get-port url))
 388                      (compressed   (http-get nar-url))
 389                      (uncompressed (http-get (string-append base "nar/"
 390                                                             (basename %item))))
 391                      (narinfo      (recutils->alist body)))
 392                 (list (file-exists? nar)
 393                       (filter (lambda (item)
 394                                 (match item
 395                                   (("Compression" . _) #t)
 396                                   (("StorePath" . _)  #t)
 397                                   (("URL" . _) #t)
 398                                   (_ #f)))
 399                               narinfo)
 400                       (response-code compressed)
 401                       (= (response-content-length compressed)
 402                          (stat:size (stat nar)))
 403                       (= (string->number
 404                           (assoc-ref narinfo "FileSize"))
 405                          (stat:size (stat nar)))
 406                       (response-code uncompressed)))))))))
 407
 408 (unless (zlib-available?)
 409   (test-skip 1))
 410 (let ((item (add-text-to-store %store "fake-compressed-thing.tar.gz"
 411                                (random-text))))
 412   (test-equal "with cache, uncompressed"
 413     (list #t
 414           `(("StorePath" . ,item)
 415             ("URL" . ,(string-append "nar/" (basename item)))
 416             ("Compression" . "none"))
 417           200                                     ;nar/…
 418           (path-info-nar-size
 419            (query-path-info %store item))         ;FileSize
 420           404)                                    ;nar/gzip/…
 421     (call-with-temporary-directory
 422      (lambda (cache)
 423        (let ((thread (with-separate-output-ports
 424                       (call-with-new-thread
 425                        (lambda ()
 426                          (guix-publish "--port=6796" "-C2"
 427                                        (string-append "--cache=" cache)))))))
 428          (wait-until-ready 6796)
 429          (let* ((base     "http://localhost:6796/")
 430                 (part     (store-path-hash-part item))
 431                 (url      (string-append base part ".narinfo"))
 432                 (cached   (string-append cache "/none/"
 433                                          (basename item) ".narinfo"))
 434                 (nar      (string-append cache "/none/"
 435                                          (basename item) ".nar"))
 436                 (response (http-get url)))
 437            (and (= 404 (response-code response))
 438
 439                 (wait-for-file cached)
 440                 (let* ((body         (http-get-port url))
 441                        (compressed   (http-get (string-append base "nar/gzip/"
 442                                                               (basename item))))
 443                        (uncompressed (http-get (string-append base "nar/"
 444                                                               (basename item))))
 445                        (narinfo      (recutils->alist body)))
 446                   (list (file-exists? nar)
 447                         (filter (lambda (item)
 448                                   (match item
 449                                     (("Compression" . _) #t)
 450                                     (("StorePath" . _)  #t)
 451                                     (("URL" . _) #t)
 452                                     (_ #f)))
 453                                 narinfo)
 454                         (response-code uncompressed)
 455                         (string->number
 456                          (assoc-ref narinfo "FileSize"))
 457                         (response-code compressed))))))))))
 458
 459 (test-equal "/log/NAME"
 460   `(200 #t application/x-bzip2)
 461   (let ((drv (run-with-store %store
 462                (gexp->derivation "with-log"
 463                                  #~(call-with-output-file #$output
 464                                      (lambda (port)
 465                                        (display "Hello, build log!"
 466                                                 (current-error-port))
 467                                        (display "" port)))))))
 468     (build-derivations %store (list drv))
 469     (let* ((response (http-get
 470                       (publish-uri (string-append "/log/"
 471                                                   (basename (derivation->output-path drv))))
 472                       #:decode-body? #f))
 473            (base     (basename (derivation-file-name drv)))
 474            (log      (string-append (dirname %state-directory)
 475                                     "/log/guix/drvs/" (string-take base 2)
 476                                     "/" (string-drop base 2) ".bz2")))
 477       (list (response-code response)
 478             (= (response-content-length response) (stat:size (stat log)))
 479             (first (response-content-type response))))))
 480
 481 (test-equal "/log/NAME not found"
 482   404
 483   (let ((uri (publish-uri "/log/does-not-exist")))
 484     (response-code (http-get uri))))
 485
 486 (test-end "publish")