Commit | Line | Data |
---|---|---|
0f41c26f | 1 | ;;; GNU Guix --- Functional package management for GNU |
cd4027fa | 2 | ;;; Copyright © 2012, 2013, 2014 Ludovic Courtès <ludo@gnu.org> |
d2d8779b | 3 | ;;; Copyright © 2014 Mark H Weaver <mhw@netris.org> |
0f41c26f LC |
4 | ;;; |
5 | ;;; This file is part of GNU Guix. | |
6 | ;;; | |
7 | ;;; GNU Guix is free software; you can redistribute it and/or modify it | |
8 | ;;; under the terms of the GNU General Public License as published by | |
9 | ;;; the Free Software Foundation; either version 3 of the License, or (at | |
10 | ;;; your option) any later version. | |
11 | ;;; | |
12 | ;;; GNU Guix is distributed in the hope that it will be useful, but | |
13 | ;;; WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | ;;; GNU General Public License for more details. | |
16 | ;;; | |
17 | ;;; You should have received a copy of the GNU General Public License | |
18 | ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. | |
19 | ||
20 | (define-module (guix nar) | |
21 | #:use-module (guix utils) | |
22 | #:use-module (guix serialization) | |
cd4027fa LC |
23 | #:use-module ((guix build utils) |
24 | #:select (delete-file-recursively with-directory-excursion)) | |
25 | #:use-module (guix store) | |
26 | #:use-module (guix ui) ; for '_' | |
27 | #:use-module (guix hash) | |
28 | #:use-module (guix pki) | |
29 | #:use-module (guix pk-crypto) | |
0f41c26f LC |
30 | #:use-module (rnrs bytevectors) |
31 | #:use-module (rnrs io ports) | |
32 | #:use-module (srfi srfi-1) | |
cd4027fa | 33 | #:use-module (srfi srfi-11) |
0f41c26f | 34 | #:use-module (srfi srfi-26) |
53c63ee9 LC |
35 | #:use-module (srfi srfi-34) |
36 | #:use-module (srfi srfi-35) | |
0f41c26f | 37 | #:use-module (ice-9 ftw) |
53c63ee9 LC |
38 | #:use-module (ice-9 match) |
39 | #:export (nar-error? | |
cd4027fa LC |
40 | nar-error-port |
41 | nar-error-file | |
42 | ||
53c63ee9 | 43 | nar-read-error? |
53c63ee9 LC |
44 | nar-read-error-token |
45 | ||
cd4027fa LC |
46 | nar-invalid-hash-error? |
47 | nar-invalid-hash-error-expected | |
48 | nar-invalid-hash-error-actual | |
49 | ||
50 | nar-signature-error? | |
51 | nar-signature-error-signature | |
52 | ||
53c63ee9 | 53 | write-file |
cd4027fa LC |
54 | restore-file |
55 | ||
56 | restore-file-set)) | |
0f41c26f LC |
57 | |
58 | ;;; Comment: | |
59 | ;;; | |
60 | ;;; Read and write Nix archives, aka. ‘nar’. | |
61 | ;;; | |
62 | ;;; Code: | |
63 | ||
53c63ee9 | 64 | (define-condition-type &nar-error &error ; XXX: inherit from &nix-error ? |
cd4027fa LC |
65 | nar-error? |
66 | (file nar-error-file) ; file we were restoring, or #f | |
67 | (port nar-error-port)) ; port from which we read | |
53c63ee9 LC |
68 | |
69 | (define-condition-type &nar-read-error &nar-error | |
70 | nar-read-error? | |
53c63ee9 LC |
71 | (token nar-read-error-token)) ; faulty token, or #f |
72 | ||
cd4027fa LC |
73 | (define-condition-type &nar-signature-error &nar-error |
74 | nar-signature-error? | |
75 | (signature nar-signature-error-signature)) ; faulty signature or #f | |
53c63ee9 | 76 | |
cd4027fa LC |
77 | (define-condition-type &nar-invalid-hash-error &nar-signature-error |
78 | nar-invalid-hash-error? | |
79 | (expected nar-invalid-hash-error-expected) ; expected hash (a bytevector) | |
80 | (actual nar-invalid-hash-error-actual)) ; actual hash | |
81 | ||
82 | \f | |
53c63ee9 LC |
83 | (define (dump in out size) |
84 | "Copy SIZE bytes from IN to OUT." | |
85 | (define buf-size 65536) | |
86 | (define buf (make-bytevector buf-size)) | |
87 | ||
88 | (let loop ((left size)) | |
89 | (if (<= left 0) | |
90 | 0 | |
91 | (let ((read (get-bytevector-n! in buf 0 (min left buf-size)))) | |
92 | (if (eof-object? read) | |
93 | left | |
94 | (begin | |
95 | (put-bytevector out buf 0 read) | |
96 | (loop (- left read)))))))) | |
97 | ||
0f41c26f LC |
98 | (define (write-contents file p size) |
99 | "Write SIZE bytes from FILE to output port P." | |
100 | (define (call-with-binary-input-file file proc) | |
101 | ;; Open FILE as a binary file. This avoids scan-for-encoding, and thus | |
102 | ;; avoids any initial buffering. Disable file name canonicalization to | |
103 | ;; avoid stat'ing like crazy. | |
104 | (with-fluids ((%file-port-name-canonicalization #f)) | |
105 | (let ((port (open-file file "rb"))) | |
48e488eb LC |
106 | (dynamic-wind |
107 | (const #t) | |
108 | (cut proc port) | |
109 | (lambda () | |
110 | (close-port port)))))) | |
0f41c26f | 111 | |
0f41c26f LC |
112 | (write-string "contents" p) |
113 | (write-long-long size p) | |
114 | (call-with-binary-input-file file | |
115 | ;; Use `sendfile' when available (Guile 2.0.8+). | |
a93e91ff LC |
116 | (if (and (compile-time-value (defined? 'sendfile)) |
117 | (file-port? p)) | |
0f41c26f | 118 | (cut sendfile p <> size 0) |
53c63ee9 | 119 | (cut dump <> p size))) |
0f41c26f LC |
120 | (write-padding size p)) |
121 | ||
53c63ee9 LC |
122 | (define (read-contents in out) |
123 | "Read the contents of a file from the Nar at IN, write it to OUT, and return | |
124 | the size in bytes." | |
125 | (define executable? | |
126 | (match (read-string in) | |
127 | ("contents" | |
128 | #f) | |
129 | ("executable" | |
130 | (match (list (read-string in) (read-string in)) | |
131 | (("" "contents") #t) | |
132 | (x (raise | |
133 | (condition (&message | |
134 | (message "unexpected executable file marker")) | |
135 | (&nar-read-error (port in) | |
136 | (file #f) | |
137 | (token x)))))) | |
138 | #t) | |
139 | (x | |
140 | (raise | |
141 | (condition (&message (message "unsupported nar file type")) | |
142 | (&nar-read-error (port in) (file #f) (token x))))))) | |
143 | ||
144 | (let ((size (read-long-long in))) | |
145 | ;; Note: `sendfile' cannot be used here because of port buffering on IN. | |
146 | (dump in out size) | |
147 | ||
148 | (when executable? | |
149 | (chmod out #o755)) | |
150 | (let ((m (modulo size 8))) | |
151 | (unless (zero? m) | |
152 | (get-bytevector-n in (- 8 m)))) | |
153 | size)) | |
154 | ||
155 | (define %archive-version-1 | |
156 | ;; Magic cookie for Nix archives. | |
157 | "nix-archive-1") | |
158 | ||
0f41c26f LC |
159 | (define (write-file file port) |
160 | "Write the contents of FILE to PORT in Nar format, recursing into | |
161 | sub-directories of FILE as needed." | |
0f41c26f LC |
162 | (define p port) |
163 | ||
164 | (write-string %archive-version-1 p) | |
165 | ||
166 | (let dump ((f file)) | |
167 | (let ((s (lstat f))) | |
168 | (write-string "(" p) | |
169 | (case (stat:type s) | |
170 | ((regular) | |
171 | (write-string "type" p) | |
172 | (write-string "regular" p) | |
173 | (if (not (zero? (logand (stat:mode s) #o100))) | |
174 | (begin | |
175 | (write-string "executable" p) | |
176 | (write-string "" p))) | |
177 | (write-contents f p (stat:size s))) | |
178 | ((directory) | |
179 | (write-string "type" p) | |
180 | (write-string "directory" p) | |
d2d8779b MW |
181 | (let ((entries |
182 | ;; NOTE: Guile 2.0.5's 'scandir' returns all subdirectories | |
183 | ;; unconditionally, including "." and "..", regardless of the | |
184 | ;; 'select?' predicate passed to it, so we have to filter | |
185 | ;; those out externally. | |
186 | (filter (negate (cut member <> '("." ".."))) | |
187 | ;; 'scandir' defaults to 'string-locale<?' to sort | |
188 | ;; files, but this happens to be case-insensitive (at | |
189 | ;; least in 'en_US' locale on libc 2.18.) Conversely, | |
190 | ;; we want files to be sorted in a case-sensitive | |
191 | ;; fashion. | |
192 | (scandir f (const #t) string<?)))) | |
0f41c26f LC |
193 | (for-each (lambda (e) |
194 | (let ((f (string-append f "/" e))) | |
195 | (write-string "entry" p) | |
196 | (write-string "(" p) | |
197 | (write-string "name" p) | |
198 | (write-string e p) | |
199 | (write-string "node" p) | |
200 | (dump f) | |
201 | (write-string ")" p))) | |
202 | entries))) | |
8f3114b7 LC |
203 | ((symlink) |
204 | (write-string "type" p) | |
205 | (write-string "symlink" p) | |
206 | (write-string "target" p) | |
207 | (write-string (readlink f) p)) | |
0f41c26f | 208 | (else |
3140f2df LC |
209 | (raise (condition (&message (message "unsupported file type")) |
210 | (&nar-error (file f) (port port)))))) | |
0f41c26f LC |
211 | (write-string ")" p)))) |
212 | ||
53c63ee9 LC |
213 | (define (restore-file port file) |
214 | "Read a file (possibly a directory structure) in Nar format from PORT. | |
215 | Restore it as FILE." | |
216 | (let ((signature (read-string port))) | |
217 | (unless (equal? signature %archive-version-1) | |
218 | (raise | |
219 | (condition (&message (message "invalid nar signature")) | |
220 | (&nar-read-error (port port) | |
221 | (token signature) | |
222 | (file #f)))))) | |
223 | ||
224 | (let restore ((file file)) | |
8f3114b7 LC |
225 | (define (read-eof-marker) |
226 | (match (read-string port) | |
227 | (")" #t) | |
228 | (x (raise | |
229 | (condition | |
230 | (&message (message "invalid nar end-of-file marker")) | |
231 | (&nar-read-error (port port) (file file) (token x))))))) | |
232 | ||
53c63ee9 LC |
233 | (match (list (read-string port) (read-string port) (read-string port)) |
234 | (("(" "type" "regular") | |
235 | (call-with-output-file file (cut read-contents port <>)) | |
8f3114b7 LC |
236 | (read-eof-marker)) |
237 | (("(" "type" "symlink") | |
238 | (match (list (read-string port) (read-string port)) | |
239 | (("target" target) | |
240 | (symlink target file) | |
241 | (read-eof-marker)) | |
53c63ee9 LC |
242 | (x (raise |
243 | (condition | |
8f3114b7 | 244 | (&message (message "invalid symlink tokens")) |
53c63ee9 LC |
245 | (&nar-read-error (port port) (file file) (token x))))))) |
246 | (("(" "type" "directory") | |
247 | (let ((dir file)) | |
248 | (mkdir dir) | |
249 | (let loop ((prefix (read-string port))) | |
250 | (match prefix | |
251 | ("entry" | |
252 | (match (list (read-string port) | |
253 | (read-string port) (read-string port) | |
254 | (read-string port)) | |
255 | (("(" "name" file "node") | |
256 | (restore (string-append dir "/" file)) | |
257 | (match (read-string port) | |
258 | (")" #t) | |
259 | (x | |
260 | (raise | |
261 | (condition | |
262 | (&message | |
263 | (message "unexpected directory entry termination")) | |
264 | (&nar-read-error (port port) | |
265 | (file file) | |
266 | (token x)))))) | |
267 | (loop (read-string port))))) | |
268 | (")" #t) ; done with DIR | |
269 | (x | |
270 | (raise | |
271 | (condition | |
272 | (&message (message "unexpected directory inter-entry marker")) | |
273 | (&nar-read-error (port port) (file file) (token x))))))))) | |
274 | (x | |
275 | (raise | |
276 | (condition | |
277 | (&message (message "unsupported nar entry type")) | |
278 | (&nar-read-error (port port) (file file) (token x)))))))) | |
279 | ||
cd4027fa LC |
280 | \f |
281 | ;;; | |
282 | ;;; Restoring a file set into the store. | |
283 | ;;; | |
284 | ||
285 | ;; The code below accesses the store directly and is meant to be run from | |
286 | ;; "build hooks", which cannot invoke the daemon's 'import-paths' RPC since | |
287 | ;; (1) the locks on the files to be restored as already held, and (2) the | |
288 | ;; $NIX_HELD_LOCKS hackish environment variable cannot be set. | |
289 | ;; | |
290 | ;; So we're really duplicating that functionality of the daemon (well, until | |
291 | ;; most of the daemon is in Scheme :-)). But note that we do use a couple of | |
292 | ;; RPCs for functionality not available otherwise, like 'valid-path?'. | |
293 | ||
294 | (define (lock-store-file file) | |
295 | "Acquire exclusive access to FILE, a store file." | |
296 | (call-with-output-file (string-append file ".lock") | |
297 | (cut fcntl-flock <> 'write-lock))) | |
298 | ||
299 | (define (unlock-store-file file) | |
300 | "Release access to FILE." | |
301 | (call-with-input-file (string-append file ".lock") | |
302 | (cut fcntl-flock <> 'unlock))) | |
303 | ||
304 | (define* (finalize-store-file source target | |
305 | #:key (references '()) deriver (lock? #t)) | |
306 | "Rename SOURCE to TARGET and register TARGET as a valid store item, with | |
307 | REFERENCES and DERIVER. When LOCK? is true, acquire exclusive locks on TARGET | |
308 | before attempting to register it; otherwise, assume TARGET's locks are already | |
309 | held." | |
310 | ||
311 | ;; XXX: Currently we have to call out to the daemon to check whether TARGET | |
312 | ;; is valid. | |
313 | (with-store store | |
314 | (unless (valid-path? store target) | |
315 | (when lock? | |
316 | (lock-store-file target)) | |
317 | ||
318 | (unless (valid-path? store target) | |
319 | ;; If FILE already exists, delete it (it's invalid anyway.) | |
320 | (when (file-exists? target) | |
321 | (delete-file-recursively target)) | |
322 | ||
323 | ;; Install the new TARGET. | |
324 | (rename-file source target) | |
325 | ||
326 | ;; Register TARGET. As a side effect, it resets the timestamps of all | |
327 | ;; its files, recursively. However, it doesn't attempt to deduplicate | |
328 | ;; its files like 'importPaths' does (FIXME). | |
329 | (register-path target | |
330 | #:references references | |
331 | #:deriver deriver)) | |
332 | ||
333 | (when lock? | |
334 | (unlock-store-file target))))) | |
335 | ||
336 | (define (temporary-store-directory) | |
337 | "Return the file name of a temporary directory created in the store that is | |
338 | protected from garbage collection." | |
339 | (let* ((template (string-append (%store-prefix) "/guix-XXXXXX")) | |
340 | (port (mkstemp! template))) | |
341 | (close-port port) | |
9132b9bd LC |
342 | |
343 | ;; Make sure TEMPLATE is not collected while we populate it. | |
cd4027fa | 344 | (with-store store |
9132b9bd | 345 | (add-indirect-root store template)) |
cd4027fa LC |
346 | |
347 | ;; There's a small window during which the GC could delete the file. Try | |
348 | ;; again if that happens. | |
349 | (if (file-exists? template) | |
350 | (begin | |
351 | ;; It's up to the caller to create that file or directory. | |
352 | (delete-file template) | |
353 | template) | |
354 | (temporary-store-directory)))) | |
355 | ||
356 | (define* (restore-file-set port | |
357 | #:key (verify-signature? #t) (lock? #t) | |
358 | (log-port (current-error-port))) | |
359 | "Restore the file set read from PORT to the store. The format of the data | |
360 | on PORT must be as created by 'export-paths'---i.e., a series of Nar-formatted | |
361 | archives with interspersed meta-data joining them together, possibly with a | |
362 | digital signature at the end. Log progress to LOG-PORT. Return the list of | |
363 | files restored. | |
364 | ||
365 | When LOCK? is #f, assume locks for the files to be restored are already held. | |
366 | This is the case when the daemon calls a build hook. | |
367 | ||
368 | Note that this procedure accesses the store directly, so it's only meant to be | |
369 | used by the daemon's build hooks since they cannot call back to the daemon | |
370 | while the locks are held." | |
371 | (define %export-magic | |
372 | ;; Number used to identify genuine file set archives. | |
373 | #x4558494e) | |
374 | ||
375 | (define port* | |
376 | ;; Keep that one around, for error conditions. | |
377 | port) | |
378 | ||
379 | (define (assert-valid-signature signature hash file) | |
24194b6b NK |
380 | ;; Bail out if SIGNATURE, which must be a string as produced by |
381 | ;; 'canonical-sexp->string', doesn't match HASH, a bytevector containing | |
382 | ;; the expected hash for FILE. | |
e4687a5e LC |
383 | (let ((signature (catch 'gcry-error |
384 | (lambda () | |
385 | (string->canonical-sexp signature)) | |
386 | (lambda (err . _) | |
387 | (raise (condition | |
388 | (&message | |
389 | (message "signature is not a valid \ | |
cd4027fa | 390 | s-expression")) |
e4687a5e LC |
391 | (&nar-signature-error |
392 | (file file) | |
393 | (signature signature) (port port)))))))) | |
394 | (signature-case (signature hash (current-acl)) | |
395 | (valid-signature #t) | |
396 | (invalid-signature | |
397 | (raise (condition | |
398 | (&message (message "invalid signature")) | |
399 | (&nar-signature-error | |
400 | (file file) (signature signature) (port port))))) | |
401 | (hash-mismatch | |
402 | (raise (condition (&message (message "invalid hash")) | |
403 | (&nar-invalid-hash-error | |
404 | (port port) (file file) | |
405 | (signature signature) | |
406 | (expected (hash-data->bytevector | |
407 | (signature-signed-data signature))) | |
408 | (actual hash))))) | |
409 | (unauthorized-key | |
410 | (raise (condition (&message (message "unauthorized public key")) | |
411 | (&nar-signature-error | |
412 | (signature signature) (file file) (port port))))) | |
413 | (corrupt-signature | |
414 | (raise (condition | |
415 | (&message (message "corrupt signature data")) | |
416 | (&nar-signature-error | |
417 | (signature signature) (file file) (port port)))))))) | |
cd4027fa LC |
418 | |
419 | (let loop ((n (read-long-long port)) | |
420 | (files '())) | |
421 | (case n | |
422 | ((0) | |
423 | (reverse files)) | |
424 | ((1) | |
425 | (let-values (((port get-hash) | |
426 | (open-sha256-input-port port))) | |
427 | (let ((temp (temporary-store-directory))) | |
428 | (restore-file port temp) | |
429 | (let ((magic (read-int port))) | |
430 | (unless (= magic %export-magic) | |
431 | (raise (condition | |
432 | (&message (message "corrupt file set archive")) | |
433 | (&nar-read-error | |
434 | (port port*) (file #f) (token #f)))))) | |
435 | ||
436 | (let ((file (read-store-path port)) | |
437 | (refs (read-store-path-list port)) | |
438 | (deriver (read-string port)) | |
439 | (hash (get-hash)) | |
440 | (has-sig? (= 1 (read-int port)))) | |
441 | (format log-port | |
442 | (_ "importing file or directory '~a'...~%") | |
443 | file) | |
444 | ||
445 | (let ((sig (and has-sig? (read-string port)))) | |
446 | (when verify-signature? | |
447 | (if sig | |
448 | (begin | |
449 | (assert-valid-signature sig hash file) | |
450 | (format log-port | |
451 | (_ "found valid signature for '~a'~%") | |
452 | file) | |
453 | (finalize-store-file temp file | |
454 | #:references refs | |
455 | #:deriver deriver | |
456 | #:lock? lock?) | |
457 | (loop (read-long-long port) | |
458 | (cons file files))) | |
459 | (raise (condition | |
460 | (&message (message "imported file lacks \ | |
461 | a signature")) | |
462 | (&nar-signature-error | |
463 | (port port*) (file file) (signature #f))))))))))) | |
464 | (else | |
465 | ;; Neither 0 nor 1. | |
466 | (raise (condition | |
467 | (&message (message "invalid inter-file archive mark")) | |
468 | (&nar-read-error | |
469 | (port port) (file #f) (token #f)))))))) | |
470 | ||
0f41c26f | 471 | ;;; nar.scm ends here |