Commit | Line | Data |
---|---|---|
233e7676 | 1 | ;;; GNU Guix --- Functional package management for GNU |
7df3ab0f | 2 | ;;; Copyright © 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Ludovic Courtès <ludo@gnu.org> |
f3a42251 | 3 | ;;; Copyright © 2018 Jan Nieuwenhuizen <janneke@gnu.org> |
fdae0fa5 | 4 | ;;; Copyright © 2019, 2020 Mathieu Othacehe <m.othacehe@gmail.com> |
771c5e15 | 5 | ;;; Copyright © 2020 Florian Pelz <pelzflorian@pelzflorian.de> |
7a45b5d5 | 6 | ;;; Copyright © 2020 Lars-Dominik Braun <ldb@leibniz-psychology.org> |
77d3cf08 | 7 | ;;; |
233e7676 | 8 | ;;; This file is part of GNU Guix. |
77d3cf08 | 9 | ;;; |
233e7676 | 10 | ;;; GNU Guix is free software; you can redistribute it and/or modify it |
77d3cf08 LC |
11 | ;;; under the terms of the GNU General Public License as published by |
12 | ;;; the Free Software Foundation; either version 3 of the License, or (at | |
13 | ;;; your option) any later version. | |
14 | ;;; | |
233e7676 | 15 | ;;; GNU Guix is distributed in the hope that it will be useful, but |
77d3cf08 LC |
16 | ;;; WITHOUT ANY WARRANTY; without even the implied warranty of |
17 | ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
18 | ;;; GNU General Public License for more details. | |
19 | ;;; | |
20 | ;;; You should have received a copy of the GNU General Public License | |
233e7676 | 21 | ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>. |
77d3cf08 LC |
22 | |
23 | (define-module (guix store) | |
82058eff | 24 | #:use-module (guix utils) |
d8eea3d2 | 25 | #:use-module (guix config) |
de9fbe9c | 26 | #:use-module (guix deprecation) |
f9704f17 | 27 | #:use-module (guix memoization) |
0f41c26f | 28 | #:use-module (guix serialization) |
e87f0591 | 29 | #:use-module (guix monads) |
c6080c32 | 30 | #:use-module (guix records) |
4c0c4db0 | 31 | #:use-module (guix base16) |
cd041b26 | 32 | #:use-module (guix base32) |
ca719424 | 33 | #:use-module (gcrypt hash) |
03870da8 | 34 | #:use-module (guix profiling) |
b0a6a971 | 35 | #:autoload (guix build syscalls) (terminal-columns) |
77d3cf08 | 36 | #:use-module (rnrs bytevectors) |
2535635f | 37 | #:use-module (ice-9 binary-ports) |
c6080c32 | 38 | #:use-module ((ice-9 control) #:select (let/ec)) |
d9d7b9ec | 39 | #:use-module (ice-9 atomic) |
77d3cf08 LC |
40 | #:use-module (srfi srfi-1) |
41 | #:use-module (srfi srfi-9) | |
bf8e7fc5 | 42 | #:use-module (srfi srfi-9 gnu) |
6581ec9a | 43 | #:use-module (srfi srfi-11) |
77d3cf08 | 44 | #:use-module (srfi srfi-26) |
e87088c9 LC |
45 | #:use-module (srfi srfi-34) |
46 | #:use-module (srfi srfi-35) | |
26bbbb95 | 47 | #:use-module (srfi srfi-39) |
77d3cf08 | 48 | #:use-module (ice-9 match) |
3f1e6939 | 49 | #:use-module (ice-9 vlist) |
6bfec3ed | 50 | #:use-module (ice-9 popen) |
d9d7b9ec | 51 | #:autoload (ice-9 threads) (current-processor-count) |
1f3ea898 | 52 | #:use-module (ice-9 format) |
1397b422 LC |
53 | #:use-module (web uri) |
54 | #:export (%daemon-socket-uri | |
a9d2a105 | 55 | %gc-roots-directory |
f8a8e0fe | 56 | %default-substitute-urls |
9fd72fb1 | 57 | |
de9fbe9c LC |
58 | store-connection? |
59 | store-connection-version | |
60 | store-connection-major-version | |
61 | store-connection-minor-version | |
62 | store-connection-socket | |
63 | ||
64 | ;; Deprecated forms for 'store-connection'. | |
9fd72fb1 | 65 | nix-server? |
dc0f74e5 | 66 | nix-server-version |
77d3cf08 LC |
67 | nix-server-major-version |
68 | nix-server-minor-version | |
69 | nix-server-socket | |
de9fbe9c | 70 | |
dc0f74e5 | 71 | current-store-protocol-version ;for internal use |
0a3c723e | 72 | cache-lookup-recorder ;for internal use |
c6080c32 | 73 | mcached |
77d3cf08 | 74 | |
f9e8a123 LC |
75 | &store-error store-error? |
76 | &store-connection-error store-connection-error? | |
77 | store-connection-error-file | |
78 | store-connection-error-code | |
79 | &store-protocol-error store-protocol-error? | |
80 | store-protocol-error-message | |
81 | store-protocol-error-status | |
82 | ||
83 | ;; Deprecated forms for '&store-error' et al. | |
e87088c9 | 84 | &nix-error nix-error? |
ef86c39f LC |
85 | &nix-connection-error nix-connection-error? |
86 | nix-connection-error-file | |
87 | nix-connection-error-code | |
e87088c9 LC |
88 | &nix-protocol-error nix-protocol-error? |
89 | nix-protocol-error-message | |
90 | nix-protocol-error-status | |
91 | ||
d9d7b9ec LC |
92 | allocate-store-connection-cache |
93 | store-connection-cache | |
94 | set-store-connection-cache | |
95 | set-store-connection-cache! | |
96 | ||
26bbbb95 | 97 | hash-algo |
07e70f48 | 98 | build-mode |
26bbbb95 | 99 | |
dd14678b | 100 | connect-to-daemon |
77d3cf08 | 101 | open-connection |
2f608c14 | 102 | port->connection |
3abaf0c4 | 103 | close-connection |
ce4a4829 | 104 | with-store |
77d3cf08 | 105 | set-build-options |
4f740b67 | 106 | set-build-options* |
31ef99a8 | 107 | valid-path? |
82058eff | 108 | query-path-hash |
11e7a6cf | 109 | hash-part->path |
533d1768 | 110 | query-path-info |
0d268c5d | 111 | add-data-to-store |
77d3cf08 LC |
112 | add-text-to-store |
113 | add-to-store | |
7f11efba | 114 | add-file-tree-to-store |
68dbd5c9 | 115 | file-mapping->tree |
f3a42251 | 116 | binary-file |
041b340d | 117 | with-build-handler |
c40bf581 LC |
118 | map/accumulate-builds |
119 | mapm/accumulate-builds | |
abac874b LC |
120 | build-things |
121 | build | |
16748d80 LC |
122 | query-failed-paths |
123 | clear-failed-paths | |
2a7e1121 | 124 | ensure-path |
7df3ab0f | 125 | find-roots |
3259877d | 126 | add-temp-root |
34811f02 | 127 | add-indirect-root |
a9d2a105 LC |
128 | add-permanent-root |
129 | remove-permanent-root | |
26bbbb95 | 130 | |
0f3d2504 LC |
131 | substitutable? |
132 | substitutable-path | |
133 | substitutable-deriver | |
134 | substitutable-references | |
135 | substitutable-download-size | |
136 | substitutable-nar-size | |
137 | has-substitutes? | |
138 | substitutable-paths | |
139 | substitutable-path-info | |
140 | ||
533d1768 DT |
141 | path-info? |
142 | path-info-deriver | |
143 | path-info-hash | |
144 | path-info-references | |
145 | path-info-registration-time | |
146 | path-info-nar-size | |
147 | ||
f9aefa2d | 148 | built-in-builders |
fae31edc | 149 | references |
c70cf1a7 | 150 | references/cached |
e74f64b9 | 151 | references* |
0744a9f0 | 152 | query-path-info* |
3f1e6939 | 153 | requisites |
fae31edc | 154 | referrers |
e3fd0ce6 | 155 | optimize-store |
c63d9403 | 156 | verify-store |
50add477 | 157 | topologically-sorted |
fae31edc LC |
158 | valid-derivers |
159 | query-derivation-outputs | |
3259877d LC |
160 | live-paths |
161 | dead-paths | |
162 | collect-garbage | |
163 | delete-paths | |
526382ff LC |
164 | import-paths |
165 | export-paths | |
3259877d | 166 | |
dcee50c1 LC |
167 | current-build-output-port |
168 | ||
e87f0591 LC |
169 | %store-monad |
170 | store-bind | |
171 | store-return | |
172 | store-lift | |
023d9892 | 173 | store-lower |
e87f0591 LC |
174 | run-with-store |
175 | %guile-for-build | |
98a7b528 LC |
176 | current-system |
177 | set-current-system | |
91be09de | 178 | current-target-system |
fdae0fa5 | 179 | set-current-target |
e87f0591 LC |
180 | text-file |
181 | interned-file | |
7f11efba | 182 | interned-file-tree |
e87f0591 | 183 | |
26bbbb95 | 184 | %store-prefix |
cd041b26 LC |
185 | store-path |
186 | output-path | |
187 | fixed-output-path | |
26bbbb95 | 188 | store-path? |
9336e5b5 | 189 | direct-store-path? |
e3d74106 | 190 | derivation-path? |
771c5e15 | 191 | store-path-base |
2c6ab6cc | 192 | store-path-package-name |
eddd4077 | 193 | store-path-hash-part |
cdb5b075 | 194 | direct-store-path |
fb94d82b | 195 | derivation-log-file |
eddd4077 | 196 | log-file)) |
77d3cf08 | 197 | |
6ef61cc4 | 198 | (define %protocol-version #x163) |
77d3cf08 | 199 | |
d66b704b LC |
200 | (define %worker-magic-1 #x6e697863) ; "nixc" |
201 | (define %worker-magic-2 #x6478696f) ; "dxio" | |
77d3cf08 LC |
202 | |
203 | (define (protocol-major magic) | |
204 | (logand magic #xff00)) | |
205 | (define (protocol-minor magic) | |
206 | (logand magic #x00ff)) | |
dc0f74e5 LC |
207 | (define (protocol-version major minor) |
208 | (logior major minor)) | |
77d3cf08 LC |
209 | |
210 | (define-syntax define-enumerate-type | |
211 | (syntax-rules () | |
212 | ((_ name->int (name id) ...) | |
213 | (define-syntax name->int | |
214 | (syntax-rules (name ...) | |
215 | ((_ name) id) ...))))) | |
216 | ||
217 | (define-enumerate-type operation-id | |
218 | ;; operation numbers from worker-protocol.hh | |
219 | (quit 0) | |
220 | (valid-path? 1) | |
221 | (has-substitutes? 3) | |
222 | (query-path-hash 4) | |
223 | (query-references 5) | |
224 | (query-referrers 6) | |
225 | (add-to-store 7) | |
226 | (add-text-to-store 8) | |
abac874b | 227 | (build-things 9) |
77d3cf08 LC |
228 | (ensure-path 10) |
229 | (add-temp-root 11) | |
230 | (add-indirect-root 12) | |
231 | (sync-with-gc 13) | |
232 | (find-roots 14) | |
233 | (export-path 16) | |
234 | (query-deriver 18) | |
235 | (set-options 19) | |
236 | (collect-garbage 20) | |
63193ebf | 237 | ;;(query-substitutable-path-info 21) ; obsolete as of #x10c |
77d3cf08 | 238 | (query-derivation-outputs 22) |
63193ebf | 239 | (query-all-valid-paths 23) |
77d3cf08 LC |
240 | (query-failed-paths 24) |
241 | (clear-failed-paths 25) | |
242 | (query-path-info 26) | |
243 | (import-paths 27) | |
63193ebf LC |
244 | (query-derivation-output-names 28) |
245 | (query-path-from-hash-part 29) | |
246 | (query-substitutable-path-infos 30) | |
247 | (query-valid-paths 31) | |
fae31edc | 248 | (query-substitutable-paths 32) |
e3fd0ce6 | 249 | (query-valid-derivers 33) |
c63d9403 | 250 | (optimize-store 34) |
f9aefa2d LC |
251 | (verify-store 35) |
252 | (built-in-builders 80)) | |
77d3cf08 LC |
253 | |
254 | (define-enumerate-type hash-algo | |
255 | ;; hash.hh | |
256 | (md5 1) | |
257 | (sha1 2) | |
258 | (sha256 3)) | |
259 | ||
07e70f48 LC |
260 | (define-enumerate-type build-mode |
261 | ;; store-api.hh | |
262 | (normal 0) | |
263 | (repair 1) | |
264 | (check 2)) | |
265 | ||
3259877d LC |
266 | (define-enumerate-type gc-action |
267 | ;; store-api.hh | |
268 | (return-live 0) | |
269 | (return-dead 1) | |
270 | (delete-dead 2) | |
271 | (delete-specific 3)) | |
272 | ||
77d3cf08 | 273 | (define %default-socket-path |
80d0447c | 274 | (string-append %state-directory "/daemon-socket/socket")) |
77d3cf08 | 275 | |
1397b422 LC |
276 | (define %daemon-socket-uri |
277 | ;; URI or file name of the socket the daemon listens too. | |
9fd72fb1 LC |
278 | (make-parameter (or (getenv "GUIX_DAEMON_SOCKET") |
279 | %default-socket-path))) | |
280 | ||
281 | ||
77d3cf08 | 282 | \f |
0f3d2504 LC |
283 | ;; Information about a substitutable store path. |
284 | (define-record-type <substitutable> | |
285 | (substitutable path deriver refs dl-size nar-size) | |
286 | substitutable? | |
287 | (path substitutable-path) | |
288 | (deriver substitutable-deriver) | |
289 | (refs substitutable-references) | |
290 | (dl-size substitutable-download-size) | |
291 | (nar-size substitutable-nar-size)) | |
292 | ||
293 | (define (read-substitutable-path-list p) | |
294 | (let loop ((len (read-int p)) | |
295 | (result '())) | |
296 | (if (zero? len) | |
297 | (reverse result) | |
298 | (let ((path (read-store-path p)) | |
299 | (deriver (read-store-path p)) | |
300 | (refs (read-store-path-list p)) | |
301 | (dl-size (read-long-long p)) | |
302 | (nar-size (read-long-long p))) | |
303 | (loop (- len 1) | |
304 | (cons (substitutable path deriver refs dl-size nar-size) | |
305 | result)))))) | |
306 | ||
533d1768 DT |
307 | ;; Information about a store path. |
308 | (define-record-type <path-info> | |
309 | (path-info deriver hash references registration-time nar-size) | |
310 | path-info? | |
22572d56 | 311 | (deriver path-info-deriver) ;string | #f |
533d1768 DT |
312 | (hash path-info-hash) |
313 | (references path-info-references) | |
314 | (registration-time path-info-registration-time) | |
315 | (nar-size path-info-nar-size)) | |
316 | ||
317 | (define (read-path-info p) | |
22572d56 LC |
318 | (let ((deriver (match (read-store-path p) |
319 | ("" #f) | |
320 | (x x))) | |
533d1768 DT |
321 | (hash (base16-string->bytevector (read-string p))) |
322 | (refs (read-store-path-list p)) | |
323 | (registration-time (read-int p)) | |
324 | (nar-size (read-long-long p))) | |
325 | (path-info deriver hash refs registration-time nar-size))) | |
326 | ||
77d3cf08 | 327 | (define-syntax write-arg |
0d268c5d LC |
328 | (syntax-rules (integer boolean bytevector |
329 | string string-list string-pairs | |
3259877d | 330 | store-path store-path-list base16) |
77d3cf08 LC |
331 | ((_ integer arg p) |
332 | (write-int arg p)) | |
333 | ((_ boolean arg p) | |
334 | (write-int (if arg 1 0) p)) | |
0d268c5d LC |
335 | ((_ bytevector arg p) |
336 | (write-bytevector arg p)) | |
77d3cf08 LC |
337 | ((_ string arg p) |
338 | (write-string arg p)) | |
339 | ((_ string-list arg p) | |
82058eff | 340 | (write-string-list arg p)) |
6c20d1d0 LC |
341 | ((_ string-pairs arg p) |
342 | (write-string-pairs arg p)) | |
3259877d LC |
343 | ((_ store-path arg p) |
344 | (write-store-path arg p)) | |
345 | ((_ store-path-list arg p) | |
346 | (write-store-path-list arg p)) | |
82058eff LC |
347 | ((_ base16 arg p) |
348 | (write-string (bytevector->base16-string arg) p)))) | |
77d3cf08 LC |
349 | |
350 | (define-syntax read-arg | |
7df3ab0f LC |
351 | (syntax-rules (integer boolean string store-path |
352 | store-path-list string-list string-pairs | |
533d1768 | 353 | substitutable-path-list path-info base16) |
77d3cf08 LC |
354 | ((_ integer p) |
355 | (read-int p)) | |
356 | ((_ boolean p) | |
357 | (not (zero? (read-int p)))) | |
358 | ((_ string p) | |
359 | (read-string p)) | |
360 | ((_ store-path p) | |
82058eff | 361 | (read-store-path p)) |
3259877d LC |
362 | ((_ store-path-list p) |
363 | (read-store-path-list p)) | |
f9aefa2d LC |
364 | ((_ string-list p) |
365 | (read-string-list p)) | |
7df3ab0f LC |
366 | ((_ string-pairs p) |
367 | (read-string-pairs p)) | |
0f3d2504 LC |
368 | ((_ substitutable-path-list p) |
369 | (read-substitutable-path-list p)) | |
533d1768 DT |
370 | ((_ path-info p) |
371 | (read-path-info p)) | |
0f3d2504 | 372 | ((_ base16 p) |
82058eff | 373 | (base16-string->bytevector (read-string p))))) |
77d3cf08 LC |
374 | |
375 | \f | |
376 | ;; remote-store.cc | |
377 | ||
de9fbe9c LC |
378 | (define-record-type* <store-connection> store-connection %make-store-connection |
379 | store-connection? | |
380 | (socket store-connection-socket) | |
381 | (major store-connection-major-version) | |
382 | (minor store-connection-minor-version) | |
2c3f47ee | 383 | |
de9fbe9c LC |
384 | (buffer store-connection-output-port) ;output port |
385 | (flush store-connection-flush-output) ;thunk | |
e037e9db | 386 | |
2c3f47ee LC |
387 | ;; Caches. We keep them per-connection, because store paths build |
388 | ;; during the session are temporary GC roots kept for the duration of | |
389 | ;; the session. | |
de9fbe9c LC |
390 | (ats-cache store-connection-add-to-store-cache) |
391 | (atts-cache store-connection-add-text-to-store-cache) | |
d9d7b9ec LC |
392 | (caches store-connection-caches |
393 | (default '#())) ;vector | |
3961edf2 LC |
394 | (built-in-builders store-connection-built-in-builders |
395 | (default (delay '())))) ;promise | |
77d3cf08 | 396 | |
de9fbe9c | 397 | (set-record-type-printer! <store-connection> |
bf8e7fc5 | 398 | (lambda (obj port) |
de9fbe9c LC |
399 | (format port "#<store-connection ~a.~a ~a>" |
400 | (store-connection-major-version obj) | |
401 | (store-connection-minor-version obj) | |
bf8e7fc5 LC |
402 | (number->string (object-address obj) |
403 | 16)))) | |
404 | ||
de9fbe9c LC |
405 | (define-deprecated/alias nix-server? store-connection?) |
406 | (define-deprecated/alias nix-server-major-version | |
407 | store-connection-major-version) | |
408 | (define-deprecated/alias nix-server-minor-version | |
409 | store-connection-minor-version) | |
410 | (define-deprecated/alias nix-server-socket store-connection-socket) | |
411 | ||
412 | ||
f9e8a123 LC |
413 | (define-condition-type &store-error &error |
414 | store-error?) | |
e87088c9 | 415 | |
f9e8a123 LC |
416 | (define-condition-type &store-connection-error &store-error |
417 | store-connection-error? | |
418 | (file store-connection-error-file) | |
419 | (errno store-connection-error-code)) | |
420 | ||
421 | (define-condition-type &store-protocol-error &store-error | |
422 | store-protocol-error? | |
423 | (message store-protocol-error-message) | |
424 | (status store-protocol-error-status)) | |
425 | ||
426 | (define-deprecated/alias &nix-error &store-error) | |
427 | (define-deprecated/alias nix-error? store-error?) | |
428 | (define-deprecated/alias &nix-connection-error &store-connection-error) | |
429 | (define-deprecated/alias nix-connection-error? store-connection-error?) | |
430 | (define-deprecated/alias nix-connection-error-file | |
431 | store-connection-error-file) | |
432 | (define-deprecated/alias nix-connection-error-code | |
433 | store-connection-error-code) | |
434 | (define-deprecated/alias &nix-protocol-error &store-protocol-error) | |
435 | (define-deprecated/alias nix-protocol-error? store-protocol-error?) | |
436 | (define-deprecated/alias nix-protocol-error-message | |
437 | store-protocol-error-message) | |
438 | (define-deprecated/alias nix-protocol-error-status | |
439 | store-protocol-error-status) | |
ef86c39f | 440 | |
e87088c9 | 441 | |
3b5cd17a LC |
442 | (define-syntax-rule (system-error-to-connection-error file exp ...) |
443 | "Catch 'system-error' exceptions and translate them to | |
f9e8a123 | 444 | '&store-connection-error'." |
3b5cd17a LC |
445 | (catch 'system-error |
446 | (lambda () | |
447 | exp ...) | |
448 | (lambda args | |
449 | (let ((errno (system-error-errno args))) | |
f9e8a123 | 450 | (raise (condition (&store-connection-error |
3b5cd17a LC |
451 | (file file) |
452 | (errno errno)))))))) | |
453 | ||
6230d6f0 LC |
454 | (define (open-unix-domain-socket file) |
455 | "Connect to the Unix-domain socket at FILE and return it. Raise a | |
f9e8a123 | 456 | '&store-connection-error' upon error." |
77d3cf08 LC |
457 | (let ((s (with-fluids ((%default-port-encoding #f)) |
458 | ;; This trick allows use of the `scm_c_read' optimization. | |
459 | (socket PF_UNIX SOCK_STREAM 0))) | |
460 | (a (make-socket-address PF_UNIX file))) | |
df1fab58 | 461 | |
3b5cd17a LC |
462 | (system-error-to-connection-error file |
463 | (connect s a) | |
464 | s))) | |
ef86c39f | 465 | |
5df1395a LC |
466 | (define %default-guix-port |
467 | ;; Default port when connecting to a daemon over TCP/IP. | |
468 | 44146) | |
469 | ||
3dff90ce LC |
470 | (define (open-inet-socket host port) |
471 | "Connect to the Unix-domain socket at HOST:PORT and return it. Raise a | |
f9e8a123 | 472 | '&store-connection-error' upon error." |
3dff90ce LC |
473 | (let ((sock (with-fluids ((%default-port-encoding #f)) |
474 | ;; This trick allows use of the `scm_c_read' optimization. | |
475 | (socket PF_UNIX SOCK_STREAM 0)))) | |
476 | (define addresses | |
477 | (getaddrinfo host | |
478 | (if (number? port) (number->string port) port) | |
479 | (if (number? port) | |
480 | (logior AI_ADDRCONFIG AI_NUMERICSERV) | |
7ae97a4c LC |
481 | AI_ADDRCONFIG) |
482 | 0 ;any address family | |
483 | SOCK_STREAM)) ;TCP only | |
3dff90ce LC |
484 | |
485 | (let loop ((addresses addresses)) | |
486 | (match addresses | |
487 | ((ai rest ...) | |
488 | (let ((s (socket (addrinfo:fam ai) | |
489 | ;; TCP/IP only | |
490 | SOCK_STREAM IPPROTO_IP))) | |
491 | ||
492 | (catch 'system-error | |
493 | (lambda () | |
494 | (connect s (addrinfo:addr ai)) | |
950d51c9 LC |
495 | |
496 | ;; Setting this option makes a dramatic difference because it | |
497 | ;; avoids the "ACK delay" on our RPC messages. | |
498 | (setsockopt s IPPROTO_TCP TCP_NODELAY 1) | |
3dff90ce LC |
499 | s) |
500 | (lambda args | |
501 | ;; Connection failed, so try one of the other addresses. | |
502 | (close s) | |
503 | (if (null? rest) | |
f9e8a123 | 504 | (raise (condition (&store-connection-error |
3dff90ce LC |
505 | (file host) |
506 | (errno (system-error-errno args))))) | |
507 | (loop rest)))))))))) | |
508 | ||
1397b422 LC |
509 | (define (connect-to-daemon uri) |
510 | "Connect to the daemon at URI, a string that may be an actual URI or a file | |
dd14678b LC |
511 | name, and return an input/output port. |
512 | ||
513 | This is a low-level procedure that does not perform the initial handshake with | |
514 | the daemon. Use 'open-connection' for that." | |
285f63e8 | 515 | (define (not-supported) |
f9e8a123 | 516 | (raise (condition (&store-connection-error |
285f63e8 LC |
517 | (file uri) |
518 | (errno ENOTSUP))))) | |
519 | ||
1397b422 LC |
520 | (define connect |
521 | (match (string->uri uri) | |
522 | (#f ;URI is a file name | |
523 | open-unix-domain-socket) | |
524 | ((? uri? uri) | |
525 | (match (uri-scheme uri) | |
526 | ((or #f 'file 'unix) | |
527 | (lambda (_) | |
528 | (open-unix-domain-socket (uri-path uri)))) | |
3dff90ce LC |
529 | ('guix |
530 | (lambda (_) | |
5df1395a LC |
531 | (open-inet-socket (uri-host uri) |
532 | (or (uri-port uri) %default-guix-port)))) | |
285f63e8 LC |
533 | ((? symbol? scheme) |
534 | ;; Try to dynamically load a module for SCHEME. | |
535 | ;; XXX: Errors are swallowed. | |
536 | (match (false-if-exception | |
537 | (resolve-interface `(guix store ,scheme))) | |
538 | ((? module? module) | |
539 | (match (false-if-exception | |
540 | (module-ref module 'connect-to-daemon)) | |
541 | ((? procedure? connect) | |
542 | (lambda (_) | |
543 | (connect uri))) | |
544 | (x (not-supported)))) | |
545 | (#f (not-supported)))) | |
1397b422 | 546 | (x |
285f63e8 | 547 | (not-supported)))))) |
1397b422 LC |
548 | |
549 | (connect uri)) | |
550 | ||
551 | (define* (open-connection #:optional (uri (%daemon-socket-uri)) | |
6230d6f0 | 552 | #:key port (reserve-space? #t) cpu-affinity) |
1397b422 LC |
553 | "Connect to the daemon at URI (a string), or, if PORT is not #f, use it as |
554 | the I/O port over which to communicate to a build daemon. | |
6230d6f0 LC |
555 | |
556 | When RESERVE-SPACE? is true, instruct it to reserve a little bit of extra | |
557 | space on the file system so that the garbage collector can still operate, | |
558 | should the disk become full. When CPU-AFFINITY is true, it must be an integer | |
559 | corresponding to an OS-level CPU number to which the daemon's worker process | |
560 | for this connection will be pinned. Return a server object." | |
e3e0886c LC |
561 | (define (handshake-error) |
562 | (raise (condition | |
563 | (&store-connection-error (file (or port uri)) | |
564 | (errno EPROTO)) | |
565 | (&message (message "build daemon handshake failed"))))) | |
566 | ||
13d5e8da LC |
567 | (guard (c ((nar-error? c) |
568 | ;; One of the 'write-' or 'read-' calls below failed, but this is | |
569 | ;; really a connection error. | |
e3e0886c | 570 | (handshake-error))) |
e037e9db LC |
571 | (let*-values (((port) |
572 | (or port (connect-to-daemon uri))) | |
573 | ((output flush) | |
574 | (buffering-output-port port | |
575 | (make-bytevector 8192)))) | |
13d5e8da LC |
576 | (write-int %worker-magic-1 port) |
577 | (let ((r (read-int port))) | |
e3e0886c LC |
578 | (unless (= r %worker-magic-2) |
579 | (handshake-error)) | |
580 | ||
581 | (let ((v (read-int port))) | |
582 | (unless (= (protocol-major %protocol-version) | |
583 | (protocol-major v)) | |
584 | (handshake-error)) | |
585 | ||
586 | (write-int %protocol-version port) | |
587 | (when (>= (protocol-minor v) 14) | |
588 | (write-int (if cpu-affinity 1 0) port) | |
589 | (when cpu-affinity | |
590 | (write-int cpu-affinity port))) | |
591 | (when (>= (protocol-minor v) 11) | |
592 | (write-int (if reserve-space? 1 0) port)) | |
593 | (letrec* ((built-in-builders | |
594 | (delay (%built-in-builders conn))) | |
d9d7b9ec LC |
595 | (caches |
596 | (make-vector | |
597 | (atomic-box-ref %store-connection-caches) | |
598 | vlist-null)) | |
e3e0886c LC |
599 | (conn |
600 | (%make-store-connection port | |
601 | (protocol-major v) | |
602 | (protocol-minor v) | |
603 | output flush | |
604 | (make-hash-table 100) | |
605 | (make-hash-table 100) | |
d9d7b9ec | 606 | caches |
e3e0886c LC |
607 | built-in-builders))) |
608 | (let loop ((done? (process-stderr conn))) | |
609 | (or done? (process-stderr conn))) | |
610 | conn)))))) | |
77d3cf08 | 611 | |
2f608c14 LC |
612 | (define* (port->connection port |
613 | #:key (version %protocol-version)) | |
614 | "Assimilate PORT, an input/output port, and return a connection to the | |
615 | daemon, assuming the given protocol VERSION. | |
616 | ||
617 | Warning: this procedure assumes that the initial handshake with the daemon has | |
618 | already taken place on PORT and that we're just continuing on this established | |
619 | connection. Use with care." | |
620 | (let-values (((output flush) | |
621 | (buffering-output-port port (make-bytevector 8192)))) | |
3961edf2 LC |
622 | (define connection |
623 | (%make-store-connection port | |
624 | (protocol-major version) | |
625 | (protocol-minor version) | |
626 | output flush | |
627 | (make-hash-table 100) | |
628 | (make-hash-table 100) | |
d9d7b9ec LC |
629 | (make-vector |
630 | (atomic-box-ref %store-connection-caches) | |
631 | vlist-null) | |
3961edf2 LC |
632 | (delay (%built-in-builders connection)))) |
633 | ||
634 | connection)) | |
de9fbe9c LC |
635 | |
636 | (define (store-connection-version store) | |
dc0f74e5 | 637 | "Return the protocol version of STORE as an integer." |
de9fbe9c LC |
638 | (protocol-version (store-connection-major-version store) |
639 | (store-connection-minor-version store))) | |
640 | ||
641 | (define-deprecated/alias nix-server-version store-connection-version) | |
dc0f74e5 | 642 | |
e037e9db LC |
643 | (define (write-buffered-output server) |
644 | "Flush SERVER's output port." | |
de9fbe9c LC |
645 | (force-output (store-connection-output-port server)) |
646 | ((store-connection-flush-output server))) | |
e037e9db | 647 | |
3abaf0c4 LC |
648 | (define (close-connection server) |
649 | "Close the connection to SERVER." | |
de9fbe9c | 650 | (close (store-connection-socket server))) |
3abaf0c4 | 651 | |
95207e70 LC |
652 | (define (call-with-store proc) |
653 | "Call PROC with an open store connection." | |
ce4a4829 | 654 | (let ((store (open-connection))) |
376ba0ce LC |
655 | (define (thunk) |
656 | (parameterize ((current-store-protocol-version | |
657 | (store-connection-version store))) | |
3d9ea605 LC |
658 | (call-with-values (lambda () (proc store)) |
659 | (lambda results | |
660 | (close-connection store) | |
661 | (apply values results))))) | |
376ba0ce | 662 | |
82d8ab01 LC |
663 | (with-exception-handler (lambda (exception) |
664 | (close-connection store) | |
665 | (raise-exception exception)) | |
666 | thunk))) | |
ce4a4829 | 667 | |
95207e70 LC |
668 | (define-syntax-rule (with-store store exp ...) |
669 | "Bind STORE to an open connection to the store and evaluate EXPs; | |
670 | automatically close the store when the dynamic extent of EXP is left." | |
671 | (call-with-store (lambda (store) exp ...))) | |
672 | ||
dc0f74e5 LC |
673 | (define current-store-protocol-version |
674 | ;; Protocol version of the store currently used. XXX: This is a hack to | |
675 | ;; communicate the protocol version to the build output port. It's a hack | |
676 | ;; because it could be inaccurrate, for instance if there's code that | |
677 | ;; manipulates several store connections at once; it works well for the | |
678 | ;; purposes of (guix status) though. | |
679 | (make-parameter #f)) | |
680 | ||
dcee50c1 LC |
681 | (define current-build-output-port |
682 | ;; The port where build output is sent. | |
683 | (make-parameter (current-error-port))) | |
684 | ||
526382ff LC |
685 | (define* (dump-port in out |
686 | #:optional len | |
687 | #:key (buffer-size 16384)) | |
688 | "Read LEN bytes from IN (or as much as possible if LEN is #f) and write it | |
689 | to OUT, using chunks of BUFFER-SIZE bytes." | |
690 | (define buffer | |
691 | (make-bytevector buffer-size)) | |
692 | ||
693 | (let loop ((total 0) | |
694 | (bytes (get-bytevector-n! in buffer 0 | |
695 | (if len | |
696 | (min len buffer-size) | |
697 | buffer-size)))) | |
698 | (or (eof-object? bytes) | |
699 | (and len (= total len)) | |
700 | (let ((total (+ total bytes))) | |
701 | (put-bytevector out buffer 0 bytes) | |
702 | (loop total | |
703 | (get-bytevector-n! in buffer 0 | |
704 | (if len | |
705 | (min (- len total) buffer-size) | |
706 | buffer-size))))))) | |
707 | ||
d28869af LC |
708 | (define %newlines |
709 | ;; Newline characters triggering a flush of 'current-build-output-port'. | |
76832d34 | 710 | ;; Unlike Guile's 'line, we flush upon #\return so that progress reports |
d28869af LC |
711 | ;; that use that trick are correctly displayed. |
712 | (char-set #\newline #\return)) | |
713 | ||
526382ff | 714 | (define* (process-stderr server #:optional user-port) |
dcee50c1 LC |
715 | "Read standard output and standard error from SERVER, writing it to |
716 | CURRENT-BUILD-OUTPUT-PORT. Return #t when SERVER is done sending data, and | |
717 | #f otherwise; in the latter case, the caller should call `process-stderr' | |
bbdb3ffa LC |
718 | again until #t is returned or an error is raised. |
719 | ||
720 | Since the build process's output cannot be assumed to be UTF-8, we | |
721 | conservatively consider it to be Latin-1, thereby avoiding possible | |
722 | encoding conversion errors." | |
77d3cf08 | 723 | (define p |
de9fbe9c | 724 | (store-connection-socket server)) |
77d3cf08 LC |
725 | |
726 | ;; magic cookies from worker-protocol.hh | |
5674a3fd LC |
727 | (define %stderr-next #x6f6c6d67) ; "olmg", build log |
728 | (define %stderr-read #x64617461) ; "data", data needed from source | |
729 | (define %stderr-write #x64617416) ; "dat\x16", data for sink | |
730 | (define %stderr-last #x616c7473) ; "alts", we're done | |
731 | (define %stderr-error #x63787470) ; "cxtp", error reporting | |
77d3cf08 LC |
732 | |
733 | (let ((k (read-int p))) | |
734 | (cond ((= k %stderr-write) | |
526382ff LC |
735 | ;; Write a byte stream to USER-PORT. |
736 | (let* ((len (read-int p)) | |
737 | (m (modulo len 8))) | |
6374633b LC |
738 | (dump-port p user-port len |
739 | #:buffer-size (if (<= len 16384) 16384 65536)) | |
526382ff LC |
740 | (unless (zero? m) |
741 | ;; Consume padding, as for strings. | |
742 | (get-bytevector-n p (- 8 m)))) | |
dcee50c1 | 743 | #f) |
77d3cf08 | 744 | ((= k %stderr-read) |
526382ff | 745 | ;; Read a byte stream from USER-PORT. |
5895f244 LC |
746 | ;; Note: Avoid 'get-bytevector-n' to work around |
747 | ;; <http://bugs.gnu.org/17591> in Guile up to 2.0.11. | |
526382ff | 748 | (let* ((max-len (read-int p)) |
5895f244 LC |
749 | (data (make-bytevector max-len)) |
750 | (len (get-bytevector-n! user-port data 0 max-len))) | |
39d1e965 | 751 | (write-bytevector data p len) |
dcee50c1 | 752 | #f)) |
77d3cf08 | 753 | ((= k %stderr-next) |
ce72c780 LC |
754 | ;; Log a string. Build logs are usually UTF-8-encoded, but they |
755 | ;; may also contain arbitrary byte sequences that should not cause | |
756 | ;; this to fail. Thus, use the permissive | |
757 | ;; 'read-maybe-utf8-string'. | |
758 | (let ((s (read-maybe-utf8-string p))) | |
dcee50c1 | 759 | (display s (current-build-output-port)) |
d28869af | 760 | (when (string-any %newlines s) |
2535635f | 761 | (force-output (current-build-output-port))) |
dcee50c1 | 762 | #f)) |
77d3cf08 | 763 | ((= k %stderr-error) |
526382ff | 764 | ;; Report an error. |
ce72c780 | 765 | (let ((error (read-maybe-utf8-string p)) |
0ff3e3aa LC |
766 | ;; Currently the daemon fails to send a status code for early |
767 | ;; errors like DB schema version mismatches, so check for EOF. | |
de9fbe9c | 768 | (status (if (and (>= (store-connection-minor-version server) 8) |
0ff3e3aa | 769 | (not (eof-object? (lookahead-u8 p)))) |
77d3cf08 LC |
770 | (read-int p) |
771 | 1))) | |
f9e8a123 | 772 | (raise (condition (&store-protocol-error |
e87088c9 LC |
773 | (message error) |
774 | (status status)))))) | |
77d3cf08 | 775 | ((= k %stderr-last) |
dcee50c1 | 776 | ;; The daemon is done (see `stopWork' in `nix-worker.cc'.) |
77d3cf08 LC |
777 | #t) |
778 | (else | |
f9e8a123 | 779 | (raise (condition (&store-protocol-error |
e87088c9 LC |
780 | (message "invalid error code") |
781 | (status k)))))))) | |
77d3cf08 | 782 | |
4d581220 | 783 | (define %default-substitute-urls |
d70533cb LC |
784 | ;; Default list of substituters. This is *not* the list baked in |
785 | ;; 'guix-daemon', but it is used by 'guix-service-type' and and a couple of | |
786 | ;; clients ('guix build --log-file' uses it.) | |
df061d07 LC |
787 | (map (if (false-if-exception (resolve-interface '(gnutls))) |
788 | (cut string-append "https://" <>) | |
789 | (cut string-append "http://" <>)) | |
757e633d | 790 | '("ci.guix.gnu.org"))) |
4d581220 | 791 | |
81c580c8 LC |
792 | (define (current-user-name) |
793 | "Return the name of the calling user." | |
794 | (catch #t | |
795 | (lambda () | |
796 | (passwd:name (getpwuid (getuid)))) | |
797 | (lambda _ | |
798 | (getenv "USER")))) | |
799 | ||
77d3cf08 | 800 | (define* (set-build-options server |
c3eb878f | 801 | #:key keep-failed? keep-going? fallback? |
77d3cf08 | 802 | (verbosity 0) |
2fba87ac | 803 | rounds ;number of build rounds |
deac976d | 804 | max-build-jobs |
6c20d1d0 | 805 | timeout |
deac976d | 806 | max-silent-time |
7f44ab48 LC |
807 | (offload? #t) |
808 | (use-build-hook? *unspecified*) ;deprecated | |
77d3cf08 LC |
809 | (build-verbosity 0) |
810 | (log-type 0) | |
e036c31b | 811 | (print-build-trace #t) |
81c580c8 | 812 | (user-name (current-user-name)) |
dc0f74e5 LC |
813 | |
814 | ;; When true, provide machine-readable "build | |
815 | ;; traces" for use by (guix status). Old clients | |
816 | ;; are unable to make sense, which is why it's | |
817 | ;; disabled by default. | |
818 | print-extended-build-trace? | |
819 | ||
6ef61cc4 LC |
820 | ;; When true, the daemon prefixes builder output |
821 | ;; with "@ build-log" traces so we can | |
822 | ;; distinguish it from daemon output, and we can | |
823 | ;; distinguish each builder's output | |
824 | ;; (PRINT-BUILD-TRACE must be true as well.) The | |
825 | ;; latter is particularly useful when | |
826 | ;; MAX-BUILD-JOBS > 1. | |
827 | multiplexed-build-output? | |
828 | ||
deac976d | 829 | build-cores |
63193ebf | 830 | (use-substitutes? #t) |
4d581220 | 831 | |
fb4bf72b LC |
832 | ;; Client-provided substitute URLs. If it is #f, |
833 | ;; the daemon's settings are used. Otherwise, it | |
834 | ;; overrides the daemons settings; see 'guix | |
835 | ;; substitute'. | |
b0a6a971 LC |
836 | (substitute-urls #f) |
837 | ||
838 | ;; Number of columns in the client's terminal. | |
38f50f49 LC |
839 | (terminal-columns (terminal-columns)) |
840 | ||
841 | ;; Locale of the client. | |
739f6309 | 842 | (locale (false-if-exception (setlocale LC_MESSAGES)))) |
77d3cf08 LC |
843 | ;; Must be called after `open-connection'. |
844 | ||
7a45b5d5 LDB |
845 | (define buffered |
846 | (store-connection-output-port server)) | |
77d3cf08 | 847 | |
7f44ab48 LC |
848 | (unless (unspecified? use-build-hook?) |
849 | (warn-about-deprecation #:use-build-hook? #f | |
850 | #:replacement #:offload?)) | |
851 | ||
77d3cf08 | 852 | (let-syntax ((send (syntax-rules () |
e036c31b LC |
853 | ((_ (type option) ...) |
854 | (begin | |
7a45b5d5 | 855 | (write-arg type option buffered) |
e036c31b | 856 | ...))))) |
7a45b5d5 | 857 | (write-int (operation-id set-options) buffered) |
e036c31b | 858 | (send (boolean keep-failed?) (boolean keep-going?) |
deac976d | 859 | (boolean fallback?) (integer verbosity)) |
de9fbe9c | 860 | (when (< (store-connection-minor-version server) #x61) |
deac976d LC |
861 | (let ((max-build-jobs (or max-build-jobs 1)) |
862 | (max-silent-time (or max-silent-time 3600))) | |
863 | (send (integer max-build-jobs) (integer max-silent-time)))) | |
de9fbe9c | 864 | (when (>= (store-connection-minor-version server) 2) |
7f44ab48 LC |
865 | (send (boolean (if (unspecified? use-build-hook?) |
866 | offload? | |
867 | use-build-hook?)))) | |
de9fbe9c | 868 | (when (>= (store-connection-minor-version server) 4) |
371e87d2 LC |
869 | (send (integer build-verbosity) (integer log-type) |
870 | (boolean print-build-trace))) | |
de9fbe9c LC |
871 | (when (and (>= (store-connection-minor-version server) 6) |
872 | (< (store-connection-minor-version server) #x61)) | |
deac976d LC |
873 | (let ((build-cores (or build-cores (current-processor-count)))) |
874 | (send (integer build-cores)))) | |
de9fbe9c | 875 | (when (>= (store-connection-minor-version server) 10) |
371e87d2 | 876 | (send (boolean use-substitutes?))) |
de9fbe9c | 877 | (when (>= (store-connection-minor-version server) 12) |
dc0f74e5 LC |
878 | (let ((pairs `(;; This option is honored by 'guix substitute' et al. |
879 | ,@(if print-build-trace | |
880 | `(("print-extended-build-trace" | |
881 | . ,(if print-extended-build-trace? "1" "0"))) | |
882 | '()) | |
6ef61cc4 LC |
883 | ,@(if multiplexed-build-output? |
884 | `(("multiplexed-build-output" | |
885 | . ,(if multiplexed-build-output? "true" "false"))) | |
886 | '()) | |
dc0f74e5 | 887 | ,@(if timeout |
41c45e78 LC |
888 | `(("build-timeout" . ,(number->string timeout))) |
889 | '()) | |
deac976d LC |
890 | ,@(if max-silent-time |
891 | `(("build-max-silent-time" | |
892 | . ,(number->string max-silent-time))) | |
893 | '()) | |
894 | ,@(if max-build-jobs | |
895 | `(("build-max-jobs" | |
896 | . ,(number->string max-build-jobs))) | |
897 | '()) | |
898 | ,@(if build-cores | |
899 | `(("build-cores" . ,(number->string build-cores))) | |
900 | '()) | |
fb4bf72b LC |
901 | ,@(if substitute-urls |
902 | `(("substitute-urls" | |
903 | . ,(string-join substitute-urls))) | |
2fba87ac LC |
904 | '()) |
905 | ,@(if rounds | |
906 | `(("build-repeat" | |
907 | . ,(number->string (max 0 (1- rounds))))) | |
b0a6a971 | 908 | '()) |
81c580c8 LC |
909 | ,@(if user-name |
910 | `(("user-name" . ,user-name)) | |
911 | '()) | |
b0a6a971 LC |
912 | ,@(if terminal-columns |
913 | `(("terminal-columns" | |
914 | . ,(number->string terminal-columns))) | |
38f50f49 LC |
915 | '()) |
916 | ,@(if locale | |
917 | `(("locale" . ,locale)) | |
fb4bf72b | 918 | '())))) |
f401b1e9 | 919 | (send (string-pairs pairs)))) |
7a45b5d5 | 920 | (write-buffered-output server) |
dcee50c1 LC |
921 | (let loop ((done? (process-stderr server))) |
922 | (or done? (process-stderr server))))) | |
77d3cf08 | 923 | |
e037e9db LC |
924 | (define (buffering-output-port port buffer) |
925 | "Return two value: an output port wrapped around PORT that uses BUFFER (a | |
926 | bytevector) as its internal buffer, and a thunk to flush this output port." | |
927 | ;; Note: In Guile 2.2.2, custom binary output ports already have their own | |
928 | ;; 4K internal buffer. | |
929 | (define size | |
930 | (bytevector-length buffer)) | |
931 | ||
932 | (define total 0) | |
933 | ||
934 | (define (flush) | |
935 | (put-bytevector port buffer 0 total) | |
38098241 | 936 | (force-output port) |
e037e9db LC |
937 | (set! total 0)) |
938 | ||
939 | (define (write bv offset count) | |
940 | (if (zero? count) ;end of file | |
941 | (flush) | |
942 | (let loop ((offset offset) | |
943 | (count count) | |
944 | (written 0)) | |
945 | (cond ((= total size) | |
946 | (flush) | |
947 | (loop offset count written)) | |
948 | ((zero? count) | |
949 | written) | |
950 | (else | |
951 | (let ((to-copy (min count (- size total)))) | |
952 | (bytevector-copy! bv offset buffer total to-copy) | |
953 | (set! total (+ total to-copy)) | |
954 | (loop (+ offset to-copy) (- count to-copy) | |
955 | (+ written to-copy)))))))) | |
956 | ||
957 | ;; Note: We need to return FLUSH because the custom binary port has no way | |
958 | ;; to be notified of a 'force-output' call on itself. | |
959 | (values (make-custom-binary-output-port "buffering-output-port" | |
960 | write #f #f flush) | |
961 | flush)) | |
962 | ||
73b0ebdd LC |
963 | (define profiled? |
964 | (let ((profiled | |
965 | (or (and=> (getenv "GUIX_PROFILING") string-tokenize) | |
966 | '()))) | |
967 | (lambda (component) | |
968 | "Return true if COMPONENT profiling is active." | |
969 | (member component profiled)))) | |
970 | ||
f4453df9 LC |
971 | (define %rpc-calls |
972 | ;; Mapping from RPC names (symbols) to invocation counts. | |
973 | (make-hash-table)) | |
974 | ||
975 | (define* (show-rpc-profile #:optional (port (current-error-port))) | |
976 | "Write to PORT a summary of the RPCs that have been made." | |
977 | (let ((profile (sort (hash-fold alist-cons '() %rpc-calls) | |
978 | (lambda (rpc1 rpc2) | |
979 | (< (cdr rpc1) (cdr rpc2)))))) | |
980 | (format port "Remote procedure call summary: ~a RPCs~%" | |
981 | (match profile | |
982 | (((names . counts) ...) | |
983 | (reduce + 0 counts)))) | |
984 | (for-each (match-lambda | |
985 | ((rpc . count) | |
986 | (format port " ~30a ... ~5@a~%" rpc count))) | |
987 | profile))) | |
988 | ||
989 | (define record-operation | |
990 | ;; Optionally, increment the number of calls of the given RPC. | |
03870da8 LC |
991 | (if (profiled? "rpc") |
992 | (begin | |
993 | (register-profiling-hook! "rpc" show-rpc-profile) | |
994 | (lambda (name) | |
995 | (let ((count (or (hashq-ref %rpc-calls name) 0))) | |
996 | (hashq-set! %rpc-calls name (+ count 1))))) | |
997 | (lambda (_) | |
998 | #t))) | |
f4453df9 | 999 | |
fd060fd3 | 1000 | (define-syntax operation |
77d3cf08 | 1001 | (syntax-rules () |
fd060fd3 | 1002 | "Define a client-side RPC stub for the given operation." |
3259877d | 1003 | ((_ (name (type arg) ...) docstring return ...) |
fd060fd3 | 1004 | (lambda (server arg ...) |
77d3cf08 | 1005 | docstring |
de9fbe9c LC |
1006 | (let* ((s (store-connection-socket server)) |
1007 | (buffered (store-connection-output-port server))) | |
f4453df9 | 1008 | (record-operation 'name) |
e037e9db LC |
1009 | (write-int (operation-id name) buffered) |
1010 | (write-arg type arg buffered) | |
77d3cf08 | 1011 | ... |
e037e9db LC |
1012 | (write-buffered-output server) |
1013 | ||
dcee50c1 LC |
1014 | ;; Loop until the server is done sending error output. |
1015 | (let loop ((done? (process-stderr server))) | |
1016 | (or done? (loop (process-stderr server)))) | |
3259877d | 1017 | (values (read-arg return s) ...)))))) |
77d3cf08 | 1018 | |
fd060fd3 LC |
1019 | (define-syntax-rule (define-operation (name args ...) |
1020 | docstring return ...) | |
1021 | (define name | |
1022 | (operation (name args ...) docstring return ...))) | |
1023 | ||
31ef99a8 | 1024 | (define-operation (valid-path? (string path)) |
06b76acc LC |
1025 | "Return #t when PATH designates a valid store item and #f otherwise (an |
1026 | invalid item may exist on disk but still be invalid, for instance because it | |
1027 | is the result of an aborted or failed build.) | |
1028 | ||
f9e8a123 | 1029 | A '&store-protocol-error' condition is raised if PATH is not prefixed by the |
06b76acc | 1030 | store directory (/gnu/store)." |
31ef99a8 LC |
1031 | boolean) |
1032 | ||
63193ebf | 1033 | (define-operation (query-path-hash (store-path path)) |
aa8fff0c | 1034 | "Return the SHA256 hash of the nar serialization of PATH as a bytevector." |
82058eff LC |
1035 | base16) |
1036 | ||
11e7a6cf LC |
1037 | (define hash-part->path |
1038 | (let ((query-path-from-hash-part | |
1039 | (operation (query-path-from-hash-part (string hash)) | |
1040 | #f | |
1041 | store-path))) | |
1042 | (lambda (server hash-part) | |
1043 | "Return the store path whose hash part is HASH-PART (a nix-base32 | |
7fcb86da | 1044 | string). Return the empty string if no such path exists." |
11e7a6cf LC |
1045 | ;; This RPC is primarily used by Hydra to reply to HTTP GETs of |
1046 | ;; /HASH.narinfo. | |
1047 | (query-path-from-hash-part server hash-part)))) | |
1048 | ||
533d1768 DT |
1049 | (define-operation (query-path-info (store-path path)) |
1050 | "Return the info (hash, references, etc.) for PATH." | |
1051 | path-info) | |
1052 | ||
0d268c5d | 1053 | (define add-data-to-store |
fd060fd3 LC |
1054 | ;; A memoizing version of `add-to-store', to avoid repeated RPCs with |
1055 | ;; the very same arguments during a given session. | |
1056 | (let ((add-text-to-store | |
0d268c5d | 1057 | (operation (add-text-to-store (string name) (bytevector text) |
fd060fd3 LC |
1058 | (string-list references)) |
1059 | #f | |
d1f7748a LC |
1060 | store-path)) |
1061 | (lookup (if (profiled? "add-data-to-store-cache") | |
1062 | (let ((lookups 0) | |
1063 | (hits 0) | |
1064 | (drv 0) | |
1065 | (scheme 0)) | |
1066 | (define (show-stats) | |
1067 | (define (% n) | |
1068 | (if (zero? lookups) | |
1069 | 100. | |
1070 | (* 100. (/ n lookups)))) | |
1071 | ||
1072 | (format (current-error-port) " | |
1073 | 'add-data-to-store' cache: | |
1074 | lookups: ~5@a | |
1075 | hits: ~5@a (~,1f%) | |
1076 | .drv files: ~5@a (~,1f%) | |
1077 | Scheme files: ~5@a (~,1f%)~%" | |
1078 | lookups hits (% hits) | |
1079 | drv (% drv) | |
1080 | scheme (% scheme))) | |
1081 | ||
1082 | (register-profiling-hook! "add-data-to-store-cache" | |
1083 | show-stats) | |
1084 | (lambda (cache args) | |
1085 | (let ((result (hash-ref cache args))) | |
1086 | (set! lookups (+ 1 lookups)) | |
1087 | (when result | |
1088 | (set! hits (+ 1 hits))) | |
1089 | (match args | |
1090 | ((_ name _) | |
1091 | (cond ((string-suffix? ".drv" name) | |
1092 | (set! drv (+ drv 1))) | |
1093 | ((string-suffix? "-builder" name) | |
1094 | (set! scheme (+ scheme 1))) | |
1095 | ((string-suffix? ".scm" name) | |
1096 | (set! scheme (+ scheme 1)))))) | |
1097 | result))) | |
1098 | hash-ref))) | |
0d268c5d LC |
1099 | (lambda* (server name bytes #:optional (references '())) |
1100 | "Add BYTES under file NAME in the store, and return its store path. | |
bdcf35a6 LC |
1101 | REFERENCES is the list of store paths referred to by the resulting store |
1102 | path." | |
0d268c5d | 1103 | (let* ((args `(,bytes ,name ,references)) |
de9fbe9c | 1104 | (cache (store-connection-add-text-to-store-cache server))) |
d1f7748a | 1105 | (or (lookup cache args) |
0d268c5d | 1106 | (let ((path (add-text-to-store server name bytes references))) |
bdcf35a6 LC |
1107 | (hash-set! cache args path) |
1108 | path)))))) | |
1109 | ||
0d268c5d LC |
1110 | (define* (add-text-to-store store name text #:optional (references '())) |
1111 | "Add TEXT under file NAME in the store, and return its store path. | |
1112 | REFERENCES is the list of store paths referred to by the resulting store | |
1113 | path." | |
1114 | (add-data-to-store store name (string->utf8 text) references)) | |
1115 | ||
1ec32f4a LC |
1116 | (define true |
1117 | ;; Define it once and for all since we use it as a default value for | |
1118 | ;; 'add-to-store' and want to make sure two default values are 'eq?' for the | |
1119 | ;; purposes or memoization. | |
1120 | (lambda (file stat) | |
1121 | #t)) | |
1122 | ||
fd060fd3 | 1123 | (define add-to-store |
a7b6ffee LC |
1124 | ;; A memoizing version of `add-to-store'. This is important because |
1125 | ;; `add-to-store' leads to huge data transfers to the server, and | |
1126 | ;; because it's often called many times with the very same argument. | |
1ec32f4a LC |
1127 | (let ((add-to-store |
1128 | (lambda* (server basename recursive? hash-algo file-name | |
1129 | #:key (select? true)) | |
1130 | ;; We don't use the 'operation' macro so we can pass SELECT? to | |
1131 | ;; 'write-file'. | |
0ca3d556 | 1132 | (record-operation 'add-to-store) |
7a45b5d5 LDB |
1133 | (let ((port (store-connection-socket server)) |
1134 | (buffered (store-connection-output-port server))) | |
1135 | (write-int (operation-id add-to-store) buffered) | |
1136 | (write-string basename buffered) | |
1137 | (write-int 1 buffered) ;obsolete, must be #t | |
1138 | (write-int (if recursive? 1 0) buffered) | |
1139 | (write-string hash-algo buffered) | |
1140 | (write-file file-name buffered #:select? select?) | |
38098241 | 1141 | (write-buffered-output server) |
1ec32f4a LC |
1142 | (let loop ((done? (process-stderr server))) |
1143 | (or done? (loop (process-stderr server)))) | |
1144 | (read-store-path port))))) | |
1145 | (lambda* (server basename recursive? hash-algo file-name | |
1146 | #:key (select? true)) | |
a9ebd9ef | 1147 | "Add the contents of FILE-NAME under BASENAME to the store. When |
69792b28 LC |
1148 | RECURSIVE? is false, FILE-NAME must designate a regular file--not a directory |
1149 | nor a symlink. When RECURSIVE? is true and FILE-NAME designates a directory, | |
1150 | the contents of FILE-NAME are added recursively; if FILE-NAME designates a | |
1151 | flat file and RECURSIVE? is true, its contents are added, and its permission | |
1ec32f4a LC |
1152 | bits are kept. HASH-ALGO must be a string such as \"sha256\". |
1153 | ||
1154 | When RECURSIVE? is true, call (SELECT? FILE STAT) for each directory entry, | |
1155 | where FILE is the entry's absolute file name and STAT is the result of | |
1156 | 'lstat'; exclude entries for which SELECT? does not return true." | |
b4671215 LC |
1157 | ;; Note: We don't stat FILE-NAME at each call, and thus we assume that |
1158 | ;; the file remains unchanged for the lifetime of SERVER. | |
1159 | (let* ((args `(,file-name ,basename ,recursive? ,hash-algo ,select?)) | |
de9fbe9c | 1160 | (cache (store-connection-add-to-store-cache server))) |
b4671215 | 1161 | (or (hash-ref cache args) |
1ec32f4a LC |
1162 | (let ((path (add-to-store server basename recursive? |
1163 | hash-algo file-name | |
1164 | #:select? select?))) | |
a7b6ffee LC |
1165 | (hash-set! cache args path) |
1166 | path)))))) | |
1167 | ||
7f11efba LC |
1168 | (define %not-slash |
1169 | (char-set-complement (char-set #\/))) | |
1170 | ||
1171 | (define* (add-file-tree-to-store server tree | |
1172 | #:key | |
1173 | (hash-algo "sha256") | |
1174 | (recursive? #t)) | |
1175 | "Add the given TREE to the store on SERVER. TREE must be an entry such as: | |
1176 | ||
1177 | (\"my-tree\" directory | |
1178 | (\"a\" regular (data \"hello\")) | |
1179 | (\"b\" symlink \"a\") | |
1180 | (\"c\" directory | |
1181 | (\"d\" executable (file \"/bin/sh\")))) | |
1182 | ||
1183 | This is a generalized version of 'add-to-store'. It allows you to reproduce | |
1184 | an arbitrary directory layout in the store without creating a derivation." | |
1185 | ||
1186 | ;; Note: The format of TREE was chosen to allow trees to be compared with | |
1187 | ;; 'equal?', which in turn allows us to memoize things. | |
1188 | ||
1189 | (define root | |
1190 | ;; TREE is a single entry. | |
1191 | (list tree)) | |
1192 | ||
1193 | (define basename | |
1194 | (match tree | |
1195 | ((name . _) name))) | |
1196 | ||
1197 | (define (lookup file) | |
1198 | (let loop ((components (string-tokenize file %not-slash)) | |
1199 | (tree root)) | |
1200 | (match components | |
1201 | ((basename) | |
1202 | (assoc basename tree)) | |
1203 | ((head . rest) | |
1204 | (loop rest | |
1205 | (match (assoc-ref tree head) | |
1206 | (('directory . entries) entries))))))) | |
1207 | ||
1208 | (define (file-type+size file) | |
1209 | (match (lookup file) | |
1210 | ((_ (and type (or 'directory 'symlink)) . _) | |
1211 | (values type 0)) | |
1212 | ((_ type ('file file)) | |
1213 | (values type (stat:size (stat file)))) | |
1214 | ((_ type ('data (? string? data))) | |
1215 | (values type (string-length data))) | |
1216 | ((_ type ('data (? bytevector? data))) | |
1217 | (values type (bytevector-length data))))) | |
1218 | ||
1219 | (define (file-port file) | |
1220 | (match (lookup file) | |
1221 | ((_ (or 'regular 'executable) content) | |
1222 | (match content | |
1223 | (('file (? string? file)) | |
1224 | (open-file file "r0b")) | |
1225 | (('data (? string? str)) | |
1226 | (open-input-string str)) | |
1227 | (('data (? bytevector? bv)) | |
1228 | (open-bytevector-input-port bv)))))) | |
1229 | ||
1230 | (define (symlink-target file) | |
1231 | (match (lookup file) | |
1232 | ((_ 'symlink target) target))) | |
1233 | ||
1234 | (define (directory-entries directory) | |
1235 | (match (lookup directory) | |
1236 | ((_ 'directory (names . _) ...) names))) | |
1237 | ||
1238 | (define cache | |
de9fbe9c | 1239 | (store-connection-add-to-store-cache server)) |
7f11efba LC |
1240 | |
1241 | (or (hash-ref cache tree) | |
1242 | (begin | |
1243 | ;; We don't use the 'operation' macro so we can use 'write-file-tree' | |
1244 | ;; instead of 'write-file'. | |
1245 | (record-operation 'add-to-store/tree) | |
7a45b5d5 LDB |
1246 | (let ((port (store-connection-socket server)) |
1247 | (buffered (store-connection-output-port server))) | |
1248 | (write-int (operation-id add-to-store) buffered) | |
1249 | (write-string basename buffered) | |
1250 | (write-int 1 buffered) ;obsolete, must be #t | |
1251 | (write-int (if recursive? 1 0) buffered) | |
1252 | (write-string hash-algo buffered) | |
1253 | (write-file-tree basename buffered | |
7f11efba LC |
1254 | #:file-type+size file-type+size |
1255 | #:file-port file-port | |
1256 | #:symlink-target symlink-target | |
1257 | #:directory-entries directory-entries) | |
000bbe02 | 1258 | (write-buffered-output server) |
7f11efba LC |
1259 | (let loop ((done? (process-stderr server))) |
1260 | (or done? (loop (process-stderr server)))) | |
1261 | (let ((result (read-store-path port))) | |
1262 | (hash-set! cache tree result) | |
1263 | result))))) | |
1264 | ||
68dbd5c9 LC |
1265 | (define (file-mapping->tree mapping) |
1266 | "Convert MAPPING, an alist like: | |
1267 | ||
1268 | ((\"guix/build/utils.scm\" . \"…/utils.scm\")) | |
1269 | ||
1270 | to a tree suitable for 'add-file-tree-to-store' and 'interned-file-tree'." | |
1271 | (let ((mapping (map (match-lambda | |
1272 | ((destination . source) | |
1273 | (cons (string-tokenize destination %not-slash) | |
1274 | source))) | |
1275 | mapping))) | |
1276 | (fold (lambda (pair result) | |
1277 | (match pair | |
1278 | ((destination . source) | |
1279 | (let loop ((destination destination) | |
1280 | (result result)) | |
1281 | (match destination | |
1282 | ((file) | |
1283 | (let* ((mode (stat:mode (stat source))) | |
1284 | (type (if (zero? (logand mode #o100)) | |
1285 | 'regular | |
1286 | 'executable))) | |
1287 | (alist-cons file | |
1288 | `(,type (file ,source)) | |
1289 | result))) | |
1290 | ((file rest ...) | |
1291 | (let ((directory (assoc-ref result file))) | |
1292 | (alist-cons file | |
1293 | `(directory | |
1294 | ,@(loop rest | |
1295 | (match directory | |
1296 | (('directory . entries) entries) | |
1297 | (#f '())))) | |
1298 | (if directory | |
1299 | (alist-delete file result) | |
1300 | result))))))))) | |
1301 | '() | |
1302 | mapping))) | |
1303 | ||
041b340d LC |
1304 | (define current-build-prompt |
1305 | ;; When true, this is the prompt to abort to when 'build-things' is called. | |
1306 | (make-parameter #f)) | |
1307 | ||
1308 | (define (call-with-build-handler handler thunk) | |
1309 | "Register HANDLER as a \"build handler\" and invoke THUNK." | |
1310 | (define tag | |
1311 | (make-prompt-tag "build handler")) | |
1312 | ||
1313 | (parameterize ((current-build-prompt tag)) | |
1314 | (call-with-prompt tag | |
1315 | thunk | |
1316 | (lambda (k . args) | |
1317 | ;; Since HANDLER may call K, which in turn may call 'build-things' | |
1318 | ;; again, reinstate a prompt (thus, it's not a tail call.) | |
1319 | (call-with-build-handler handler | |
1320 | (lambda () | |
1321 | (apply handler k args))))))) | |
1322 | ||
1323 | (define (invoke-build-handler store things mode) | |
1324 | "Abort to 'current-build-prompt' if it is set." | |
1325 | (or (not (current-build-prompt)) | |
1326 | (abort-to-prompt (current-build-prompt) store things mode))) | |
1327 | ||
1328 | (define-syntax-rule (with-build-handler handler exp ...) | |
1329 | "Register HANDLER as a \"build handler\" and invoke THUNK. When | |
1330 | 'build-things' is called within the dynamic extent of the call to THUNK, | |
1331 | HANDLER is invoked like so: | |
1332 | ||
1333 | (HANDLER CONTINUE STORE THINGS MODE) | |
1334 | ||
1335 | where CONTINUE is the continuation, and the remaining arguments are those that | |
1336 | were passed to 'build-things'. | |
1337 | ||
1338 | Build handlers are useful to announce a build plan with 'show-what-to-build' | |
1339 | and to implement dry runs (by not invoking CONTINUE) in a way that gracefully | |
1340 | deals with \"dynamic dependencies\" such as grafts---derivations that depend | |
1341 | on the build output of a previous derivation." | |
1342 | (call-with-build-handler handler (lambda () exp ...))) | |
1343 | ||
c40bf581 LC |
1344 | ;; Unresolved dynamic dependency. |
1345 | (define-record-type <unresolved> | |
1346 | (unresolved things continuation) | |
1347 | unresolved? | |
1348 | (things unresolved-things) | |
1349 | (continuation unresolved-continuation)) | |
1350 | ||
1351 | (define (build-accumulator continue store things mode) | |
1352 | "This build handler accumulates THINGS and returns an <unresolved> object." | |
1353 | (if (= mode (build-mode normal)) | |
1354 | (unresolved things continue) | |
1355 | (continue #t))) | |
1356 | ||
1357 | (define (map/accumulate-builds store proc lst) | |
1358 | "Apply PROC over each element of LST, accumulating 'build-things' calls and | |
1359 | coalescing them into a single call." | |
1360 | (define result | |
1361 | (map (lambda (obj) | |
1362 | (with-build-handler build-accumulator | |
1363 | (proc obj))) | |
1364 | lst)) | |
1365 | ||
1366 | (match (append-map (lambda (obj) | |
1367 | (if (unresolved? obj) | |
1368 | (unresolved-things obj) | |
1369 | '())) | |
1370 | result) | |
1371 | (() | |
1372 | result) | |
1373 | (to-build | |
1374 | ;; We've accumulated things TO-BUILD. Actually build them and resume the | |
1375 | ;; corresponding continuations. | |
1376 | (build-things store (delete-duplicates to-build)) | |
1377 | (map/accumulate-builds store | |
1378 | (lambda (obj) | |
1379 | (if (unresolved? obj) | |
1380 | ;; Pass #f because 'build-things' is now | |
1381 | ;; unnecessary. | |
1382 | ((unresolved-continuation obj) #f) | |
1383 | obj)) | |
1384 | result)))) | |
1385 | ||
07e70f48 LC |
1386 | (define build-things |
1387 | (let ((build (operation (build-things (string-list things) | |
1388 | (integer mode)) | |
1389 | "Do it!" | |
2734cbb8 LC |
1390 | boolean)) |
1391 | (build/old (operation (build-things (string-list things)) | |
1392 | "Do it!" | |
1393 | boolean))) | |
07e70f48 LC |
1394 | (lambda* (store things #:optional (mode (build-mode normal))) |
1395 | "Build THINGS, a list of store items which may be either '.drv' files or | |
abac874b LC |
1396 | outputs, and return when the worker is done building them. Elements of THINGS |
1397 | that are not derivations can only be substituted and not built locally. | |
f8a9f99c LC |
1398 | Alternately, an element of THING can be a derivation/output name pair, in |
1399 | which case the daemon will attempt to substitute just the requested output of | |
041b340d LC |
1400 | the derivation. Return #t on success. |
1401 | ||
1402 | When a handler is installed with 'with-build-handler', it is called any time | |
1403 | 'build-things' is called." | |
1404 | (or (not (invoke-build-handler store things mode)) | |
1405 | (let ((things (map (match-lambda | |
1406 | ((drv . output) (string-append drv "!" output)) | |
1407 | (thing thing)) | |
1408 | things))) | |
1409 | (parameterize ((current-store-protocol-version | |
1410 | (store-connection-version store))) | |
1411 | (if (>= (store-connection-minor-version store) 15) | |
1412 | (build store things mode) | |
1413 | (if (= mode (build-mode normal)) | |
1414 | (build/old store things) | |
1415 | (raise (condition (&store-protocol-error | |
1416 | (message "unsupported build mode") | |
1417 | (status 1)))))))))))) | |
26bbbb95 | 1418 | |
2a7e1121 | 1419 | (define-operation (ensure-path (store-path path)) |
d9e57db7 MO |
1420 | "Ensure that a path is valid. If it is not valid, it may be made valid by |
1421 | running a substitute. As a GC root is not created by the daemon, you may want | |
1422 | to call ADD-TEMP-ROOT on that store path." | |
2a7e1121 MO |
1423 | boolean) |
1424 | ||
7df3ab0f LC |
1425 | (define-operation (find-roots) |
1426 | "Return a list of root/target pairs: for each pair, the first element is the | |
1427 | GC root file name and the second element is its target in the store. | |
1428 | ||
1429 | When talking to a local daemon, this operation is equivalent to the 'gc-roots' | |
1430 | procedure in (guix store roots), except that the 'find-roots' excludes | |
1431 | potential roots that do not point to store items." | |
1432 | string-pairs) | |
1433 | ||
d3648e01 LC |
1434 | (define-operation (add-temp-root (store-path path)) |
1435 | "Make PATH a temporary root for the duration of the current session. | |
1436 | Return #t." | |
1437 | boolean) | |
1438 | ||
34811f02 | 1439 | (define-operation (add-indirect-root (string file-name)) |
a9d2a105 LC |
1440 | "Make the symlink FILE-NAME an indirect root for the garbage collector: |
1441 | whatever store item FILE-NAME points to will not be collected. Return #t on | |
1442 | success. | |
1443 | ||
1444 | FILE-NAME can be anywhere on the file system, but it must be an absolute file | |
1445 | name--it is the caller's responsibility to ensure that it is an absolute file | |
1446 | name." | |
34811f02 LC |
1447 | boolean) |
1448 | ||
a9d2a105 LC |
1449 | (define %gc-roots-directory |
1450 | ;; The place where garbage collector roots (symlinks) are kept. | |
1451 | (string-append %state-directory "/gcroots")) | |
1452 | ||
1453 | (define (add-permanent-root target) | |
1454 | "Add a garbage collector root pointing to TARGET, an element of the store, | |
1455 | preventing TARGET from even being collected. This can also be used if TARGET | |
1456 | does not exist yet. | |
1457 | ||
1458 | Raise an error if the caller does not have write access to the GC root | |
1459 | directory." | |
1460 | (let* ((root (string-append %gc-roots-directory "/" (basename target)))) | |
1461 | (catch 'system-error | |
1462 | (lambda () | |
1463 | (symlink target root)) | |
1464 | (lambda args | |
1465 | ;; If ROOT already exists, this is fine; otherwise, re-throw. | |
1466 | (unless (= EEXIST (system-error-errno args)) | |
1467 | (apply throw args)))))) | |
1468 | ||
1469 | (define (remove-permanent-root target) | |
1470 | "Remove the permanent garbage collector root pointing to TARGET. Raise an | |
1471 | error if there is no such root." | |
1472 | (delete-file (string-append %gc-roots-directory "/" (basename target)))) | |
1473 | ||
fae31edc LC |
1474 | (define references |
1475 | (operation (query-references (store-path path)) | |
1476 | "Return the list of references of PATH." | |
1477 | store-path-list)) | |
1478 | ||
f6fee16e | 1479 | (define* (fold-path store proc seed paths |
3f1e6939 | 1480 | #:optional (relatives (cut references store <>))) |
f6fee16e | 1481 | "Call PROC for each of the RELATIVES of PATHS, exactly once, and return the |
3f1e6939 LC |
1482 | result formed from the successive calls to PROC, the first of which is passed |
1483 | SEED." | |
f6fee16e | 1484 | (let loop ((paths paths) |
3f1e6939 LC |
1485 | (result seed) |
1486 | (seen vlist-null)) | |
1487 | (match paths | |
1488 | ((path rest ...) | |
1489 | (if (vhash-assoc path seen) | |
1490 | (loop rest result seen) | |
1491 | (let ((seen (vhash-cons path #t seen)) | |
1492 | (rest (append rest (relatives path))) | |
1493 | (result (proc path result))) | |
1494 | (loop rest result seen)))) | |
1495 | (() | |
1496 | result)))) | |
1497 | ||
f6fee16e LC |
1498 | (define (requisites store paths) |
1499 | "Return the requisites of PATHS, including PATHS---i.e., their closures (all | |
1500 | its references, recursively)." | |
1501 | (fold-path store cons '() paths)) | |
3f1e6939 | 1502 | |
50add477 LC |
1503 | (define (topologically-sorted store paths) |
1504 | "Return a list containing PATHS and all their references sorted in | |
1505 | topological order." | |
1506 | (define (traverse) | |
1507 | ;; Do a simple depth-first traversal of all of PATHS. | |
1508 | (let loop ((paths paths) | |
1509 | (visited vlist-null) | |
1510 | (result '())) | |
1511 | (define (visit n) | |
1512 | (vhash-cons n #t visited)) | |
1513 | ||
1514 | (define (visited? n) | |
1515 | (vhash-assoc n visited)) | |
1516 | ||
1517 | (match paths | |
1518 | ((head tail ...) | |
1519 | (if (visited? head) | |
1520 | (loop tail visited result) | |
1521 | (call-with-values | |
1522 | (lambda () | |
1523 | (loop (references store head) | |
1524 | (visit head) | |
1525 | result)) | |
1526 | (lambda (visited result) | |
1527 | (loop tail | |
1528 | visited | |
1529 | (cons head result)))))) | |
1530 | (() | |
1531 | (values visited result))))) | |
1532 | ||
1533 | (call-with-values traverse | |
1534 | (lambda (_ result) | |
1535 | (reverse result)))) | |
1536 | ||
fae31edc LC |
1537 | (define referrers |
1538 | (operation (query-referrers (store-path path)) | |
1539 | "Return the list of path that refer to PATH." | |
1540 | store-path-list)) | |
1541 | ||
1542 | (define valid-derivers | |
1543 | (operation (query-valid-derivers (store-path path)) | |
1544 | "Return the list of valid \"derivers\" of PATH---i.e., all the | |
1545 | .drv present in the store that have PATH among their outputs." | |
1546 | store-path-list)) | |
1547 | ||
1548 | (define query-derivation-outputs ; avoid name clash with `derivation-outputs' | |
1549 | (operation (query-derivation-outputs (store-path path)) | |
1550 | "Return the list of outputs of PATH, a .drv file." | |
1551 | store-path-list)) | |
1552 | ||
0f3d2504 LC |
1553 | (define-operation (has-substitutes? (store-path path)) |
1554 | "Return #t if binary substitutes are available for PATH, and #f otherwise." | |
1555 | boolean) | |
1556 | ||
1557 | (define substitutable-paths | |
1558 | (operation (query-substitutable-paths (store-path-list paths)) | |
1559 | "Return the subset of PATHS that is substitutable." | |
1560 | store-path-list)) | |
1561 | ||
1562 | (define substitutable-path-info | |
f65cf81a | 1563 | (operation (query-substitutable-path-infos (store-path-list paths)) |
0f3d2504 LC |
1564 | "Return information about the subset of PATHS that is |
1565 | substitutable. For each substitutable path, a `substitutable?' object is | |
dd78e90a LC |
1566 | returned; thus, the resulting list can be shorter than PATHS. Furthermore, |
1567 | that there is no guarantee that the order of the resulting list matches the | |
1568 | order of PATHS." | |
0f3d2504 LC |
1569 | substitutable-path-list)) |
1570 | ||
3961edf2 | 1571 | (define %built-in-builders |
f9aefa2d LC |
1572 | (let ((builders (operation (built-in-builders) |
1573 | "Return the built-in builders." | |
1574 | string-list))) | |
1575 | (lambda (store) | |
1576 | "Return the names of the supported built-in derivation builders | |
3961edf2 | 1577 | supported by STORE. The result is memoized for STORE." |
f9aefa2d LC |
1578 | ;; Check whether STORE's version supports this RPC and built-in |
1579 | ;; derivation builders in general, which appeared in Guix > 0.11.0. | |
1580 | ;; Return the empty list if it doesn't. Note that this RPC does not | |
1581 | ;; exist in 'nix-daemon'. | |
de9fbe9c LC |
1582 | (if (or (> (store-connection-major-version store) #x100) |
1583 | (and (= (store-connection-major-version store) #x100) | |
1584 | (>= (store-connection-minor-version store) #x60))) | |
f9aefa2d LC |
1585 | (builders store) |
1586 | '())))) | |
1587 | ||
3961edf2 LC |
1588 | (define (built-in-builders store) |
1589 | "Return the names of the supported built-in derivation builders | |
1590 | supported by STORE." | |
1591 | (force (store-connection-built-in-builders store))) | |
1592 | ||
e3fd0ce6 LC |
1593 | (define-operation (optimize-store) |
1594 | "Optimize the store by hard-linking identical files (\"deduplication\".) | |
1595 | Return #t on success." | |
1596 | ;; Note: the daemon in Guix <= 0.8.2 does not implement this RPC. | |
1597 | boolean) | |
1598 | ||
c63d9403 LC |
1599 | (define verify-store |
1600 | (let ((verify (operation (verify-store (boolean check-contents?) | |
1601 | (boolean repair?)) | |
1602 | "Verify the store." | |
1603 | boolean))) | |
1604 | (lambda* (store #:key check-contents? repair?) | |
1605 | "Verify the integrity of the store and return false if errors remain, | |
1606 | and true otherwise. When REPAIR? is true, repair any missing or altered store | |
1607 | items by substituting them (this typically requires root privileges because it | |
1608 | is not an atomic operation.) When CHECK-CONTENTS? is true, check the contents | |
1609 | of store items; this can take a lot of time." | |
1610 | (not (verify store check-contents? repair?))))) | |
1611 | ||
3259877d LC |
1612 | (define (run-gc server action to-delete min-freed) |
1613 | "Perform the garbage-collector operation ACTION, one of the | |
1614 | `gc-action' values. When ACTION is `delete-specific', the TO-DELETE is | |
1615 | the list of store paths to delete. IGNORE-LIVENESS? should always be | |
1616 | #f. MIN-FREED is the minimum amount of disk space to be freed, in | |
1617 | bytes, before the GC can stop. Return the list of store paths delete, | |
1618 | and the number of bytes freed." | |
7a45b5d5 LDB |
1619 | (let ((s (store-connection-socket server)) |
1620 | (buffered (store-connection-output-port server))) | |
1621 | (write-int (operation-id collect-garbage) buffered) | |
1622 | (write-int action buffered) | |
1623 | (write-store-path-list to-delete buffered) | |
1624 | (write-arg boolean #f buffered) ; ignore-liveness? | |
1625 | (write-long-long min-freed buffered) | |
1626 | (write-int 0 buffered) ; obsolete | |
de9fbe9c | 1627 | (when (>= (store-connection-minor-version server) 5) |
3259877d | 1628 | ;; Obsolete `use-atime' and `max-atime' parameters. |
7a45b5d5 LDB |
1629 | (write-int 0 buffered) |
1630 | (write-int 0 buffered)) | |
1631 | (write-buffered-output server) | |
3259877d LC |
1632 | |
1633 | ;; Loop until the server is done sending error output. | |
1634 | (let loop ((done? (process-stderr server))) | |
1635 | (or done? (loop (process-stderr server)))) | |
1636 | ||
1637 | (let ((paths (read-store-path-list s)) | |
1638 | (freed (read-long-long s)) | |
1639 | (obsolete (read-long-long s))) | |
000c59b6 LC |
1640 | (unless (null? paths) |
1641 | ;; To be on the safe side, completely invalidate both caches. | |
1642 | ;; Otherwise we could end up returning store paths that are no longer | |
1643 | ;; valid. | |
de9fbe9c LC |
1644 | (hash-clear! (store-connection-add-to-store-cache server)) |
1645 | (hash-clear! (store-connection-add-text-to-store-cache server))) | |
000c59b6 | 1646 | |
3259877d LC |
1647 | (values paths freed)))) |
1648 | ||
1649 | (define-syntax-rule (%long-long-max) | |
1650 | ;; Maximum unsigned 64-bit integer. | |
1651 | (- (expt 2 64) 1)) | |
1652 | ||
1653 | (define (live-paths server) | |
1654 | "Return the list of live store paths---i.e., store paths still | |
1655 | referenced, and thus not subject to being garbage-collected." | |
1656 | (run-gc server (gc-action return-live) '() (%long-long-max))) | |
1657 | ||
1658 | (define (dead-paths server) | |
1659 | "Return the list of dead store paths---i.e., store paths no longer | |
1660 | referenced, and thus subject to being garbage-collected." | |
1661 | (run-gc server (gc-action return-dead) '() (%long-long-max))) | |
1662 | ||
1663 | (define* (collect-garbage server #:optional (min-freed (%long-long-max))) | |
1664 | "Collect garbage from the store at SERVER. If MIN-FREED is non-zero, | |
1665 | then collect at least MIN-FREED bytes. Return the paths that were | |
1666 | collected, and the number of bytes freed." | |
1667 | (run-gc server (gc-action delete-dead) '() min-freed)) | |
1668 | ||
1669 | (define* (delete-paths server paths #:optional (min-freed (%long-long-max))) | |
1670 | "Delete PATHS from the store at SERVER, if they are no longer | |
1671 | referenced. If MIN-FREED is non-zero, then stop after at least | |
1672 | MIN-FREED bytes have been collected. Return the paths that were | |
1673 | collected, and the number of bytes freed." | |
1674 | (run-gc server (gc-action delete-specific) paths min-freed)) | |
1675 | ||
526382ff LC |
1676 | (define (import-paths server port) |
1677 | "Import the set of store paths read from PORT into SERVER's store. An error | |
1678 | is raised if the set of paths read from PORT is not signed (as per | |
1679 | 'export-path #:sign? #t'.) Return the list of store paths imported." | |
de9fbe9c | 1680 | (let ((s (store-connection-socket server))) |
526382ff LC |
1681 | (write-int (operation-id import-paths) s) |
1682 | (let loop ((done? (process-stderr server port))) | |
1683 | (or done? (loop (process-stderr server port)))) | |
1684 | (read-store-path-list s))) | |
1685 | ||
1686 | (define* (export-path server path port #:key (sign? #t)) | |
1687 | "Export PATH to PORT. When SIGN? is true, sign it." | |
7a45b5d5 LDB |
1688 | (let ((s (store-connection-socket server)) |
1689 | (buffered (store-connection-output-port server))) | |
1690 | (write-int (operation-id export-path) buffered) | |
1691 | (write-store-path path buffered) | |
1692 | (write-arg boolean sign? buffered) | |
1693 | (write-buffered-output server) | |
526382ff LC |
1694 | (let loop ((done? (process-stderr server port))) |
1695 | (or done? (loop (process-stderr server port)))) | |
1696 | (= 1 (read-int s)))) | |
1697 | ||
b03267df LC |
1698 | (define* (export-paths server paths port #:key (sign? #t) recursive? |
1699 | (start (const #f)) | |
1700 | (progress (const #f)) | |
1701 | (finish (const #f))) | |
99fbddf9 | 1702 | "Export the store paths listed in PATHS to PORT, in topological order, |
5b3d863f | 1703 | signing them if SIGN? is true. When RECURSIVE? is true, export the closure of |
b03267df LC |
1704 | PATHS---i.e., PATHS and all their dependencies. |
1705 | ||
1706 | START, PROGRESS, and FINISH are used to track progress of the data transfer. | |
1707 | START is a one-argument that is passed the list of store items that will be | |
1708 | transferred; it returns values that are then used as the initial state | |
1709 | threaded through PROGRESS calls. PROGRESS is passed the store item about to | |
1710 | be sent, along with the values previously return by START or by PROGRESS | |
1711 | itself. FINISH is called when the last store item has been called." | |
cafb92d8 | 1712 | (define ordered |
5b3d863f LC |
1713 | (let ((sorted (topologically-sorted server paths))) |
1714 | ;; When RECURSIVE? is #f, filter out the references of PATHS. | |
1715 | (if recursive? | |
1716 | sorted | |
1717 | (filter (cut member <> paths) sorted)))) | |
cafb92d8 | 1718 | |
b03267df LC |
1719 | (let loop ((paths ordered) |
1720 | (state (call-with-values (lambda () (start ordered)) | |
1721 | list))) | |
1d506993 LC |
1722 | (match paths |
1723 | (() | |
b03267df | 1724 | (apply finish state) |
1d506993 LC |
1725 | (write-int 0 port)) |
1726 | ((head tail ...) | |
1727 | (write-int 1 port) | |
1728 | (and (export-path server head port #:sign? sign?) | |
b03267df LC |
1729 | (loop tail |
1730 | (call-with-values | |
1731 | (lambda () (apply progress head state)) | |
1732 | list))))))) | |
526382ff | 1733 | |
16748d80 LC |
1734 | (define-operation (query-failed-paths) |
1735 | "Return the list of store items for which a build failure is cached. | |
1736 | ||
1737 | The result is always the empty list unless the daemon was started with | |
1738 | '--cache-failures'." | |
1739 | store-path-list) | |
1740 | ||
1741 | (define-operation (clear-failed-paths (store-path-list items)) | |
1742 | "Remove ITEMS from the list of cached build failures. | |
1743 | ||
1744 | This makes sense only when the daemon was started with '--cache-failures'." | |
1745 | boolean) | |
1746 | ||
26bbbb95 | 1747 | \f |
d9d7b9ec LC |
1748 | ;;; |
1749 | ;;; Per-connection caches. | |
1750 | ;;; | |
1751 | ||
1752 | ;; Number of currently allocated store connection caches--things that go in | |
1753 | ;; the 'caches' vector of <store-connection>. | |
1754 | (define %store-connection-caches (make-atomic-box 0)) | |
1755 | ||
1756 | (define (allocate-store-connection-cache name) | |
1757 | "Allocate a new cache for store connections and return its identifier. Said | |
1758 | identifier can be passed as an argument to " | |
1759 | (let loop ((current (atomic-box-ref %store-connection-caches))) | |
1760 | (let ((previous (atomic-box-compare-and-swap! %store-connection-caches | |
1761 | current (+ current 1)))) | |
1762 | (if (= previous current) | |
1763 | current | |
1764 | (loop current))))) | |
1765 | ||
1766 | (define %object-cache-id | |
1767 | ;; The "object cache", mapping lowerable objects such as <package> records | |
1768 | ;; to derivations. | |
1769 | (allocate-store-connection-cache 'object-cache)) | |
1770 | ||
1771 | (define (vector-set vector index value) | |
1772 | (let ((new (vector-copy vector))) | |
1773 | (vector-set! new index value) | |
1774 | new)) | |
1775 | ||
1776 | (define (store-connection-cache store cache) | |
1777 | "Return the cache of STORE identified by CACHE, an identifier as returned by | |
1778 | 'allocate-store-connection-cache'." | |
1779 | (vector-ref (store-connection-caches store) cache)) | |
1780 | ||
1781 | (define (set-store-connection-cache store cache value) | |
1782 | "Return a copy of STORE where CACHE has the given VALUE. CACHE must be a | |
1783 | value returned by 'allocate-store-connection-cache'." | |
1784 | (store-connection | |
1785 | (inherit store) | |
1786 | (caches (vector-set (store-connection-caches store) cache value)))) | |
1787 | ||
1788 | (define set-store-connection-caches! ;private | |
1789 | (record-modifier <store-connection> 'caches)) | |
1790 | ||
1791 | (define (set-store-connection-cache! store cache value) | |
1792 | "Set STORE's CACHE to VALUE. | |
1793 | ||
1794 | This is a mutating version that should be avoided. Prefer the functional | |
1795 | 'set-store-connection-cache' instead, together with using %STORE-MONAD." | |
1796 | (vector-set! (store-connection-caches store) cache value)) | |
1797 | ||
fde3c349 LC |
1798 | |
1799 | (define %reference-cache-id | |
1800 | ;; Cache mapping store items to their list of references. Caching matters | |
1801 | ;; because when building a profile in the presence of grafts, we keep | |
1802 | ;; calling 'graft-derivation', which in turn calls 'references/cached' many | |
1803 | ;; times with the same arguments. | |
1804 | (allocate-store-connection-cache 'reference-cache)) | |
1805 | ||
1806 | (define (references/cached store item) | |
1807 | "Like 'references', but cache results." | |
1808 | (let ((cache (store-connection-cache store %reference-cache-id))) | |
1809 | (match (vhash-assoc item cache) | |
1810 | ((_ . references) | |
1811 | references) | |
1812 | (#f | |
1813 | (let* ((references (references store item)) | |
1814 | (cache (vhash-cons item references cache))) | |
1815 | (set-store-connection-cache! store %reference-cache-id cache) | |
1816 | references))))) | |
1817 | ||
d9d7b9ec | 1818 | \f |
e87f0591 LC |
1819 | ;;; |
1820 | ;;; Store monad. | |
1821 | ;;; | |
1822 | ||
4e190c28 LC |
1823 | (define-syntax-rule (define-alias new old) |
1824 | (define-syntax new (identifier-syntax old))) | |
e87f0591 | 1825 | |
4e190c28 LC |
1826 | ;; The store monad allows us to (1) build sequences of operations in the |
1827 | ;; store, and (2) make the store an implicit part of the execution context, | |
1828 | ;; rather than a parameter of every single function. | |
1829 | (define-alias %store-monad %state-monad) | |
1830 | (define-alias store-return state-return) | |
1831 | (define-alias store-bind state-bind) | |
e87f0591 | 1832 | |
dcb95c1f LC |
1833 | ;; Instantiate templates for %STORE-MONAD since it's syntactically different |
1834 | ;; from %STATE-MONAD. | |
1835 | (template-directory instantiations %store-monad) | |
1836 | ||
c57e417e | 1837 | (define* (cache-object-mapping object keys result |
d9d7b9ec LC |
1838 | #:key |
1839 | (cache %object-cache-id) | |
1840 | (vhash-cons vhash-consq)) | |
c6080c32 LC |
1841 | "Augment the store's object cache with a mapping from OBJECT/KEYS to RESULT. |
1842 | KEYS is a list of additional keys to match against, for instance a (SYSTEM | |
c57e417e | 1843 | TARGET) tuple. Use VHASH-CONS to insert OBJECT into the cache. |
c6080c32 LC |
1844 | |
1845 | OBJECT is typically a high-level object such as a <package> or an <origin>, | |
1846 | and RESULT is typically its derivation." | |
1847 | (lambda (store) | |
1848 | (values result | |
d9d7b9ec LC |
1849 | (set-store-connection-cache |
1850 | store cache | |
1851 | (vhash-cons object (cons result keys) | |
1852 | (store-connection-cache store cache)))))) | |
c6080c32 | 1853 | |
0a3c723e LC |
1854 | (define (cache-lookup-recorder component title) |
1855 | "Return a procedure of two arguments to record cache lookups, hits, and | |
1856 | misses for COMPONENT. The procedure must be passed a Boolean indicating | |
1857 | whether the cache lookup was a hit, and the actual cache (a vhash)." | |
1858 | (if (profiled? component) | |
73b0ebdd LC |
1859 | (let ((fresh 0) |
1860 | (lookups 0) | |
70b33d88 LC |
1861 | (hits 0) |
1862 | (size 0)) | |
73b0ebdd | 1863 | (register-profiling-hook! |
0a3c723e | 1864 | component |
73b0ebdd | 1865 | (lambda () |
0a3c723e | 1866 | (format (current-error-port) "~a: |
73b0ebdd LC |
1867 | fresh caches: ~5@a |
1868 | lookups: ~5@a | |
70b33d88 LC |
1869 | hits: ~5@a (~,1f%) |
1870 | cache size: ~5@a entries~%" | |
0a3c723e | 1871 | title fresh lookups hits |
73b0ebdd LC |
1872 | (if (zero? lookups) |
1873 | 100. | |
70b33d88 LC |
1874 | (* 100. (/ hits lookups))) |
1875 | size))) | |
73b0ebdd LC |
1876 | |
1877 | (lambda (hit? cache) | |
1878 | (set! fresh | |
0a3c723e LC |
1879 | (if (eq? cache vlist-null) |
1880 | (+ 1 fresh) | |
1881 | fresh)) | |
73b0ebdd | 1882 | (set! lookups (+ 1 lookups)) |
70b33d88 LC |
1883 | (set! hits (if hit? (+ hits 1) hits)) |
1884 | (set! size (+ (if hit? 0 1) | |
1885 | (vlist-length cache))))) | |
73b0ebdd LC |
1886 | (lambda (x y) |
1887 | #t))) | |
1888 | ||
0a3c723e LC |
1889 | (define record-cache-lookup! |
1890 | (cache-lookup-recorder "object-cache" "Store object cache")) | |
1891 | ||
0a912b3a | 1892 | (define-inlinable (lookup-cached-object object keys vhash-fold*) |
c6080c32 | 1893 | "Return the cached object in the store connection corresponding to OBJECT |
c57e417e LC |
1894 | and KEYS; use VHASH-FOLD* to look for OBJECT in the cache. KEYS is a list of |
1895 | additional keys to match against, and which are compared with 'equal?'. | |
1896 | Return #f on failure and the cached result otherwise." | |
c6080c32 | 1897 | (lambda (store) |
d9d7b9ec | 1898 | (let* ((cache (store-connection-cache store %object-cache-id)) |
73b0ebdd LC |
1899 | |
1900 | ;; Escape as soon as we find the result. This avoids traversing | |
1901 | ;; the whole vlist chain and significantly reduces the number of | |
1902 | ;; 'hashq' calls. | |
1903 | (value (let/ec return | |
c57e417e LC |
1904 | (vhash-fold* (lambda (item result) |
1905 | (match item | |
1906 | ((value . keys*) | |
1907 | (if (equal? keys keys*) | |
1908 | (return value) | |
1909 | result)))) | |
1910 | #f object | |
1911 | cache)))) | |
73b0ebdd LC |
1912 | (record-cache-lookup! value cache) |
1913 | (values value store)))) | |
c6080c32 | 1914 | |
c57e417e LC |
1915 | (define* (%mcached mthunk object #:optional (keys '()) |
1916 | #:key | |
1917 | (vhash-cons vhash-consq) | |
1918 | (vhash-fold* vhash-foldq*)) | |
c6080c32 | 1919 | "Bind the monadic value returned by MTHUNK, which supposedly corresponds to |
c57e417e LC |
1920 | OBJECT/KEYS, or return its cached value. Use VHASH-CONS to insert OBJECT into |
1921 | the cache, and VHASH-FOLD* to look it up." | |
1922 | (mlet %store-monad ((cached (lookup-cached-object object keys | |
0a912b3a | 1923 | vhash-fold*))) |
c6080c32 LC |
1924 | (if cached |
1925 | (return cached) | |
1926 | (>>= (mthunk) | |
1927 | (lambda (result) | |
c57e417e LC |
1928 | (cache-object-mapping object keys result |
1929 | #:vhash-cons vhash-cons)))))) | |
c6080c32 | 1930 | |
c57e417e LC |
1931 | (define-syntax mcached |
1932 | (syntax-rules (eq? equal?) | |
1933 | "Run MVALUE, which corresponds to OBJECT/KEYS, and cache it; or return the | |
c6080c32 LC |
1934 | value associated with OBJECT/KEYS in the store's object cache if there is |
1935 | one." | |
c57e417e LC |
1936 | ((_ eq? mvalue object keys ...) |
1937 | (%mcached (lambda () mvalue) | |
1938 | object (list keys ...) | |
1939 | #:vhash-cons vhash-consq | |
1940 | #:vhash-fold* vhash-foldq*)) | |
1941 | ((_ equal? mvalue object keys ...) | |
1942 | (%mcached (lambda () mvalue) | |
1943 | object (list keys ...) | |
1944 | #:vhash-cons vhash-cons | |
1945 | #:vhash-fold* vhash-fold*)) | |
1946 | ((_ mvalue object keys ...) | |
1947 | (mcached eq? mvalue object keys ...)))) | |
c6080c32 | 1948 | |
5808dcc2 LC |
1949 | (define (preserve-documentation original proc) |
1950 | "Return PROC with documentation taken from ORIGINAL." | |
1951 | (set-object-property! proc 'documentation | |
1952 | (procedure-property original 'documentation)) | |
1953 | proc) | |
1954 | ||
e87f0591 LC |
1955 | (define (store-lift proc) |
1956 | "Lift PROC, a procedure whose first argument is a connection to the store, | |
1957 | in the store monad." | |
5808dcc2 LC |
1958 | (preserve-documentation proc |
1959 | (lambda args | |
1960 | (lambda (store) | |
1961 | (values (apply proc store args) store))))) | |
e87f0591 | 1962 | |
023d9892 LC |
1963 | (define (store-lower proc) |
1964 | "Lower PROC, a monadic procedure in %STORE-MONAD, to a \"normal\" procedure | |
1965 | taking the store as its first argument." | |
5808dcc2 LC |
1966 | (preserve-documentation proc |
1967 | (lambda (store . args) | |
1968 | (run-with-store store (apply proc args))))) | |
023d9892 | 1969 | |
c40bf581 LC |
1970 | (define (mapm/accumulate-builds mproc lst) |
1971 | "Like 'mapm' in %STORE-MONAD, but accumulate 'build-things' calls and | |
1972 | coalesce them into a single call." | |
1973 | (lambda (store) | |
1974 | (values (map/accumulate-builds store | |
1975 | (lambda (obj) | |
1976 | (run-with-store store | |
80963744 LC |
1977 | (mproc obj) |
1978 | #:system (%current-system) | |
1979 | #:target (%current-target-system))) | |
c40bf581 LC |
1980 | lst) |
1981 | store))) | |
1982 | ||
1983 | ||
e87f0591 LC |
1984 | ;; |
1985 | ;; Store monad operators. | |
1986 | ;; | |
1987 | ||
f3a42251 JN |
1988 | (define* (binary-file name |
1989 | data ;bytevector | |
1990 | #:optional (references '())) | |
1991 | "Return as a monadic value the absolute file name in the store of the file | |
1992 | containing DATA, a bytevector. REFERENCES is a list of store items that the | |
1993 | resulting text file refers to; it defaults to the empty list." | |
1994 | (lambda (store) | |
1995 | (values (add-data-to-store store name data references) | |
1996 | store))) | |
1997 | ||
1998 | (define* (text-file name | |
1999 | text ;string | |
ad372953 | 2000 | #:optional (references '())) |
e87f0591 | 2001 | "Return as a monadic value the absolute file name in the store of the file |
ad372953 LC |
2002 | containing TEXT, a string. REFERENCES is a list of store items that the |
2003 | resulting text file refers to; it defaults to the empty list." | |
e87f0591 | 2004 | (lambda (store) |
ad372953 | 2005 | (values (add-text-to-store store name text references) |
4e190c28 | 2006 | store))) |
e87f0591 LC |
2007 | |
2008 | (define* (interned-file file #:optional name | |
1ec32f4a | 2009 | #:key (recursive? #t) (select? true)) |
e87f0591 LC |
2010 | "Return the name of FILE once interned in the store. Use NAME as its store |
2011 | name, or the basename of FILE if NAME is omitted. | |
2012 | ||
2013 | When RECURSIVE? is true, the contents of FILE are added recursively; if FILE | |
2014 | designates a flat file and RECURSIVE? is true, its contents are added, and its | |
1ec32f4a LC |
2015 | permission bits are kept. |
2016 | ||
2017 | When RECURSIVE? is true, call (SELECT? FILE STAT) for each directory entry, | |
2018 | where FILE is the entry's absolute file name and STAT is the result of | |
2019 | 'lstat'; exclude entries for which SELECT? does not return true." | |
e87f0591 | 2020 | (lambda (store) |
4e190c28 | 2021 | (values (add-to-store store (or name (basename file)) |
1ec32f4a LC |
2022 | recursive? "sha256" file |
2023 | #:select? select?) | |
4e190c28 | 2024 | store))) |
e87f0591 | 2025 | |
7f11efba LC |
2026 | (define interned-file-tree |
2027 | (store-lift add-file-tree-to-store)) | |
2028 | ||
abac874b LC |
2029 | (define build |
2030 | ;; Monadic variant of 'build-things'. | |
2031 | (store-lift build-things)) | |
2032 | ||
4f740b67 AK |
2033 | (define set-build-options* |
2034 | (store-lift set-build-options)) | |
2035 | ||
e74f64b9 LC |
2036 | (define references* |
2037 | (store-lift references)) | |
2038 | ||
0744a9f0 LC |
2039 | (define (query-path-info* item) |
2040 | "Monadic version of 'query-path-info' that returns #f when ITEM is not in | |
2041 | the store." | |
2042 | (lambda (store) | |
f9e8a123 | 2043 | (guard (c ((store-protocol-error? c) |
0744a9f0 LC |
2044 | ;; ITEM is not in the store; return #f. |
2045 | (values #f store))) | |
2046 | (values (query-path-info store item) store)))) | |
2047 | ||
98a7b528 LC |
2048 | (define-inlinable (current-system) |
2049 | ;; Consult the %CURRENT-SYSTEM fluid at bind time. This is equivalent to | |
2050 | ;; (lift0 %current-system %store-monad), but inlinable, thus avoiding | |
2051 | ;; closure allocation in some cases. | |
2052 | (lambda (state) | |
2053 | (values (%current-system) state))) | |
2054 | ||
2055 | (define-inlinable (set-current-system system) | |
2056 | ;; Set the %CURRENT-SYSTEM fluid at bind time. | |
2057 | (lambda (state) | |
2058 | (values (%current-system system) state))) | |
2059 | ||
91be09de MO |
2060 | (define-inlinable (current-target-system) |
2061 | ;; Consult the %CURRENT-TARGET-SYSTEM fluid at bind time. | |
2062 | (lambda (state) | |
2063 | (values (%current-target-system) state))) | |
2064 | ||
fdae0fa5 MO |
2065 | (define-inlinable (set-current-target target) |
2066 | ;; Set the %CURRENT-TARGET-SYSTEM fluid at bind time. | |
2067 | (lambda (state) | |
2068 | (values (%current-target-system target) state))) | |
2069 | ||
e87f0591 LC |
2070 | (define %guile-for-build |
2071 | ;; The derivation of the Guile to be used within the build environment, | |
2072 | ;; when using 'gexp->derivation' and co. | |
2073 | (make-parameter #f)) | |
2074 | ||
2075 | (define* (run-with-store store mval | |
2076 | #:key | |
2077 | (guile-for-build (%guile-for-build)) | |
45bba475 LC |
2078 | (system (%current-system)) |
2079 | (target #f)) | |
e87f0591 | 2080 | "Run MVAL, a monadic value in the store monad, in STORE, an open store |
3698f524 | 2081 | connection, and return the result." |
a8afb9ae LC |
2082 | ;; Initialize the dynamic bindings here to avoid bad surprises. The |
2083 | ;; difficulty lies in the fact that dynamic bindings are resolved at | |
2084 | ;; bind-time and not at call time, which can be disconcerting. | |
e87f0591 | 2085 | (parameterize ((%guile-for-build guile-for-build) |
a8afb9ae | 2086 | (%current-system system) |
45bba475 | 2087 | (%current-target-system target)) |
3698f524 LC |
2088 | (call-with-values (lambda () |
2089 | (run-with-state mval store)) | |
dd0ee954 | 2090 | (lambda (result new-store) |
728a4ab1 LC |
2091 | (when (and store new-store) |
2092 | ;; Copy the object cache from NEW-STORE so we don't fully discard | |
2093 | ;; the state. | |
d9d7b9ec LC |
2094 | (let ((caches (store-connection-caches new-store))) |
2095 | (set-store-connection-caches! store caches))) | |
728a4ab1 | 2096 | result)))) |
e87f0591 LC |
2097 | |
2098 | \f | |
26bbbb95 LC |
2099 | ;;; |
2100 | ;;; Store paths. | |
2101 | ;;; | |
2102 | ||
2103 | (define %store-prefix | |
2104 | ;; Absolute path to the Nix store. | |
1d6816f9 | 2105 | (make-parameter %store-directory)) |
26bbbb95 | 2106 | |
cd041b26 LC |
2107 | (define (compressed-hash bv size) ; `compressHash' |
2108 | "Given the hash stored in BV, return a compressed version thereof that fits | |
2109 | in SIZE bytes." | |
2110 | (define new (make-bytevector size 0)) | |
2111 | (define old-size (bytevector-length bv)) | |
2112 | (let loop ((i 0)) | |
2113 | (if (= i old-size) | |
2114 | new | |
2115 | (let* ((j (modulo i size)) | |
2116 | (o (bytevector-u8-ref new j))) | |
2117 | (bytevector-u8-set! new j | |
2118 | (logxor o (bytevector-u8-ref bv i))) | |
2119 | (loop (+ 1 i)))))) | |
2120 | ||
2121 | (define (store-path type hash name) ; makeStorePath | |
2122 | "Return the store path for NAME/HASH/TYPE." | |
2123 | (let* ((s (string-append type ":sha256:" | |
2124 | (bytevector->base16-string hash) ":" | |
2125 | (%store-prefix) ":" name)) | |
2126 | (h (sha256 (string->utf8 s))) | |
2127 | (c (compressed-hash h 20))) | |
2128 | (string-append (%store-prefix) "/" | |
2129 | (bytevector->nix-base32-string c) "-" | |
2130 | name))) | |
2131 | ||
2132 | (define (output-path output hash name) ; makeOutputPath | |
2133 | "Return an output path for OUTPUT (the name of the output as a string) of | |
2134 | the derivation called NAME with hash HASH." | |
2135 | (store-path (string-append "output:" output) hash | |
2136 | (if (string=? output "out") | |
2137 | name | |
2138 | (string-append name "-" output)))) | |
2139 | ||
2140 | (define* (fixed-output-path name hash | |
2141 | #:key | |
2142 | (output "out") | |
2143 | (hash-algo 'sha256) | |
2144 | (recursive? #t)) | |
2145 | "Return an output path for the fixed output OUTPUT defined by HASH of type | |
2146 | HASH-ALGO, of the derivation NAME. RECURSIVE? has the same meaning as for | |
2147 | 'add-to-store'." | |
2148 | (if (and recursive? (eq? hash-algo 'sha256)) | |
2149 | (store-path "source" hash name) | |
2150 | (let ((tag (string-append "fixed:" output ":" | |
2151 | (if recursive? "r:" "") | |
2152 | (symbol->string hash-algo) ":" | |
2153 | (bytevector->base16-string hash) ":"))) | |
2154 | (store-path (string-append "output:" output) | |
2155 | (sha256 (string->utf8 tag)) | |
2156 | name)))) | |
2157 | ||
f39bd08a LC |
2158 | (define (store-path? path) |
2159 | "Return #t if PATH is a store path." | |
2160 | ;; This is a lightweight check, compared to using a regexp, but this has to | |
2161 | ;; be fast as it's called often in `derivation', for instance. | |
2162 | ;; `isStorePath' in Nix does something similar. | |
2163 | (string-prefix? (%store-prefix) path)) | |
26bbbb95 | 2164 | |
9336e5b5 LC |
2165 | (define (direct-store-path? path) |
2166 | "Return #t if PATH is a store path, and not a sub-directory of a store path. | |
2167 | This predicate is sometimes needed because files *under* a store path are not | |
2168 | valid inputs." | |
2169 | (and (store-path? path) | |
eee21271 | 2170 | (not (string=? path (%store-prefix))) |
9336e5b5 LC |
2171 | (let ((len (+ 1 (string-length (%store-prefix))))) |
2172 | (not (string-index (substring path len) #\/))))) | |
2173 | ||
cdb5b075 CS |
2174 | (define (direct-store-path path) |
2175 | "Return the direct store path part of PATH, stripping components after | |
2176 | '/gnu/store/xxxx-foo'." | |
2177 | (let ((prefix-length (+ (string-length (%store-prefix)) 35))) | |
2178 | (if (> (string-length path) prefix-length) | |
2179 | (let ((slash (string-index path #\/ prefix-length))) | |
2180 | (if slash (string-take path slash) path)) | |
2181 | path))) | |
2182 | ||
26bbbb95 LC |
2183 | (define (derivation-path? path) |
2184 | "Return #t if PATH is a derivation path." | |
2185 | (and (store-path? path) (string-suffix? ".drv" path))) | |
e3d74106 | 2186 | |
771c5e15 FP |
2187 | (define (store-path-base path) |
2188 | "Return the base path of a path in the store." | |
2189 | (and (string-prefix? (%store-prefix) path) | |
2190 | (let ((base (string-drop path (+ 1 (string-length (%store-prefix)))))) | |
2191 | (and (> (string-length base) 33) | |
2192 | (not (string-index base #\/)) | |
2193 | base)))) | |
5c0f1845 | 2194 | |
e3d74106 LC |
2195 | (define (store-path-package-name path) |
2196 | "Return the package name part of PATH, a file name in the store." | |
771c5e15 FP |
2197 | (let ((base (store-path-base path))) |
2198 | (string-drop base (+ 32 1)))) ;32 hash part + 1 hyphen | |
2c6ab6cc LC |
2199 | |
2200 | (define (store-path-hash-part path) | |
2201 | "Return the hash part of PATH as a base32 string, or #f if PATH is not a | |
2202 | syntactically valid store path." | |
814ee99d LC |
2203 | (match (store-path-base path) |
2204 | (#f #f) | |
2205 | (base | |
2206 | (let ((hash (string-take base 32))) | |
2207 | (and (string-every %nix-base32-charset hash) | |
2208 | hash))))) | |
eddd4077 | 2209 | |
fb94d82b LC |
2210 | (define (derivation-log-file drv) |
2211 | "Return the build log file for DRV, a derivation file name, or #f if it | |
2212 | could not be found." | |
2213 | (let* ((base (basename drv)) | |
7c4700e9 LC |
2214 | (log (string-append (or (getenv "GUIX_LOG_DIRECTORY") |
2215 | (string-append %localstatedir "/log/guix")) | |
2216 | "/drvs/" | |
fb94d82b LC |
2217 | (string-take base 2) "/" |
2218 | (string-drop base 2))) | |
2219 | (log.gz (string-append log ".gz")) | |
2220 | (log.bz2 (string-append log ".bz2"))) | |
2221 | (cond ((file-exists? log.gz) log.gz) | |
2222 | ((file-exists? log.bz2) log.bz2) | |
2223 | ((file-exists? log) log) | |
2224 | (else #f)))) | |
2225 | ||
eddd4077 LC |
2226 | (define (log-file store file) |
2227 | "Return the build log file for FILE, or #f if none could be found. FILE | |
2228 | must be an absolute store file name, or a derivation file name." | |
eddd4077 | 2229 | (cond ((derivation-path? file) |
fb94d82b | 2230 | (derivation-log-file file)) |
eddd4077 LC |
2231 | (else |
2232 | (match (valid-derivers store file) | |
2233 | ((derivers ...) | |
2234 | ;; Return the first that works. | |
2235 | (any (cut log-file store <>) derivers)) | |
2236 | (_ #f))))) | |
3b5cd17a LC |
2237 | |
2238 | ;;; Local Variables: | |
2239 | ;;; eval: (put 'system-error-to-connection-error 'scheme-indent-function 1) | |
2240 | ;;; End: |