3 #include "local-store.hh"
19 static string gcLockName
= "gc.lock";
20 static string tempRootsDir
= "temproots";
21 static string gcRootsDir
= "gcroots";
24 /* Acquire the global GC lock. This is used to prevent new Nix
25 processes from starting after the temporary root files have been
26 read. To be precise: when they try to create a new temporary root
27 file, they will block until the garbage collector has finished /
28 yielded the GC lock. */
29 int LocalStore::openGCLock(LockType lockType
)
31 Path fnGCLock
= (format("%1%/%2%")
32 % settings
.nixStateDir
% gcLockName
).str();
34 debug(format("acquiring global GC lock `%1%'") % fnGCLock
);
36 AutoCloseFD fdGCLock
= open(fnGCLock
.c_str(), O_RDWR
| O_CREAT
, 0600);
38 throw SysError(format("opening global GC lock `%1%'") % fnGCLock
);
39 closeOnExec(fdGCLock
);
41 if (!lockFile(fdGCLock
, lockType
, false)) {
42 printMsg(lvlError
, format("waiting for the big garbage collector lock..."));
43 lockFile(fdGCLock
, lockType
, true);
46 /* !!! Restrict read permission on the GC root. Otherwise any
47 process that can open the file for reading can DoS the
50 return fdGCLock
.borrow();
54 static void makeSymlink(const Path
& link
, const Path
& target
)
56 /* Create directories up to `gcRoot'. */
57 createDirs(dirOf(link
));
59 /* Create the new symlink. */
60 Path tempLink
= (format("%1%.tmp-%2%-%3%")
61 % link
% getpid() % rand()).str();
62 createSymlink(target
, tempLink
);
64 /* Atomically replace the old one. */
65 if (rename(tempLink
.c_str(), link
.c_str()) == -1)
66 throw SysError(format("cannot rename `%1%' to `%2%'")
71 void LocalStore::syncWithGC()
73 AutoCloseFD fdGCLock
= openGCLock(ltRead
);
77 void LocalStore::addIndirectRoot(const Path
& path
)
79 string hash
= printHash32(hashString(htSHA1
, path
));
80 Path realRoot
= canonPath((format("%1%/%2%/auto/%3%")
81 % settings
.nixStateDir
% gcRootsDir
% hash
).str());
82 makeSymlink(realRoot
, path
);
86 Path
addPermRoot(StoreAPI
& store
, const Path
& _storePath
,
87 const Path
& _gcRoot
, bool indirect
, bool allowOutsideRootsDir
)
89 Path
storePath(canonPath(_storePath
));
90 Path
gcRoot(canonPath(_gcRoot
));
91 assertStorePath(storePath
);
93 if (isInStore(gcRoot
))
95 "creating a garbage collector root (%1%) in the Nix store is forbidden "
96 "(are you running nix-build inside the store?)") % gcRoot
);
99 /* Don't clobber the link if it already exists and doesn't
100 point to the Nix store. */
101 if (pathExists(gcRoot
) && (!isLink(gcRoot
) || !isInStore(readLink(gcRoot
))))
102 throw Error(format("cannot create symlink `%1%'; already exists") % gcRoot
);
103 makeSymlink(gcRoot
, storePath
);
104 store
.addIndirectRoot(gcRoot
);
108 if (!allowOutsideRootsDir
) {
109 Path rootsDir
= canonPath((format("%1%/%2%") % settings
.nixStateDir
% gcRootsDir
).str());
111 if (string(gcRoot
, 0, rootsDir
.size() + 1) != rootsDir
+ "/")
113 "path `%1%' is not a valid garbage collector root; "
114 "it's not in the directory `%2%'")
115 % gcRoot
% rootsDir
);
118 if (baseNameOf(gcRoot
) == baseNameOf(storePath
))
119 writeFile(gcRoot
, "");
121 makeSymlink(gcRoot
, storePath
);
124 /* Check that the root can be found by the garbage collector.
125 !!! This can be very slow on machines that have many roots.
126 Instead of reading all the roots, it would be more efficient to
127 check if the root is in a directory in or linked from the
128 gcroots directory. */
129 if (settings
.checkRootReachability
) {
130 Roots roots
= store
.findRoots();
131 if (roots
.find(gcRoot
) == roots
.end())
134 "warning: `%1%' is not in a directory where the garbage collector looks for roots; "
135 "therefore, `%2%' might be removed by the garbage collector")
136 % gcRoot
% storePath
);
139 /* Grab the global GC root, causing us to block while a GC is in
140 progress. This prevents the set of permanent roots from
141 increasing while a GC is in progress. */
148 void LocalStore::addTempRoot(const Path
& path
)
150 /* Create the temporary roots file for this process. */
151 if (fdTempRoots
== -1) {
154 Path dir
= (format("%1%/%2%") % settings
.nixStateDir
% tempRootsDir
).str();
157 fnTempRoots
= (format("%1%/%2%")
158 % dir
% getpid()).str();
160 AutoCloseFD fdGCLock
= openGCLock(ltRead
);
162 if (pathExists(fnTempRoots
))
163 /* It *must* be stale, since there can be no two
164 processes with the same pid. */
165 unlink(fnTempRoots
.c_str());
167 fdTempRoots
= openLockFile(fnTempRoots
, true);
171 debug(format("acquiring read lock on `%1%'") % fnTempRoots
);
172 lockFile(fdTempRoots
, ltRead
, true);
174 /* Check whether the garbage collector didn't get in our
177 if (fstat(fdTempRoots
, &st
) == -1)
178 throw SysError(format("statting `%1%'") % fnTempRoots
);
179 if (st
.st_size
== 0) break;
181 /* The garbage collector deleted this file before we could
182 get a lock. (It won't delete the file after we get a
188 /* Upgrade the lock to a write lock. This will cause us to block
189 if the garbage collector is holding our lock. */
190 debug(format("acquiring write lock on `%1%'") % fnTempRoots
);
191 lockFile(fdTempRoots
, ltWrite
, true);
193 string s
= path
+ '\0';
194 writeFull(fdTempRoots
, s
);
196 /* Downgrade to a read lock. */
197 debug(format("downgrading to read lock on `%1%'") % fnTempRoots
);
198 lockFile(fdTempRoots
, ltRead
, true);
202 typedef std::shared_ptr
<AutoCloseFD
> FDPtr
;
203 typedef list
<FDPtr
> FDs
;
206 static void readTempRoots(PathSet
& tempRoots
, FDs
& fds
)
208 /* Read the `temproots' directory for per-process temporary root
210 DirEntries tempRootFiles
= readDirectory(
211 (format("%1%/%2%") % settings
.nixStateDir
% tempRootsDir
).str());
213 for (auto & i
: tempRootFiles
) {
214 Path path
= (format("%1%/%2%/%3%") % settings
.nixStateDir
% tempRootsDir
% i
.name
).str();
216 debug(format("reading temporary root file `%1%'") % path
);
217 FDPtr
fd(new AutoCloseFD(open(path
.c_str(), O_RDWR
, 0666)));
219 /* It's okay if the file has disappeared. */
220 if (errno
== ENOENT
) continue;
221 throw SysError(format("opening temporary roots file `%1%'") % path
);
224 /* This should work, but doesn't, for some reason. */
225 //FDPtr fd(new AutoCloseFD(openLockFile(path, false)));
226 //if (*fd == -1) continue;
228 /* Try to acquire a write lock without blocking. This can
229 only succeed if the owning process has died. In that case
230 we don't care about its temporary roots. */
231 if (lockFile(*fd
, ltWrite
, false)) {
232 printMsg(lvlError
, format("removing stale temporary roots file `%1%'") % path
);
233 unlink(path
.c_str());
238 /* Acquire a read lock. This will prevent the owning process
239 from upgrading to a write lock, therefore it will block in
241 debug(format("waiting for read lock on `%1%'") % path
);
242 lockFile(*fd
, ltRead
, true);
244 /* Read the entire file. */
245 string contents
= readFile(*fd
);
247 /* Extract the roots. */
248 string::size_type pos
= 0, end
;
250 while ((end
= contents
.find((char) 0, pos
)) != string::npos
) {
251 Path
root(contents
, pos
, end
- pos
);
252 debug(format("got temporary root `%1%'") % root
);
253 assertStorePath(root
);
254 tempRoots
.insert(root
);
258 fds
.push_back(fd
); /* keep open */
263 static void foundRoot(StoreAPI
& store
,
264 const Path
& path
, const Path
& target
, Roots
& roots
)
266 Path storePath
= toStorePath(target
);
267 if (store
.isValidPath(storePath
))
268 roots
[path
] = storePath
;
270 printMsg(lvlInfo
, format("skipping invalid root from `%1%' to `%2%'") % path
% storePath
);
274 static void findRoots(StoreAPI
& store
, const Path
& path
, unsigned char type
, Roots
& roots
)
278 if (type
== DT_UNKNOWN
)
279 type
= getFileType(path
);
281 if (type
== DT_DIR
) {
282 for (auto & i
: readDirectory(path
))
283 findRoots(store
, path
+ "/" + i
.name
, i
.type
, roots
);
286 else if (type
== DT_LNK
) {
287 Path target
= readLink(path
);
288 if (isInStore(target
))
289 foundRoot(store
, path
, target
, roots
);
291 /* Handle indirect roots. */
293 target
= absPath(target
, dirOf(path
));
294 if (!pathExists(target
)) {
295 if (isInDir(path
, settings
.nixStateDir
+ "/" + gcRootsDir
+ "/auto")) {
296 printMsg(lvlInfo
, format("removing stale link from `%1%' to `%2%'") % path
% target
);
297 unlink(path
.c_str());
300 struct stat st2
= lstat(target
);
301 if (!S_ISLNK(st2
.st_mode
)) return;
302 Path target2
= readLink(target
);
303 if (isInStore(target2
)) foundRoot(store
, target
, target2
, roots
);
308 else if (type
== DT_REG
) {
309 Path storePath
= settings
.nixStore
+ "/" + baseNameOf(path
);
310 if (store
.isValidPath(storePath
))
311 roots
[path
] = storePath
;
316 catch (SysError
& e
) {
317 /* We only ignore permanent failures. */
318 if (e
.errNo
== EACCES
|| e
.errNo
== ENOENT
|| e
.errNo
== ENOTDIR
)
319 printMsg(lvlInfo
, format("cannot read potential root `%1%'") % path
);
326 Roots
LocalStore::findRoots()
330 /* Process direct roots in {gcroots,manifests,profiles}. */
331 nix::findRoots(*this, settings
.nixStateDir
+ "/" + gcRootsDir
, DT_UNKNOWN
, roots
);
332 if (pathExists(settings
.nixStateDir
+ "/manifests"))
333 nix::findRoots(*this, settings
.nixStateDir
+ "/manifests", DT_UNKNOWN
, roots
);
334 nix::findRoots(*this, settings
.nixStateDir
+ "/profiles", DT_UNKNOWN
, roots
);
340 static void addAdditionalRoots(StoreAPI
& store
, PathSet
& roots
)
342 Path rootFinder
= getEnv("NIX_ROOT_FINDER",
343 settings
.nixLibexecDir
+ "/guix/list-runtime-roots");
345 if (rootFinder
.empty()) return;
347 debug(format("executing `%1%' to find additional roots") % rootFinder
);
349 string result
= runProgram(rootFinder
);
351 StringSet paths
= tokenizeString
<StringSet
>(result
, "\n");
353 foreach (StringSet::iterator
, i
, paths
) {
355 Path path
= toStorePath(*i
);
356 if (roots
.find(path
) == roots
.end() && store
.isValidPath(path
)) {
357 debug(format("got additional root `%1%'") % path
);
365 struct GCLimitReached
{ };
368 struct LocalStore::GCState
377 bool gcKeepDerivations
;
378 unsigned long long bytesInvalidated
;
379 bool moveToTrash
= true;
382 GCState(GCResults
& results_
) : results(results_
), bytesInvalidated(0) { }
386 bool LocalStore::isActiveTempFile(const GCState
& state
,
387 const Path
& path
, const string
& suffix
)
389 return hasSuffix(path
, suffix
)
390 && state
.tempRoots
.find(string(path
, 0, path
.size() - suffix
.size())) != state
.tempRoots
.end();
394 void LocalStore::deleteGarbage(GCState
& state
, const Path
& path
)
396 unsigned long long bytesFreed
;
397 deletePath(path
, bytesFreed
);
398 state
.results
.bytesFreed
+= bytesFreed
;
402 void LocalStore::deletePathRecursive(GCState
& state
, const Path
& path
)
406 unsigned long long size
= 0;
408 if (isValidPath(path
)) {
410 queryReferrers(path
, referrers
);
411 foreach (PathSet::iterator
, i
, referrers
)
412 if (*i
!= path
) deletePathRecursive(state
, *i
);
413 size
= queryPathInfo(path
).narSize
;
414 invalidatePathChecked(path
);
418 if (lstat(path
.c_str(), &st
)) {
419 if (errno
== ENOENT
) return;
420 throw SysError(format("getting status of %1%") % path
);
423 printMsg(lvlInfo
, format("deleting `%1%'") % path
);
425 state
.results
.paths
.insert(path
);
427 /* If the path is not a regular file or symlink, move it to the
428 trash directory. The move is to ensure that later (when we're
429 not holding the global GC lock) we can delete the path without
430 being afraid that the path has become alive again. Otherwise
431 delete it right away. */
432 if (state
.moveToTrash
&& S_ISDIR(st
.st_mode
)) {
433 // Estimate the amount freed using the narSize field. FIXME:
434 // if the path was not valid, need to determine the actual
437 if (chmod(path
.c_str(), st
.st_mode
| S_IWUSR
) == -1)
438 throw SysError(format("making `%1%' writable") % path
);
439 Path tmp
= state
.trashDir
+ "/" + baseNameOf(path
);
440 if (rename(path
.c_str(), tmp
.c_str()))
441 throw SysError(format("unable to rename `%1%' to `%2%'") % path
% tmp
);
442 state
.bytesInvalidated
+= size
;
443 } catch (SysError
& e
) {
444 if (e
.errNo
== ENOSPC
) {
445 printMsg(lvlInfo
, format("note: can't create move `%1%': %2%") % path
% e
.msg());
446 deleteGarbage(state
, path
);
450 deleteGarbage(state
, path
);
452 if (state
.results
.bytesFreed
+ state
.bytesInvalidated
> state
.options
.maxFreed
) {
453 printMsg(lvlInfo
, format("deleted or invalidated more than %1% bytes; stopping") % state
.options
.maxFreed
);
454 throw GCLimitReached();
459 bool LocalStore::canReachRoot(GCState
& state
, PathSet
& visited
, const Path
& path
)
461 if (visited
.find(path
) != visited
.end()) return false;
463 if (state
.alive
.find(path
) != state
.alive
.end()) {
467 if (state
.dead
.find(path
) != state
.dead
.end()) {
471 if (state
.roots
.find(path
) != state
.roots
.end()) {
472 printMsg(lvlDebug
, format("cannot delete `%1%' because it's a root") % path
);
473 state
.alive
.insert(path
);
477 visited
.insert(path
);
479 if (!isValidPath(path
)) return false;
483 /* Don't delete this path if any of its referrers are alive. */
484 queryReferrers(path
, incoming
);
486 /* If gc-keep-derivations is set and this is a derivation, then
487 don't delete the derivation if any of the outputs are alive. */
488 if (state
.gcKeepDerivations
&& isDerivation(path
)) {
489 PathSet outputs
= queryDerivationOutputs(path
);
490 foreach (PathSet::iterator
, i
, outputs
)
491 if (isValidPath(*i
) && queryDeriver(*i
) == path
)
495 /* If gc-keep-outputs is set, then don't delete this path if there
496 are derivers of this path that are not garbage. */
497 if (state
.gcKeepOutputs
) {
498 PathSet derivers
= queryValidDerivers(path
);
499 foreach (PathSet::iterator
, i
, derivers
)
503 foreach (PathSet::iterator
, i
, incoming
)
505 if (canReachRoot(state
, visited
, *i
)) {
506 state
.alive
.insert(path
);
514 void LocalStore::tryToDelete(GCState
& state
, const Path
& path
)
518 if (path
== linksDir
|| path
== state
.trashDir
) return;
520 startNest(nest
, lvlDebug
, format("considering whether to delete `%1%'") % path
);
522 if (!isValidPath(path
)) {
523 /* A lock file belonging to a path that we're building right
524 now isn't garbage. */
525 if (isActiveTempFile(state
, path
, ".lock")) return;
527 /* Don't delete .chroot directories for derivations that are
528 currently being built. */
529 if (isActiveTempFile(state
, path
, ".chroot")) return;
534 if (canReachRoot(state
, visited
, path
)) {
535 printMsg(lvlDebug
, format("cannot delete `%1%' because it's still reachable") % path
);
537 /* No path we visited was a root, so everything is garbage.
538 But we only delete ‘path’ and its referrers here so that
539 ‘nix-store --delete’ doesn't have the unexpected effect of
540 recursing into derivations and outputs. */
541 state
.dead
.insert(visited
.begin(), visited
.end());
542 if (state
.shouldDelete
)
543 deletePathRecursive(state
, path
);
548 /* Unlink all files in /nix/store/.links that have a link count of 1,
549 which indicates that there are no other links and so they can be
550 safely deleted. FIXME: race condition with optimisePath(): we
551 might see a link count of 1 just before optimisePath() increases
553 void LocalStore::removeUnusedLinks(const GCState
& state
)
555 AutoCloseDir dir
= opendir(linksDir
.c_str());
556 if (!dir
) throw SysError(format("opening directory `%1%'") % linksDir
);
558 long long actualSize
= 0, unsharedSize
= 0;
560 struct dirent
* dirent
;
561 while (errno
= 0, dirent
= readdir(dir
)) {
563 string name
= dirent
->d_name
;
564 if (name
== "." || name
== "..") continue;
565 Path path
= linksDir
+ "/" + name
;
568 if (lstat(path
.c_str(), &st
) == -1)
569 throw SysError(format("statting `%1%'") % path
);
571 if (st
.st_nlink
!= 1) {
572 unsigned long long size
= st
.st_blocks
* 512ULL;
574 unsharedSize
+= (st
.st_nlink
- 1) * size
;
578 printMsg(lvlTalkative
, format("deleting unused link `%1%'") % path
);
580 if (unlink(path
.c_str()) == -1)
581 throw SysError(format("deleting `%1%'") % path
);
583 state
.results
.bytesFreed
+= st
.st_blocks
* 512;
587 if (stat(linksDir
.c_str(), &st
) == -1)
588 throw SysError(format("statting `%1%'") % linksDir
);
589 long long overhead
= st
.st_blocks
* 512ULL;
591 printMsg(lvlInfo
, format("note: currently hard linking saves %.2f MiB")
592 % ((unsharedSize
- actualSize
- overhead
) / (1024.0 * 1024.0)));
596 void LocalStore::collectGarbage(const GCOptions
& options
, GCResults
& results
)
598 GCState
state(results
);
599 state
.options
= options
;
600 state
.trashDir
= settings
.nixStore
+ "/trash";
601 state
.gcKeepOutputs
= settings
.gcKeepOutputs
;
602 state
.gcKeepDerivations
= settings
.gcKeepDerivations
;
604 /* Using `--ignore-liveness' with `--delete' can have unintended
605 consequences if `gc-keep-outputs' or `gc-keep-derivations' are
606 true (the garbage collector will recurse into deleting the
607 outputs or derivers, respectively). So disable them. */
608 if (options
.action
== GCOptions::gcDeleteSpecific
&& options
.ignoreLiveness
) {
609 state
.gcKeepOutputs
= false;
610 state
.gcKeepDerivations
= false;
613 state
.shouldDelete
= options
.action
== GCOptions::gcDeleteDead
|| options
.action
== GCOptions::gcDeleteSpecific
;
615 /* Acquire the global GC root. This prevents
616 a) New roots from being added.
617 b) Processes from creating new temporary root files. */
618 AutoCloseFD fdGCLock
= openGCLock(ltWrite
);
620 /* Find the roots. Since we've grabbed the GC lock, the set of
621 permanent roots cannot increase now. */
622 printMsg(lvlError
, format("finding garbage collector roots..."));
623 Roots rootMap
= options
.ignoreLiveness
? Roots() : findRoots();
625 foreach (Roots::iterator
, i
, rootMap
) state
.roots
.insert(i
->second
);
627 /* Add additional roots returned by the program specified by the
628 NIX_ROOT_FINDER environment variable. This is typically used
629 to add running programs to the set of roots (to prevent them
630 from being garbage collected). */
631 if (!options
.ignoreLiveness
)
632 addAdditionalRoots(*this, state
.roots
);
634 /* Read the temporary roots. This acquires read locks on all
635 per-process temporary root files. So after this point no paths
636 can be added to the set of temporary roots. */
638 readTempRoots(state
.tempRoots
, fds
);
639 state
.roots
.insert(state
.tempRoots
.begin(), state
.tempRoots
.end());
641 /* After this point the set of roots or temporary roots cannot
642 increase, since we hold locks on everything. So everything
643 that is not reachable from `roots' is garbage. */
645 if (state
.shouldDelete
) {
646 if (pathExists(state
.trashDir
)) deleteGarbage(state
, state
.trashDir
);
648 createDirs(state
.trashDir
);
649 } catch (SysError
& e
) {
650 if (e
.errNo
== ENOSPC
) {
651 printMsg(lvlInfo
, format("note: can't create trash directory: %1%") % e
.msg());
652 state
.moveToTrash
= false;
657 /* Now either delete all garbage paths, or just the specified
658 paths (for gcDeleteSpecific). */
660 if (options
.action
== GCOptions::gcDeleteSpecific
) {
662 foreach (PathSet::iterator
, i
, options
.pathsToDelete
) {
664 tryToDelete(state
, *i
);
665 if (state
.dead
.find(*i
) == state
.dead
.end())
666 throw Error(format("cannot delete path `%1%' since it is still alive") % *i
);
669 } else if (options
.maxFreed
> 0) {
671 if (state
.shouldDelete
)
672 printMsg(lvlError
, format("deleting garbage..."));
674 printMsg(lvlError
, format("determining live/dead paths..."));
678 AutoCloseDir dir
= opendir(settings
.nixStore
.c_str());
679 if (!dir
) throw SysError(format("opening directory `%1%'") % settings
.nixStore
);
681 /* Read the store and immediately delete all paths that
682 aren't valid. When using --max-freed etc., deleting
683 invalid paths is preferred over deleting unreachable
684 paths, since unreachable paths could become reachable
685 again. We don't use readDirectory() here so that GCing
688 struct dirent
* dirent
;
689 while (errno
= 0, dirent
= readdir(dir
)) {
691 string name
= dirent
->d_name
;
692 if (name
== "." || name
== "..") continue;
693 Path path
= settings
.nixStore
+ "/" + name
;
694 if (isValidPath(path
))
695 entries
.push_back(path
);
697 tryToDelete(state
, path
);
702 /* Now delete the unreachable valid paths. Randomise the
703 order in which we delete entries to make the collector
704 less biased towards deleting paths that come
705 alphabetically first (e.g. /nix/store/000...). This
706 matters when using --max-freed etc. */
707 vector
<Path
> entries_(entries
.begin(), entries
.end());
708 random_shuffle(entries_
.begin(), entries_
.end());
710 foreach (vector
<Path
>::iterator
, i
, entries_
)
711 tryToDelete(state
, *i
);
713 } catch (GCLimitReached
& e
) {
717 if (state
.options
.action
== GCOptions::gcReturnLive
) {
718 state
.results
.paths
= state
.alive
;
722 if (state
.options
.action
== GCOptions::gcReturnDead
) {
723 state
.results
.paths
= state
.dead
;
727 /* Allow other processes to add to the store from here on. */
731 /* Delete the trash directory. */
732 printMsg(lvlInfo
, format("deleting `%1%'") % state
.trashDir
);
733 deleteGarbage(state
, state
.trashDir
);
735 /* Clean up the links directory. */
736 if (options
.action
== GCOptions::gcDeleteDead
|| options
.action
== GCOptions::gcDeleteSpecific
) {
737 printMsg(lvlError
, format("deleting unused links..."));
738 removeUnusedLinks(state
);
741 /* While we're at it, vacuum the database. */
742 //if (options.action == GCOptions::gcDeleteDead) vacuumDB();