Commit | Line | Data |
---|---|---|
36457566 LC |
1 | #include "config.h" |
2 | ||
3 | #include "util.hh" | |
4 | #include "local-store.hh" | |
5 | #include "globals.hh" | |
6 | ||
54c260e6 | 7 | #include <cstdlib> |
638ccde1 | 8 | #include <cstring> |
36457566 LC |
9 | #include <sys/types.h> |
10 | #include <sys/stat.h> | |
11 | #include <unistd.h> | |
12 | #include <errno.h> | |
13 | #include <stdio.h> | |
14 | ||
15 | ||
16 | namespace nix { | |
17 | ||
18 | ||
19 | static void makeWritable(const Path & path) | |
20 | { | |
21 | struct stat st; | |
22 | if (lstat(path.c_str(), &st)) | |
23 | throw SysError(format("getting attributes of path `%1%'") % path); | |
24 | if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) | |
25 | throw SysError(format("changing writability of `%1%'") % path); | |
26 | } | |
27 | ||
28 | ||
29 | struct MakeReadOnly | |
30 | { | |
31 | Path path; | |
32 | MakeReadOnly(const Path & path) : path(path) { } | |
33 | ~MakeReadOnly() | |
34 | { | |
35 | try { | |
36 | /* This will make the path read-only. */ | |
37 | if (path != "") canonicaliseTimestampAndPermissions(path); | |
38 | } catch (...) { | |
39 | ignoreException(); | |
40 | } | |
41 | } | |
42 | }; | |
43 | ||
44 | ||
15ddeff5 LC |
45 | LocalStore::InodeHash LocalStore::loadInodeHash() |
46 | { | |
47 | printMsg(lvlDebug, "loading hash inodes in memory"); | |
48 | InodeHash inodeHash; | |
49 | ||
50 | AutoCloseDir dir = opendir(linksDir.c_str()); | |
51 | if (!dir) throw SysError(format("opening directory `%1%'") % linksDir); | |
52 | ||
53 | struct dirent * dirent; | |
54 | while (errno = 0, dirent = readdir(dir)) { /* sic */ | |
55 | checkInterrupt(); | |
56 | // We don't care if we hit non-hash files, anything goes | |
57 | inodeHash.insert(dirent->d_ino); | |
58 | } | |
59 | if (errno) throw SysError(format("reading directory `%1%'") % linksDir); | |
60 | ||
61 | printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); | |
62 | ||
63 | return inodeHash; | |
64 | } | |
65 | ||
66 | ||
67 | Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash) | |
68 | { | |
69 | Strings names; | |
70 | ||
71 | AutoCloseDir dir = opendir(path.c_str()); | |
72 | if (!dir) throw SysError(format("opening directory `%1%'") % path); | |
73 | ||
74 | struct dirent * dirent; | |
75 | while (errno = 0, dirent = readdir(dir)) { /* sic */ | |
76 | checkInterrupt(); | |
77 | ||
78 | if (inodeHash.count(dirent->d_ino)) { | |
79 | printMsg(lvlDebug, format("`%1%' is already linked") % dirent->d_name); | |
80 | continue; | |
81 | } | |
82 | ||
83 | string name = dirent->d_name; | |
84 | if (name == "." || name == "..") continue; | |
85 | names.push_back(name); | |
86 | } | |
87 | if (errno) throw SysError(format("reading directory `%1%'") % path); | |
88 | ||
89 | return names; | |
90 | } | |
91 | ||
92 | ||
93 | void LocalStore::optimisePath_(OptimiseStats & stats, const Path & path, InodeHash & inodeHash) | |
36457566 LC |
94 | { |
95 | checkInterrupt(); | |
15ddeff5 | 96 | |
36457566 LC |
97 | struct stat st; |
98 | if (lstat(path.c_str(), &st)) | |
99 | throw SysError(format("getting attributes of path `%1%'") % path); | |
100 | ||
101 | if (S_ISDIR(st.st_mode)) { | |
15ddeff5 | 102 | Strings names = readDirectoryIgnoringInodes(path, inodeHash); |
36457566 | 103 | foreach (Strings::iterator, i, names) |
15ddeff5 | 104 | optimisePath_(stats, path + "/" + *i, inodeHash); |
36457566 LC |
105 | return; |
106 | } | |
107 | ||
108 | /* We can hard link regular files and maybe symlinks. */ | |
109 | if (!S_ISREG(st.st_mode) | |
110 | #if CAN_LINK_SYMLINK | |
111 | && !S_ISLNK(st.st_mode) | |
112 | #endif | |
113 | ) return; | |
114 | ||
8327e733 | 115 | /* Sometimes SNAFUs can cause files in the store to be |
36457566 | 116 | modified, in particular when running programs as root under |
59e80445 | 117 | Guix System (example: $fontconfig/var/cache being modified). Skip |
36457566 LC |
118 | those files. FIXME: check the modification time. */ |
119 | if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { | |
120 | printMsg(lvlError, format("skipping suspicious writable file `%1%'") % path); | |
121 | return; | |
122 | } | |
123 | ||
e134baae | 124 | /* This can still happen on top-level files. */ |
15ddeff5 LC |
125 | if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) { |
126 | printMsg(lvlDebug, format("`%1%' is already linked, with %2% other file(s).") % path % (st.st_nlink - 2)); | |
127 | return; | |
128 | } | |
129 | ||
36457566 LC |
130 | /* Hash the file. Note that hashPath() returns the hash over the |
131 | NAR serialisation, which includes the execute bit on the file. | |
132 | Thus, executable and non-executable files with the same | |
133 | contents *won't* be linked (which is good because otherwise the | |
134 | permissions would be screwed up). | |
135 | ||
136 | Also note that if `path' is a symlink, then we're hashing the | |
137 | contents of the symlink (i.e. the result of readlink()), not | |
138 | the contents of the target (which may not even exist). */ | |
139 | Hash hash = hashPath(htSHA256, path).first; | |
36457566 LC |
140 | printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); |
141 | ||
142 | /* Check if this is a known hash. */ | |
143 | Path linkPath = linksDir + "/" + printHash32(hash); | |
144 | ||
e134baae | 145 | retry: |
36457566 LC |
146 | if (!pathExists(linkPath)) { |
147 | /* Nope, create a hard link in the links directory. */ | |
15ddeff5 LC |
148 | if (link(path.c_str(), linkPath.c_str()) == 0) { |
149 | inodeHash.insert(st.st_ino); | |
150 | return; | |
151 | } | |
12b6c951 LC |
152 | |
153 | switch (errno) { | |
154 | case EEXIST: | |
155 | /* Fall through if another process created ‘linkPath’ before | |
156 | we did. */ | |
157 | break; | |
158 | ||
159 | case ENOSPC: | |
160 | /* On ext4, that probably means the directory index is full. When | |
161 | that happens, it's fine to ignore it: we just effectively | |
162 | disable deduplication of this file. */ | |
638ccde1 LC |
163 | printMsg(lvlInfo, format("cannot link `%1%' to `%2%': %3%") |
164 | % linkPath % path % strerror(ENOSPC)); | |
12b6c951 LC |
165 | return; |
166 | ||
167 | default: | |
36457566 | 168 | throw SysError(format("cannot link `%1%' to `%2%'") % linkPath % path); |
12b6c951 | 169 | } |
36457566 LC |
170 | } |
171 | ||
172 | /* Yes! We've seen a file with the same contents. Replace the | |
173 | current file with a hard link to that file. */ | |
174 | struct stat stLink; | |
175 | if (lstat(linkPath.c_str(), &stLink)) | |
176 | throw SysError(format("getting attributes of path `%1%'") % linkPath); | |
177 | ||
36457566 LC |
178 | if (st.st_ino == stLink.st_ino) { |
179 | printMsg(lvlDebug, format("`%1%' is already linked to `%2%'") % path % linkPath); | |
180 | return; | |
181 | } | |
182 | ||
e134baae ED |
183 | if (st.st_size != stLink.st_size) { |
184 | printMsg(lvlError, format("removing corrupted link ‘%1%’") % linkPath); | |
185 | unlink(linkPath.c_str()); | |
186 | goto retry; | |
187 | } | |
188 | ||
189 | printMsg(lvlTalkative, format("linking ‘%1%’ to ‘%2%’") % path % linkPath); | |
36457566 LC |
190 | |
191 | /* Make the containing directory writable, but only if it's not | |
192 | the store itself (we don't want or need to mess with its | |
193 | permissions). */ | |
194 | bool mustToggle = !isStorePath(path); | |
195 | if (mustToggle) makeWritable(dirOf(path)); | |
196 | ||
197 | /* When we're done, make the directory read-only again and reset | |
198 | its timestamp back to 0. */ | |
199 | MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); | |
200 | ||
201 | Path tempLink = (format("%1%/.tmp-link-%2%-%3%") | |
202 | % settings.nixStore % getpid() % rand()).str(); | |
203 | ||
204 | if (link(linkPath.c_str(), tempLink.c_str()) == -1) { | |
205 | if (errno == EMLINK) { | |
206 | /* Too many links to the same file (>= 32000 on most file | |
207 | systems). This is likely to happen with empty files. | |
208 | Just shrug and ignore. */ | |
209 | if (st.st_size) | |
210 | printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); | |
211 | return; | |
212 | } | |
12b6c951 LC |
213 | throw SysError(format("cannot link `%1%' to `%2%'") % tempLink % linkPath); |
214 | } | |
36457566 LC |
215 | |
216 | /* Atomically replace the old file with the new hard link. */ | |
217 | if (rename(tempLink.c_str(), path.c_str()) == -1) { | |
b930f0ba | 218 | int renameErrno = errno; |
36457566 LC |
219 | if (unlink(tempLink.c_str()) == -1) |
220 | printMsg(lvlError, format("unable to unlink `%1%'") % tempLink); | |
b930f0ba | 221 | if (renameErrno == EMLINK) { |
36457566 LC |
222 | /* Some filesystems generate too many links on the rename, |
223 | rather than on the original link. (Probably it | |
224 | temporarily increases the st_nlink field before | |
225 | decreasing it again.) */ | |
226 | if (st.st_size) | |
227 | printMsg(lvlInfo, format("`%1%' has maximum number of links") % linkPath); | |
228 | return; | |
229 | } | |
230 | throw SysError(format("cannot rename `%1%' to `%2%'") % tempLink % path); | |
231 | } | |
232 | ||
233 | stats.filesLinked++; | |
234 | stats.bytesFreed += st.st_size; | |
235 | stats.blocksFreed += st.st_blocks; | |
236 | } | |
237 | ||
238 | ||
239 | void LocalStore::optimiseStore(OptimiseStats & stats) | |
240 | { | |
241 | PathSet paths = queryAllValidPaths(); | |
15ddeff5 | 242 | InodeHash inodeHash = loadInodeHash(); |
36457566 LC |
243 | |
244 | foreach (PathSet::iterator, i, paths) { | |
245 | addTempRoot(*i); | |
246 | if (!isValidPath(*i)) continue; /* path was GC'ed, probably */ | |
247 | startNest(nest, lvlChatty, format("hashing files in `%1%'") % *i); | |
15ddeff5 | 248 | optimisePath_(stats, *i, inodeHash); |
36457566 LC |
249 | } |
250 | } | |
251 | ||
2bb04905 LC |
252 | static string showBytes(unsigned long long bytes) |
253 | { | |
254 | return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); | |
255 | } | |
256 | ||
257 | void LocalStore::optimiseStore() | |
258 | { | |
259 | OptimiseStats stats; | |
260 | ||
261 | optimiseStore(stats); | |
262 | ||
263 | printMsg(lvlError, | |
264 | format("%1% freed by hard-linking %2% files") | |
265 | % showBytes(stats.bytesFreed) | |
266 | % stats.filesLinked); | |
267 | } | |
36457566 LC |
268 | |
269 | void LocalStore::optimisePath(const Path & path) | |
270 | { | |
271 | OptimiseStats stats; | |
15ddeff5 LC |
272 | InodeHash inodeHash; |
273 | ||
274 | if (settings.autoOptimiseStore) optimisePath_(stats, path, inodeHash); | |
36457566 LC |
275 | } |
276 | ||
277 | ||
278 | } |