nix/libstore/local-store.cc

   1 #include "config.h"
   2 #include "local-store.hh"
   3 #include "globals.hh"
   4 #include "archive.hh"
   5 #include "pathlocks.hh"
   6 #include "worker-protocol.hh"
   7 #include "derivations.hh"
   8 #include "affinity.hh"
   9
  10 #include <iostream>
  11 #include <algorithm>
  12 #include <cstring>
  13
  14 #include <sys/types.h>
  15 #include <sys/stat.h>
  16 #include <sys/time.h>
  17 #include <unistd.h>
  18 #include <utime.h>
  19 #include <fcntl.h>
  20 #include <errno.h>
  21 #include <stdio.h>
  22 #include <time.h>
  23 #include <grp.h>
  24
  25 #if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H
  26 #include <sched.h>
  27 #include <sys/statvfs.h>
  28 #include <sys/mount.h>
  29 #endif
  30
  31 #include <sys/ioctl.h>
  32 #include <errno.h>
  33
  34 #include <sqlite3.h>
  35
  36
  37 namespace nix {
  38
  39
  40 void checkStoreNotSymlink()
  41 {
  42     if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return;
  43     Path path = settings.nixStore;
  44     struct stat st;
  45     while (path != "/") {
  46         if (lstat(path.c_str(), &st))
  47             throw SysError(format("getting status of `%1%'") % path);
  48         if (S_ISLNK(st.st_mode))
  49             throw Error(format(
  50                 "the path `%1%' is a symlink; "
  51                 "this is not allowed for the store and its parent directories")
  52                 % path);
  53         path = dirOf(path);
  54     }
  55 }
  56
  57
  58 LocalStore::LocalStore(bool reserveSpace)
  59     : didSetSubstituterEnv(false)
  60 {
  61     schemaPath = settings.nixDBPath + "/schema";
  62
  63     if (settings.readOnlyMode) {
  64         openDB(false);
  65         return;
  66     }
  67
  68     /* Create missing state directories if they don't already exist. */
  69     createDirs(settings.nixStore);
  70     makeStoreWritable();
  71     createDirs(linksDir = settings.nixStore + "/.links");
  72     Path profilesDir = settings.nixStateDir + "/profiles";
  73     createDirs(profilesDir);
  74     createDirs(settings.nixStateDir + "/temproots");
  75     createDirs(settings.nixDBPath);
  76     Path gcRootsDir = settings.nixStateDir + "/gcroots";
  77     if (!pathExists(gcRootsDir)) {
  78         createDirs(gcRootsDir);
  79         createSymlink(profilesDir, gcRootsDir + "/profiles");
  80     }
  81
  82     /* Optionally, create directories and set permissions for a
  83        multi-user install. */
  84     if (getuid() == 0 && settings.buildUsersGroup != "") {
  85
  86         Path perUserDir = profilesDir + "/per-user";
  87         createDirs(perUserDir);
  88         if (chmod(perUserDir.c_str(), 0755) == -1)
  89             throw SysError(format("could not set permissions on '%1%' to 755")
  90                            % perUserDir);
  91
  92         mode_t perm = 01775;
  93
  94         struct group * gr = getgrnam(settings.buildUsersGroup.c_str());
  95         if (!gr)
  96             throw Error(format("the group `%1%' specified in `build-users-group' does not exist")
  97                 % settings.buildUsersGroup);
  98         else {
  99             struct stat st;
 100             if (stat(settings.nixStore.c_str(), &st))
 101                 throw SysError(format("getting attributes of path '%1%'") % settings.nixStore);
 102
 103             if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
 104                 if (chown(settings.nixStore.c_str(), 0, gr->gr_gid) == -1)
 105                     throw SysError(format("changing ownership of path '%1%'") % settings.nixStore);
 106                 if (chmod(settings.nixStore.c_str(), perm) == -1)
 107                     throw SysError(format("changing permissions on path '%1%'") % settings.nixStore);
 108             }
 109         }
 110     }
 111
 112     checkStoreNotSymlink();
 113
 114     /* We can't open a SQLite database if the disk is full.  Since
 115        this prevents the garbage collector from running when it's most
 116        needed, we reserve some dummy space that we can free just
 117        before doing a garbage collection. */
 118     try {
 119         Path reservedPath = settings.nixDBPath + "/reserved";
 120         if (reserveSpace) {
 121             struct stat st;
 122             if (stat(reservedPath.c_str(), &st) == -1 ||
 123                 st.st_size != settings.reservedSize)
 124             {
 125                 AutoCloseFD fd = open(reservedPath.c_str(), O_WRONLY | O_CREAT, 0600);
 126                 int res = -1;
 127 #if HAVE_POSIX_FALLOCATE
 128                 res = posix_fallocate(fd, 0, settings.reservedSize);
 129 #endif
 130                 if (res == -1) {
 131                     writeFull(fd, string(settings.reservedSize, 'X'));
 132                     ftruncate(fd, settings.reservedSize);
 133                 }
 134             }
 135         }
 136         else
 137             deletePath(reservedPath);
 138     } catch (SysError & e) { /* don't care about errors */
 139     }
 140
 141     /* Acquire the big fat lock in shared mode to make sure that no
 142        schema upgrade is in progress. */
 143     try {
 144         Path globalLockPath = settings.nixDBPath + "/big-lock";
 145         globalLock = openLockFile(globalLockPath.c_str(), true);
 146     } catch (SysError & e) {
 147         if (e.errNo != EACCES) throw;
 148         settings.readOnlyMode = true;
 149         openDB(false);
 150         return;
 151     }
 152
 153     if (!lockFile(globalLock, ltRead, false)) {
 154         printMsg(lvlError, "waiting for the big store lock...");
 155         lockFile(globalLock, ltRead, true);
 156     }
 157
 158     /* Check the current database schema and if necessary do an
 159        upgrade.  */
 160     int curSchema = getSchema();
 161     if (curSchema > nixSchemaVersion)
 162         throw Error(format("current store schema is version %1%, but I only support %2%")
 163             % curSchema % nixSchemaVersion);
 164
 165     else if (curSchema == 0) { /* new store */
 166         curSchema = nixSchemaVersion;
 167         openDB(true);
 168         writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str());
 169     }
 170
 171     else if (curSchema < nixSchemaVersion) {
 172         /* Guix always used version 7 of the schema.  */
 173         throw Error(
 174             format("Your store database uses an implausibly old schema, version %1%.")
 175             % curSchema);
 176     }
 177
 178     else openDB(false);
 179 }
 180
 181
 182 LocalStore::~LocalStore()
 183 {
 184     try {
 185         if (runningSubstituter) {
 186             RunningSubstituter &i = *runningSubstituter;
 187             if (!i.disabled) {
 188                 i.to.close();
 189                 i.from.close();
 190                 i.error.close();
 191                 if (i.pid != -1)
 192                     i.pid.wait(true);
 193             }
 194         }
 195     } catch (...) {
 196         ignoreException();
 197     }
 198
 199     try {
 200         if (fdTempRoots != -1) {
 201             fdTempRoots.close();
 202             unlink(fnTempRoots.c_str());
 203         }
 204     } catch (...) {
 205         ignoreException();
 206     }
 207 }
 208
 209
 210 int LocalStore::getSchema()
 211 {
 212     int curSchema = 0;
 213     if (pathExists(schemaPath)) {
 214         string s = readFile(schemaPath);
 215         if (!string2Int(s, curSchema))
 216             throw Error(format("`%1%' is corrupt") % schemaPath);
 217     }
 218     return curSchema;
 219 }
 220
 221
 222 void LocalStore::openDB(bool create)
 223 {
 224     if (access(settings.nixDBPath.c_str(), R_OK | W_OK))
 225         throw SysError(format("store database directory `%1%' is not writable") % settings.nixDBPath);
 226
 227     /* Open the store database. */
 228     string dbPath = settings.nixDBPath + "/db.sqlite";
 229     if (sqlite3_open_v2(dbPath.c_str(), &db.db,
 230             SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
 231         throw Error(format("cannot open store database `%1%'") % dbPath);
 232
 233     if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
 234         throwSQLiteError(db, "setting timeout");
 235
 236     if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK)
 237         throwSQLiteError(db, "enabling foreign keys");
 238
 239     /* !!! check whether sqlite has been built with foreign key
 240        support */
 241
 242     /* Whether SQLite should fsync().  "Normal" synchronous mode
 243        should be safe enough.  If the user asks for it, don't sync at
 244        all.  This can cause database corruption if the system
 245        crashes. */
 246     string syncMode = settings.fsyncMetadata ? "normal" : "off";
 247     if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
 248         throwSQLiteError(db, "setting synchronous mode");
 249
 250     /* Set the SQLite journal mode.  WAL mode is fastest, so it's the
 251        default. */
 252     string mode = settings.useSQLiteWAL ? "wal" : "truncate";
 253     string prevMode;
 254     {
 255         SQLiteStmt stmt;
 256         stmt.create(db, "pragma main.journal_mode;");
 257         if (sqlite3_step(stmt) != SQLITE_ROW)
 258             throwSQLiteError(db, "querying journal mode");
 259         prevMode = string((const char *) sqlite3_column_text(stmt, 0));
 260     }
 261     if (prevMode != mode &&
 262         sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
 263         throwSQLiteError(db, "setting journal mode");
 264
 265     /* Increase the auto-checkpoint interval to 40000 pages.  This
 266        seems enough to ensure that instantiating the NixOS system
 267        derivation is done in a single fsync(). */
 268     if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 40000;", 0, 0, 0) != SQLITE_OK)
 269         throwSQLiteError(db, "setting autocheckpoint interval");
 270
 271     /* Initialise the database schema, if necessary. */
 272     if (create) {
 273         const char * schema =
 274 #include "schema.sql.hh"
 275             ;
 276         if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK)
 277             throwSQLiteError(db, "initialising database schema");
 278     }
 279
 280     /* Prepare SQL statements. */
 281     stmtRegisterValidPath.create(db,
 282         "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);");
 283     stmtUpdatePathInfo.create(db,
 284         "update ValidPaths set narSize = ?, hash = ? where path = ?;");
 285     stmtAddReference.create(db,
 286         "insert or replace into Refs (referrer, reference) values (?, ?);");
 287     stmtQueryPathInfo.create(db,
 288         "select id, hash, registrationTime, deriver, narSize from ValidPaths where path = ?;");
 289     stmtQueryReferences.create(db,
 290         "select path from Refs join ValidPaths on reference = id where referrer = ?;");
 291     stmtQueryReferrers.create(db,
 292         "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);");
 293     stmtInvalidatePath.create(db,
 294         "delete from ValidPaths where path = ?;");
 295     stmtRegisterFailedPath.create(db,
 296         "insert or ignore into FailedPaths (path, time) values (?, ?);");
 297     stmtHasPathFailed.create(db,
 298         "select time from FailedPaths where path = ?;");
 299     stmtQueryFailedPaths.create(db,
 300         "select path from FailedPaths;");
 301     // If the path is a derivation, then clear its outputs.
 302     stmtClearFailedPath.create(db,
 303         "delete from FailedPaths where ?1 = '*' or path = ?1 "
 304         "or path in (select d.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where v.path = ?1);");
 305     stmtAddDerivationOutput.create(db,
 306         "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);");
 307     stmtQueryValidDerivers.create(db,
 308         "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;");
 309     stmtQueryDerivationOutputs.create(db,
 310         "select id, path from DerivationOutputs where drv = ?;");
 311     // Use "path >= ?" with limit 1 rather than "path like '?%'" to
 312     // ensure efficient lookup.
 313     stmtQueryPathFromHashPart.create(db,
 314         "select path from ValidPaths where path >= ? limit 1;");
 315     stmtQueryValidPaths.create(db, "select path from ValidPaths");
 316 }
 317
 318
 319 /* To improve purity, users may want to make the store a read-only
 320    bind mount.  So make the store writable for this process. */
 321 void LocalStore::makeStoreWritable()
 322 {
 323 #if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_REMOUNT)
 324     if (getuid() != 0) return;
 325     /* Check if /nix/store is on a read-only mount. */
 326     struct statvfs stat;
 327     if (statvfs(settings.nixStore.c_str(), &stat) != 0)
 328         throw SysError("getting info about the store mount point");
 329
 330     if (stat.f_flag & ST_RDONLY) {
 331         if (unshare(CLONE_NEWNS) == -1)
 332             throw SysError("setting up a private mount namespace");
 333
 334         if (mount(0, settings.nixStore.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
 335             throw SysError(format("remounting %1% writable") % settings.nixStore);
 336     }
 337 #endif
 338 }
 339
 340
 341 const time_t mtimeStore = 1; /* 1 second into the epoch */
 342
 343
 344 static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st)
 345 {
 346     if (!S_ISLNK(st.st_mode)) {
 347
 348         /* Mask out all type related bits. */
 349         mode_t mode = st.st_mode & ~S_IFMT;
 350
 351         if (mode != 0444 && mode != 0555) {
 352             mode = (st.st_mode & S_IFMT)
 353                  | 0444
 354                  | (st.st_mode & S_IXUSR ? 0111 : 0);
 355             if (chmod(path.c_str(), mode) == -1)
 356                 throw SysError(format("changing mode of `%1%' to %2$o") % path % mode);
 357         }
 358
 359     }
 360
 361     if (st.st_mtime != mtimeStore) {
 362         struct timeval times[2];
 363         times[0].tv_sec = st.st_atime;
 364         times[0].tv_usec = 0;
 365         times[1].tv_sec = mtimeStore;
 366         times[1].tv_usec = 0;
 367 #if HAVE_LUTIMES
 368         if (lutimes(path.c_str(), times) == -1)
 369             if (errno != ENOSYS ||
 370                 (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1))
 371 #else
 372         if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
 373 #endif
 374             throw SysError(format("changing modification time of `%1%'") % path);
 375     }
 376 }
 377
 378
 379 void canonicaliseTimestampAndPermissions(const Path & path)
 380 {
 381     struct stat st;
 382     if (lstat(path.c_str(), &st))
 383         throw SysError(format("getting attributes of path `%1%'") % path);
 384     canonicaliseTimestampAndPermissions(path, st);
 385 }
 386
 387
 388 static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSeen & inodesSeen)
 389 {
 390     checkInterrupt();
 391
 392     struct stat st;
 393     if (lstat(path.c_str(), &st))
 394         throw SysError(format("getting attributes of path `%1%'") % path);
 395
 396     /* Really make sure that the path is of a supported type. */
 397     if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
 398         throw Error(format("file ‘%1%’ has an unsupported type") % path);
 399
 400     /* Fail if the file is not owned by the build user.  This prevents
 401        us from messing up the ownership/permissions of files
 402        hard-linked into the output (e.g. "ln /etc/shadow $out/foo").
 403        However, ignore files that we chown'ed ourselves previously to
 404        ensure that we don't fail on hard links within the same build
 405        (i.e. "touch $out/foo; ln $out/foo $out/bar"). */
 406     if (fromUid != (uid_t) -1 && st.st_uid != fromUid) {
 407         assert(!S_ISDIR(st.st_mode));
 408         if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end())
 409             throw BuildError(format("invalid ownership on file `%1%'") % path);
 410         mode_t mode = st.st_mode & ~S_IFMT;
 411         assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
 412         return;
 413     }
 414
 415     inodesSeen.insert(Inode(st.st_dev, st.st_ino));
 416
 417     canonicaliseTimestampAndPermissions(path, st);
 418
 419     /* Change ownership to the current uid.  If it's a symlink, use
 420        lchown if available, otherwise don't bother.  Wrong ownership
 421        of a symlink doesn't matter, since the owning user can't change
 422        the symlink and can't delete it because the directory is not
 423        writable.  The only exception is top-level paths in the
 424        store (since that directory is group-writable for the build
 425        users group); we check for this case below. */
 426     if (st.st_uid != geteuid()) {
 427 #if HAVE_LCHOWN
 428         if (lchown(path.c_str(), geteuid(), getegid()) == -1)
 429 #else
 430         if (!S_ISLNK(st.st_mode) &&
 431             chown(path.c_str(), geteuid(), getegid()) == -1)
 432 #endif
 433             throw SysError(format("changing owner of `%1%' to %2%")
 434                 % path % geteuid());
 435     }
 436
 437     if (S_ISDIR(st.st_mode)) {
 438         DirEntries entries = readDirectory(path);
 439         for (auto & i : entries)
 440             canonicalisePathMetaData_(path + "/" + i.name, fromUid, inodesSeen);
 441     }
 442 }
 443
 444
 445 void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen)
 446 {
 447     canonicalisePathMetaData_(path, fromUid, inodesSeen);
 448
 449     /* On platforms that don't have lchown(), the top-level path can't
 450        be a symlink, since we can't change its ownership. */
 451     struct stat st;
 452     if (lstat(path.c_str(), &st))
 453         throw SysError(format("getting attributes of path `%1%'") % path);
 454
 455     if (st.st_uid != geteuid()) {
 456         assert(S_ISLNK(st.st_mode));
 457         throw Error(format("wrong ownership of top-level store path `%1%'") % path);
 458     }
 459 }
 460
 461
 462 void canonicalisePathMetaData(const Path & path, uid_t fromUid)
 463 {
 464     InodesSeen inodesSeen;
 465     canonicalisePathMetaData(path, fromUid, inodesSeen);
 466 }
 467
 468
 469 void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & drv)
 470 {
 471     string drvName = storePathToName(drvPath);
 472     assert(isDerivation(drvName));
 473     drvName = string(drvName, 0, drvName.size() - drvExtension.size());
 474
 475     if (isFixedOutputDrv(drv)) {
 476         DerivationOutputs::const_iterator out = drv.outputs.find("out");
 477         if (out == drv.outputs.end())
 478             throw Error(format("derivation `%1%' does not have an output named `out'") % drvPath);
 479
 480         bool recursive; HashType ht; Hash h;
 481         out->second.parseHashInfo(recursive, ht, h);
 482         Path outPath = makeFixedOutputPath(recursive, ht, h, drvName);
 483
 484         StringPairs::const_iterator j = drv.env.find("out");
 485         if (out->second.path != outPath || j == drv.env.end() || j->second != outPath)
 486             throw Error(format("derivation `%1%' has incorrect output `%2%', should be `%3%'")
 487                 % drvPath % out->second.path % outPath);
 488     }
 489
 490     else {
 491         Derivation drvCopy(drv);
 492         foreach (DerivationOutputs::iterator, i, drvCopy.outputs) {
 493             i->second.path = "";
 494             drvCopy.env[i->first] = "";
 495         }
 496
 497         Hash h = hashDerivationModulo(*this, drvCopy);
 498
 499         foreach (DerivationOutputs::const_iterator, i, drv.outputs) {
 500             Path outPath = makeOutputPath(i->first, h, drvName);
 501             StringPairs::const_iterator j = drv.env.find(i->first);
 502             if (i->second.path != outPath || j == drv.env.end() || j->second != outPath)
 503                 throw Error(format("derivation `%1%' has incorrect output `%2%', should be `%3%'")
 504                     % drvPath % i->second.path % outPath);
 505         }
 506     }
 507 }
 508
 509
 510 uint64_t LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs)
 511 {
 512     stmtRegisterValidPath.use()
 513         (info.path)
 514         ("sha256:" + printHash(info.hash))
 515         (info.registrationTime == 0 ? time(0) : info.registrationTime)
 516         (info.deriver, info.deriver != "")
 517         (info.narSize, info.narSize != 0)
 518         .exec();
 519     uint64_t id = sqlite3_last_insert_rowid(db);
 520
 521     /* If this is a derivation, then store the derivation outputs in
 522        the database.  This is useful for the garbage collector: it can
 523        efficiently query whether a path is an output of some
 524        derivation. */
 525     if (isDerivation(info.path)) {
 526         Derivation drv = readDerivation(info.path);
 527
 528         /* Verify that the output paths in the derivation are correct
 529            (i.e., follow the scheme for computing output paths from
 530            derivations).  Note that if this throws an error, then the
 531            DB transaction is rolled back, so the path validity
 532            registration above is undone. */
 533         if (checkOutputs) checkDerivationOutputs(info.path, drv);
 534
 535         for (auto & i : drv.outputs) {
 536             stmtAddDerivationOutput.use()
 537                 (id)
 538                 (i.first)
 539                 (i.second.path)
 540                 .exec();
 541         }
 542     }
 543
 544     return id;
 545 }
 546
 547
 548 void LocalStore::addReference(uint64_t referrer, uint64_t reference)
 549 {
 550     stmtAddReference.use()(referrer)(reference).exec();
 551 }
 552
 553
 554 void LocalStore::registerFailedPath(const Path & path)
 555 {
 556     retrySQLite<void>([&]() {
 557         stmtRegisterFailedPath.use()(path)(time(0)).step();
 558     });
 559 }
 560
 561
 562 bool LocalStore::hasPathFailed(const Path & path)
 563 {
 564     return retrySQLite<bool>([&]() {
 565         return stmtHasPathFailed.use()(path).next();
 566     });
 567 }
 568
 569
 570 PathSet LocalStore::queryFailedPaths()
 571 {
 572     return retrySQLite<PathSet>([&]() {
 573         auto useQueryFailedPaths(stmtQueryFailedPaths.use());
 574
 575         PathSet res;
 576         while (useQueryFailedPaths.next())
 577             res.insert(useQueryFailedPaths.getStr(0));
 578
 579         return res;
 580     });
 581 }
 582
 583
 584 void LocalStore::clearFailedPaths(const PathSet & paths)
 585 {
 586     retrySQLite<void>([&]() {
 587         SQLiteTxn txn(db);
 588
 589         for (auto & path : paths)
 590             stmtClearFailedPath.use()(path).exec();
 591
 592         txn.commit();
 593     });
 594 }
 595
 596
 597 Hash parseHashField(const Path & path, const string & s)
 598 {
 599     string::size_type colon = s.find(':');
 600     if (colon == string::npos)
 601         throw Error(format("corrupt hash `%1%' in valid-path entry for `%2%'")
 602             % s % path);
 603     HashType ht = parseHashType(string(s, 0, colon));
 604     if (ht == htUnknown)
 605         throw Error(format("unknown hash type `%1%' in valid-path entry for `%2%'")
 606             % string(s, 0, colon) % path);
 607     return parseHash(ht, string(s, colon + 1));
 608 }
 609
 610
 611 ValidPathInfo LocalStore::queryPathInfo(const Path & path)
 612 {
 613     ValidPathInfo info;
 614     info.path = path;
 615
 616     assertStorePath(path);
 617
 618     return retrySQLite<ValidPathInfo>([&]() {
 619
 620         /* Get the path info. */
 621         auto useQueryPathInfo(stmtQueryPathInfo.use()(path));
 622
 623         if (!useQueryPathInfo.next())
 624             throw Error(format("path `%1%' is not valid") % path);
 625
 626         info.id = useQueryPathInfo.getInt(0);
 627
 628         info.hash = parseHashField(path, useQueryPathInfo.getStr(1));
 629
 630         info.registrationTime = useQueryPathInfo.getInt(2);
 631
 632         auto s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3);
 633         if (s) info.deriver = s;
 634
 635         /* Note that narSize = NULL yields 0. */
 636         info.narSize = useQueryPathInfo.getInt(4);
 637
 638         /* Get the references. */
 639         auto useQueryReferences(stmtQueryReferences.use()(info.id));
 640
 641         while (useQueryReferences.next())
 642             info.references.insert(useQueryReferences.getStr(0));
 643
 644         return info;
 645     });
 646 }
 647
 648
 649 /* Update path info in the database.  Currently only updates the
 650    narSize field. */
 651 void LocalStore::updatePathInfo(const ValidPathInfo & info)
 652 {
 653     stmtUpdatePathInfo.use()
 654         (info.narSize, info.narSize != 0)
 655         ("sha256:" + printHash(info.hash))
 656         (info.path)
 657         .exec();
 658 }
 659
 660
 661 uint64_t LocalStore::queryValidPathId(const Path & path)
 662 {
 663     auto use(stmtQueryPathInfo.use()(path));
 664     if (!use.next())
 665         throw Error(format("path ‘%1%’ is not valid") % path);
 666     return use.getInt(0);
 667 }
 668
 669
 670 bool LocalStore::isValidPath_(const Path & path)
 671 {
 672     return stmtQueryPathInfo.use()(path).next();
 673 }
 674
 675
 676 bool LocalStore::isValidPath(const Path & path)
 677 {
 678     return retrySQLite<bool>([&]() {
 679         return isValidPath_(path);
 680     });
 681 }
 682
 683
 684 PathSet LocalStore::queryValidPaths(const PathSet & paths)
 685 {
 686     return retrySQLite<PathSet>([&]() {
 687         PathSet res;
 688         foreach (PathSet::const_iterator, i, paths)
 689             if (isValidPath_(*i)) res.insert(*i);
 690         return res;
 691     });
 692 }
 693
 694
 695 PathSet LocalStore::queryAllValidPaths()
 696 {
 697     return retrySQLite<PathSet>([&]() {
 698         auto use(stmtQueryValidPaths.use());
 699         PathSet res;
 700         while (use.next()) res.insert(use.getStr(0));
 701         return res;
 702     });
 703 }
 704
 705
 706 void LocalStore::queryReferences(const Path & path,
 707     PathSet & references)
 708 {
 709     ValidPathInfo info = queryPathInfo(path);
 710     references.insert(info.references.begin(), info.references.end());
 711 }
 712
 713
 714 void LocalStore::queryReferrers_(const Path & path, PathSet & referrers)
 715 {
 716     auto useQueryReferrers(stmtQueryReferrers.use()(path));
 717
 718     while (useQueryReferrers.next())
 719         referrers.insert(useQueryReferrers.getStr(0));
 720 }
 721
 722
 723 void LocalStore::queryReferrers(const Path & path, PathSet & referrers)
 724 {
 725     assertStorePath(path);
 726     return retrySQLite<void>([&]() {
 727         queryReferrers_(path, referrers);
 728     });
 729 }
 730
 731
 732 Path LocalStore::queryDeriver(const Path & path)
 733 {
 734     return queryPathInfo(path).deriver;
 735 }
 736
 737
 738 PathSet LocalStore::queryValidDerivers(const Path & path)
 739 {
 740     assertStorePath(path);
 741
 742     return retrySQLite<PathSet>([&]() {
 743         auto useQueryValidDerivers(stmtQueryValidDerivers.use()(path));
 744
 745         PathSet derivers;
 746         while (useQueryValidDerivers.next())
 747             derivers.insert(useQueryValidDerivers.getStr(1));
 748
 749         return derivers;
 750     });
 751 }
 752
 753
 754 PathSet LocalStore::queryDerivationOutputs(const Path & path)
 755 {
 756     return retrySQLite<PathSet>([&]() {
 757         auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path)));
 758
 759         PathSet outputs;
 760         while (useQueryDerivationOutputs.next())
 761             outputs.insert(useQueryDerivationOutputs.getStr(1));
 762
 763         return outputs;
 764     });
 765 }
 766
 767
 768 StringSet LocalStore::queryDerivationOutputNames(const Path & path)
 769 {
 770     return retrySQLite<StringSet>([&]() {
 771         auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path)));
 772
 773         StringSet outputNames;
 774         while (useQueryDerivationOutputs.next())
 775             outputNames.insert(useQueryDerivationOutputs.getStr(0));
 776
 777         return outputNames;
 778     });
 779 }
 780
 781
 782 Path LocalStore::queryPathFromHashPart(const string & hashPart)
 783 {
 784     if (hashPart.size() != 32) throw Error("invalid hash part");
 785
 786     Path prefix = settings.nixStore + "/" + hashPart;
 787
 788     return retrySQLite<Path>([&]() -> Path {
 789         auto useQueryPathFromHashPart(stmtQueryPathFromHashPart.use()(prefix));
 790
 791         if (!useQueryPathFromHashPart.next()) return "";
 792
 793         const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0);
 794         return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : "";
 795     });
 796 }
 797
 798
 799 void LocalStore::setSubstituterEnv()
 800 {
 801     if (didSetSubstituterEnv) return;
 802
 803     /* Pass configuration options (including those overridden with
 804        --option) to substituters. */
 805     setenv("_NIX_OPTIONS", settings.pack().c_str(), 1);
 806
 807     didSetSubstituterEnv = true;
 808 }
 809
 810
 811 void LocalStore::startSubstituter(RunningSubstituter & run)
 812 {
 813     if (run.disabled || run.pid != -1) return;
 814
 815     debug(format("starting substituter program `%1% substitute'")
 816           % settings.guixProgram);
 817
 818     Pipe toPipe, fromPipe, errorPipe;
 819
 820     toPipe.create();
 821     fromPipe.create();
 822     errorPipe.create();
 823
 824     setSubstituterEnv();
 825
 826     run.pid = startProcess([&]() {
 827         if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
 828             throw SysError("dupping stdin");
 829         if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
 830             throw SysError("dupping stdout");
 831         if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1)
 832             throw SysError("dupping stderr");
 833         execl(settings.guixProgram.c_str(), "guix", "substitute", "--query", NULL);
 834         throw SysError(format("executing `%1%'") % settings.guixProgram);
 835     });
 836
 837     run.to = toPipe.writeSide.borrow();
 838     run.from = run.fromBuf.fd = fromPipe.readSide.borrow();
 839     run.error = errorPipe.readSide.borrow();
 840
 841     toPipe.readSide.close();
 842     fromPipe.writeSide.close();
 843     errorPipe.writeSide.close();
 844
 845     /* The substituter may exit right away if it's disabled in any way
 846        (e.g. copy-from-other-stores.pl will exit if no other stores
 847        are configured). */
 848     try {
 849         getLineFromSubstituter(run);
 850     } catch (EndOfFile & e) {
 851         run.to.close();
 852         run.from.close();
 853         run.error.close();
 854         run.disabled = true;
 855         if (run.pid.wait(true) != 0) throw;
 856     }
 857 }
 858
 859
 860 /* Read a line from the substituter's stdout, while also processing
 861    its stderr. */
 862 string LocalStore::getLineFromSubstituter(RunningSubstituter & run)
 863 {
 864     string res, err;
 865
 866     /* We might have stdout data left over from the last time. */
 867     if (run.fromBuf.hasData()) goto haveData;
 868
 869     while (1) {
 870         checkInterrupt();
 871
 872         fd_set fds;
 873         FD_ZERO(&fds);
 874         FD_SET(run.from, &fds);
 875         FD_SET(run.error, &fds);
 876
 877         /* Wait for data to appear on the substituter's stdout or
 878            stderr. */
 879         if (select(run.from > run.error ? run.from + 1 : run.error + 1, &fds, 0, 0, 0) == -1) {
 880             if (errno == EINTR) continue;
 881             throw SysError("waiting for input from the substituter");
 882         }
 883
 884         /* Completely drain stderr before dealing with stdout. */
 885         if (FD_ISSET(run.error, &fds)) {
 886             char buf[4096];
 887             ssize_t n = read(run.error, (unsigned char *) buf, sizeof(buf));
 888             if (n == -1) {
 889                 if (errno == EINTR) continue;
 890                 throw SysError("reading from substituter's stderr");
 891             }
 892             if (n == 0) throw EndOfFile(format("`%1% substitute' died unexpectedly")
 893                                         % settings.guixProgram);
 894             err.append(buf, n);
 895             string::size_type p;
 896             while (((p = err.find('\n')) != string::npos)
 897                    || ((p = err.find('\r')) != string::npos)) {
 898                 string thing(err, 0, p + 1);
 899                 writeToStderr("substitute: " + thing);
 900                 err = string(err, p + 1);
 901             }
 902         }
 903
 904         /* Read from stdout until we get a newline or the buffer is empty. */
 905         else if (run.fromBuf.hasData() || FD_ISSET(run.from, &fds)) {
 906         haveData:
 907             do {
 908                 unsigned char c;
 909                 run.fromBuf(&c, 1);
 910                 if (c == '\n') {
 911                     if (!err.empty()) printMsg(lvlError, "substitute: " + err);
 912                     return res;
 913                 }
 914                 res += c;
 915             } while (run.fromBuf.hasData());
 916         }
 917     }
 918 }
 919
 920
 921 template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & run)
 922 {
 923     string s = getLineFromSubstituter(run);
 924     T res;
 925     if (!string2Int(s, res)) throw Error("integer expected from stream");
 926     return res;
 927 }
 928
 929
 930 PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
 931 {
 932     PathSet res;
 933
 934     if (!settings.useSubstitutes || paths.empty()) return res;
 935
 936     if (!runningSubstituter) {
 937         std::unique_ptr<RunningSubstituter>fresh(new RunningSubstituter);
 938         runningSubstituter.swap(fresh);
 939     }
 940
 941     RunningSubstituter & run = *runningSubstituter;
 942     startSubstituter(run);
 943
 944     if (!run.disabled) {
 945         string s = "have ";
 946         foreach (PathSet::const_iterator, j, paths)
 947             if (res.find(*j) == res.end()) { s += *j; s += " "; }
 948         writeLine(run.to, s);
 949         while (true) {
 950             /* FIXME: we only read stderr when an error occurs, so
 951                substituters should only write (short) messages to
 952                stderr when they fail.  I.e. they shouldn't write debug
 953                output. */
 954             Path path = getLineFromSubstituter(run);
 955             if (path == "") break;
 956             res.insert(path);
 957         }
 958     }
 959
 960     return res;
 961 }
 962
 963
 964 void LocalStore::querySubstitutablePathInfos(PathSet & paths, SubstitutablePathInfos & infos)
 965 {
 966     if (!settings.useSubstitutes) return;
 967
 968     if (!runningSubstituter) {
 969         std::unique_ptr<RunningSubstituter>fresh(new RunningSubstituter);
 970         runningSubstituter.swap(fresh);
 971     }
 972
 973     RunningSubstituter & run = *runningSubstituter;
 974     startSubstituter(run);
 975     if (run.disabled) return;
 976
 977     string s = "info ";
 978     foreach (PathSet::const_iterator, i, paths)
 979         if (infos.find(*i) == infos.end()) { s += *i; s += " "; }
 980     writeLine(run.to, s);
 981
 982     while (true) {
 983         Path path = getLineFromSubstituter(run);
 984         if (path == "") break;
 985         if (paths.find(path) == paths.end())
 986             throw Error(format("got unexpected path `%1%' from substituter") % path);
 987         paths.erase(path);
 988         SubstitutablePathInfo & info(infos[path]);
 989         info.deriver = getLineFromSubstituter(run);
 990         if (info.deriver != "") assertStorePath(info.deriver);
 991         int nrRefs = getIntLineFromSubstituter<int>(run);
 992         while (nrRefs--) {
 993             Path p = getLineFromSubstituter(run);
 994             assertStorePath(p);
 995             info.references.insert(p);
 996         }
 997         info.downloadSize = getIntLineFromSubstituter<long long>(run);
 998         info.narSize = getIntLineFromSubstituter<long long>(run);
 999     }
1000 }
1001
1002
1003 void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
1004     SubstitutablePathInfos & infos)
1005 {
1006     if (!paths.empty()) {
1007         PathSet todo = paths;
1008         querySubstitutablePathInfos(todo, infos);
1009     }
1010 }
1011
1012
1013 Hash LocalStore::queryPathHash(const Path & path)
1014 {
1015     return queryPathInfo(path).hash;
1016 }
1017
1018
1019 void LocalStore::registerValidPath(const ValidPathInfo & info)
1020 {
1021     ValidPathInfos infos;
1022     infos.push_back(info);
1023     registerValidPaths(infos);
1024 }
1025
1026
1027 void LocalStore::registerValidPaths(const ValidPathInfos & infos)
1028 {
1029     /* SQLite will fsync by default, but the new valid paths may not be fsync-ed.
1030      * So some may want to fsync them before registering the validity, at the
1031      * expense of some speed of the path registering operation. */
1032     if (settings.syncBeforeRegistering) sync();
1033
1034     return retrySQLite<void>([&]() {
1035         SQLiteTxn txn(db);
1036         PathSet paths;
1037
1038         foreach (ValidPathInfos::const_iterator, i, infos) {
1039             assert(i->hash.type == htSHA256);
1040             if (isValidPath_(i->path))
1041                 updatePathInfo(*i);
1042             else
1043                 addValidPath(*i, false);
1044             paths.insert(i->path);
1045         }
1046
1047         for (auto & i : infos) {
1048             auto referrer = queryValidPathId(i.path);
1049             for (auto & j : i.references)
1050                 addReference(referrer, queryValidPathId(j));
1051         }
1052
1053         /* Check that the derivation outputs are correct.  We can't do
1054            this in addValidPath() above, because the references might
1055            not be valid yet. */
1056         foreach (ValidPathInfos::const_iterator, i, infos)
1057             if (isDerivation(i->path)) {
1058                 // FIXME: inefficient; we already loaded the
1059                 // derivation in addValidPath().
1060                 Derivation drv = readDerivation(i->path);
1061                 checkDerivationOutputs(i->path, drv);
1062             }
1063
1064         /* Do a topological sort of the paths.  This will throw an
1065            error if a cycle is detected and roll back the
1066            transaction.  Cycles can only occur when a derivation
1067            has multiple outputs. */
1068         topoSortPaths(*this, paths);
1069
1070         txn.commit();
1071     });
1072 }
1073
1074
1075 /* Invalidate a path.  The caller is responsible for checking that
1076    there are no referrers. */
1077 void LocalStore::invalidatePath(const Path & path)
1078 {
1079     debug(format("invalidating path `%1%'") % path);
1080
1081     drvHashes.erase(path);
1082
1083     stmtInvalidatePath.use()(path).exec();
1084
1085     /* Note that the foreign key constraints on the Refs table take
1086        care of deleting the references entries for `path'. */
1087 }
1088
1089
1090 Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
1091     bool recursive, HashType hashAlgo, bool repair)
1092 {
1093     Hash h = hashString(hashAlgo, dump);
1094
1095     Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name);
1096
1097     addTempRoot(dstPath);
1098
1099     if (repair || !isValidPath(dstPath)) {
1100
1101         /* The first check above is an optimisation to prevent
1102            unnecessary lock acquisition. */
1103
1104         PathLocks outputLock(singleton<PathSet, Path>(dstPath));
1105
1106         if (repair || !isValidPath(dstPath)) {
1107
1108             if (pathExists(dstPath)) deletePath(dstPath);
1109
1110             if (recursive) {
1111                 StringSource source(dump);
1112                 restorePath(dstPath, source);
1113             } else
1114                 writeFile(dstPath, dump);
1115
1116             canonicalisePathMetaData(dstPath, -1);
1117
1118             /* Register the SHA-256 hash of the NAR serialisation of
1119                the path in the database.  We may just have computed it
1120                above (if called with recursive == true and hashAlgo ==
1121                sha256); otherwise, compute it here. */
1122             HashResult hash;
1123             if (recursive) {
1124                 hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
1125                 hash.second = dump.size();
1126             } else
1127                 hash = hashPath(htSHA256, dstPath);
1128
1129             optimisePath(dstPath); // FIXME: combine with hashPath()
1130
1131             ValidPathInfo info;
1132             info.path = dstPath;
1133             info.hash = hash.first;
1134             info.narSize = hash.second;
1135             registerValidPath(info);
1136         }
1137
1138         outputLock.setDeletion(true);
1139     }
1140
1141     return dstPath;
1142 }
1143
1144
1145 Path LocalStore::addToStore(const string & name, const Path & _srcPath,
1146     bool recursive, HashType hashAlgo, PathFilter & filter, bool repair)
1147 {
1148     Path srcPath(absPath(_srcPath));
1149     debug(format("adding `%1%' to the store") % srcPath);
1150
1151     /* Read the whole path into memory. This is not a very scalable
1152        method for very large paths, but `copyPath' is mainly used for
1153        small files. */
1154     StringSink sink;
1155     if (recursive)
1156         dumpPath(srcPath, sink, filter);
1157     else
1158         sink.s = readFile(srcPath);
1159
1160     return addToStoreFromDump(sink.s, name, recursive, hashAlgo, repair);
1161 }
1162
1163
1164 Path LocalStore::addTextToStore(const string & name, const string & s,
1165     const PathSet & references, bool repair)
1166 {
1167     Path dstPath = computeStorePathForText(name, s, references);
1168
1169     addTempRoot(dstPath);
1170
1171     if (repair || !isValidPath(dstPath)) {
1172
1173         PathLocks outputLock(singleton<PathSet, Path>(dstPath));
1174
1175         if (repair || !isValidPath(dstPath)) {
1176
1177             if (pathExists(dstPath)) deletePath(dstPath);
1178
1179             writeFile(dstPath, s);
1180
1181             canonicalisePathMetaData(dstPath, -1);
1182
1183             HashResult hash = hashPath(htSHA256, dstPath);
1184
1185             optimisePath(dstPath);
1186
1187             ValidPathInfo info;
1188             info.path = dstPath;
1189             info.hash = hash.first;
1190             info.narSize = hash.second;
1191             info.references = references;
1192             registerValidPath(info);
1193         }
1194
1195         outputLock.setDeletion(true);
1196     }
1197
1198     return dstPath;
1199 }
1200
1201
1202 struct HashAndWriteSink : Sink
1203 {
1204     Sink & writeSink;
1205     HashSink hashSink;
1206     HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
1207     {
1208     }
1209     virtual void operator () (const unsigned char * data, size_t len)
1210     {
1211         writeSink(data, len);
1212         hashSink(data, len);
1213     }
1214     Hash currentHash()
1215     {
1216         return hashSink.currentHash().first;
1217     }
1218 };
1219
1220
1221 #define EXPORT_MAGIC 0x4558494e
1222
1223
1224 static void checkSecrecy(const Path & path)
1225 {
1226     struct stat st;
1227     if (stat(path.c_str(), &st))
1228         throw SysError(format("getting status of `%1%'") % path);
1229     if ((st.st_mode & (S_IRWXG | S_IRWXO)) != 0)
1230         throw Error(format("file `%1%' should be secret (inaccessible to everybody else)!") % path);
1231 }
1232
1233
1234 static std::string runAuthenticationProgram(const Strings & args)
1235 {
1236     Strings fullArgs = { "authenticate" };
1237     fullArgs.insert(fullArgs.end(), args.begin(), args.end()); // append
1238     return runProgram(settings.guixProgram, false, fullArgs);
1239 }
1240
1241 /* Sign HASH with the key stored in file SECRETKEY.  Return the signature as a
1242    string, or raise an exception upon error.  */
1243 static std::string signHash(const string &secretKey, const Hash &hash)
1244 {
1245     Strings args;
1246     args.push_back("sign");
1247     args.push_back(secretKey);
1248     args.push_back(printHash(hash));
1249
1250     return runAuthenticationProgram(args);
1251 }
1252
1253 /* Verify SIGNATURE and return the base16-encoded hash over which it was
1254    computed.  */
1255 static std::string verifySignature(const string &signature)
1256 {
1257     Path tmpDir = createTempDir("", "guix", true, true, 0700);
1258     AutoDelete delTmp(tmpDir);
1259
1260     Path sigFile = tmpDir + "/sig";
1261     writeFile(sigFile, signature);
1262
1263     Strings args;
1264     args.push_back("verify");
1265     args.push_back(sigFile);
1266     return runAuthenticationProgram(args);
1267 }
1268
1269 void LocalStore::exportPath(const Path & path, bool sign,
1270     Sink & sink)
1271 {
1272     assertStorePath(path);
1273
1274     printMsg(lvlInfo, format("exporting path `%1%'") % path);
1275
1276     if (!isValidPath(path))
1277         throw Error(format("path `%1%' is not valid") % path);
1278
1279     HashAndWriteSink hashAndWriteSink(sink);
1280
1281     dumpPath(path, hashAndWriteSink);
1282
1283     /* Refuse to export paths that have changed.  This prevents
1284        filesystem corruption from spreading to other machines.
1285        Don't complain if the stored hash is zero (unknown). */
1286     Hash hash = hashAndWriteSink.currentHash();
1287     Hash storedHash = queryPathHash(path);
1288     if (hash != storedHash && storedHash != Hash(storedHash.type))
1289         throw Error(format("hash of path `%1%' has changed from `%2%' to `%3%'!") % path
1290             % printHash(storedHash) % printHash(hash));
1291
1292     writeInt(EXPORT_MAGIC, hashAndWriteSink);
1293
1294     writeString(path, hashAndWriteSink);
1295
1296     PathSet references;
1297     queryReferences(path, references);
1298     writeStrings(references, hashAndWriteSink);
1299
1300     Path deriver = queryDeriver(path);
1301     writeString(deriver, hashAndWriteSink);
1302
1303     if (sign) {
1304         Hash hash = hashAndWriteSink.currentHash();
1305
1306         writeInt(1, hashAndWriteSink);
1307
1308         Path secretKey = settings.nixConfDir + "/signing-key.sec";
1309         checkSecrecy(secretKey);
1310
1311         string signature = signHash(secretKey, hash);
1312
1313         writeString(signature, hashAndWriteSink);
1314
1315     } else
1316         writeInt(0, hashAndWriteSink);
1317 }
1318
1319
1320 struct HashAndReadSource : Source
1321 {
1322     Source & readSource;
1323     HashSink hashSink;
1324     bool hashing;
1325     HashAndReadSource(Source & readSource) : readSource(readSource), hashSink(htSHA256)
1326     {
1327         hashing = true;
1328     }
1329     size_t read(unsigned char * data, size_t len)
1330     {
1331         size_t n = readSource.read(data, len);
1332         if (hashing) hashSink(data, n);
1333         return n;
1334     }
1335 };
1336
1337
1338 /* Create a temporary directory in the store that won't be
1339    garbage-collected. */
1340 Path LocalStore::createTempDirInStore()
1341 {
1342     Path tmpDir;
1343     do {
1344         /* There is a slight possibility that `tmpDir' gets deleted by
1345            the GC between createTempDir() and addTempRoot(), so repeat
1346            until `tmpDir' exists. */
1347         tmpDir = createTempDir(settings.nixStore);
1348         addTempRoot(tmpDir);
1349     } while (!pathExists(tmpDir));
1350     return tmpDir;
1351 }
1352
1353
1354 Path LocalStore::importPath(bool requireSignature, Source & source)
1355 {
1356     HashAndReadSource hashAndReadSource(source);
1357
1358     /* We don't yet know what store path this archive contains (the
1359        store path follows the archive data proper), and besides, we
1360        don't know yet whether the signature is valid. */
1361     Path tmpDir = createTempDirInStore();
1362     AutoDelete delTmp(tmpDir);
1363     Path unpacked = tmpDir + "/unpacked";
1364
1365     restorePath(unpacked, hashAndReadSource);
1366
1367     unsigned int magic = readInt(hashAndReadSource);
1368     if (magic != EXPORT_MAGIC)
1369         throw Error("normalized archive cannot be imported; wrong format");
1370
1371     Path dstPath = readStorePath(hashAndReadSource);
1372
1373     PathSet references = readStorePaths<PathSet>(hashAndReadSource);
1374
1375     Path deriver = readString(hashAndReadSource);
1376     if (deriver != "") assertStorePath(deriver);
1377
1378     Hash hash = hashAndReadSource.hashSink.finish().first;
1379     hashAndReadSource.hashing = false;
1380
1381     bool haveSignature = readInt(hashAndReadSource) == 1;
1382
1383     if (requireSignature && !haveSignature)
1384         throw Error(format("imported archive of `%1%' lacks a signature") % dstPath);
1385
1386     if (haveSignature) {
1387         string signature = readString(hashAndReadSource);
1388
1389         if (requireSignature) {
1390             string hash2 = verifySignature(signature);
1391
1392             /* Note: runProgram() throws an exception if the signature
1393                is invalid. */
1394
1395             if (printHash(hash) != hash2)
1396                 throw Error(
1397                     "signed hash doesn't match actual contents of imported "
1398                     "archive; archive could be corrupt, or someone is trying "
1399                     "to import a Trojan horse");
1400         }
1401     }
1402
1403     /* Do the actual import. */
1404
1405     /* !!! way too much code duplication with addTextToStore() etc. */
1406     addTempRoot(dstPath);
1407
1408     if (!isValidPath(dstPath)) {
1409
1410         PathLocks outputLock;
1411
1412         /* Lock the output path.  But don't lock if we're being called
1413            from a build hook (whose parent process already acquired a
1414            lock on this path). */
1415         Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS"));
1416         if (find(locksHeld.begin(), locksHeld.end(), dstPath) == locksHeld.end())
1417             outputLock.lockPaths(singleton<PathSet, Path>(dstPath));
1418
1419         if (!isValidPath(dstPath)) {
1420
1421             if (pathExists(dstPath)) deletePath(dstPath);
1422
1423             if (rename(unpacked.c_str(), dstPath.c_str()) == -1)
1424                 throw SysError(format("cannot move `%1%' to `%2%'")
1425                     % unpacked % dstPath);
1426
1427             canonicalisePathMetaData(dstPath, -1);
1428
1429             /* !!! if we were clever, we could prevent the hashPath()
1430                here. */
1431             HashResult hash = hashPath(htSHA256, dstPath);
1432
1433             optimisePath(dstPath); // FIXME: combine with hashPath()
1434
1435             ValidPathInfo info;
1436             info.path = dstPath;
1437             info.hash = hash.first;
1438             info.narSize = hash.second;
1439             info.references = references;
1440             info.deriver = deriver != "" && isValidPath(deriver) ? deriver : "";
1441             registerValidPath(info);
1442         }
1443
1444         outputLock.setDeletion(true);
1445     }
1446
1447     return dstPath;
1448 }
1449
1450
1451 Paths LocalStore::importPaths(bool requireSignature, Source & source)
1452 {
1453     Paths res;
1454     while (true) {
1455         unsigned long long n = readLongLong(source);
1456         if (n == 0) break;
1457         if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'");
1458         res.push_back(importPath(requireSignature, source));
1459     }
1460     return res;
1461 }
1462
1463
1464 void LocalStore::invalidatePathChecked(const Path & path)
1465 {
1466     assertStorePath(path);
1467
1468     retrySQLite<void>([&]() {
1469         SQLiteTxn txn(db);
1470
1471         if (isValidPath_(path)) {
1472             PathSet referrers; queryReferrers_(path, referrers);
1473             referrers.erase(path); /* ignore self-references */
1474             if (!referrers.empty())
1475                 throw PathInUse(format("cannot delete path `%1%' because it is in use by %2%")
1476                     % path % showPaths(referrers));
1477             invalidatePath(path);
1478         }
1479
1480         txn.commit();
1481     });
1482 }
1483
1484
1485 bool LocalStore::verifyStore(bool checkContents, bool repair)
1486 {
1487     printMsg(lvlError, format("reading the store..."));
1488
1489     bool errors = false;
1490
1491     /* Acquire the global GC lock to prevent a garbage collection. */
1492     AutoCloseFD fdGCLock = openGCLock(ltWrite);
1493
1494     PathSet store;
1495     for (auto & i : readDirectory(settings.nixStore)) store.insert(i.name);
1496
1497     /* Check whether all valid paths actually exist. */
1498     printMsg(lvlInfo, "checking path existence...");
1499
1500     PathSet validPaths2 = queryAllValidPaths(), validPaths, done;
1501
1502     foreach (PathSet::iterator, i, validPaths2)
1503         verifyPath(*i, store, done, validPaths, repair, errors);
1504
1505     /* Release the GC lock so that checking content hashes (which can
1506        take ages) doesn't block the GC or builds. */
1507     fdGCLock.close();
1508
1509     /* Optionally, check the content hashes (slow). */
1510     if (checkContents) {
1511         printMsg(lvlInfo, "checking hashes...");
1512
1513         Hash nullHash(htSHA256);
1514
1515         foreach (PathSet::iterator, i, validPaths) {
1516             try {
1517                 ValidPathInfo info = queryPathInfo(*i);
1518
1519                 /* Check the content hash (optionally - slow). */
1520                 printMsg(lvlTalkative, format("checking contents of `%1%'") % *i);
1521                 HashResult current = hashPath(info.hash.type, *i);
1522
1523                 if (info.hash != nullHash && info.hash != current.first) {
1524                     printMsg(lvlError, format("path `%1%' was modified! "
1525                             "expected hash `%2%', got `%3%'")
1526                         % *i % printHash(info.hash) % printHash(current.first));
1527                     if (repair) repairPath(*i); else errors = true;
1528                 } else {
1529
1530                     bool update = false;
1531
1532                     /* Fill in missing hashes. */
1533                     if (info.hash == nullHash) {
1534                         printMsg(lvlError, format("fixing missing hash on `%1%'") % *i);
1535                         info.hash = current.first;
1536                         update = true;
1537                     }
1538
1539                     /* Fill in missing narSize fields (from old stores). */
1540                     if (info.narSize == 0) {
1541                         printMsg(lvlError, format("updating size field on `%1%' to %2%") % *i % current.second);
1542                         info.narSize = current.second;
1543                         update = true;
1544                     }
1545
1546                     if (update) updatePathInfo(info);
1547
1548                 }
1549
1550             } catch (Error & e) {
1551                 /* It's possible that the path got GC'ed, so ignore
1552                    errors on invalid paths. */
1553                 if (isValidPath(*i))
1554                     printMsg(lvlError, format("error: %1%") % e.msg());
1555                 else
1556                     printMsg(lvlError, format("warning: %1%") % e.msg());
1557                 errors = true;
1558             }
1559         }
1560     }
1561
1562     return errors;
1563 }
1564
1565
1566 void LocalStore::verifyPath(const Path & path, const PathSet & store,
1567     PathSet & done, PathSet & validPaths, bool repair, bool & errors)
1568 {
1569     checkInterrupt();
1570
1571     if (done.find(path) != done.end()) return;
1572     done.insert(path);
1573
1574     if (!isStorePath(path)) {
1575         printMsg(lvlError, format("path `%1%' is not in the store") % path);
1576         invalidatePath(path);
1577         return;
1578     }
1579
1580     if (store.find(baseNameOf(path)) == store.end()) {
1581         /* Check any referrers first.  If we can invalidate them
1582            first, then we can invalidate this path as well. */
1583         bool canInvalidate = true;
1584         PathSet referrers; queryReferrers(path, referrers);
1585         foreach (PathSet::iterator, i, referrers)
1586             if (*i != path) {
1587                 verifyPath(*i, store, done, validPaths, repair, errors);
1588                 if (validPaths.find(*i) != validPaths.end())
1589                     canInvalidate = false;
1590             }
1591
1592         if (canInvalidate) {
1593             printMsg(lvlError, format("path `%1%' disappeared, removing from database...") % path);
1594             invalidatePath(path);
1595         } else {
1596             printMsg(lvlError, format("path `%1%' disappeared, but it still has valid referrers!") % path);
1597             if (repair)
1598                 try {
1599                     repairPath(path);
1600                 } catch (Error & e) {
1601                     printMsg(lvlError, format("warning: %1%") % e.msg());
1602                     errors = true;
1603                 }
1604             else errors = true;
1605         }
1606
1607         return;
1608     }
1609
1610     validPaths.insert(path);
1611 }
1612
1613
1614 bool LocalStore::pathContentsGood(const Path & path)
1615 {
1616     std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path);
1617     if (i != pathContentsGoodCache.end()) return i->second;
1618     printMsg(lvlInfo, format("checking path `%1%'...") % path);
1619     ValidPathInfo info = queryPathInfo(path);
1620     bool res;
1621     if (!pathExists(path))
1622         res = false;
1623     else {
1624         HashResult current = hashPath(info.hash.type, path);
1625         Hash nullHash(htSHA256);
1626         res = info.hash == nullHash || info.hash == current.first;
1627     }
1628     pathContentsGoodCache[path] = res;
1629     if (!res) printMsg(lvlError, format("path `%1%' is corrupted or missing!") % path);
1630     return res;
1631 }
1632
1633
1634 void LocalStore::markContentsGood(const Path & path)
1635 {
1636     pathContentsGoodCache[path] = true;
1637 }
1638
1639
1640 void LocalStore::vacuumDB()
1641 {
1642     if (sqlite3_exec(db, "vacuum;", 0, 0, 0) != SQLITE_OK)
1643         throwSQLiteError(db, "vacuuming SQLite database");
1644 }
1645
1646
1647 void LocalStore::createUser(const std::string & userName, uid_t userId)
1648 {
1649     auto dir = settings.nixStateDir + "/profiles/per-user/" + userName;
1650
1651     createDirs(dir);
1652     if (chmod(dir.c_str(), 0755) == -1)
1653         throw SysError(format("changing permissions of directory '%s'") % dir);
1654     if (chown(dir.c_str(), userId, -1) == -1)
1655         throw SysError(format("changing owner of directory '%s'") % dir);
1656 }
1657
1658
1659 }