nix/libstore/local-store.cc

   1 #include "config.h"
   2 #include "local-store.hh"
   3 #include "globals.hh"
   4 #include "archive.hh"
   5 #include "pathlocks.hh"
   6 #include "worker-protocol.hh"
   7 #include "derivations.hh"
   8 #include "affinity.hh"
   9
  10 #include <iostream>
  11 #include <algorithm>
  12 #include <cstring>
  13
  14 #include <sys/types.h>
  15 #include <sys/stat.h>
  16 #include <sys/time.h>
  17 #include <unistd.h>
  18 #include <utime.h>
  19 #include <fcntl.h>
  20 #include <errno.h>
  21 #include <stdio.h>
  22 #include <time.h>
  23 #include <grp.h>
  24
  25 #if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H
  26 #include <sched.h>
  27 #include <sys/statvfs.h>
  28 #include <sys/mount.h>
  29 #endif
  30
  31 #include <sys/ioctl.h>
  32 #include <errno.h>
  33
  34 #include <sqlite3.h>
  35
  36
  37 namespace nix {
  38
  39
  40 void checkStoreNotSymlink()
  41 {
  42     if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return;
  43     Path path = settings.nixStore;
  44     struct stat st;
  45     while (path != "/") {
  46         if (lstat(path.c_str(), &st))
  47             throw SysError(format("getting status of `%1%'") % path);
  48         if (S_ISLNK(st.st_mode))
  49             throw Error(format(
  50                 "the path `%1%' is a symlink; "
  51                 "this is not allowed for the store and its parent directories")
  52                 % path);
  53         path = dirOf(path);
  54     }
  55 }
  56
  57
  58 LocalStore::LocalStore(bool reserveSpace)
  59     : didSetSubstituterEnv(false)
  60 {
  61     schemaPath = settings.nixDBPath + "/schema";
  62
  63     if (settings.readOnlyMode) {
  64         openDB(false);
  65         return;
  66     }
  67
  68     /* Create missing state directories if they don't already exist. */
  69     createDirs(settings.nixStore);
  70     makeStoreWritable();
  71     createDirs(linksDir = settings.nixStore + "/.links");
  72     Path profilesDir = settings.nixStateDir + "/profiles";
  73     createDirs(profilesDir);
  74     createDirs(settings.nixStateDir + "/temproots");
  75     createDirs(settings.nixDBPath);
  76     Path gcRootsDir = settings.nixStateDir + "/gcroots";
  77     if (!pathExists(gcRootsDir)) {
  78         createDirs(gcRootsDir);
  79         createSymlink(profilesDir, gcRootsDir + "/profiles");
  80     }
  81
  82     /* Optionally, create directories and set permissions for a
  83        multi-user install. */
  84     if (getuid() == 0 && settings.buildUsersGroup != "") {
  85
  86         Path perUserDir = profilesDir + "/per-user";
  87         createDirs(perUserDir);
  88         if (chmod(perUserDir.c_str(), 0755) == -1)
  89             throw SysError(format("could not set permissions on '%1%' to 755")
  90                            % perUserDir);
  91
  92         mode_t perm = 01775;
  93
  94         struct group * gr = getgrnam(settings.buildUsersGroup.c_str());
  95         if (!gr)
  96             throw Error(format("the group `%1%' specified in `build-users-group' does not exist")
  97                 % settings.buildUsersGroup);
  98         else {
  99             struct stat st;
 100             if (stat(settings.nixStore.c_str(), &st))
 101                 throw SysError(format("getting attributes of path '%1%'") % settings.nixStore);
 102
 103             if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
 104                 if (chown(settings.nixStore.c_str(), 0, gr->gr_gid) == -1)
 105                     throw SysError(format("changing ownership of path '%1%'") % settings.nixStore);
 106                 if (chmod(settings.nixStore.c_str(), perm) == -1)
 107                     throw SysError(format("changing permissions on path '%1%'") % settings.nixStore);
 108             }
 109         }
 110     }
 111
 112     checkStoreNotSymlink();
 113
 114     /* We can't open a SQLite database if the disk is full.  Since
 115        this prevents the garbage collector from running when it's most
 116        needed, we reserve some dummy space that we can free just
 117        before doing a garbage collection. */
 118     try {
 119         Path reservedPath = settings.nixDBPath + "/reserved";
 120         if (reserveSpace) {
 121             struct stat st;
 122             if (stat(reservedPath.c_str(), &st) == -1 ||
 123                 st.st_size != settings.reservedSize)
 124             {
 125                 AutoCloseFD fd = open(reservedPath.c_str(), O_WRONLY | O_CREAT, 0600);
 126                 int res = -1;
 127 #if HAVE_POSIX_FALLOCATE
 128                 res = posix_fallocate(fd, 0, settings.reservedSize);
 129 #endif
 130                 if (res == -1) {
 131                     writeFull(fd, string(settings.reservedSize, 'X'));
 132                     ftruncate(fd, settings.reservedSize);
 133                 }
 134             }
 135         }
 136         else
 137             deletePath(reservedPath);
 138     } catch (SysError & e) { /* don't care about errors */
 139     }
 140
 141     /* Acquire the big fat lock in shared mode to make sure that no
 142        schema upgrade is in progress. */
 143     try {
 144         Path globalLockPath = settings.nixDBPath + "/big-lock";
 145         globalLock = openLockFile(globalLockPath.c_str(), true);
 146     } catch (SysError & e) {
 147         if (e.errNo != EACCES) throw;
 148         settings.readOnlyMode = true;
 149         openDB(false);
 150         return;
 151     }
 152
 153     if (!lockFile(globalLock, ltRead, false)) {
 154         printMsg(lvlError, "waiting for the big store lock...");
 155         lockFile(globalLock, ltRead, true);
 156     }
 157
 158     /* Check the current database schema and if necessary do an
 159        upgrade.  */
 160     int curSchema = getSchema();
 161     if (curSchema > nixSchemaVersion)
 162         throw Error(format("current store schema is version %1%, but I only support %2%")
 163             % curSchema % nixSchemaVersion);
 164
 165     else if (curSchema == 0) { /* new store */
 166         curSchema = nixSchemaVersion;
 167         openDB(true);
 168         writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str());
 169     }
 170
 171     else if (curSchema < nixSchemaVersion) {
 172         /* Guix always used version 7 of the schema.  */
 173         throw Error(
 174             format("Your store database uses an implausibly old schema, version %1%.")
 175             % curSchema);
 176     }
 177
 178     else openDB(false);
 179 }
 180
 181
 182 LocalStore::~LocalStore()
 183 {
 184     try {
 185         if (runningSubstituter) {
 186             RunningSubstituter &i = *runningSubstituter;
 187             if (!i.disabled) {
 188                 i.to.close();
 189                 i.from.close();
 190                 i.error.close();
 191                 if (i.pid != -1)
 192                     i.pid.wait(true);
 193             }
 194         }
 195     } catch (...) {
 196         ignoreException();
 197     }
 198
 199     try {
 200         if (fdTempRoots != -1) {
 201             fdTempRoots.close();
 202             unlink(fnTempRoots.c_str());
 203         }
 204     } catch (...) {
 205         ignoreException();
 206     }
 207 }
 208
 209
 210 int LocalStore::getSchema()
 211 {
 212     int curSchema = 0;
 213     if (pathExists(schemaPath)) {
 214         string s = readFile(schemaPath);
 215         if (!string2Int(s, curSchema))
 216             throw Error(format("`%1%' is corrupt") % schemaPath);
 217     }
 218     return curSchema;
 219 }
 220
 221
 222 void LocalStore::openDB(bool create)
 223 {
 224     if (access(settings.nixDBPath.c_str(), R_OK | W_OK))
 225         throw SysError(format("store database directory `%1%' is not writable") % settings.nixDBPath);
 226
 227     /* Open the store database. */
 228     string dbPath = settings.nixDBPath + "/db.sqlite";
 229     if (sqlite3_open_v2(dbPath.c_str(), &db.db,
 230             SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
 231         throw Error(format("cannot open store database `%1%'") % dbPath);
 232
 233     if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
 234         throwSQLiteError(db, "setting timeout");
 235
 236     if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK)
 237         throwSQLiteError(db, "enabling foreign keys");
 238
 239     /* !!! check whether sqlite has been built with foreign key
 240        support */
 241
 242     /* Whether SQLite should fsync().  "Normal" synchronous mode
 243        should be safe enough.  If the user asks for it, don't sync at
 244        all.  This can cause database corruption if the system
 245        crashes. */
 246     string syncMode = settings.fsyncMetadata ? "normal" : "off";
 247     if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
 248         throwSQLiteError(db, "setting synchronous mode");
 249
 250     /* Set the SQLite journal mode.  WAL mode is fastest, so it's the
 251        default. */
 252     string mode = settings.useSQLiteWAL ? "wal" : "truncate";
 253     string prevMode;
 254     {
 255         SQLiteStmt stmt;
 256         stmt.create(db, "pragma main.journal_mode;");
 257         if (sqlite3_step(stmt) != SQLITE_ROW)
 258             throwSQLiteError(db, "querying journal mode");
 259         prevMode = string((const char *) sqlite3_column_text(stmt, 0));
 260     }
 261     if (prevMode != mode &&
 262         sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
 263         throwSQLiteError(db, "setting journal mode");
 264
 265     /* Increase the auto-checkpoint interval to 40000 pages.  This
 266        seems enough to ensure that instantiating the NixOS system
 267        derivation is done in a single fsync(). */
 268     if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 40000;", 0, 0, 0) != SQLITE_OK)
 269         throwSQLiteError(db, "setting autocheckpoint interval");
 270
 271     /* Initialise the database schema, if necessary. */
 272     if (create) {
 273         const char * schema =
 274 #include "schema.sql.hh"
 275             ;
 276         if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK)
 277             throwSQLiteError(db, "initialising database schema");
 278     }
 279
 280     /* Prepare SQL statements. */
 281     stmtRegisterValidPath.create(db,
 282         "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);");
 283     stmtUpdatePathInfo.create(db,
 284         "update ValidPaths set narSize = ?, hash = ? where path = ?;");
 285     stmtAddReference.create(db,
 286         "insert or replace into Refs (referrer, reference) values (?, ?);");
 287     stmtQueryPathInfo.create(db,
 288         "select id, hash, registrationTime, deriver, narSize from ValidPaths where path = ?;");
 289     stmtQueryReferences.create(db,
 290         "select path from Refs join ValidPaths on reference = id where referrer = ?;");
 291     stmtQueryReferrers.create(db,
 292         "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);");
 293     stmtInvalidatePath.create(db,
 294         "delete from ValidPaths where path = ?;");
 295     stmtRegisterFailedPath.create(db,
 296         "insert or ignore into FailedPaths (path, time) values (?, ?);");
 297     stmtHasPathFailed.create(db,
 298         "select time from FailedPaths where path = ?;");
 299     stmtQueryFailedPaths.create(db,
 300         "select path from FailedPaths;");
 301     // If the path is a derivation, then clear its outputs.
 302     stmtClearFailedPath.create(db,
 303         "delete from FailedPaths where ?1 = '*' or path = ?1 "
 304         "or path in (select d.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where v.path = ?1);");
 305     stmtAddDerivationOutput.create(db,
 306         "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);");
 307     stmtQueryValidDerivers.create(db,
 308         "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;");
 309     stmtQueryDerivationOutputs.create(db,
 310         "select id, path from DerivationOutputs where drv = ?;");
 311     // Use "path >= ?" with limit 1 rather than "path like '?%'" to
 312     // ensure efficient lookup.
 313     stmtQueryPathFromHashPart.create(db,
 314         "select path from ValidPaths where path >= ? limit 1;");
 315     stmtQueryValidPaths.create(db, "select path from ValidPaths");
 316 }
 317
 318
 319 /* To improve purity, users may want to make the store a read-only
 320    bind mount.  So make the store writable for this process. */
 321 void LocalStore::makeStoreWritable()
 322 {
 323 #if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_REMOUNT)
 324     if (getuid() != 0) return;
 325     /* Check if /nix/store is on a read-only mount. */
 326     struct statvfs stat;
 327     if (statvfs(settings.nixStore.c_str(), &stat) != 0)
 328         throw SysError("getting info about the store mount point");
 329
 330     if (stat.f_flag & ST_RDONLY) {
 331         if (unshare(CLONE_NEWNS) == -1)
 332             throw SysError("setting up a private mount namespace");
 333
 334         if (mount(0, settings.nixStore.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
 335             throw SysError(format("remounting %1% writable") % settings.nixStore);
 336     }
 337 #endif
 338 }
 339
 340
 341 const time_t mtimeStore = 1; /* 1 second into the epoch */
 342
 343
 344 static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st)
 345 {
 346     if (!S_ISLNK(st.st_mode)) {
 347
 348         /* Mask out all type related bits. */
 349         mode_t mode = st.st_mode & ~S_IFMT;
 350
 351         if (mode != 0444 && mode != 0555) {
 352             mode = (st.st_mode & S_IFMT)
 353                  | 0444
 354                  | (st.st_mode & S_IXUSR ? 0111 : 0);
 355             if (chmod(path.c_str(), mode) == -1)
 356                 throw SysError(format("changing mode of `%1%' to %2$o") % path % mode);
 357         }
 358
 359     }
 360
 361     if (st.st_mtime != mtimeStore) {
 362         struct timeval times[2];
 363         times[0].tv_sec = st.st_atime;
 364         times[0].tv_usec = 0;
 365         times[1].tv_sec = mtimeStore;
 366         times[1].tv_usec = 0;
 367 #if HAVE_LUTIMES
 368         if (lutimes(path.c_str(), times) == -1)
 369             if (errno != ENOSYS ||
 370                 (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1))
 371 #else
 372         if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
 373 #endif
 374             throw SysError(format("changing modification time of `%1%'") % path);
 375     }
 376 }
 377
 378
 379 void canonicaliseTimestampAndPermissions(const Path & path)
 380 {
 381     struct stat st;
 382     if (lstat(path.c_str(), &st))
 383         throw SysError(format("getting attributes of path `%1%'") % path);
 384     canonicaliseTimestampAndPermissions(path, st);
 385 }
 386
 387
 388 static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSeen & inodesSeen)
 389 {
 390     checkInterrupt();
 391
 392     struct stat st;
 393     if (lstat(path.c_str(), &st))
 394         throw SysError(format("getting attributes of path `%1%'") % path);
 395
 396     /* Really make sure that the path is of a supported type. */
 397     if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
 398         throw Error(format("file ‘%1%’ has an unsupported type") % path);
 399
 400     /* Fail if the file is not owned by the build user.  This prevents
 401        us from messing up the ownership/permissions of files
 402        hard-linked into the output (e.g. "ln /etc/shadow $out/foo").
 403        However, ignore files that we chown'ed ourselves previously to
 404        ensure that we don't fail on hard links within the same build
 405        (i.e. "touch $out/foo; ln $out/foo $out/bar"). */
 406     if (fromUid != (uid_t) -1 && st.st_uid != fromUid) {
 407         assert(!S_ISDIR(st.st_mode));
 408         if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end())
 409             throw BuildError(format("invalid ownership on file `%1%'") % path);
 410         mode_t mode = st.st_mode & ~S_IFMT;
 411         assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
 412         return;
 413     }
 414
 415     inodesSeen.insert(Inode(st.st_dev, st.st_ino));
 416
 417     canonicaliseTimestampAndPermissions(path, st);
 418
 419     /* Change ownership to the current uid.  If it's a symlink, use
 420        lchown if available, otherwise don't bother.  Wrong ownership
 421        of a symlink doesn't matter, since the owning user can't change
 422        the symlink and can't delete it because the directory is not
 423        writable.  The only exception is top-level paths in the
 424        store (since that directory is group-writable for the build
 425        users group); we check for this case below. */
 426     if (st.st_uid != geteuid()) {
 427 #if HAVE_LCHOWN
 428         if (lchown(path.c_str(), geteuid(), getegid()) == -1)
 429 #else
 430         if (!S_ISLNK(st.st_mode) &&
 431             chown(path.c_str(), geteuid(), getegid()) == -1)
 432 #endif
 433             throw SysError(format("changing owner of `%1%' to %2%")
 434                 % path % geteuid());
 435     }
 436
 437     if (S_ISDIR(st.st_mode)) {
 438         DirEntries entries = readDirectory(path);
 439         for (auto & i : entries)
 440             canonicalisePathMetaData_(path + "/" + i.name, fromUid, inodesSeen);
 441     }
 442 }
 443
 444
 445 void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen)
 446 {
 447     canonicalisePathMetaData_(path, fromUid, inodesSeen);
 448
 449     /* On platforms that don't have lchown(), the top-level path can't
 450        be a symlink, since we can't change its ownership. */
 451     struct stat st;
 452     if (lstat(path.c_str(), &st))
 453         throw SysError(format("getting attributes of path `%1%'") % path);
 454
 455     if (st.st_uid != geteuid()) {
 456         assert(S_ISLNK(st.st_mode));
 457         throw Error(format("wrong ownership of top-level store path `%1%'") % path);
 458     }
 459 }
 460
 461
 462 void canonicalisePathMetaData(const Path & path, uid_t fromUid)
 463 {
 464     InodesSeen inodesSeen;
 465     canonicalisePathMetaData(path, fromUid, inodesSeen);
 466 }
 467
 468
 469 void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & drv)
 470 {
 471     string drvName = storePathToName(drvPath);
 472     assert(isDerivation(drvName));
 473     drvName = string(drvName, 0, drvName.size() - drvExtension.size());
 474
 475     if (isFixedOutputDrv(drv)) {
 476         DerivationOutputs::const_iterator out = drv.outputs.find("out");
 477         if (out == drv.outputs.end())
 478             throw Error(format("derivation `%1%' does not have an output named `out'") % drvPath);
 479
 480         bool recursive; HashType ht; Hash h;
 481         out->second.parseHashInfo(recursive, ht, h);
 482         Path outPath = makeFixedOutputPath(recursive, ht, h, drvName);
 483
 484         StringPairs::const_iterator j = drv.env.find("out");
 485         if (out->second.path != outPath || j == drv.env.end() || j->second != outPath)
 486             throw Error(format("derivation `%1%' has incorrect output `%2%', should be `%3%'")
 487                 % drvPath % out->second.path % outPath);
 488     }
 489
 490     else {
 491         Derivation drvCopy(drv);
 492         foreach (DerivationOutputs::iterator, i, drvCopy.outputs) {
 493             i->second.path = "";
 494             drvCopy.env[i->first] = "";
 495         }
 496
 497         Hash h = hashDerivationModulo(*this, drvCopy);
 498
 499         foreach (DerivationOutputs::const_iterator, i, drv.outputs) {
 500             Path outPath = makeOutputPath(i->first, h, drvName);
 501             StringPairs::const_iterator j = drv.env.find(i->first);
 502             if (i->second.path != outPath || j == drv.env.end() || j->second != outPath)
 503                 throw Error(format("derivation `%1%' has incorrect output `%2%', should be `%3%'")
 504                     % drvPath % i->second.path % outPath);
 505         }
 506     }
 507 }
 508
 509
 510 uint64_t LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs)
 511 {
 512     stmtRegisterValidPath.use()
 513         (info.path)
 514         ("sha256:" + printHash(info.hash))
 515         (info.registrationTime == 0 ? time(0) : info.registrationTime)
 516         (info.deriver, info.deriver != "")
 517         (info.narSize, info.narSize != 0)
 518         .exec();
 519     uint64_t id = sqlite3_last_insert_rowid(db);
 520
 521     /* If this is a derivation, then store the derivation outputs in
 522        the database.  This is useful for the garbage collector: it can
 523        efficiently query whether a path is an output of some
 524        derivation. */
 525     if (isDerivation(info.path)) {
 526         Derivation drv = readDerivation(info.path);
 527
 528         /* Verify that the output paths in the derivation are correct
 529            (i.e., follow the scheme for computing output paths from
 530            derivations).  Note that if this throws an error, then the
 531            DB transaction is rolled back, so the path validity
 532            registration above is undone. */
 533         if (checkOutputs) checkDerivationOutputs(info.path, drv);
 534
 535         for (auto & i : drv.outputs) {
 536             stmtAddDerivationOutput.use()
 537                 (id)
 538                 (i.first)
 539                 (i.second.path)
 540                 .exec();
 541         }
 542     }
 543
 544     return id;
 545 }
 546
 547
 548 void LocalStore::addReference(uint64_t referrer, uint64_t reference)
 549 {
 550     stmtAddReference.use()(referrer)(reference).exec();
 551 }
 552
 553
 554 void LocalStore::registerFailedPath(const Path & path)
 555 {
 556     retrySQLite<void>([&]() {
 557         stmtRegisterFailedPath.use()(path)(time(0)).step();
 558     });
 559 }
 560
 561
 562 bool LocalStore::hasPathFailed(const Path & path)
 563 {
 564     return retrySQLite<bool>([&]() {
 565         return stmtHasPathFailed.use()(path).next();
 566     });
 567 }
 568
 569
 570 PathSet LocalStore::queryFailedPaths()
 571 {
 572     return retrySQLite<PathSet>([&]() {
 573         auto useQueryFailedPaths(stmtQueryFailedPaths.use());
 574
 575         PathSet res;
 576         while (useQueryFailedPaths.next())
 577             res.insert(useQueryFailedPaths.getStr(0));
 578
 579         return res;
 580     });
 581 }
 582
 583
 584 void LocalStore::clearFailedPaths(const PathSet & paths)
 585 {
 586     retrySQLite<void>([&]() {
 587         SQLiteTxn txn(db);
 588
 589         for (auto & path : paths)
 590             stmtClearFailedPath.use()(path).exec();
 591
 592         txn.commit();
 593     });
 594 }
 595
 596
 597 Hash parseHashField(const Path & path, const string & s)
 598 {
 599     string::size_type colon = s.find(':');
 600     if (colon == string::npos)
 601         throw Error(format("corrupt hash `%1%' in valid-path entry for `%2%'")
 602             % s % path);
 603     HashType ht = parseHashType(string(s, 0, colon));
 604     if (ht == htUnknown)
 605         throw Error(format("unknown hash type `%1%' in valid-path entry for `%2%'")
 606             % string(s, 0, colon) % path);
 607     return parseHash(ht, string(s, colon + 1));
 608 }
 609
 610
 611 ValidPathInfo LocalStore::queryPathInfo(const Path & path)
 612 {
 613     ValidPathInfo info;
 614     info.path = path;
 615
 616     assertStorePath(path);
 617
 618     return retrySQLite<ValidPathInfo>([&]() {
 619
 620         /* Get the path info. */
 621         auto useQueryPathInfo(stmtQueryPathInfo.use()(path));
 622
 623         if (!useQueryPathInfo.next())
 624             throw Error(format("path `%1%' is not valid") % path);
 625
 626         info.id = useQueryPathInfo.getInt(0);
 627
 628         info.hash = parseHashField(path, useQueryPathInfo.getStr(1));
 629
 630         info.registrationTime = useQueryPathInfo.getInt(2);
 631
 632         auto s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3);
 633         if (s) info.deriver = s;
 634
 635         /* Note that narSize = NULL yields 0. */
 636         info.narSize = useQueryPathInfo.getInt(4);
 637
 638         /* Get the references. */
 639         auto useQueryReferences(stmtQueryReferences.use()(info.id));
 640
 641         while (useQueryReferences.next())
 642             info.references.insert(useQueryReferences.getStr(0));
 643
 644         return info;
 645     });
 646 }
 647
 648
 649 /* Update path info in the database.  Currently only updates the
 650    narSize field. */
 651 void LocalStore::updatePathInfo(const ValidPathInfo & info)
 652 {
 653     stmtUpdatePathInfo.use()
 654         (info.narSize, info.narSize != 0)
 655         ("sha256:" + printHash(info.hash))
 656         (info.path)
 657         .exec();
 658 }
 659
 660
 661 uint64_t LocalStore::queryValidPathId(const Path & path)
 662 {
 663     auto use(stmtQueryPathInfo.use()(path));
 664     if (!use.next())
 665         throw Error(format("path ‘%1%’ is not valid") % path);
 666     return use.getInt(0);
 667 }
 668
 669
 670 bool LocalStore::isValidPath_(const Path & path)
 671 {
 672     return stmtQueryPathInfo.use()(path).next();
 673 }
 674
 675
 676 bool LocalStore::isValidPath(const Path & path)
 677 {
 678     return retrySQLite<bool>([&]() {
 679         return isValidPath_(path);
 680     });
 681 }
 682
 683
 684 PathSet LocalStore::queryValidPaths(const PathSet & paths)
 685 {
 686     return retrySQLite<PathSet>([&]() {
 687         PathSet res;
 688         foreach (PathSet::const_iterator, i, paths)
 689             if (isValidPath_(*i)) res.insert(*i);
 690         return res;
 691     });
 692 }
 693
 694
 695 PathSet LocalStore::queryAllValidPaths()
 696 {
 697     return retrySQLite<PathSet>([&]() {
 698         auto use(stmtQueryValidPaths.use());
 699         PathSet res;
 700         while (use.next()) res.insert(use.getStr(0));
 701         return res;
 702     });
 703 }
 704
 705
 706 void LocalStore::queryReferences(const Path & path,
 707     PathSet & references)
 708 {
 709     ValidPathInfo info = queryPathInfo(path);
 710     references.insert(info.references.begin(), info.references.end());
 711 }
 712
 713
 714 void LocalStore::queryReferrers_(const Path & path, PathSet & referrers)
 715 {
 716     auto useQueryReferrers(stmtQueryReferrers.use()(path));
 717
 718     while (useQueryReferrers.next())
 719         referrers.insert(useQueryReferrers.getStr(0));
 720 }
 721
 722
 723 void LocalStore::queryReferrers(const Path & path, PathSet & referrers)
 724 {
 725     assertStorePath(path);
 726     return retrySQLite<void>([&]() {
 727         queryReferrers_(path, referrers);
 728     });
 729 }
 730
 731
 732 Path LocalStore::queryDeriver(const Path & path)
 733 {
 734     return queryPathInfo(path).deriver;
 735 }
 736
 737
 738 PathSet LocalStore::queryValidDerivers(const Path & path)
 739 {
 740     assertStorePath(path);
 741
 742     return retrySQLite<PathSet>([&]() {
 743         auto useQueryValidDerivers(stmtQueryValidDerivers.use()(path));
 744
 745         PathSet derivers;
 746         while (useQueryValidDerivers.next())
 747             derivers.insert(useQueryValidDerivers.getStr(1));
 748
 749         return derivers;
 750     });
 751 }
 752
 753
 754 PathSet LocalStore::queryDerivationOutputs(const Path & path)
 755 {
 756     return retrySQLite<PathSet>([&]() {
 757         auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path)));
 758
 759         PathSet outputs;
 760         while (useQueryDerivationOutputs.next())
 761             outputs.insert(useQueryDerivationOutputs.getStr(1));
 762
 763         return outputs;
 764     });
 765 }
 766
 767
 768 StringSet LocalStore::queryDerivationOutputNames(const Path & path)
 769 {
 770     return retrySQLite<StringSet>([&]() {
 771         auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path)));
 772
 773         StringSet outputNames;
 774         while (useQueryDerivationOutputs.next())
 775             outputNames.insert(useQueryDerivationOutputs.getStr(0));
 776
 777         return outputNames;
 778     });
 779 }
 780
 781
 782 Path LocalStore::queryPathFromHashPart(const string & hashPart)
 783 {
 784     if (hashPart.size() != 32) throw Error("invalid hash part");
 785
 786     Path prefix = settings.nixStore + "/" + hashPart;
 787
 788     return retrySQLite<Path>([&]() -> Path {
 789         auto useQueryPathFromHashPart(stmtQueryPathFromHashPart.use()(prefix));
 790
 791         if (!useQueryPathFromHashPart.next()) return "";
 792
 793         const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0);
 794         return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : "";
 795     });
 796 }
 797
 798
 799 void LocalStore::setSubstituterEnv()
 800 {
 801     if (didSetSubstituterEnv) return;
 802
 803     /* Pass configuration options (including those overridden with
 804        --option) to substituters. */
 805     setenv("_NIX_OPTIONS", settings.pack().c_str(), 1);
 806
 807     didSetSubstituterEnv = true;
 808 }
 809
 810
 811 void LocalStore::startSubstituter(RunningSubstituter & run)
 812 {
 813     if (run.disabled || run.pid != -1) return;
 814
 815     debug(format("starting substituter program `%1% substitute'")
 816           % settings.guixProgram);
 817
 818     Pipe toPipe, fromPipe, errorPipe;
 819
 820     toPipe.create();
 821     fromPipe.create();
 822     errorPipe.create();
 823
 824     setSubstituterEnv();
 825
 826     run.pid = startProcess([&]() {
 827         if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
 828             throw SysError("dupping stdin");
 829         if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
 830             throw SysError("dupping stdout");
 831         if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1)
 832             throw SysError("dupping stderr");
 833         execl(settings.guixProgram.c_str(), "guix", "substitute", "--query", NULL);
 834         throw SysError(format("executing `%1%'") % settings.guixProgram);
 835     });
 836
 837     run.to = toPipe.writeSide.borrow();
 838     run.from = run.fromBuf.fd = fromPipe.readSide.borrow();
 839     run.error = errorPipe.readSide.borrow();
 840
 841     toPipe.readSide.close();
 842     fromPipe.writeSide.close();
 843     errorPipe.writeSide.close();
 844
 845     /* The substituter may exit right away if it's disabled in any way
 846        (e.g. copy-from-other-stores.pl will exit if no other stores
 847        are configured). */
 848     try {
 849         getLineFromSubstituter(run);
 850     } catch (EndOfFile & e) {
 851         run.to.close();
 852         run.from.close();
 853         run.error.close();
 854         run.disabled = true;
 855         if (run.pid.wait(true) != 0) throw;
 856     }
 857 }
 858
 859
 860 /* Read a line from the substituter's stdout, while also processing
 861    its stderr. */
 862 string LocalStore::getLineFromSubstituter(RunningSubstituter & run)
 863 {
 864     string res, err;
 865
 866     /* We might have stdout data left over from the last time. */
 867     if (run.fromBuf.hasData()) goto haveData;
 868
 869     while (1) {
 870         checkInterrupt();
 871
 872         fd_set fds;
 873         FD_ZERO(&fds);
 874         FD_SET(run.from, &fds);
 875         FD_SET(run.error, &fds);
 876
 877         /* Wait for data to appear on the substituter's stdout or
 878            stderr. */
 879         if (select(run.from > run.error ? run.from + 1 : run.error + 1, &fds, 0, 0, 0) == -1) {
 880             if (errno == EINTR) continue;
 881             throw SysError("waiting for input from the substituter");
 882         }
 883
 884         /* Completely drain stderr before dealing with stdout. */
 885         if (FD_ISSET(run.error, &fds)) {
 886             char buf[4096];
 887             ssize_t n = read(run.error, (unsigned char *) buf, sizeof(buf));
 888             if (n == -1) {
 889                 if (errno == EINTR) continue;
 890                 throw SysError("reading from substituter's stderr");
 891             }
 892             if (n == 0) throw EndOfFile(format("`%1% substitute' died unexpectedly")
 893                                         % settings.guixProgram);
 894             err.append(buf, n);
 895             string::size_type p;
 896             while (((p = err.find('\n')) != string::npos)
 897                    || ((p = err.find('\r')) != string::npos)) {
 898                 string thing(err, 0, p + 1);
 899                 writeToStderr("substitute: " + thing);
 900                 err = string(err, p + 1);
 901             }
 902         }
 903
 904         /* Read from stdout until we get a newline or the buffer is empty. */
 905         else if (run.fromBuf.hasData() || FD_ISSET(run.from, &fds)) {
 906         haveData:
 907             do {
 908                 unsigned char c;
 909                 run.fromBuf(&c, 1);
 910                 if (c == '\n') {
 911                     if (!err.empty()) printMsg(lvlError, "substitute: " + err);
 912                     return res;
 913                 }
 914                 res += c;
 915             } while (run.fromBuf.hasData());
 916         }
 917     }
 918 }
 919
 920
 921 template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & run)
 922 {
 923     string s = getLineFromSubstituter(run);
 924     T res;
 925     if (!string2Int(s, res)) throw Error("integer expected from stream");
 926     return res;
 927 }
 928
 929
 930 PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
 931 {
 932     PathSet res;
 933
 934     if (!settings.useSubstitutes || paths.empty()) return res;
 935
 936     if (!runningSubstituter) {
 937         std::unique_ptr<RunningSubstituter>fresh(new RunningSubstituter);
 938         runningSubstituter.swap(fresh);
 939     }
 940
 941     RunningSubstituter & run = *runningSubstituter;
 942     startSubstituter(run);
 943
 944     if (!run.disabled) {
 945         string s = "have ";
 946         foreach (PathSet::const_iterator, j, paths)
 947             if (res.find(*j) == res.end()) { s += *j; s += " "; }
 948         writeLine(run.to, s);
 949         while (true) {
 950             /* FIXME: we only read stderr when an error occurs, so
 951                substituters should only write (short) messages to
 952                stderr when they fail.  I.e. they shouldn't write debug
 953                output. */
 954             Path path = getLineFromSubstituter(run);
 955             if (path == "") break;
 956             res.insert(path);
 957         }
 958     }
 959
 960     return res;
 961 }
 962
 963
 964 void LocalStore::querySubstitutablePathInfos(PathSet & paths, SubstitutablePathInfos & infos)
 965 {
 966     if (!settings.useSubstitutes) return;
 967
 968     if (!runningSubstituter) {
 969         std::unique_ptr<RunningSubstituter>fresh(new RunningSubstituter);
 970         runningSubstituter.swap(fresh);
 971     }
 972
 973     RunningSubstituter & run = *runningSubstituter;
 974     startSubstituter(run);
 975     if (run.disabled) return;
 976
 977     string s = "info ";
 978     foreach (PathSet::const_iterator, i, paths)
 979         if (infos.find(*i) == infos.end()) { s += *i; s += " "; }
 980     writeLine(run.to, s);
 981
 982     while (true) {
 983         Path path = getLineFromSubstituter(run);
 984         if (path == "") break;
 985         if (paths.find(path) == paths.end())
 986             throw Error(format("got unexpected path `%1%' from substituter") % path);
 987         paths.erase(path);
 988         SubstitutablePathInfo & info(infos[path]);
 989         info.deriver = getLineFromSubstituter(run);
 990         if (info.deriver != "") assertStorePath(info.deriver);
 991         int nrRefs = getIntLineFromSubstituter<int>(run);
 992         while (nrRefs--) {
 993             Path p = getLineFromSubstituter(run);
 994             assertStorePath(p);
 995             info.references.insert(p);
 996         }
 997         info.downloadSize = getIntLineFromSubstituter<long long>(run);
 998         info.narSize = getIntLineFromSubstituter<long long>(run);
 999     }
1000 }
1001
1002
1003 void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
1004     SubstitutablePathInfos & infos)
1005 {
1006     if (!paths.empty()) {
1007         PathSet todo = paths;
1008         querySubstitutablePathInfos(todo, infos);
1009     }
1010 }
1011
1012
1013 Hash LocalStore::queryPathHash(const Path & path)
1014 {
1015     return queryPathInfo(path).hash;
1016 }
1017
1018
1019 void LocalStore::registerValidPath(const ValidPathInfo & info)
1020 {
1021     ValidPathInfos infos;
1022     infos.push_back(info);
1023     registerValidPaths(infos);
1024 }
1025
1026
1027 void LocalStore::registerValidPaths(const ValidPathInfos & infos)
1028 {
1029     /* SQLite will fsync by default, but the new valid paths may not be fsync-ed.
1030      * So some may want to fsync them before registering the validity, at the
1031      * expense of some speed of the path registering operation. */
1032     if (settings.syncBeforeRegistering) sync();
1033
1034     return retrySQLite<void>([&]() {
1035         SQLiteTxn txn(db);
1036         PathSet paths;
1037
1038         foreach (ValidPathInfos::const_iterator, i, infos) {
1039             assert(i->hash.type == htSHA256);
1040             if (isValidPath_(i->path))
1041                 updatePathInfo(*i);
1042             else
1043                 addValidPath(*i, false);
1044             paths.insert(i->path);
1045         }
1046
1047         for (auto & i : infos) {
1048             auto referrer = queryValidPathId(i.path);
1049             for (auto & j : i.references)
1050                 addReference(referrer, queryValidPathId(j));
1051         }
1052
1053         /* Check that the derivation outputs are correct.  We can't do
1054            this in addValidPath() above, because the references might
1055            not be valid yet. */
1056         foreach (ValidPathInfos::const_iterator, i, infos)
1057             if (isDerivation(i->path)) {
1058                 // FIXME: inefficient; we already loaded the
1059                 // derivation in addValidPath().
1060                 Derivation drv = readDerivation(i->path);
1061                 checkDerivationOutputs(i->path, drv);
1062             }
1063
1064         /* Do a topological sort of the paths.  This will throw an
1065            error if a cycle is detected and roll back the
1066            transaction.  Cycles can only occur when a derivation
1067            has multiple outputs. */
1068         topoSortPaths(*this, paths);
1069
1070         txn.commit();
1071     });
1072 }
1073
1074
1075 /* Invalidate a path.  The caller is responsible for checking that
1076    there are no referrers. */
1077 void LocalStore::invalidatePath(const Path & path)
1078 {
1079     debug(format("invalidating path `%1%'") % path);
1080
1081     drvHashes.erase(path);
1082
1083     stmtInvalidatePath.use()(path).exec();
1084
1085     /* Note that the foreign key constraints on the Refs table take
1086        care of deleting the references entries for `path'. */
1087 }
1088
1089
1090 Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
1091     bool recursive, HashType hashAlgo, bool repair)
1092 {
1093     Hash h = hashString(hashAlgo, dump);
1094
1095     Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name);
1096
1097     addTempRoot(dstPath);
1098
1099     if (repair || !isValidPath(dstPath)) {
1100
1101         /* The first check above is an optimisation to prevent
1102            unnecessary lock acquisition. */
1103
1104         PathLocks outputLock(singleton<PathSet, Path>(dstPath));
1105
1106         if (repair || !isValidPath(dstPath)) {
1107
1108             if (pathExists(dstPath)) deletePath(dstPath);
1109
1110             if (recursive) {
1111                 StringSource source(dump);
1112                 restorePath(dstPath, source);
1113             } else
1114                 writeFile(dstPath, dump);
1115
1116             canonicalisePathMetaData(dstPath, -1);
1117
1118             /* Register the SHA-256 hash of the NAR serialisation of
1119                the path in the database.  We may just have computed it
1120                above (if called with recursive == true and hashAlgo ==
1121                sha256); otherwise, compute it here. */
1122             HashResult hash;
1123             if (recursive) {
1124                 hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
1125                 hash.second = dump.size();
1126             } else
1127                 hash = hashPath(htSHA256, dstPath);
1128
1129             optimisePath(dstPath); // FIXME: combine with hashPath()
1130
1131             ValidPathInfo info;
1132             info.path = dstPath;
1133             info.hash = hash.first;
1134             info.narSize = hash.second;
1135             registerValidPath(info);
1136         }
1137
1138         outputLock.setDeletion(true);
1139     }
1140
1141     return dstPath;
1142 }
1143
1144
1145 Path LocalStore::addToStore(const string & name, const Path & _srcPath,
1146     bool recursive, HashType hashAlgo, PathFilter & filter, bool repair)
1147 {
1148     Path srcPath(absPath(_srcPath));
1149     debug(format("adding `%1%' to the store") % srcPath);
1150
1151     /* Read the whole path into memory. This is not a very scalable
1152        method for very large paths, but `copyPath' is mainly used for
1153        small files. */
1154     StringSink sink;
1155     if (recursive)
1156         dumpPath(srcPath, sink, filter);
1157     else
1158         sink.s = readFile(srcPath);
1159
1160     return addToStoreFromDump(sink.s, name, recursive, hashAlgo, repair);
1161 }
1162
1163
1164 Path LocalStore::addTextToStore(const string & name, const string & s,
1165     const PathSet & references, bool repair)
1166 {
1167     Path dstPath = computeStorePathForText(name, s, references);
1168
1169     addTempRoot(dstPath);
1170
1171     if (repair || !isValidPath(dstPath)) {
1172
1173         PathLocks outputLock(singleton<PathSet, Path>(dstPath));
1174
1175         if (repair || !isValidPath(dstPath)) {
1176
1177             if (pathExists(dstPath)) deletePath(dstPath);
1178
1179             writeFile(dstPath, s);
1180
1181             canonicalisePathMetaData(dstPath, -1);
1182
1183             HashResult hash = hashPath(htSHA256, dstPath);
1184
1185             optimisePath(dstPath);
1186
1187             ValidPathInfo info;
1188             info.path = dstPath;
1189             info.hash = hash.first;
1190             info.narSize = hash.second;
1191             info.references = references;
1192             registerValidPath(info);
1193         }
1194
1195         outputLock.setDeletion(true);
1196     }
1197
1198     return dstPath;
1199 }
1200
1201
1202 struct HashAndWriteSink : Sink
1203 {
1204     Sink & writeSink;
1205     HashSink hashSink;
1206     HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
1207     {
1208     }
1209     virtual void operator () (const unsigned char * data, size_t len)
1210     {
1211         writeSink(data, len);
1212         hashSink(data, len);
1213     }
1214     Hash currentHash()
1215     {
1216         return hashSink.currentHash().first;
1217     }
1218 };
1219
1220
1221 #define EXPORT_MAGIC 0x4558494e
1222
1223
1224 static void checkSecrecy(const Path & path)
1225 {
1226     struct stat st;
1227     if (stat(path.c_str(), &st))
1228         throw SysError(format("getting status of `%1%'") % path);
1229     if ((st.st_mode & (S_IRWXG | S_IRWXO)) != 0)
1230         throw Error(format("file `%1%' should be secret (inaccessible to everybody else)!") % path);
1231 }
1232
1233
1234 static std::string runAuthenticationProgram(const Strings & args)
1235 {
1236     Strings fullArgs = { "authenticate" };
1237     fullArgs.insert(fullArgs.end(), args.begin(), args.end()); // append
1238     return runProgram(settings.guixProgram, false, fullArgs);
1239 }
1240
1241 void LocalStore::exportPath(const Path & path, bool sign,
1242     Sink & sink)
1243 {
1244     assertStorePath(path);
1245
1246     printMsg(lvlInfo, format("exporting path `%1%'") % path);
1247
1248     if (!isValidPath(path))
1249         throw Error(format("path `%1%' is not valid") % path);
1250
1251     HashAndWriteSink hashAndWriteSink(sink);
1252
1253     dumpPath(path, hashAndWriteSink);
1254
1255     /* Refuse to export paths that have changed.  This prevents
1256        filesystem corruption from spreading to other machines.
1257        Don't complain if the stored hash is zero (unknown). */
1258     Hash hash = hashAndWriteSink.currentHash();
1259     Hash storedHash = queryPathHash(path);
1260     if (hash != storedHash && storedHash != Hash(storedHash.type))
1261         throw Error(format("hash of path `%1%' has changed from `%2%' to `%3%'!") % path
1262             % printHash(storedHash) % printHash(hash));
1263
1264     writeInt(EXPORT_MAGIC, hashAndWriteSink);
1265
1266     writeString(path, hashAndWriteSink);
1267
1268     PathSet references;
1269     queryReferences(path, references);
1270     writeStrings(references, hashAndWriteSink);
1271
1272     Path deriver = queryDeriver(path);
1273     writeString(deriver, hashAndWriteSink);
1274
1275     if (sign) {
1276         Hash hash = hashAndWriteSink.currentHash();
1277
1278         writeInt(1, hashAndWriteSink);
1279
1280         Path tmpDir = createTempDir();
1281         AutoDelete delTmp(tmpDir);
1282         Path hashFile = tmpDir + "/hash";
1283         writeFile(hashFile, printHash(hash));
1284
1285         Path secretKey = settings.nixConfDir + "/signing-key.sec";
1286         checkSecrecy(secretKey);
1287
1288         Strings args;
1289         args.push_back("rsautl");
1290         args.push_back("-sign");
1291         args.push_back("-inkey");
1292         args.push_back(secretKey);
1293         args.push_back("-in");
1294         args.push_back(hashFile);
1295
1296         string signature = runAuthenticationProgram(args);
1297
1298         writeString(signature, hashAndWriteSink);
1299
1300     } else
1301         writeInt(0, hashAndWriteSink);
1302 }
1303
1304
1305 struct HashAndReadSource : Source
1306 {
1307     Source & readSource;
1308     HashSink hashSink;
1309     bool hashing;
1310     HashAndReadSource(Source & readSource) : readSource(readSource), hashSink(htSHA256)
1311     {
1312         hashing = true;
1313     }
1314     size_t read(unsigned char * data, size_t len)
1315     {
1316         size_t n = readSource.read(data, len);
1317         if (hashing) hashSink(data, n);
1318         return n;
1319     }
1320 };
1321
1322
1323 /* Create a temporary directory in the store that won't be
1324    garbage-collected. */
1325 Path LocalStore::createTempDirInStore()
1326 {
1327     Path tmpDir;
1328     do {
1329         /* There is a slight possibility that `tmpDir' gets deleted by
1330            the GC between createTempDir() and addTempRoot(), so repeat
1331            until `tmpDir' exists. */
1332         tmpDir = createTempDir(settings.nixStore);
1333         addTempRoot(tmpDir);
1334     } while (!pathExists(tmpDir));
1335     return tmpDir;
1336 }
1337
1338
1339 Path LocalStore::importPath(bool requireSignature, Source & source)
1340 {
1341     HashAndReadSource hashAndReadSource(source);
1342
1343     /* We don't yet know what store path this archive contains (the
1344        store path follows the archive data proper), and besides, we
1345        don't know yet whether the signature is valid. */
1346     Path tmpDir = createTempDirInStore();
1347     AutoDelete delTmp(tmpDir);
1348     Path unpacked = tmpDir + "/unpacked";
1349
1350     restorePath(unpacked, hashAndReadSource);
1351
1352     unsigned int magic = readInt(hashAndReadSource);
1353     if (magic != EXPORT_MAGIC)
1354         throw Error("normalized archive cannot be imported; wrong format");
1355
1356     Path dstPath = readStorePath(hashAndReadSource);
1357
1358     PathSet references = readStorePaths<PathSet>(hashAndReadSource);
1359
1360     Path deriver = readString(hashAndReadSource);
1361     if (deriver != "") assertStorePath(deriver);
1362
1363     Hash hash = hashAndReadSource.hashSink.finish().first;
1364     hashAndReadSource.hashing = false;
1365
1366     bool haveSignature = readInt(hashAndReadSource) == 1;
1367
1368     if (requireSignature && !haveSignature)
1369         throw Error(format("imported archive of `%1%' lacks a signature") % dstPath);
1370
1371     if (haveSignature) {
1372         string signature = readString(hashAndReadSource);
1373
1374         if (requireSignature) {
1375             Path sigFile = tmpDir + "/sig";
1376             writeFile(sigFile, signature);
1377
1378             Strings args;
1379             args.push_back("rsautl");
1380             args.push_back("-verify");
1381             args.push_back("-inkey");
1382             args.push_back(settings.nixConfDir + "/signing-key.pub");
1383             args.push_back("-pubin");
1384             args.push_back("-in");
1385             args.push_back(sigFile);
1386             string hash2 = runAuthenticationProgram(args);
1387
1388             /* Note: runProgram() throws an exception if the signature
1389                is invalid. */
1390
1391             if (printHash(hash) != hash2)
1392                 throw Error(
1393                     "signed hash doesn't match actual contents of imported "
1394                     "archive; archive could be corrupt, or someone is trying "
1395                     "to import a Trojan horse");
1396         }
1397     }
1398
1399     /* Do the actual import. */
1400
1401     /* !!! way too much code duplication with addTextToStore() etc. */
1402     addTempRoot(dstPath);
1403
1404     if (!isValidPath(dstPath)) {
1405
1406         PathLocks outputLock;
1407
1408         /* Lock the output path.  But don't lock if we're being called
1409            from a build hook (whose parent process already acquired a
1410            lock on this path). */
1411         Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS"));
1412         if (find(locksHeld.begin(), locksHeld.end(), dstPath) == locksHeld.end())
1413             outputLock.lockPaths(singleton<PathSet, Path>(dstPath));
1414
1415         if (!isValidPath(dstPath)) {
1416
1417             if (pathExists(dstPath)) deletePath(dstPath);
1418
1419             if (rename(unpacked.c_str(), dstPath.c_str()) == -1)
1420                 throw SysError(format("cannot move `%1%' to `%2%'")
1421                     % unpacked % dstPath);
1422
1423             canonicalisePathMetaData(dstPath, -1);
1424
1425             /* !!! if we were clever, we could prevent the hashPath()
1426                here. */
1427             HashResult hash = hashPath(htSHA256, dstPath);
1428
1429             optimisePath(dstPath); // FIXME: combine with hashPath()
1430
1431             ValidPathInfo info;
1432             info.path = dstPath;
1433             info.hash = hash.first;
1434             info.narSize = hash.second;
1435             info.references = references;
1436             info.deriver = deriver != "" && isValidPath(deriver) ? deriver : "";
1437             registerValidPath(info);
1438         }
1439
1440         outputLock.setDeletion(true);
1441     }
1442
1443     return dstPath;
1444 }
1445
1446
1447 Paths LocalStore::importPaths(bool requireSignature, Source & source)
1448 {
1449     Paths res;
1450     while (true) {
1451         unsigned long long n = readLongLong(source);
1452         if (n == 0) break;
1453         if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'");
1454         res.push_back(importPath(requireSignature, source));
1455     }
1456     return res;
1457 }
1458
1459
1460 void LocalStore::invalidatePathChecked(const Path & path)
1461 {
1462     assertStorePath(path);
1463
1464     retrySQLite<void>([&]() {
1465         SQLiteTxn txn(db);
1466
1467         if (isValidPath_(path)) {
1468             PathSet referrers; queryReferrers_(path, referrers);
1469             referrers.erase(path); /* ignore self-references */
1470             if (!referrers.empty())
1471                 throw PathInUse(format("cannot delete path `%1%' because it is in use by %2%")
1472                     % path % showPaths(referrers));
1473             invalidatePath(path);
1474         }
1475
1476         txn.commit();
1477     });
1478 }
1479
1480
1481 bool LocalStore::verifyStore(bool checkContents, bool repair)
1482 {
1483     printMsg(lvlError, format("reading the store..."));
1484
1485     bool errors = false;
1486
1487     /* Acquire the global GC lock to prevent a garbage collection. */
1488     AutoCloseFD fdGCLock = openGCLock(ltWrite);
1489
1490     PathSet store;
1491     for (auto & i : readDirectory(settings.nixStore)) store.insert(i.name);
1492
1493     /* Check whether all valid paths actually exist. */
1494     printMsg(lvlInfo, "checking path existence...");
1495
1496     PathSet validPaths2 = queryAllValidPaths(), validPaths, done;
1497
1498     foreach (PathSet::iterator, i, validPaths2)
1499         verifyPath(*i, store, done, validPaths, repair, errors);
1500
1501     /* Release the GC lock so that checking content hashes (which can
1502        take ages) doesn't block the GC or builds. */
1503     fdGCLock.close();
1504
1505     /* Optionally, check the content hashes (slow). */
1506     if (checkContents) {
1507         printMsg(lvlInfo, "checking hashes...");
1508
1509         Hash nullHash(htSHA256);
1510
1511         foreach (PathSet::iterator, i, validPaths) {
1512             try {
1513                 ValidPathInfo info = queryPathInfo(*i);
1514
1515                 /* Check the content hash (optionally - slow). */
1516                 printMsg(lvlTalkative, format("checking contents of `%1%'") % *i);
1517                 HashResult current = hashPath(info.hash.type, *i);
1518
1519                 if (info.hash != nullHash && info.hash != current.first) {
1520                     printMsg(lvlError, format("path `%1%' was modified! "
1521                             "expected hash `%2%', got `%3%'")
1522                         % *i % printHash(info.hash) % printHash(current.first));
1523                     if (repair) repairPath(*i); else errors = true;
1524                 } else {
1525
1526                     bool update = false;
1527
1528                     /* Fill in missing hashes. */
1529                     if (info.hash == nullHash) {
1530                         printMsg(lvlError, format("fixing missing hash on `%1%'") % *i);
1531                         info.hash = current.first;
1532                         update = true;
1533                     }
1534
1535                     /* Fill in missing narSize fields (from old stores). */
1536                     if (info.narSize == 0) {
1537                         printMsg(lvlError, format("updating size field on `%1%' to %2%") % *i % current.second);
1538                         info.narSize = current.second;
1539                         update = true;
1540                     }
1541
1542                     if (update) updatePathInfo(info);
1543
1544                 }
1545
1546             } catch (Error & e) {
1547                 /* It's possible that the path got GC'ed, so ignore
1548                    errors on invalid paths. */
1549                 if (isValidPath(*i))
1550                     printMsg(lvlError, format("error: %1%") % e.msg());
1551                 else
1552                     printMsg(lvlError, format("warning: %1%") % e.msg());
1553                 errors = true;
1554             }
1555         }
1556     }
1557
1558     return errors;
1559 }
1560
1561
1562 void LocalStore::verifyPath(const Path & path, const PathSet & store,
1563     PathSet & done, PathSet & validPaths, bool repair, bool & errors)
1564 {
1565     checkInterrupt();
1566
1567     if (done.find(path) != done.end()) return;
1568     done.insert(path);
1569
1570     if (!isStorePath(path)) {
1571         printMsg(lvlError, format("path `%1%' is not in the store") % path);
1572         invalidatePath(path);
1573         return;
1574     }
1575
1576     if (store.find(baseNameOf(path)) == store.end()) {
1577         /* Check any referrers first.  If we can invalidate them
1578            first, then we can invalidate this path as well. */
1579         bool canInvalidate = true;
1580         PathSet referrers; queryReferrers(path, referrers);
1581         foreach (PathSet::iterator, i, referrers)
1582             if (*i != path) {
1583                 verifyPath(*i, store, done, validPaths, repair, errors);
1584                 if (validPaths.find(*i) != validPaths.end())
1585                     canInvalidate = false;
1586             }
1587
1588         if (canInvalidate) {
1589             printMsg(lvlError, format("path `%1%' disappeared, removing from database...") % path);
1590             invalidatePath(path);
1591         } else {
1592             printMsg(lvlError, format("path `%1%' disappeared, but it still has valid referrers!") % path);
1593             if (repair)
1594                 try {
1595                     repairPath(path);
1596                 } catch (Error & e) {
1597                     printMsg(lvlError, format("warning: %1%") % e.msg());
1598                     errors = true;
1599                 }
1600             else errors = true;
1601         }
1602
1603         return;
1604     }
1605
1606     validPaths.insert(path);
1607 }
1608
1609
1610 bool LocalStore::pathContentsGood(const Path & path)
1611 {
1612     std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path);
1613     if (i != pathContentsGoodCache.end()) return i->second;
1614     printMsg(lvlInfo, format("checking path `%1%'...") % path);
1615     ValidPathInfo info = queryPathInfo(path);
1616     bool res;
1617     if (!pathExists(path))
1618         res = false;
1619     else {
1620         HashResult current = hashPath(info.hash.type, path);
1621         Hash nullHash(htSHA256);
1622         res = info.hash == nullHash || info.hash == current.first;
1623     }
1624     pathContentsGoodCache[path] = res;
1625     if (!res) printMsg(lvlError, format("path `%1%' is corrupted or missing!") % path);
1626     return res;
1627 }
1628
1629
1630 void LocalStore::markContentsGood(const Path & path)
1631 {
1632     pathContentsGoodCache[path] = true;
1633 }
1634
1635
1636 void LocalStore::vacuumDB()
1637 {
1638     if (sqlite3_exec(db, "vacuum;", 0, 0, 0) != SQLITE_OK)
1639         throwSQLiteError(db, "vacuuming SQLite database");
1640 }
1641
1642
1643 void LocalStore::createUser(const std::string & userName, uid_t userId)
1644 {
1645     auto dir = settings.nixStateDir + "/profiles/per-user/" + userName;
1646
1647     createDirs(dir);
1648     if (chmod(dir.c_str(), 0755) == -1)
1649         throw SysError(format("changing permissions of directory '%s'") % dir);
1650     if (chown(dir.c_str(), userId, -1) == -1)
1651         throw SysError(format("changing owner of directory '%s'") % dir);
1652 }
1653
1654
1655 }