nix/libstore/local-store.cc

   1 #include "config.h"
   2 #include "local-store.hh"
   3 #include "globals.hh"
   4 #include "archive.hh"
   5 #include "pathlocks.hh"
   6 #include "worker-protocol.hh"
   7 #include "derivations.hh"
   8 #include "affinity.hh"
   9
  10 #include <iostream>
  11 #include <algorithm>
  12 #include <cstring>
  13
  14 #include <sys/types.h>
  15 #include <sys/stat.h>
  16 #include <sys/time.h>
  17 #include <unistd.h>
  18 #include <utime.h>
  19 #include <fcntl.h>
  20 #include <errno.h>
  21 #include <stdio.h>
  22 #include <time.h>
  23 #include <grp.h>
  24 #include <ctype.h>
  25
  26 #if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H
  27 #include <sched.h>
  28 #include <sys/statvfs.h>
  29 #include <sys/mount.h>
  30 #endif
  31
  32 #include <sys/ioctl.h>
  33 #include <errno.h>
  34
  35 #include <sqlite3.h>
  36
  37
  38 namespace nix {
  39
  40
  41 void checkStoreNotSymlink()
  42 {
  43     if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return;
  44     Path path = settings.nixStore;
  45     struct stat st;
  46     while (path != "/") {
  47         if (lstat(path.c_str(), &st))
  48             throw SysError(format("getting status of `%1%'") % path);
  49         if (S_ISLNK(st.st_mode))
  50             throw Error(format(
  51                 "the path `%1%' is a symlink; "
  52                 "this is not allowed for the store and its parent directories")
  53                 % path);
  54         path = dirOf(path);
  55     }
  56 }
  57
  58
  59 LocalStore::LocalStore(bool reserveSpace)
  60     : didSetSubstituterEnv(false)
  61 {
  62     schemaPath = settings.nixDBPath + "/schema";
  63
  64     if (settings.readOnlyMode) {
  65         openDB(false);
  66         return;
  67     }
  68
  69     /* Create missing state directories if they don't already exist. */
  70     createDirs(settings.nixStore);
  71     makeStoreWritable();
  72     createDirs(linksDir = settings.nixStore + "/.links");
  73     Path profilesDir = settings.nixStateDir + "/profiles";
  74     createDirs(profilesDir);
  75     createDirs(settings.nixStateDir + "/temproots");
  76     createDirs(settings.nixDBPath);
  77     Path gcRootsDir = settings.nixStateDir + "/gcroots";
  78     if (!pathExists(gcRootsDir)) {
  79         createDirs(gcRootsDir);
  80         createSymlink(profilesDir, gcRootsDir + "/profiles");
  81     }
  82
  83     /* Optionally, create directories and set permissions for a
  84        multi-user install. */
  85     if (getuid() == 0 && settings.buildUsersGroup != "") {
  86
  87         Path perUserDir = profilesDir + "/per-user";
  88         createDirs(perUserDir);
  89         if (chmod(perUserDir.c_str(), 0755) == -1)
  90             throw SysError(format("could not set permissions on '%1%' to 755")
  91                            % perUserDir);
  92
  93         mode_t perm = 01775;
  94
  95         struct group * gr = getgrnam(settings.buildUsersGroup.c_str());
  96         if (!gr)
  97             throw Error(format("the group `%1%' specified in `build-users-group' does not exist")
  98                 % settings.buildUsersGroup);
  99         else {
 100             struct stat st;
 101             if (stat(settings.nixStore.c_str(), &st))
 102                 throw SysError(format("getting attributes of path '%1%'") % settings.nixStore);
 103
 104             if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) {
 105                 if (chown(settings.nixStore.c_str(), 0, gr->gr_gid) == -1)
 106                     throw SysError(format("changing ownership of path '%1%'") % settings.nixStore);
 107                 if (chmod(settings.nixStore.c_str(), perm) == -1)
 108                     throw SysError(format("changing permissions on path '%1%'") % settings.nixStore);
 109             }
 110         }
 111     }
 112
 113     checkStoreNotSymlink();
 114
 115     /* We can't open a SQLite database if the disk is full.  Since
 116        this prevents the garbage collector from running when it's most
 117        needed, we reserve some dummy space that we can free just
 118        before doing a garbage collection. */
 119     try {
 120         Path reservedPath = settings.nixDBPath + "/reserved";
 121         if (reserveSpace) {
 122             struct stat st;
 123             if (stat(reservedPath.c_str(), &st) == -1 ||
 124                 st.st_size != settings.reservedSize)
 125             {
 126                 AutoCloseFD fd = open(reservedPath.c_str(), O_WRONLY | O_CREAT, 0600);
 127                 int res = -1;
 128 #if HAVE_POSIX_FALLOCATE
 129                 res = posix_fallocate(fd, 0, settings.reservedSize);
 130 #endif
 131                 if (res == -1) {
 132                     writeFull(fd, string(settings.reservedSize, 'X'));
 133                     ftruncate(fd, settings.reservedSize);
 134                 }
 135             }
 136         }
 137         else
 138             deletePath(reservedPath);
 139     } catch (SysError & e) { /* don't care about errors */
 140     }
 141
 142     /* Acquire the big fat lock in shared mode to make sure that no
 143        schema upgrade is in progress. */
 144     try {
 145         Path globalLockPath = settings.nixDBPath + "/big-lock";
 146         globalLock = openLockFile(globalLockPath.c_str(), true);
 147     } catch (SysError & e) {
 148         if (e.errNo != EACCES) throw;
 149         settings.readOnlyMode = true;
 150         openDB(false);
 151         return;
 152     }
 153
 154     if (!lockFile(globalLock, ltRead, false)) {
 155         printMsg(lvlError, "waiting for the big store lock...");
 156         lockFile(globalLock, ltRead, true);
 157     }
 158
 159     /* Check the current database schema and if necessary do an
 160        upgrade.  */
 161     int curSchema = getSchema();
 162     if (curSchema > nixSchemaVersion)
 163         throw Error(format("current store schema is version %1%, but I only support %2%")
 164             % curSchema % nixSchemaVersion);
 165
 166     else if (curSchema == 0) { /* new store */
 167         curSchema = nixSchemaVersion;
 168         openDB(true);
 169         writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str());
 170     }
 171
 172     else if (curSchema < nixSchemaVersion) {
 173         /* Guix always used version 7 of the schema.  */
 174         throw Error(
 175             format("Your store database uses an implausibly old schema, version %1%.")
 176             % curSchema);
 177     }
 178
 179     else openDB(false);
 180 }
 181
 182
 183 LocalStore::~LocalStore()
 184 {
 185     try {
 186         if (runningSubstituter) {
 187             RunningSubstituter &i = *runningSubstituter;
 188             if (!i.disabled) {
 189                 i.to.close();
 190                 i.from.close();
 191                 i.error.close();
 192                 if (i.pid != -1)
 193                     i.pid.wait(true);
 194             }
 195         }
 196     } catch (...) {
 197         ignoreException();
 198     }
 199
 200     try {
 201         if (fdTempRoots != -1) {
 202             fdTempRoots.close();
 203             unlink(fnTempRoots.c_str());
 204         }
 205     } catch (...) {
 206         ignoreException();
 207     }
 208 }
 209
 210
 211 int LocalStore::getSchema()
 212 {
 213     int curSchema = 0;
 214     if (pathExists(schemaPath)) {
 215         string s = readFile(schemaPath);
 216         if (!string2Int(s, curSchema))
 217             throw Error(format("`%1%' is corrupt") % schemaPath);
 218     }
 219     return curSchema;
 220 }
 221
 222
 223 void LocalStore::openDB(bool create)
 224 {
 225     if (access(settings.nixDBPath.c_str(), R_OK | W_OK))
 226         throw SysError(format("store database directory `%1%' is not writable") % settings.nixDBPath);
 227
 228     /* Open the store database. */
 229     string dbPath = settings.nixDBPath + "/db.sqlite";
 230     if (sqlite3_open_v2(dbPath.c_str(), &db.db,
 231             SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
 232         throw Error(format("cannot open store database `%1%'") % dbPath);
 233
 234     if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
 235         throwSQLiteError(db, "setting timeout");
 236
 237     if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK)
 238         throwSQLiteError(db, "enabling foreign keys");
 239
 240     /* !!! check whether sqlite has been built with foreign key
 241        support */
 242
 243     /* Whether SQLite should fsync().  "Normal" synchronous mode
 244        should be safe enough.  If the user asks for it, don't sync at
 245        all.  This can cause database corruption if the system
 246        crashes. */
 247     string syncMode = settings.fsyncMetadata ? "normal" : "off";
 248     if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
 249         throwSQLiteError(db, "setting synchronous mode");
 250
 251     /* Set the SQLite journal mode.  WAL mode is fastest, so it's the
 252        default. */
 253     string mode = settings.useSQLiteWAL ? "wal" : "truncate";
 254     string prevMode;
 255     {
 256         SQLiteStmt stmt;
 257         stmt.create(db, "pragma main.journal_mode;");
 258         if (sqlite3_step(stmt) != SQLITE_ROW)
 259             throwSQLiteError(db, "querying journal mode");
 260         prevMode = string((const char *) sqlite3_column_text(stmt, 0));
 261     }
 262     if (prevMode != mode &&
 263         sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
 264         throwSQLiteError(db, "setting journal mode");
 265
 266     /* Increase the auto-checkpoint interval to 40000 pages.  This
 267        seems enough to ensure that instantiating the NixOS system
 268        derivation is done in a single fsync(). */
 269     if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 40000;", 0, 0, 0) != SQLITE_OK)
 270         throwSQLiteError(db, "setting autocheckpoint interval");
 271
 272     /* Initialise the database schema, if necessary. */
 273     if (create) {
 274         const char * schema =
 275 #include "schema.sql.hh"
 276             ;
 277         if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK)
 278             throwSQLiteError(db, "initialising database schema");
 279     }
 280
 281     /* Prepare SQL statements. */
 282     stmtRegisterValidPath.create(db,
 283         "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);");
 284     stmtUpdatePathInfo.create(db,
 285         "update ValidPaths set narSize = ?, hash = ? where path = ?;");
 286     stmtAddReference.create(db,
 287         "insert or replace into Refs (referrer, reference) values (?, ?);");
 288     stmtQueryPathInfo.create(db,
 289         "select id, hash, registrationTime, deriver, narSize from ValidPaths where path = ?;");
 290     stmtQueryReferences.create(db,
 291         "select path from Refs join ValidPaths on reference = id where referrer = ?;");
 292     stmtQueryReferrers.create(db,
 293         "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);");
 294     stmtInvalidatePath.create(db,
 295         "delete from ValidPaths where path = ?;");
 296     stmtRegisterFailedPath.create(db,
 297         "insert or ignore into FailedPaths (path, time) values (?, ?);");
 298     stmtHasPathFailed.create(db,
 299         "select time from FailedPaths where path = ?;");
 300     stmtQueryFailedPaths.create(db,
 301         "select path from FailedPaths;");
 302     // If the path is a derivation, then clear its outputs.
 303     stmtClearFailedPath.create(db,
 304         "delete from FailedPaths where ?1 = '*' or path = ?1 "
 305         "or path in (select d.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where v.path = ?1);");
 306     stmtAddDerivationOutput.create(db,
 307         "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);");
 308     stmtQueryValidDerivers.create(db,
 309         "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;");
 310     stmtQueryDerivationOutputs.create(db,
 311         "select id, path from DerivationOutputs where drv = ?;");
 312     // Use "path >= ?" with limit 1 rather than "path like '?%'" to
 313     // ensure efficient lookup.
 314     stmtQueryPathFromHashPart.create(db,
 315         "select path from ValidPaths where path >= ? limit 1;");
 316     stmtQueryValidPaths.create(db, "select path from ValidPaths");
 317 }
 318
 319
 320 /* To improve purity, users may want to make the store a read-only
 321    bind mount.  So make the store writable for this process. */
 322 void LocalStore::makeStoreWritable()
 323 {
 324 #if HAVE_UNSHARE && HAVE_STATVFS && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_REMOUNT)
 325     if (getuid() != 0) return;
 326     /* Check if /nix/store is on a read-only mount. */
 327     struct statvfs stat;
 328     if (statvfs(settings.nixStore.c_str(), &stat) != 0)
 329         throw SysError("getting info about the store mount point");
 330
 331     if (stat.f_flag & ST_RDONLY) {
 332         if (unshare(CLONE_NEWNS) == -1)
 333             throw SysError("setting up a private mount namespace");
 334
 335         if (mount(0, settings.nixStore.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1)
 336             throw SysError(format("remounting %1% writable") % settings.nixStore);
 337     }
 338 #endif
 339 }
 340
 341
 342 const time_t mtimeStore = 1; /* 1 second into the epoch */
 343
 344
 345 static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st)
 346 {
 347     if (!S_ISLNK(st.st_mode)) {
 348
 349         /* Mask out all type related bits. */
 350         mode_t mode = st.st_mode & ~S_IFMT;
 351
 352         if (mode != 0444 && mode != 0555) {
 353             mode = (st.st_mode & S_IFMT)
 354                  | 0444
 355                  | (st.st_mode & S_IXUSR ? 0111 : 0);
 356             if (chmod(path.c_str(), mode) == -1)
 357                 throw SysError(format("changing mode of `%1%' to %2$o") % path % mode);
 358         }
 359
 360     }
 361
 362     if (st.st_mtime != mtimeStore) {
 363         struct timeval times[2];
 364         times[0].tv_sec = st.st_atime;
 365         times[0].tv_usec = 0;
 366         times[1].tv_sec = mtimeStore;
 367         times[1].tv_usec = 0;
 368 #if HAVE_LUTIMES
 369         if (lutimes(path.c_str(), times) == -1)
 370             if (errno != ENOSYS ||
 371                 (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1))
 372 #else
 373         if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)
 374 #endif
 375             throw SysError(format("changing modification time of `%1%'") % path);
 376     }
 377 }
 378
 379
 380 void canonicaliseTimestampAndPermissions(const Path & path)
 381 {
 382     struct stat st;
 383     if (lstat(path.c_str(), &st))
 384         throw SysError(format("getting attributes of path `%1%'") % path);
 385     canonicaliseTimestampAndPermissions(path, st);
 386 }
 387
 388
 389 static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSeen & inodesSeen)
 390 {
 391     checkInterrupt();
 392
 393     struct stat st;
 394     if (lstat(path.c_str(), &st))
 395         throw SysError(format("getting attributes of path `%1%'") % path);
 396
 397     /* Really make sure that the path is of a supported type. */
 398     if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode)))
 399         throw Error(format("file ‘%1%’ has an unsupported type") % path);
 400
 401     /* Fail if the file is not owned by the build user.  This prevents
 402        us from messing up the ownership/permissions of files
 403        hard-linked into the output (e.g. "ln /etc/shadow $out/foo").
 404        However, ignore files that we chown'ed ourselves previously to
 405        ensure that we don't fail on hard links within the same build
 406        (i.e. "touch $out/foo; ln $out/foo $out/bar"). */
 407     if (fromUid != (uid_t) -1 && st.st_uid != fromUid) {
 408         assert(!S_ISDIR(st.st_mode));
 409         if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end())
 410             throw BuildError(format("invalid ownership on file `%1%'") % path);
 411         mode_t mode = st.st_mode & ~S_IFMT;
 412         assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore));
 413         return;
 414     }
 415
 416     inodesSeen.insert(Inode(st.st_dev, st.st_ino));
 417
 418     canonicaliseTimestampAndPermissions(path, st);
 419
 420     /* Change ownership to the current uid.  If it's a symlink, use
 421        lchown if available, otherwise don't bother.  Wrong ownership
 422        of a symlink doesn't matter, since the owning user can't change
 423        the symlink and can't delete it because the directory is not
 424        writable.  The only exception is top-level paths in the
 425        store (since that directory is group-writable for the build
 426        users group); we check for this case below. */
 427     if (st.st_uid != geteuid()) {
 428 #if HAVE_LCHOWN
 429         if (lchown(path.c_str(), geteuid(), getegid()) == -1)
 430 #else
 431         if (!S_ISLNK(st.st_mode) &&
 432             chown(path.c_str(), geteuid(), getegid()) == -1)
 433 #endif
 434             throw SysError(format("changing owner of `%1%' to %2%")
 435                 % path % geteuid());
 436     }
 437
 438     if (S_ISDIR(st.st_mode)) {
 439         DirEntries entries = readDirectory(path);
 440         for (auto & i : entries)
 441             canonicalisePathMetaData_(path + "/" + i.name, fromUid, inodesSeen);
 442     }
 443 }
 444
 445
 446 void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen)
 447 {
 448     canonicalisePathMetaData_(path, fromUid, inodesSeen);
 449
 450     /* On platforms that don't have lchown(), the top-level path can't
 451        be a symlink, since we can't change its ownership. */
 452     struct stat st;
 453     if (lstat(path.c_str(), &st))
 454         throw SysError(format("getting attributes of path `%1%'") % path);
 455
 456     if (st.st_uid != geteuid()) {
 457         assert(S_ISLNK(st.st_mode));
 458         throw Error(format("wrong ownership of top-level store path `%1%'") % path);
 459     }
 460 }
 461
 462
 463 void canonicalisePathMetaData(const Path & path, uid_t fromUid)
 464 {
 465     InodesSeen inodesSeen;
 466     canonicalisePathMetaData(path, fromUid, inodesSeen);
 467 }
 468
 469
 470 void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & drv)
 471 {
 472     string drvName = storePathToName(drvPath);
 473     assert(isDerivation(drvName));
 474     drvName = string(drvName, 0, drvName.size() - drvExtension.size());
 475
 476     if (isFixedOutputDrv(drv)) {
 477         DerivationOutputs::const_iterator out = drv.outputs.find("out");
 478         if (out == drv.outputs.end())
 479             throw Error(format("derivation `%1%' does not have an output named `out'") % drvPath);
 480
 481         bool recursive; HashType ht; Hash h;
 482         out->second.parseHashInfo(recursive, ht, h);
 483         Path outPath = makeFixedOutputPath(recursive, ht, h, drvName);
 484
 485         StringPairs::const_iterator j = drv.env.find("out");
 486         if (out->second.path != outPath || j == drv.env.end() || j->second != outPath)
 487             throw Error(format("derivation `%1%' has incorrect output `%2%', should be `%3%'")
 488                 % drvPath % out->second.path % outPath);
 489     }
 490
 491     else {
 492         Derivation drvCopy(drv);
 493         foreach (DerivationOutputs::iterator, i, drvCopy.outputs) {
 494             i->second.path = "";
 495             drvCopy.env[i->first] = "";
 496         }
 497
 498         Hash h = hashDerivationModulo(*this, drvCopy);
 499
 500         foreach (DerivationOutputs::const_iterator, i, drv.outputs) {
 501             Path outPath = makeOutputPath(i->first, h, drvName);
 502             StringPairs::const_iterator j = drv.env.find(i->first);
 503             if (i->second.path != outPath || j == drv.env.end() || j->second != outPath)
 504                 throw Error(format("derivation `%1%' has incorrect output `%2%', should be `%3%'")
 505                     % drvPath % i->second.path % outPath);
 506         }
 507     }
 508 }
 509
 510
 511 uint64_t LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs)
 512 {
 513     stmtRegisterValidPath.use()
 514         (info.path)
 515         ("sha256:" + printHash(info.hash))
 516         (info.registrationTime == 0 ? time(0) : info.registrationTime)
 517         (info.deriver, info.deriver != "")
 518         (info.narSize, info.narSize != 0)
 519         .exec();
 520     uint64_t id = sqlite3_last_insert_rowid(db);
 521
 522     /* If this is a derivation, then store the derivation outputs in
 523        the database.  This is useful for the garbage collector: it can
 524        efficiently query whether a path is an output of some
 525        derivation. */
 526     if (isDerivation(info.path)) {
 527         Derivation drv = readDerivation(info.path);
 528
 529         /* Verify that the output paths in the derivation are correct
 530            (i.e., follow the scheme for computing output paths from
 531            derivations).  Note that if this throws an error, then the
 532            DB transaction is rolled back, so the path validity
 533            registration above is undone. */
 534         if (checkOutputs) checkDerivationOutputs(info.path, drv);
 535
 536         for (auto & i : drv.outputs) {
 537             stmtAddDerivationOutput.use()
 538                 (id)
 539                 (i.first)
 540                 (i.second.path)
 541                 .exec();
 542         }
 543     }
 544
 545     return id;
 546 }
 547
 548
 549 void LocalStore::addReference(uint64_t referrer, uint64_t reference)
 550 {
 551     stmtAddReference.use()(referrer)(reference).exec();
 552 }
 553
 554
 555 void LocalStore::registerFailedPath(const Path & path)
 556 {
 557     retrySQLite<void>([&]() {
 558         stmtRegisterFailedPath.use()(path)(time(0)).step();
 559     });
 560 }
 561
 562
 563 bool LocalStore::hasPathFailed(const Path & path)
 564 {
 565     return retrySQLite<bool>([&]() {
 566         return stmtHasPathFailed.use()(path).next();
 567     });
 568 }
 569
 570
 571 PathSet LocalStore::queryFailedPaths()
 572 {
 573     return retrySQLite<PathSet>([&]() {
 574         auto useQueryFailedPaths(stmtQueryFailedPaths.use());
 575
 576         PathSet res;
 577         while (useQueryFailedPaths.next())
 578             res.insert(useQueryFailedPaths.getStr(0));
 579
 580         return res;
 581     });
 582 }
 583
 584
 585 void LocalStore::clearFailedPaths(const PathSet & paths)
 586 {
 587     retrySQLite<void>([&]() {
 588         SQLiteTxn txn(db);
 589
 590         for (auto & path : paths)
 591             stmtClearFailedPath.use()(path).exec();
 592
 593         txn.commit();
 594     });
 595 }
 596
 597
 598 Hash parseHashField(const Path & path, const string & s)
 599 {
 600     string::size_type colon = s.find(':');
 601     if (colon == string::npos)
 602         throw Error(format("corrupt hash `%1%' in valid-path entry for `%2%'")
 603             % s % path);
 604     HashType ht = parseHashType(string(s, 0, colon));
 605     if (ht == htUnknown)
 606         throw Error(format("unknown hash type `%1%' in valid-path entry for `%2%'")
 607             % string(s, 0, colon) % path);
 608     return parseHash(ht, string(s, colon + 1));
 609 }
 610
 611
 612 ValidPathInfo LocalStore::queryPathInfo(const Path & path)
 613 {
 614     ValidPathInfo info;
 615     info.path = path;
 616
 617     assertStorePath(path);
 618
 619     return retrySQLite<ValidPathInfo>([&]() {
 620
 621         /* Get the path info. */
 622         auto useQueryPathInfo(stmtQueryPathInfo.use()(path));
 623
 624         if (!useQueryPathInfo.next())
 625             throw Error(format("path `%1%' is not valid") % path);
 626
 627         info.id = useQueryPathInfo.getInt(0);
 628
 629         info.hash = parseHashField(path, useQueryPathInfo.getStr(1));
 630
 631         info.registrationTime = useQueryPathInfo.getInt(2);
 632
 633         auto s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3);
 634         if (s) info.deriver = s;
 635
 636         /* Note that narSize = NULL yields 0. */
 637         info.narSize = useQueryPathInfo.getInt(4);
 638
 639         /* Get the references. */
 640         auto useQueryReferences(stmtQueryReferences.use()(info.id));
 641
 642         while (useQueryReferences.next())
 643             info.references.insert(useQueryReferences.getStr(0));
 644
 645         return info;
 646     });
 647 }
 648
 649
 650 /* Update path info in the database.  Currently only updates the
 651    narSize field. */
 652 void LocalStore::updatePathInfo(const ValidPathInfo & info)
 653 {
 654     stmtUpdatePathInfo.use()
 655         (info.narSize, info.narSize != 0)
 656         ("sha256:" + printHash(info.hash))
 657         (info.path)
 658         .exec();
 659 }
 660
 661
 662 uint64_t LocalStore::queryValidPathId(const Path & path)
 663 {
 664     auto use(stmtQueryPathInfo.use()(path));
 665     if (!use.next())
 666         throw Error(format("path ‘%1%’ is not valid") % path);
 667     return use.getInt(0);
 668 }
 669
 670
 671 bool LocalStore::isValidPath_(const Path & path)
 672 {
 673     return stmtQueryPathInfo.use()(path).next();
 674 }
 675
 676
 677 bool LocalStore::isValidPath(const Path & path)
 678 {
 679     return retrySQLite<bool>([&]() {
 680         return isValidPath_(path);
 681     });
 682 }
 683
 684
 685 PathSet LocalStore::queryValidPaths(const PathSet & paths)
 686 {
 687     return retrySQLite<PathSet>([&]() {
 688         PathSet res;
 689         foreach (PathSet::const_iterator, i, paths)
 690             if (isValidPath_(*i)) res.insert(*i);
 691         return res;
 692     });
 693 }
 694
 695
 696 PathSet LocalStore::queryAllValidPaths()
 697 {
 698     return retrySQLite<PathSet>([&]() {
 699         auto use(stmtQueryValidPaths.use());
 700         PathSet res;
 701         while (use.next()) res.insert(use.getStr(0));
 702         return res;
 703     });
 704 }
 705
 706
 707 void LocalStore::queryReferences(const Path & path,
 708     PathSet & references)
 709 {
 710     ValidPathInfo info = queryPathInfo(path);
 711     references.insert(info.references.begin(), info.references.end());
 712 }
 713
 714
 715 void LocalStore::queryReferrers_(const Path & path, PathSet & referrers)
 716 {
 717     auto useQueryReferrers(stmtQueryReferrers.use()(path));
 718
 719     while (useQueryReferrers.next())
 720         referrers.insert(useQueryReferrers.getStr(0));
 721 }
 722
 723
 724 void LocalStore::queryReferrers(const Path & path, PathSet & referrers)
 725 {
 726     assertStorePath(path);
 727     return retrySQLite<void>([&]() {
 728         queryReferrers_(path, referrers);
 729     });
 730 }
 731
 732
 733 Path LocalStore::queryDeriver(const Path & path)
 734 {
 735     return queryPathInfo(path).deriver;
 736 }
 737
 738
 739 PathSet LocalStore::queryValidDerivers(const Path & path)
 740 {
 741     assertStorePath(path);
 742
 743     return retrySQLite<PathSet>([&]() {
 744         auto useQueryValidDerivers(stmtQueryValidDerivers.use()(path));
 745
 746         PathSet derivers;
 747         while (useQueryValidDerivers.next())
 748             derivers.insert(useQueryValidDerivers.getStr(1));
 749
 750         return derivers;
 751     });
 752 }
 753
 754
 755 PathSet LocalStore::queryDerivationOutputs(const Path & path)
 756 {
 757     return retrySQLite<PathSet>([&]() {
 758         auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path)));
 759
 760         PathSet outputs;
 761         while (useQueryDerivationOutputs.next())
 762             outputs.insert(useQueryDerivationOutputs.getStr(1));
 763
 764         return outputs;
 765     });
 766 }
 767
 768
 769 StringSet LocalStore::queryDerivationOutputNames(const Path & path)
 770 {
 771     return retrySQLite<StringSet>([&]() {
 772         auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path)));
 773
 774         StringSet outputNames;
 775         while (useQueryDerivationOutputs.next())
 776             outputNames.insert(useQueryDerivationOutputs.getStr(0));
 777
 778         return outputNames;
 779     });
 780 }
 781
 782
 783 Path LocalStore::queryPathFromHashPart(const string & hashPart)
 784 {
 785     if (hashPart.size() != 32) throw Error("invalid hash part");
 786
 787     Path prefix = settings.nixStore + "/" + hashPart;
 788
 789     return retrySQLite<Path>([&]() -> Path {
 790         auto useQueryPathFromHashPart(stmtQueryPathFromHashPart.use()(prefix));
 791
 792         if (!useQueryPathFromHashPart.next()) return "";
 793
 794         const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0);
 795         return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : "";
 796     });
 797 }
 798
 799
 800 void LocalStore::setSubstituterEnv()
 801 {
 802     if (didSetSubstituterEnv) return;
 803
 804     /* Pass configuration options (including those overridden with
 805        --option) to substituters. */
 806     setenv("_NIX_OPTIONS", settings.pack().c_str(), 1);
 807
 808     didSetSubstituterEnv = true;
 809 }
 810
 811
 812 void LocalStore::startSubstituter(RunningSubstituter & run)
 813 {
 814     if (run.disabled || run.pid != -1) return;
 815
 816     debug(format("starting substituter program `%1% substitute'")
 817           % settings.guixProgram);
 818
 819     Pipe toPipe, fromPipe, errorPipe;
 820
 821     toPipe.create();
 822     fromPipe.create();
 823     errorPipe.create();
 824
 825     setSubstituterEnv();
 826
 827     run.pid = startProcess([&]() {
 828         if (dup2(toPipe.readSide, STDIN_FILENO) == -1)
 829             throw SysError("dupping stdin");
 830         if (dup2(fromPipe.writeSide, STDOUT_FILENO) == -1)
 831             throw SysError("dupping stdout");
 832         if (dup2(errorPipe.writeSide, STDERR_FILENO) == -1)
 833             throw SysError("dupping stderr");
 834         execl(settings.guixProgram.c_str(), "guix", "substitute", "--query", NULL);
 835         throw SysError(format("executing `%1%'") % settings.guixProgram);
 836     });
 837
 838     run.to = toPipe.writeSide.borrow();
 839     run.from = run.fromBuf.fd = fromPipe.readSide.borrow();
 840     run.error = errorPipe.readSide.borrow();
 841
 842     toPipe.readSide.close();
 843     fromPipe.writeSide.close();
 844     errorPipe.writeSide.close();
 845
 846     /* The substituter may exit right away if it's disabled in any way
 847        (e.g. copy-from-other-stores.pl will exit if no other stores
 848        are configured). */
 849     try {
 850         getLineFromSubstituter(run);
 851     } catch (EndOfFile & e) {
 852         run.to.close();
 853         run.from.close();
 854         run.error.close();
 855         run.disabled = true;
 856         if (run.pid.wait(true) != 0) throw;
 857     }
 858 }
 859
 860
 861 /* Read a line from the substituter's stdout, while also processing
 862    its stderr. */
 863 string LocalStore::getLineFromSubstituter(RunningSubstituter & run)
 864 {
 865     string res, err;
 866
 867     /* We might have stdout data left over from the last time. */
 868     if (run.fromBuf.hasData()) goto haveData;
 869
 870     while (1) {
 871         checkInterrupt();
 872
 873         fd_set fds;
 874         FD_ZERO(&fds);
 875         FD_SET(run.from, &fds);
 876         FD_SET(run.error, &fds);
 877
 878         /* Wait for data to appear on the substituter's stdout or
 879            stderr. */
 880         if (select(run.from > run.error ? run.from + 1 : run.error + 1, &fds, 0, 0, 0) == -1) {
 881             if (errno == EINTR) continue;
 882             throw SysError("waiting for input from the substituter");
 883         }
 884
 885         /* Completely drain stderr before dealing with stdout. */
 886         if (FD_ISSET(run.error, &fds)) {
 887             char buf[4096];
 888             ssize_t n = read(run.error, (unsigned char *) buf, sizeof(buf));
 889             if (n == -1) {
 890                 if (errno == EINTR) continue;
 891                 throw SysError("reading from substituter's stderr");
 892             }
 893             if (n == 0) throw EndOfFile(format("`%1% substitute' died unexpectedly")
 894                                         % settings.guixProgram);
 895             err.append(buf, n);
 896             string::size_type p;
 897             while (((p = err.find('\n')) != string::npos)
 898                    || ((p = err.find('\r')) != string::npos)) {
 899                 string thing(err, 0, p + 1);
 900                 writeToStderr("substitute: " + thing);
 901                 err = string(err, p + 1);
 902             }
 903         }
 904
 905         /* Read from stdout until we get a newline or the buffer is empty. */
 906         else if (run.fromBuf.hasData() || FD_ISSET(run.from, &fds)) {
 907         haveData:
 908             do {
 909                 unsigned char c;
 910                 run.fromBuf(&c, 1);
 911                 if (c == '\n') {
 912                     if (!err.empty()) printMsg(lvlError, "substitute: " + err);
 913                     return res;
 914                 }
 915                 res += c;
 916             } while (run.fromBuf.hasData());
 917         }
 918     }
 919 }
 920
 921
 922 template<class T> T LocalStore::getIntLineFromSubstituter(RunningSubstituter & run)
 923 {
 924     string s = getLineFromSubstituter(run);
 925     T res;
 926     if (!string2Int(s, res)) throw Error("integer expected from stream");
 927     return res;
 928 }
 929
 930
 931 PathSet LocalStore::querySubstitutablePaths(const PathSet & paths)
 932 {
 933     PathSet res;
 934
 935     if (!settings.useSubstitutes || paths.empty()) return res;
 936
 937     if (!runningSubstituter) {
 938         std::unique_ptr<RunningSubstituter>fresh(new RunningSubstituter);
 939         runningSubstituter.swap(fresh);
 940     }
 941
 942     RunningSubstituter & run = *runningSubstituter;
 943     startSubstituter(run);
 944
 945     if (!run.disabled) {
 946         string s = "have ";
 947         foreach (PathSet::const_iterator, j, paths)
 948             if (res.find(*j) == res.end()) { s += *j; s += " "; }
 949         writeLine(run.to, s);
 950         while (true) {
 951             /* FIXME: we only read stderr when an error occurs, so
 952                substituters should only write (short) messages to
 953                stderr when they fail.  I.e. they shouldn't write debug
 954                output. */
 955             Path path = getLineFromSubstituter(run);
 956             if (path == "") break;
 957             res.insert(path);
 958         }
 959     }
 960
 961     return res;
 962 }
 963
 964
 965 void LocalStore::querySubstitutablePathInfos(PathSet & paths, SubstitutablePathInfos & infos)
 966 {
 967     if (!settings.useSubstitutes) return;
 968
 969     if (!runningSubstituter) {
 970         std::unique_ptr<RunningSubstituter>fresh(new RunningSubstituter);
 971         runningSubstituter.swap(fresh);
 972     }
 973
 974     RunningSubstituter & run = *runningSubstituter;
 975     startSubstituter(run);
 976     if (run.disabled) return;
 977
 978     string s = "info ";
 979     foreach (PathSet::const_iterator, i, paths)
 980         if (infos.find(*i) == infos.end()) { s += *i; s += " "; }
 981     writeLine(run.to, s);
 982
 983     while (true) {
 984         Path path = getLineFromSubstituter(run);
 985         if (path == "") break;
 986         if (paths.find(path) == paths.end())
 987             throw Error(format("got unexpected path `%1%' from substituter") % path);
 988         paths.erase(path);
 989         SubstitutablePathInfo & info(infos[path]);
 990         info.deriver = getLineFromSubstituter(run);
 991         if (info.deriver != "") assertStorePath(info.deriver);
 992         int nrRefs = getIntLineFromSubstituter<int>(run);
 993         while (nrRefs--) {
 994             Path p = getLineFromSubstituter(run);
 995             assertStorePath(p);
 996             info.references.insert(p);
 997         }
 998         info.downloadSize = getIntLineFromSubstituter<long long>(run);
 999         info.narSize = getIntLineFromSubstituter<long long>(run);
1000     }
1001 }
1002
1003
1004 void LocalStore::querySubstitutablePathInfos(const PathSet & paths,
1005     SubstitutablePathInfos & infos)
1006 {
1007     if (!paths.empty()) {
1008         PathSet todo = paths;
1009         querySubstitutablePathInfos(todo, infos);
1010     }
1011 }
1012
1013
1014 Hash LocalStore::queryPathHash(const Path & path)
1015 {
1016     return queryPathInfo(path).hash;
1017 }
1018
1019
1020 void LocalStore::registerValidPath(const ValidPathInfo & info)
1021 {
1022     ValidPathInfos infos;
1023     infos.push_back(info);
1024     registerValidPaths(infos);
1025 }
1026
1027
1028 void LocalStore::registerValidPaths(const ValidPathInfos & infos)
1029 {
1030     /* SQLite will fsync by default, but the new valid paths may not be fsync-ed.
1031      * So some may want to fsync them before registering the validity, at the
1032      * expense of some speed of the path registering operation. */
1033     if (settings.syncBeforeRegistering) sync();
1034
1035     return retrySQLite<void>([&]() {
1036         SQLiteTxn txn(db);
1037         PathSet paths;
1038
1039         foreach (ValidPathInfos::const_iterator, i, infos) {
1040             assert(i->hash.type == htSHA256);
1041             if (isValidPath_(i->path))
1042                 updatePathInfo(*i);
1043             else
1044                 addValidPath(*i, false);
1045             paths.insert(i->path);
1046         }
1047
1048         for (auto & i : infos) {
1049             auto referrer = queryValidPathId(i.path);
1050             for (auto & j : i.references)
1051                 addReference(referrer, queryValidPathId(j));
1052         }
1053
1054         /* Check that the derivation outputs are correct.  We can't do
1055            this in addValidPath() above, because the references might
1056            not be valid yet. */
1057         foreach (ValidPathInfos::const_iterator, i, infos)
1058             if (isDerivation(i->path)) {
1059                 // FIXME: inefficient; we already loaded the
1060                 // derivation in addValidPath().
1061                 Derivation drv = readDerivation(i->path);
1062                 checkDerivationOutputs(i->path, drv);
1063             }
1064
1065         /* Do a topological sort of the paths.  This will throw an
1066            error if a cycle is detected and roll back the
1067            transaction.  Cycles can only occur when a derivation
1068            has multiple outputs. */
1069         topoSortPaths(*this, paths);
1070
1071         txn.commit();
1072     });
1073 }
1074
1075
1076 /* Invalidate a path.  The caller is responsible for checking that
1077    there are no referrers. */
1078 void LocalStore::invalidatePath(const Path & path)
1079 {
1080     debug(format("invalidating path `%1%'") % path);
1081
1082     drvHashes.erase(path);
1083
1084     stmtInvalidatePath.use()(path).exec();
1085
1086     /* Note that the foreign key constraints on the Refs table take
1087        care of deleting the references entries for `path'. */
1088 }
1089
1090
1091 Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
1092     bool recursive, HashType hashAlgo, bool repair)
1093 {
1094     Hash h = hashString(hashAlgo, dump);
1095
1096     Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name);
1097
1098     addTempRoot(dstPath);
1099
1100     if (repair || !isValidPath(dstPath)) {
1101
1102         /* The first check above is an optimisation to prevent
1103            unnecessary lock acquisition. */
1104
1105         PathLocks outputLock(singleton<PathSet, Path>(dstPath));
1106
1107         if (repair || !isValidPath(dstPath)) {
1108
1109             if (pathExists(dstPath)) deletePath(dstPath);
1110
1111             if (recursive) {
1112                 StringSource source(dump);
1113                 restorePath(dstPath, source);
1114             } else
1115                 writeFile(dstPath, dump);
1116
1117             canonicalisePathMetaData(dstPath, -1);
1118
1119             /* Register the SHA-256 hash of the NAR serialisation of
1120                the path in the database.  We may just have computed it
1121                above (if called with recursive == true and hashAlgo ==
1122                sha256); otherwise, compute it here. */
1123             HashResult hash;
1124             if (recursive) {
1125                 hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
1126                 hash.second = dump.size();
1127             } else
1128                 hash = hashPath(htSHA256, dstPath);
1129
1130             optimisePath(dstPath); // FIXME: combine with hashPath()
1131
1132             ValidPathInfo info;
1133             info.path = dstPath;
1134             info.hash = hash.first;
1135             info.narSize = hash.second;
1136             registerValidPath(info);
1137         }
1138
1139         outputLock.setDeletion(true);
1140     }
1141
1142     return dstPath;
1143 }
1144
1145
1146 Path LocalStore::addToStore(const string & name, const Path & _srcPath,
1147     bool recursive, HashType hashAlgo, PathFilter & filter, bool repair)
1148 {
1149     Path srcPath(absPath(_srcPath));
1150     debug(format("adding `%1%' to the store") % srcPath);
1151
1152     /* Read the whole path into memory. This is not a very scalable
1153        method for very large paths, but `copyPath' is mainly used for
1154        small files. */
1155     StringSink sink;
1156     if (recursive)
1157         dumpPath(srcPath, sink, filter);
1158     else
1159         sink.s = readFile(srcPath);
1160
1161     return addToStoreFromDump(sink.s, name, recursive, hashAlgo, repair);
1162 }
1163
1164
1165 Path LocalStore::addTextToStore(const string & name, const string & s,
1166     const PathSet & references, bool repair)
1167 {
1168     Path dstPath = computeStorePathForText(name, s, references);
1169
1170     addTempRoot(dstPath);
1171
1172     if (repair || !isValidPath(dstPath)) {
1173
1174         PathLocks outputLock(singleton<PathSet, Path>(dstPath));
1175
1176         if (repair || !isValidPath(dstPath)) {
1177
1178             if (pathExists(dstPath)) deletePath(dstPath);
1179
1180             writeFile(dstPath, s);
1181
1182             canonicalisePathMetaData(dstPath, -1);
1183
1184             HashResult hash = hashPath(htSHA256, dstPath);
1185
1186             optimisePath(dstPath);
1187
1188             ValidPathInfo info;
1189             info.path = dstPath;
1190             info.hash = hash.first;
1191             info.narSize = hash.second;
1192             info.references = references;
1193             registerValidPath(info);
1194         }
1195
1196         outputLock.setDeletion(true);
1197     }
1198
1199     return dstPath;
1200 }
1201
1202
1203 struct HashAndWriteSink : Sink
1204 {
1205     Sink & writeSink;
1206     HashSink hashSink;
1207     HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
1208     {
1209     }
1210     virtual void operator () (const unsigned char * data, size_t len)
1211     {
1212         writeSink(data, len);
1213         hashSink(data, len);
1214     }
1215     Hash currentHash()
1216     {
1217         return hashSink.currentHash().first;
1218     }
1219 };
1220
1221
1222 #define EXPORT_MAGIC 0x4558494e
1223
1224
1225 static void checkSecrecy(const Path & path)
1226 {
1227     struct stat st;
1228     if (stat(path.c_str(), &st))
1229         throw SysError(format("getting status of `%1%'") % path);
1230     if ((st.st_mode & (S_IRWXG | S_IRWXO)) != 0)
1231         throw Error(format("file `%1%' should be secret (inaccessible to everybody else)!") % path);
1232 }
1233
1234
1235 /* Return the authentication agent, a "guix authenticate" process started
1236    lazily.  */
1237 static std::shared_ptr<Agent> authenticationAgent()
1238 {
1239     static std::shared_ptr<Agent> agent;
1240
1241     if (!agent) {
1242         Strings args = { "authenticate" };
1243         agent = std::make_shared<Agent>(settings.guixProgram, args);
1244     }
1245
1246     return agent;
1247 }
1248
1249 /* Read an integer and the byte that immediately follows it from FD.  Return
1250    the integer.  */
1251 static int readInteger(int fd)
1252 {
1253     string str;
1254
1255     while (1) {
1256         char ch;
1257         ssize_t rd = read(fd, &ch, 1);
1258         if (rd == -1) {
1259             if (errno != EINTR)
1260                 throw SysError("reading an integer");
1261         } else if (rd == 0)
1262             throw EndOfFile("unexpected EOF reading an integer");
1263         else {
1264             if (isdigit(ch)) {
1265                 str += ch;
1266             } else {
1267                 break;
1268             }
1269         }
1270     }
1271
1272     return stoi(str);
1273 }
1274
1275 /* Read from FD a reply coming from 'guix authenticate'.  The reply has the
1276    form "CODE LEN:STR".  CODE is an integer, where zero indicates success.
1277    LEN specifies the length in bytes of the string that immediately
1278    follows.  */
1279 static std::string readAuthenticateReply(int fd)
1280 {
1281     int code = readInteger(fd);
1282     int len = readInteger(fd);
1283
1284     string str;
1285     str.resize(len);
1286     readFull(fd, (unsigned char *) &str[0], len);
1287
1288     if (code == 0)
1289         return str;
1290     else
1291         throw Error(str);
1292 }
1293
1294 /* Sign HASH with the key stored in file SECRETKEY.  Return the signature as a
1295    string, or raise an exception upon error.  */
1296 static std::string signHash(const string &secretKey, const Hash &hash)
1297 {
1298     auto agent = authenticationAgent();
1299     auto hexHash = printHash(hash);
1300
1301     writeLine(agent->toAgent.writeSide,
1302               (format("sign %1%:%2% %3%:%4%")
1303                % secretKey.size() % secretKey
1304                % hexHash.size() % hexHash).str());
1305
1306     return readAuthenticateReply(agent->fromAgent.readSide);
1307 }
1308
1309 /* Verify SIGNATURE and return the base16-encoded hash over which it was
1310    computed.  */
1311 static std::string verifySignature(const string &signature)
1312 {
1313     auto agent = authenticationAgent();
1314
1315     writeLine(agent->toAgent.writeSide,
1316               (format("verify %1%:%2%")
1317                % signature.size() % signature).str());
1318
1319     return readAuthenticateReply(agent->fromAgent.readSide);
1320 }
1321
1322 void LocalStore::exportPath(const Path & path, bool sign,
1323     Sink & sink)
1324 {
1325     assertStorePath(path);
1326
1327     printMsg(lvlInfo, format("exporting path `%1%'") % path);
1328
1329     if (!isValidPath(path))
1330         throw Error(format("path `%1%' is not valid") % path);
1331
1332     HashAndWriteSink hashAndWriteSink(sink);
1333
1334     dumpPath(path, hashAndWriteSink);
1335
1336     /* Refuse to export paths that have changed.  This prevents
1337        filesystem corruption from spreading to other machines.
1338        Don't complain if the stored hash is zero (unknown). */
1339     Hash hash = hashAndWriteSink.currentHash();
1340     Hash storedHash = queryPathHash(path);
1341     if (hash != storedHash && storedHash != Hash(storedHash.type))
1342         throw Error(format("hash of path `%1%' has changed from `%2%' to `%3%'!") % path
1343             % printHash(storedHash) % printHash(hash));
1344
1345     writeInt(EXPORT_MAGIC, hashAndWriteSink);
1346
1347     writeString(path, hashAndWriteSink);
1348
1349     PathSet references;
1350     queryReferences(path, references);
1351     writeStrings(references, hashAndWriteSink);
1352
1353     Path deriver = queryDeriver(path);
1354     writeString(deriver, hashAndWriteSink);
1355
1356     if (sign) {
1357         Hash hash = hashAndWriteSink.currentHash();
1358
1359         writeInt(1, hashAndWriteSink);
1360
1361         Path secretKey = settings.nixConfDir + "/signing-key.sec";
1362         checkSecrecy(secretKey);
1363
1364         string signature = signHash(secretKey, hash);
1365
1366         writeString(signature, hashAndWriteSink);
1367
1368     } else
1369         writeInt(0, hashAndWriteSink);
1370 }
1371
1372
1373 struct HashAndReadSource : Source
1374 {
1375     Source & readSource;
1376     HashSink hashSink;
1377     bool hashing;
1378     HashAndReadSource(Source & readSource) : readSource(readSource), hashSink(htSHA256)
1379     {
1380         hashing = true;
1381     }
1382     size_t read(unsigned char * data, size_t len)
1383     {
1384         size_t n = readSource.read(data, len);
1385         if (hashing) hashSink(data, n);
1386         return n;
1387     }
1388 };
1389
1390
1391 /* Create a temporary directory in the store that won't be
1392    garbage-collected. */
1393 Path LocalStore::createTempDirInStore()
1394 {
1395     Path tmpDir;
1396     do {
1397         /* There is a slight possibility that `tmpDir' gets deleted by
1398            the GC between createTempDir() and addTempRoot(), so repeat
1399            until `tmpDir' exists. */
1400         tmpDir = createTempDir(settings.nixStore);
1401         addTempRoot(tmpDir);
1402     } while (!pathExists(tmpDir));
1403     return tmpDir;
1404 }
1405
1406
1407 Path LocalStore::importPath(bool requireSignature, Source & source)
1408 {
1409     HashAndReadSource hashAndReadSource(source);
1410
1411     /* We don't yet know what store path this archive contains (the
1412        store path follows the archive data proper), and besides, we
1413        don't know yet whether the signature is valid. */
1414     Path tmpDir = createTempDirInStore();
1415     AutoDelete delTmp(tmpDir);
1416     Path unpacked = tmpDir + "/unpacked";
1417
1418     restorePath(unpacked, hashAndReadSource);
1419
1420     unsigned int magic = readInt(hashAndReadSource);
1421     if (magic != EXPORT_MAGIC)
1422         throw Error("normalized archive cannot be imported; wrong format");
1423
1424     Path dstPath = readStorePath(hashAndReadSource);
1425
1426     PathSet references = readStorePaths<PathSet>(hashAndReadSource);
1427
1428     Path deriver = readString(hashAndReadSource);
1429     if (deriver != "") assertStorePath(deriver);
1430
1431     Hash hash = hashAndReadSource.hashSink.finish().first;
1432     hashAndReadSource.hashing = false;
1433
1434     bool haveSignature = readInt(hashAndReadSource) == 1;
1435
1436     if (requireSignature && !haveSignature)
1437         throw Error(format("imported archive of `%1%' lacks a signature") % dstPath);
1438
1439     if (haveSignature) {
1440         string signature = readString(hashAndReadSource);
1441
1442         if (requireSignature) {
1443             string hash2 = verifySignature(signature);
1444
1445             /* Note: runProgram() throws an exception if the signature
1446                is invalid. */
1447
1448             if (printHash(hash) != hash2)
1449                 throw Error(
1450                     "signed hash doesn't match actual contents of imported "
1451                     "archive; archive could be corrupt, or someone is trying "
1452                     "to import a Trojan horse");
1453         }
1454     }
1455
1456     /* Do the actual import. */
1457
1458     /* !!! way too much code duplication with addTextToStore() etc. */
1459     addTempRoot(dstPath);
1460
1461     if (!isValidPath(dstPath)) {
1462
1463         PathLocks outputLock;
1464
1465         /* Lock the output path.  But don't lock if we're being called
1466            from a build hook (whose parent process already acquired a
1467            lock on this path). */
1468         Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS"));
1469         if (find(locksHeld.begin(), locksHeld.end(), dstPath) == locksHeld.end())
1470             outputLock.lockPaths(singleton<PathSet, Path>(dstPath));
1471
1472         if (!isValidPath(dstPath)) {
1473
1474             if (pathExists(dstPath)) deletePath(dstPath);
1475
1476             if (rename(unpacked.c_str(), dstPath.c_str()) == -1)
1477                 throw SysError(format("cannot move `%1%' to `%2%'")
1478                     % unpacked % dstPath);
1479
1480             canonicalisePathMetaData(dstPath, -1);
1481
1482             /* !!! if we were clever, we could prevent the hashPath()
1483                here. */
1484             HashResult hash = hashPath(htSHA256, dstPath);
1485
1486             optimisePath(dstPath); // FIXME: combine with hashPath()
1487
1488             ValidPathInfo info;
1489             info.path = dstPath;
1490             info.hash = hash.first;
1491             info.narSize = hash.second;
1492             info.references = references;
1493             info.deriver = deriver != "" && isValidPath(deriver) ? deriver : "";
1494             registerValidPath(info);
1495         }
1496
1497         outputLock.setDeletion(true);
1498     }
1499
1500     return dstPath;
1501 }
1502
1503
1504 Paths LocalStore::importPaths(bool requireSignature, Source & source)
1505 {
1506     Paths res;
1507     while (true) {
1508         unsigned long long n = readLongLong(source);
1509         if (n == 0) break;
1510         if (n != 1) throw Error("input doesn't look like something created by `nix-store --export'");
1511         res.push_back(importPath(requireSignature, source));
1512     }
1513     return res;
1514 }
1515
1516
1517 void LocalStore::invalidatePathChecked(const Path & path)
1518 {
1519     assertStorePath(path);
1520
1521     retrySQLite<void>([&]() {
1522         SQLiteTxn txn(db);
1523
1524         if (isValidPath_(path)) {
1525             PathSet referrers; queryReferrers_(path, referrers);
1526             referrers.erase(path); /* ignore self-references */
1527             if (!referrers.empty())
1528                 throw PathInUse(format("cannot delete path `%1%' because it is in use by %2%")
1529                     % path % showPaths(referrers));
1530             invalidatePath(path);
1531         }
1532
1533         txn.commit();
1534     });
1535 }
1536
1537
1538 bool LocalStore::verifyStore(bool checkContents, bool repair)
1539 {
1540     printMsg(lvlError, format("reading the store..."));
1541
1542     bool errors = false;
1543
1544     /* Acquire the global GC lock to prevent a garbage collection. */
1545     AutoCloseFD fdGCLock = openGCLock(ltWrite);
1546
1547     PathSet store;
1548     for (auto & i : readDirectory(settings.nixStore)) store.insert(i.name);
1549
1550     /* Check whether all valid paths actually exist. */
1551     printMsg(lvlInfo, "checking path existence...");
1552
1553     PathSet validPaths2 = queryAllValidPaths(), validPaths, done;
1554
1555     foreach (PathSet::iterator, i, validPaths2)
1556         verifyPath(*i, store, done, validPaths, repair, errors);
1557
1558     /* Release the GC lock so that checking content hashes (which can
1559        take ages) doesn't block the GC or builds. */
1560     fdGCLock.close();
1561
1562     /* Optionally, check the content hashes (slow). */
1563     if (checkContents) {
1564         printMsg(lvlInfo, "checking hashes...");
1565
1566         Hash nullHash(htSHA256);
1567
1568         foreach (PathSet::iterator, i, validPaths) {
1569             try {
1570                 ValidPathInfo info = queryPathInfo(*i);
1571
1572                 /* Check the content hash (optionally - slow). */
1573                 printMsg(lvlTalkative, format("checking contents of `%1%'") % *i);
1574                 HashResult current = hashPath(info.hash.type, *i);
1575
1576                 if (info.hash != nullHash && info.hash != current.first) {
1577                     printMsg(lvlError, format("path `%1%' was modified! "
1578                             "expected hash `%2%', got `%3%'")
1579                         % *i % printHash(info.hash) % printHash(current.first));
1580                     if (repair) repairPath(*i); else errors = true;
1581                 } else {
1582
1583                     bool update = false;
1584
1585                     /* Fill in missing hashes. */
1586                     if (info.hash == nullHash) {
1587                         printMsg(lvlError, format("fixing missing hash on `%1%'") % *i);
1588                         info.hash = current.first;
1589                         update = true;
1590                     }
1591
1592                     /* Fill in missing narSize fields (from old stores). */
1593                     if (info.narSize == 0) {
1594                         printMsg(lvlError, format("updating size field on `%1%' to %2%") % *i % current.second);
1595                         info.narSize = current.second;
1596                         update = true;
1597                     }
1598
1599                     if (update) updatePathInfo(info);
1600
1601                 }
1602
1603             } catch (Error & e) {
1604                 /* It's possible that the path got GC'ed, so ignore
1605                    errors on invalid paths. */
1606                 if (isValidPath(*i))
1607                     printMsg(lvlError, format("error: %1%") % e.msg());
1608                 else
1609                     printMsg(lvlError, format("warning: %1%") % e.msg());
1610                 errors = true;
1611             }
1612         }
1613     }
1614
1615     return errors;
1616 }
1617
1618
1619 void LocalStore::verifyPath(const Path & path, const PathSet & store,
1620     PathSet & done, PathSet & validPaths, bool repair, bool & errors)
1621 {
1622     checkInterrupt();
1623
1624     if (done.find(path) != done.end()) return;
1625     done.insert(path);
1626
1627     if (!isStorePath(path)) {
1628         printMsg(lvlError, format("path `%1%' is not in the store") % path);
1629         invalidatePath(path);
1630         return;
1631     }
1632
1633     if (store.find(baseNameOf(path)) == store.end()) {
1634         /* Check any referrers first.  If we can invalidate them
1635            first, then we can invalidate this path as well. */
1636         bool canInvalidate = true;
1637         PathSet referrers; queryReferrers(path, referrers);
1638         foreach (PathSet::iterator, i, referrers)
1639             if (*i != path) {
1640                 verifyPath(*i, store, done, validPaths, repair, errors);
1641                 if (validPaths.find(*i) != validPaths.end())
1642                     canInvalidate = false;
1643             }
1644
1645         if (canInvalidate) {
1646             printMsg(lvlError, format("path `%1%' disappeared, removing from database...") % path);
1647             invalidatePath(path);
1648         } else {
1649             printMsg(lvlError, format("path `%1%' disappeared, but it still has valid referrers!") % path);
1650             if (repair)
1651                 try {
1652                     repairPath(path);
1653                 } catch (Error & e) {
1654                     printMsg(lvlError, format("warning: %1%") % e.msg());
1655                     errors = true;
1656                 }
1657             else errors = true;
1658         }
1659
1660         return;
1661     }
1662
1663     validPaths.insert(path);
1664 }
1665
1666
1667 bool LocalStore::pathContentsGood(const Path & path)
1668 {
1669     std::map<Path, bool>::iterator i = pathContentsGoodCache.find(path);
1670     if (i != pathContentsGoodCache.end()) return i->second;
1671     printMsg(lvlInfo, format("checking path `%1%'...") % path);
1672     ValidPathInfo info = queryPathInfo(path);
1673     bool res;
1674     if (!pathExists(path))
1675         res = false;
1676     else {
1677         HashResult current = hashPath(info.hash.type, path);
1678         Hash nullHash(htSHA256);
1679         res = info.hash == nullHash || info.hash == current.first;
1680     }
1681     pathContentsGoodCache[path] = res;
1682     if (!res) printMsg(lvlError, format("path `%1%' is corrupted or missing!") % path);
1683     return res;
1684 }
1685
1686
1687 void LocalStore::markContentsGood(const Path & path)
1688 {
1689     pathContentsGoodCache[path] = true;
1690 }
1691
1692
1693 void LocalStore::vacuumDB()
1694 {
1695     if (sqlite3_exec(db, "vacuum;", 0, 0, 0) != SQLITE_OK)
1696         throwSQLiteError(db, "vacuuming SQLite database");
1697 }
1698
1699
1700 void LocalStore::createUser(const std::string & userName, uid_t userId)
1701 {
1702     auto dir = settings.nixStateDir + "/profiles/per-user/" + userName;
1703
1704     createDirs(dir);
1705     if (chmod(dir.c_str(), 0755) == -1)
1706         throw SysError(format("changing permissions of directory '%s'") % dir);
1707     if (chown(dir.c_str(), userId, -1) == -1)
1708         throw SysError(format("changing owner of directory '%s'") % dir);
1709 }
1710
1711
1712 }