#include <errno.h>
#include <stdio.h>
#include <cstring>
+#include <stdint.h>
#include <pwd.h>
#include <grp.h>
-#include <bzlib.h>
+#include <zlib.h>
+
+#if HAVE_BZLIB_H
+# include <bzlib.h>
+#endif
/* Includes required for chroot support. */
#if HAVE_SYS_PARAM_H
#include <sched.h>
#endif
-/* In GNU libc 2.11, <sys/mount.h> does not define `MS_PRIVATE', but
- <linux/fs.h> does. */
-#if !defined MS_PRIVATE && defined HAVE_LINUX_FS_H
-#include <linux/fs.h>
-#endif
-#define CHROOT_ENABLED HAVE_CHROOT && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE) && defined(CLONE_NEWNS) && defined(SYS_pivot_root)
+#define CHROOT_ENABLED HAVE_CHROOT && HAVE_SYS_MOUNT_H && defined(MS_BIND) && defined(MS_PRIVATE)
+#define CLONE_ENABLED defined(CLONE_NEWNS)
+
+#if defined(SYS_pivot_root)
+#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root,put_old))
+#endif
#if CHROOT_ENABLED
#include <sys/socket.h>
using std::map;
-static string pathNullDevice = "/dev/null";
-
-
/* Forward definition. */
class Worker;
-struct HookInstance;
/* A pointer to a goal. */
LocalStore & store;
- std::shared_ptr<HookInstance> hook;
+ std::shared_ptr<Agent> hook;
+ std::shared_ptr<Agent> substituter;
Worker(LocalStore & store);
~Worker();
//////////////////////////////////////////////////////////////////////
-/* Common initialisation performed in child processes. */
-static void commonChildInit(Pipe & logPipe)
-{
- /* Put the child in a separate session (and thus a separate
- process group) so that it has no controlling terminal (meaning
- that e.g. ssh cannot open /dev/tty) and it doesn't receive
- terminal signals. */
- if (setsid() == -1)
- throw SysError(format("creating a new session"));
-
- /* Dup the write side of the logger pipe into stderr. */
- if (dup2(logPipe.writeSide, STDERR_FILENO) == -1)
- throw SysError("cannot pipe standard error into log file");
-
- /* Dup stderr to stdout. */
- if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1)
- throw SysError("cannot dup stderr into stdout");
-
- /* Reroute stdin to /dev/null. */
- int fdDevNull = open(pathNullDevice.c_str(), O_RDWR);
- if (fdDevNull == -1)
- throw SysError(format("cannot open `%1%'") % pathNullDevice);
- if (dup2(fdDevNull, STDIN_FILENO) == -1)
- throw SysError("cannot dup null device into stdin");
- close(fdDevNull);
-}
-
/* Restore default handling of SIGPIPE, otherwise some programs will
randomly say "Broken pipe". */
static void restoreSIGPIPE()
/* Sanity check... */
if (uid == getuid() || uid == geteuid())
- throw Error(format("the Nix user should not be a member of `%1%'")
+ throw Error(format("the build user should not be a member of `%1%'")
% settings.buildUsersGroup);
/* Get the list of supplementary groups of this build user. This
killUser(uid);
}
-
-//////////////////////////////////////////////////////////////////////
-
-
-struct HookInstance
-{
- /* Pipes for talking to the build hook. */
- Pipe toHook;
-
- /* Pipe for the hook's standard output/error. */
- Pipe fromHook;
-
- /* Pipe for the builder's standard output/error. */
- Pipe builderOut;
-
- /* The process ID of the hook. */
- Pid pid;
-
- HookInstance();
-
- ~HookInstance();
-};
-
-
-HookInstance::HookInstance()
-{
- debug("starting build hook");
-
- Path buildHook = getEnv("NIX_BUILD_HOOK");
- if (string(buildHook, 0, 1) != "/") buildHook = settings.nixLibexecDir + "/nix/" + buildHook;
- buildHook = canonPath(buildHook);
-
- /* Create a pipe to get the output of the child. */
- fromHook.create();
-
- /* Create the communication pipes. */
- toHook.create();
-
- /* Create a pipe to get the output of the builder. */
- builderOut.create();
-
- /* Fork the hook. */
- pid = startProcess([&]() {
-
- commonChildInit(fromHook);
-
- if (chdir("/") == -1) throw SysError("changing into `/");
-
- /* Dup the communication pipes. */
- if (dup2(toHook.readSide, STDIN_FILENO) == -1)
- throw SysError("dupping to-hook read side");
-
- /* Use fd 4 for the builder's stdout/stderr. */
- if (dup2(builderOut.writeSide, 4) == -1)
- throw SysError("dupping builder's stdout/stderr");
-
- execl(buildHook.c_str(), buildHook.c_str(), settings.thisSystem.c_str(),
- (format("%1%") % settings.maxSilentTime).str().c_str(),
- (format("%1%") % settings.printBuildTrace).str().c_str(),
- (format("%1%") % settings.buildTimeout).str().c_str(),
- NULL);
-
- throw SysError(format("executing `%1%'") % buildHook);
- });
-
- pid.setSeparatePG(true);
- fromHook.writeSide.close();
- toHook.readSide.close();
-}
-
-
-HookInstance::~HookInstance()
-{
- try {
- toHook.writeSide.close();
- pid.kill(true);
- } catch (...) {
- ignoreException();
- }
-}
-
-
//////////////////////////////////////////////////////////////////////
/* File descriptor for the log file. */
FILE * fLogFile;
+ gzFile gzLogFile;
+#if HAVE_BZLIB_H
BZFILE * bzLogFile;
+#endif
AutoCloseFD fdLogFile;
/* Number of bytes received from the builder's stdout/stderr. */
Pipe builderOut;
/* The build hook. */
- std::shared_ptr<HookInstance> hook;
+ std::shared_ptr<Agent> hook;
/* Whether we're currently doing a chroot build. */
bool useChroot;
, needRestart(false)
, retrySubstitution(false)
, fLogFile(0)
+ , gzLogFile(0)
+#if HAVE_BZLIB_H
, bzLogFile(0)
+#endif
, useChroot(false)
, buildMode(buildMode)
{
assert(pid == -1);
}
+ /* If there was a build hook involved, remove it from the worker's
+ children. */
+ if (hook && hook->pid != -1) {
+ worker.childTerminated(hook->pid);
+ }
hook.reset();
}
trace("init");
if (settings.readOnlyMode)
- throw Error(format("cannot build derivation `%1%' - no write access to the Nix store") % drvPath);
+ throw Error(format("cannot build derivation `%1%' - no write access to the store") % drvPath);
/* The first thing to do is to make sure that the derivation
exists. If it doesn't, it may be created through a
return platform == settings.thisSystem
#if __linux__
|| (platform == "i686-linux" && settings.thisSystem == "x86_64-linux")
+ || (platform == "armhf-linux" && settings.thisSystem == "aarch64-linux")
#endif
;
}
}
/* Obtain locks on all output paths. The locks are automatically
- released when we exit this function or Nix crashes. If we
+ released when we exit this function or the client crashes. If we
can't acquire the lock, then continue; hopefully some other
goal can start a build, and if not, the main loop will sleep a
few seconds and then retry this goal. */
/* Close the read side of the logger pipe. */
if (hook) {
hook->builderOut.readSide.close();
- hook->fromHook.readSide.close();
+ hook->fromAgent.readSide.close();
}
else builderOut.readSide.close();
being valid. */
registerOutputs();
- if (buildMode == bmCheck) {
- done(BuildResult::Built);
- return;
- }
-
/* Delete unused redirected outputs (when doing hash rewriting). */
foreach (RedirectedOutputs::iterator, i, redirectedOutputs)
if (pathExists(i->second)) deletePath(i->second);
HookReply DerivationGoal::tryBuildHook()
{
- if (!settings.useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline;
+ if (!settings.useBuildHook) return rpDecline;
+
+ if (!worker.hook) {
+ Strings args = {
+ "offload",
+ settings.thisSystem.c_str(),
+ (format("%1%") % settings.maxSilentTime).str().c_str(),
+ (format("%1%") % settings.printBuildTrace).str().c_str(),
+ (format("%1%") % settings.buildTimeout).str().c_str()
+ };
- if (!worker.hook)
- worker.hook = std::shared_ptr<HookInstance>(new HookInstance);
+ worker.hook = std::make_shared<Agent>(settings.guixProgram, args);
+ }
/* Tell the hook about system features (beyond the system type)
required from the build machine. (The hook could parse the
foreach (Strings::iterator, i, features) checkStoreName(*i); /* !!! abuse */
/* Send the request to the hook. */
- writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%")
+ writeLine(worker.hook->toAgent.writeSide, (format("%1% %2% %3% %4%")
% (worker.getNrLocalBuilds() < settings.maxBuildJobs ? "1" : "0")
% drv.platform % drvPath % concatStringsSep(",", features)).str());
whether the hook wishes to perform the build. */
string reply;
while (true) {
- string s = readLine(worker.hook->fromHook.readSide);
+ string s = readLine(worker.hook->fromAgent.readSide);
if (string(s, 0, 2) == "# ") {
reply = string(s, 2);
break;
string s;
foreach (PathSet::iterator, i, allInputs) { s += *i; s += ' '; }
- writeLine(hook->toHook.writeSide, s);
+ writeLine(hook->toAgent.writeSide, s);
/* Tell the hooks the missing outputs that have to be copied back
from the remote system. */
s = "";
foreach (PathSet::iterator, i, missingPaths) { s += *i; s += ' '; }
- writeLine(hook->toHook.writeSide, s);
+ writeLine(hook->toAgent.writeSide, s);
- hook->toHook.writeSide.close();
+ hook->toAgent.writeSide.close();
/* Create the log file and pipe. */
Path logFile = openLogFile();
set<int> fds;
- fds.insert(hook->fromHook.readSide);
+ fds.insert(hook->fromAgent.readSide);
fds.insert(hook->builderOut.readSide);
- worker.childStarted(shared_from_this(), hook->pid, fds, false, false);
+ worker.childStarted(shared_from_this(), hook->pid, fds, false, true);
if (settings.printBuildTrace)
- printMsg(lvlError, format("@ build-started %1% - %2% %3%")
- % drvPath % drv.platform % logFile);
+ printMsg(lvlError, format("@ build-started %1% - %2% %3% %4%")
+ % drvPath % drv.platform % logFile % hook->pid);
return rpAccept;
}
f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit);
startNest(nest, lvlInfo, f % showPaths(missingPaths) % curRound % nrRounds);
- /* Right platform? */
- if (!canBuildLocally(drv.platform)) {
- if (settings.printBuildTrace)
- printMsg(lvlError, format("@ unsupported-platform %1% %2%") % drvPath % drv.platform);
- throw Error(
- format("a `%1%' is required to build `%3%', but I am a `%2%'")
- % drv.platform % settings.thisSystem % drvPath);
- }
-
/* Note: built-in builders are *not* running in a chroot environment so
that we can easily implement them in Guile without having it as a
derivation input (they are running under a separate build user,
Path homeDir = "/homeless-shelter";
env["HOME"] = homeDir;
- /* Tell the builder where the Nix store is. Usually they
+ /* Tell the builder where the store is. Usually they
shouldn't care, but this is useful for purity checking (e.g.,
the compiler or linker might only want to accept paths to files
in the store or in the build directory). */
getdents returns the inode of the mount point). */
env["PWD"] = tmpDirInSandbox;
- /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
- derivation, tell the builder, so that for instance `fetchurl'
- can skip checking the output. On older Nixes, this environment
- variable won't be set, so `fetchurl' will do the check. */
- if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1";
-
/* *Only* if this is a fixed-output derivation, propagate the
values of the environment variables specified in the
`impureEnvVars' attribute to the builder. This allows for
if (useChroot) {
#if CHROOT_ENABLED
/* Create a temporary directory in which we set up the chroot
- environment using bind-mounts. We put it in the Nix store
+ environment using bind-mounts. We put it in the store
to ensure that we can create hard-links to non-directory
- inputs in the fake Nix store in the chroot (see below). */
+ inputs in the fake store in the chroot (see below). */
chrootRootDir = drvPath + ".chroot";
if (pathExists(chrootRootDir)) deletePath(chrootRootDir);
dirsInChroot[tmpDirInSandbox] = tmpDir;
/* Make the closure of the inputs available in the chroot,
- rather than the whole Nix store. This prevents any access
+ rather than the whole store. This prevents any access
to undeclared dependencies. Directories are bind-mounted,
while other inputs are hard-linked (since only directories
can be bind-mounted). !!! As an extra security
- precaution, make the fake Nix store only writable by the
+ precaution, make the fake store only writable by the
build user. */
Path chrootStoreDir = chrootRootDir + settings.nixStore;
createDirs(chrootStoreDir);
- The UTS namespace ensures that builders see a hostname of
localhost rather than the actual hostname.
*/
-#if CHROOT_ENABLED
+#if __linux__
if (useChroot) {
char stack[32 * 1024];
int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | SIGCHLD;
if (!fixedOutput) flags |= CLONE_NEWNET;
- pid = clone(childEntry, stack + sizeof(stack) - 8, flags, this);
+ /* Ensure proper alignment on the stack. On aarch64, it has to be 16
+ bytes. */
+ pid = clone(childEntry,
+ (char *)(((uintptr_t)stack + sizeof(stack) - 8) & ~(uintptr_t)0xf),
+ flags, this);
if (pid == -1)
throw SysError("cloning builder process");
} else
if (!msg.empty()) throw Error(msg);
if (settings.printBuildTrace) {
- printMsg(lvlError, format("@ build-started %1% - %2% %3%")
- % drvPath % drv.platform % logFile);
+ printMsg(lvlError, format("@ build-started %1% - %2% %3% %4%")
+ % drvPath % drv.platform % logFile % pid);
}
}
+/* Return true if the operating system kernel part of SYSTEM1 and SYSTEM2 (the
+ bit that comes after the hyphen in system types such as "i686-linux") is
+ the same. */
+static bool sameOperatingSystemKernel(const std::string& system1, const std::string& system2)
+{
+ auto os1 = system1.substr(system1.find("-"));
+ auto os2 = system2.substr(system2.find("-"));
+ return os1 == os2;
+}
void DerivationGoal::runChild()
{
outside of the namespace. Making a subtree private is
local to the namespace, though, so setting MS_PRIVATE
does not affect the outside world. */
- Strings mounts = tokenizeString<Strings>(readFile("/proc/self/mountinfo", true), "\n");
- foreach (Strings::iterator, i, mounts) {
- vector<string> fields = tokenizeString<vector<string> >(*i, " ");
- string fs = decodeOctalEscaped(fields.at(4));
- if (mount(0, fs.c_str(), 0, MS_PRIVATE, 0) == -1)
- throw SysError(format("unable to make filesystem `%1%' private") % fs);
+ if (mount(0, "/", 0, MS_REC|MS_PRIVATE, 0) == -1) {
+ throw SysError("unable to make ‘/’ private mount");
}
/* Bind-mount chroot directory to itself, to treat it as a
if (mkdir("real-root", 0) == -1)
throw SysError("cannot create real-root directory");
-#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
if (pivot_root(".", "real-root") == -1)
throw SysError(format("cannot pivot old root directory onto '%1%'") % (chrootRootDir + "/real-root"));
-#undef pivot_root
if (chroot(".") == -1)
throw SysError(format("cannot change root directory to '%1%'") % chrootRootDir);
throw SysError("cannot set i686-linux personality");
}
+ if (drv.platform == "armhf-linux" &&
+ (settings.thisSystem == "aarch64-linux" ||
+ (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "aarch64")))) {
+ if (personality(PER_LINUX32) == -1)
+ throw SysError("cannot set armhf-linux personality");
+ }
+
/* Impersonate a Linux 2.6 machine to get some determinism in
builds that depend on the kernel version. */
if ((drv.platform == "i686-linux" || drv.platform == "x86_64-linux") && settings.impersonateLinux26) {
foreach (Strings::iterator, i, drv.args)
args.push_back(rewriteHashes(*i, rewritesToTmp));
- execve(drv.builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
-
+ /* If DRV targets the same operating system kernel, try to execute it:
+ there might be binfmt_misc set up for user-land emulation of other
+ architectures. However, if it targets a different operating
+ system--e.g., "i586-gnu" vs. "x86_64-linux"--do not try executing
+ it: the ELF file for that OS is likely indistinguishable from a
+ native ELF binary and it would just crash at run time. */
+ int error;
+ if (sameOperatingSystemKernel(drv.platform, settings.thisSystem)) {
+ execve(drv.builder.c_str(), stringsToCharPtrs(args).data(),
+ stringsToCharPtrs(envStrs).data());
+ error = errno;
+ } else {
+ error = ENOEXEC;
+ }
+
+ /* Right platform? Check this after we've tried 'execve' to allow for
+ transparent emulation of different platforms with binfmt_misc
+ handlers that invoke QEMU. */
+ if (error == ENOEXEC && !canBuildLocally(drv.platform)) {
+ if (settings.printBuildTrace)
+ printMsg(lvlError, format("@ unsupported-platform %1% %2%") % drvPath % drv.platform);
+ throw Error(
+ format("a `%1%' is required to build `%3%', but I am a `%2%'")
+ % drv.platform % settings.thisSystem % drvPath);
+ }
+
+ errno = error;
throw SysError(format("executing `%1%'") % drv.builder);
} catch (std::exception & e) {
else if (drv.outputs.find(*i) != drv.outputs.end())
result.insert(drv.outputs.find(*i)->second.path);
else throw BuildError(
- format("derivation contains an illegal reference specifier `%1%'")
+ format("derivation contains an invalid reference specifier `%1%'")
% *i);
}
return result;
if (useChroot) {
actualPath = chrootRootDir + path;
if (pathExists(actualPath)) {
- /* Move output paths from the chroot to the Nix store. */
+ /* Move output paths from the chroot to the store. */
if (buildMode == bmRepair)
replaceValidPath(path, actualPath);
else
if (buildMode != bmCheck && rename(actualPath.c_str(), path.c_str()) == -1)
- throw SysError(format("moving build output `%1%' from the chroot to the Nix store") % path);
+ throw SysError(format("moving build output `%1%' from the chroot to the store") % path);
}
if (buildMode != bmCheck) actualPath = path;
} else {
/* Check the hash. */
Hash h2 = recursive ? hashPath(ht, actualPath).first : hashFile(ht, actualPath);
- if (h != h2)
- throw BuildError(
- format("output path `%1%' should have %2% hash `%3%', instead has `%4%'")
- % path % i->second.hashAlgo % printHash16or32(h) % printHash16or32(h2));
+ if (h != h2) {
+ if (settings.printBuildTrace)
+ printMsg(lvlError, format("@ hash-mismatch %1% %2% %3% %4%")
+ % path % i->second.hashAlgo
+ % printHash16or32(h) % printHash16or32(h2));
+ throw BuildError(format("hash mismatch for store item '%1%'") % path);
+ }
}
/* Get rid of all weird permissions. This also checks that
throw Error(format("derivation `%1%' may not be deterministic: output `%2%' differs")
% drvPath % path);
}
+
+ if (settings.printBuildTrace)
+ printMsg(lvlError, format("@ build-succeeded %1% -") % drvPath);
+
continue;
}
infos.push_back(info);
}
- if (buildMode == bmCheck) return;
-
/* Compare the result with the previous round, and report which
path is different, if any.*/
if (curRound > 1 && prevInfos != infos) {
Path dir = (format("%1%/%2%/%3%/") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2)).str();
createDirs(dir);
- if (settings.compressLog) {
+ switch (settings.logCompression)
+ {
+ case COMPRESSION_GZIP: {
+ Path logFileName = (format("%1%/%2%.gz") % dir % string(baseName, 2)).str();
+ AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
+ if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName);
+ closeOnExec(fd);
+
+ /* Note: FD will be closed by 'gzclose'. */
+ if (!(gzLogFile = gzdopen(fd.borrow(), "w")))
+ throw Error(format("cannot open compressed log file `%1%'") % logFileName);
+
+ gzbuffer(gzLogFile, 32768);
+ gzsetparams(gzLogFile, Z_BEST_COMPRESSION, Z_DEFAULT_STRATEGY);
+ return logFileName;
+ }
+
+#if HAVE_BZLIB_H
+ case COMPRESSION_BZIP2: {
Path logFileName = (format("%1%/%2%.bz2") % dir % string(baseName, 2)).str();
AutoCloseFD fd = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
if (fd == -1) throw SysError(format("creating log file `%1%'") % logFileName);
throw Error(format("cannot open compressed log file `%1%'") % logFileName);
return logFileName;
+ }
+#endif
- } else {
+ case COMPRESSION_NONE: {
Path logFileName = (format("%1%/%2%") % dir % string(baseName, 2)).str();
fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC, 0666);
if (fdLogFile == -1) throw SysError(format("creating log file `%1%'") % logFileName);
closeOnExec(fdLogFile);
return logFileName;
+ }
}
+
+ abort();
}
void DerivationGoal::closeLogFile()
{
- if (bzLogFile) {
+ if (gzLogFile) {
+ int err;
+ err = gzclose(gzLogFile);
+ gzLogFile = NULL;
+ if (err != Z_OK) throw Error(format("cannot close compressed log file (gzip error = %1%)") % err);
+ }
+#if HAVE_BZLIB_H
+ else if (bzLogFile) {
int err;
BZ2_bzWriteClose(&err, bzLogFile, 0, 0, 0);
bzLogFile = 0;
if (err != BZ_OK) throw Error(format("cannot close compressed log file (BZip2 error = %1%)") % err);
}
+#endif
if (fLogFile) {
fclose(fLogFile);
void DerivationGoal::handleChildOutput(int fd, const string & data)
{
+ string prefix;
+
+ if (settings.multiplexedBuildOutput) {
+ /* Print a prefix that allows clients to determine whether a message
+ comes from the daemon or from a build process, and in the latter
+ case, which build process it comes from. The PID here matches the
+ one given in "@ build-started" traces; it's shorter that the
+ derivation file name, hence this choice. */
+ prefix = "@ build-log "
+ + std::to_string(pid < 0 ? hook->pid : pid)
+ + " " + std::to_string(data.size()) + "\n";
+ }
+
if ((hook && fd == hook->builderOut.readSide) ||
(!hook && fd == builderOut.readSide))
{
return;
}
if (verbosity >= settings.buildVerbosity)
- writeToStderr(data);
- if (bzLogFile) {
+ writeToStderr(prefix + data);
+
+ if (gzLogFile) {
+ if (data.size() > 0) {
+ int count, err;
+ count = gzwrite(gzLogFile, data.data(), data.size());
+ if (count == 0) throw Error(format("cannot write to compressed log file (gzip error = %1%)") % gzerror(gzLogFile, &err));
+ }
+#if HAVE_BZLIB_H
+ } else if (bzLogFile) {
int err;
BZ2_bzWrite(&err, bzLogFile, (unsigned char *) data.data(), data.size());
if (err != BZ_OK) throw Error(format("cannot write to compressed log file (BZip2 error = %1%)") % err);
+#endif
} else if (fdLogFile != -1)
writeFull(fdLogFile, data);
}
- if (hook && fd == hook->fromHook.readSide)
- writeToStderr(data);
+ if (hook && fd == hook->fromAgent.readSide)
+ writeToStderr(prefix + data);
}
/* The store path that should be realised through a substitute. */
Path storePath;
- /* The remaining substituters. */
- Paths subs;
-
- /* The current substituter. */
- Path sub;
-
- /* Whether any substituter can realise this path */
- bool hasSubstitute;
-
/* Path info returned by the substituter's query info operation. */
SubstitutablePathInfo info;
- /* Pipe for the substituter's standard output. */
- Pipe outPipe;
-
- /* Pipe for the substituter's standard error. */
- Pipe logPipe;
-
- /* The process ID of the builder. */
- Pid pid;
-
/* Lock on the store path. */
std::shared_ptr<PathLocks> outputLock;
typedef void (SubstitutionGoal::*GoalState)();
GoalState state;
+ /* The substituter. */
+ std::shared_ptr<Agent> substituter;
+
+ /* Either the empty string, or the expected hash as returned by the
+ substituter. */
+ string expectedHashStr;
+
+ /* Either the empty string, or the status phrase returned by the
+ substituter. */
+ string status;
+
+ void tryNext();
+
public:
SubstitutionGoal(const Path & storePath, Worker & worker, bool repair = false);
~SubstitutionGoal();
/* The states. */
void init();
- void tryNext();
void gotInfo();
void referencesValid();
void tryToRun();
SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, bool repair)
: Goal(worker)
- , hasSubstitute(false)
, repair(repair)
{
this->storePath = storePath;
SubstitutionGoal::~SubstitutionGoal()
{
- if (pid != -1) worker.childTerminated(pid);
+ if (substituter) worker.childTerminated(substituter->pid);
}
{
if (settings.printBuildTrace)
printMsg(lvlError, format("@ substituter-failed %1% timeout") % storePath);
- if (pid != -1) {
- pid_t savedPid = pid;
- pid.kill();
+ if (substituter) {
+ pid_t savedPid = substituter->pid;
+ substituter.reset();
worker.childTerminated(savedPid);
}
amDone(ecFailed);
}
if (settings.readOnlyMode)
- throw Error(format("cannot substitute path `%1%' - no write access to the Nix store") % storePath);
-
- subs = settings.substituters;
+ throw Error(format("cannot substitute path `%1%' - no write access to the store") % storePath);
tryNext();
}
void SubstitutionGoal::tryNext()
{
- trace("trying next substituter");
+ trace("trying substituter");
- if (subs.size() == 0) {
+ SubstitutablePathInfos infos;
+ PathSet dummy(singleton<PathSet>(storePath));
+ worker.store.querySubstitutablePathInfos(dummy, infos);
+ SubstitutablePathInfos::iterator k = infos.find(storePath);
+ if (k == infos.end()) {
/* None left. Terminate this goal and let someone else deal
with it. */
debug(format("path `%1%' is required, but there is no substituter that can build it") % storePath);
/* Hack: don't indicate failure if there were no substituters.
In that case the calling derivation should just do a
build. */
- amDone(hasSubstitute ? ecFailed : ecNoSubstituters);
- return;
+ amDone(ecNoSubstituters);
+ return;
}
- sub = subs.front();
- subs.pop_front();
-
- SubstitutablePathInfos infos;
- PathSet dummy(singleton<PathSet>(storePath));
- worker.store.querySubstitutablePathInfos(sub, dummy, infos);
- SubstitutablePathInfos::iterator k = infos.find(storePath);
- if (k == infos.end()) { tryNext(); return; }
+ /* Found a substitute. */
info = k->second;
- hasSubstitute = true;
/* To maintain the closure invariant, we first have to realise the
paths referenced by this one. */
printMsg(lvlInfo, format("fetching path `%1%'...") % storePath);
- outPipe.create();
- logPipe.create();
-
destPath = repair ? storePath + ".tmp" : storePath;
/* Remove the (stale) output path if it exists. */
if (pathExists(destPath))
deletePath(destPath);
- worker.store.setSubstituterEnv();
-
- /* Fill in the arguments. */
- Strings args;
- args.push_back(baseNameOf(sub));
- args.push_back("--substitute");
- args.push_back(storePath);
- args.push_back(destPath);
-
- /* Fork the substitute program. */
- pid = startProcess([&]() {
-
- commonChildInit(logPipe);
-
- if (dup2(outPipe.writeSide, STDOUT_FILENO) == -1)
- throw SysError("cannot dup output pipe into stdout");
+ if (!worker.substituter) {
+ const Strings args = { "substitute", "--substitute" };
+ const std::map<string, string> env = { { "_NIX_OPTIONS", settings.pack() } };
+ worker.substituter = std::make_shared<Agent>(settings.guixProgram, args, env);
+ }
- execv(sub.c_str(), stringsToCharPtrs(args).data());
+ /* Borrow the worker's substituter. */
+ if (!substituter) substituter.swap(worker.substituter);
- throw SysError(format("executing `%1%'") % sub);
- });
+ /* Send the request to the substituter. */
+ writeLine(substituter->toAgent.writeSide,
+ (format("substitute %1% %2%") % storePath % destPath).str());
- pid.setSeparatePG(true);
- pid.setKillSignal(SIGTERM);
- outPipe.writeSide.close();
- logPipe.writeSide.close();
- worker.childStarted(shared_from_this(),
- pid, singleton<set<int> >(logPipe.readSide), true, true);
+ set<int> fds;
+ fds.insert(substituter->fromAgent.readSide);
+ fds.insert(substituter->builderOut.readSide);
+ worker.childStarted(shared_from_this(), substituter->pid, fds, true, true);
state = &SubstitutionGoal::finished;
if (settings.printBuildTrace)
- printMsg(lvlError, format("@ substituter-started %1% %2%") % storePath % sub);
+ /* The second element in the message used to be the name of the
+ substituter but we're left with only one. */
+ printMsg(lvlError, format("@ substituter-started %1% substitute") % storePath);
}
{
trace("substitute finished");
- /* Since we got an EOF on the logger pipe, the substitute is
- presumed to have terminated. */
- pid_t savedPid = pid;
- int status = pid.wait(true);
-
- /* So the child is gone now. */
- worker.childTerminated(savedPid);
+ /* Remove the 'guix substitute' process from the list of children. */
+ worker.childTerminated(substituter->pid);
- /* Close the read side of the logger pipe. */
- logPipe.readSide.close();
-
- /* Get the hash info from stdout. */
- string dummy = readLine(outPipe.readSide);
- string expectedHashStr = statusOk(status) ? readLine(outPipe.readSide) : "";
- outPipe.readSide.close();
+ /* If max-jobs > 1, the worker might have created a new 'substitute'
+ process in the meantime. If that is the case, terminate ours;
+ otherwise, give it back to the worker. */
+ if (worker.substituter) {
+ substituter.reset ();
+ } else {
+ worker.substituter.swap(substituter);
+ }
/* Check the exit status and the build result. */
HashResult hash;
try {
- if (!statusOk(status))
- throw SubstError(format("fetching path `%1%' %2%")
- % storePath % statusToString(status));
+ if (status != "success")
+ throw SubstError(format("fetching path `%1%' (status: '%2%')")
+ % storePath % status);
if (!pathExists(destPath))
throw SubstError(format("substitute did not produce path `%1%'") % destPath);
+ if (expectedHashStr == "")
+ throw SubstError(format("substituter did not communicate hash for `%1'") % storePath);
+
hash = hashPath(htSHA256, destPath);
/* Verify the expected hash we got from the substituer. */
- if (expectedHashStr != "") {
- size_t n = expectedHashStr.find(':');
- if (n == string::npos)
- throw Error(format("bad hash from substituter: %1%") % expectedHashStr);
- HashType hashType = parseHashType(string(expectedHashStr, 0, n));
- if (hashType == htUnknown)
- throw Error(format("unknown hash algorithm in `%1%'") % expectedHashStr);
- Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1));
- Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, destPath).first;
- if (expectedHash != actualHash)
- throw SubstError(format("hash mismatch in downloaded path `%1%': expected %2%, got %3%")
- % storePath % printHash(expectedHash) % printHash(actualHash));
- }
+ size_t n = expectedHashStr.find(':');
+ if (n == string::npos)
+ throw Error(format("bad hash from substituter: %1%") % expectedHashStr);
+ HashType hashType = parseHashType(string(expectedHashStr, 0, n));
+ if (hashType == htUnknown)
+ throw Error(format("unknown hash algorithm in `%1%'") % expectedHashStr);
+ Hash expectedHash = parseHash16or32(hashType, string(expectedHashStr, n + 1));
+ Hash actualHash = hashType == htSHA256 ? hash.first : hashPath(hashType, destPath).first;
+ if (expectedHash != actualHash) {
+ if (settings.printBuildTrace)
+ printMsg(lvlError, format("@ hash-mismatch %1% %2% %3% %4%")
+ % storePath % "sha256"
+ % printHash16or32(expectedHash)
+ % printHash16or32(actualHash));
+ throw SubstError(format("hash mismatch for substituted item `%1%'") % storePath);
+ }
} catch (SubstError & e) {
% storePath % status % e.msg());
}
- /* Try the next substitute. */
- state = &SubstitutionGoal::tryNext;
- worker.wakeUp(shared_from_this());
+ amDone(ecFailed);
return;
}
void SubstitutionGoal::handleChildOutput(int fd, const string & data)
{
- assert(fd == logPipe.readSide);
- if (verbosity >= settings.buildVerbosity) writeToStderr(data);
- /* Don't write substitution output to a log file for now. We
- probably should, though. */
+ if (verbosity >= settings.buildVerbosity
+ && fd == substituter->builderOut.readSide) {
+ writeToStderr(data);
+ /* Don't write substitution output to a log file for now. We
+ probably should, though. */
+ }
+
+ if (fd == substituter->fromAgent.readSide) {
+ /* DATA may consist of several lines. Process them one by one. */
+ string input = data;
+ while (!input.empty()) {
+ /* Process up to the first newline. */
+ size_t end = input.find_first_of("\n");
+ string trimmed = (end != string::npos) ? input.substr(0, end) : input;
+
+ /* Update the goal's state accordingly. */
+ if (expectedHashStr == "") {
+ expectedHashStr = trimmed;
+ } else if (status == "") {
+ status = trimmed;
+ worker.wakeUp(shared_from_this());
+ } else {
+ printMsg(lvlError, format("unexpected substituter message '%1%'") % input);
+ }
+
+ input = (end != string::npos) ? input.substr(end + 1) : "";
+ }
+ }
}
void SubstitutionGoal::handleEOF(int fd)
{
- if (fd == logPipe.readSide) worker.wakeUp(shared_from_this());
+ worker.wakeUp(shared_from_this());
}
HCoop / jackhill / guix / guix.git / blobdiff