From c98525235f5b8f1ed02fd1b3849b42e5f669d364 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 20 Nov 2023 16:38:41 -0500 Subject: [PATCH 1/2] Revert "Revert "Adapt scheduler to work with dynamic derivations"" This fixes dynamic derivations, reverting #9081. I believe that this time around, #9052 is fixed. When I first rebased this, tests were failing (which wasn't the case before). The cause of those test failures were due to the crude job in which the outer goal tried to exit with the inner goal's status. Now, that error handling has been reworked to be more faithful. The exit exit status and exception of the inner goal is returned by the outer goal. The exception was what was causing the test failures, but I believe it was not having the right error code (there is more than one for failure) that caused #9081. The only cost of doing things the "right way" was that I had to introduce a hacky `preserveException` boolean. I don't like this, but, then again, none of us like anything about how the scheduler works. Issue #11927 is still there to clean everything up, subsuming the need for any `preserveException` because I doubt we will be fishing information out of state machines like this at all. This reverts commit 8440afbed756254784d9fea3eaab06649dffd390. Co-Authored-By: Eelco Dolstra --- ...erivation-creation-and-realisation-goal.cc | 126 ++++++++++++++++++ ...erivation-creation-and-realisation-goal.hh | 89 +++++++++++++ src/libstore/build/derivation-goal.cc | 26 +--- src/libstore/build/derivation-goal.hh | 4 + src/libstore/build/entry-points.cc | 5 +- src/libstore/build/goal.cc | 2 +- src/libstore/build/goal.hh | 21 +++ src/libstore/build/worker.cc | 92 ++++++++++--- src/libstore/build/worker.hh | 24 ++++ src/libstore/derived-path-map.cc | 4 + src/libstore/derived-path-map.hh | 7 +- src/libstore/meson.build | 2 + tests/functional/dyn-drv/build-built-drv.sh | 7 +- tests/functional/dyn-drv/dep-built-drv.sh | 7 +- tests/functional/dyn-drv/failing-outer.sh | 12 ++ tests/functional/dyn-drv/meson.build | 1 + .../functional/dyn-drv/text-hashed-output.nix | 10 ++ 17 files changed, 397 insertions(+), 42 deletions(-) create mode 100644 src/libstore/build/derivation-creation-and-realisation-goal.cc create mode 100644 src/libstore/build/derivation-creation-and-realisation-goal.hh create mode 100644 tests/functional/dyn-drv/failing-outer.sh diff --git a/src/libstore/build/derivation-creation-and-realisation-goal.cc b/src/libstore/build/derivation-creation-and-realisation-goal.cc new file mode 100644 index 00000000000..c33b7571f04 --- /dev/null +++ b/src/libstore/build/derivation-creation-and-realisation-goal.cc @@ -0,0 +1,126 @@ +#include "derivation-creation-and-realisation-goal.hh" +#include "worker.hh" + +namespace nix { + +DerivationCreationAndRealisationGoal::DerivationCreationAndRealisationGoal( + ref drvReq, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) + : Goal(worker, DerivedPath::Built{.drvPath = drvReq, .outputs = wantedOutputs}) + , drvReq(drvReq) + , wantedOutputs(wantedOutputs) + , buildMode(buildMode) +{ + name = + fmt("outer obtaining drv from '%s' and then building outputs %s", + drvReq->to_string(worker.store), + std::visit( + overloaded{ + [&](const OutputsSpec::All) -> std::string { return "* (all of them)"; }, + [&](const OutputsSpec::Names os) { return concatStringsSep(", ", quoteStrings(os)); }, + }, + wantedOutputs.raw)); + trace("created outer"); + + worker.updateProgress(); +} + +DerivationCreationAndRealisationGoal::~DerivationCreationAndRealisationGoal() {} + +static StorePath pathPartOfReq(const SingleDerivedPath & req) +{ + return std::visit( + overloaded{ + [&](const SingleDerivedPath::Opaque & bo) { return bo.path; }, + [&](const SingleDerivedPath::Built & bfd) { return pathPartOfReq(*bfd.drvPath); }, + }, + req.raw()); +} + +std::string DerivationCreationAndRealisationGoal::key() +{ + /* Ensure that derivations get built in order of their name, + i.e. a derivation named "aardvark" always comes before "baboon". And + substitution goals and inner derivation goals always happen before + derivation goals (due to "b$"). */ + return "c$" + std::string(pathPartOfReq(*drvReq).name()) + "$" + drvReq->to_string(worker.store); +} + +void DerivationCreationAndRealisationGoal::timedOut(Error && ex) {} + +void DerivationCreationAndRealisationGoal::addWantedOutputs(const OutputsSpec & outputs) +{ + /* If we already want all outputs, there is nothing to do. */ + auto newWanted = wantedOutputs.union_(outputs); + bool needRestart = !newWanted.isSubsetOf(wantedOutputs); + wantedOutputs = newWanted; + + if (!needRestart) + return; + + if (!optDrvPath) + // haven't started steps where the outputs matter yet + return; + worker.makeDerivationGoal(*optDrvPath, outputs, buildMode); +} + +Goal::Co DerivationCreationAndRealisationGoal::init() +{ + trace("outer init"); + + /* The first thing to do is to make sure that the derivation + exists. If it doesn't, it may be created through a + substitute. */ + if (auto optDrvPath = [this]() -> std::optional { + if (buildMode != bmNormal) + return std::nullopt; + + auto drvPath = StorePath::dummy; + try { + drvPath = resolveDerivedPath(worker.store, *drvReq); + } catch (MissingRealisation &) { + return std::nullopt; + } + auto cond = worker.evalStore.isValidPath(drvPath) || worker.store.isValidPath(drvPath); + return cond ? std::optional{drvPath} : std::nullopt; + }()) { + trace( + fmt("already have drv '%s' for '%s', can go straight to building", + worker.store.printStorePath(*optDrvPath), + drvReq->to_string(worker.store))); + } else { + trace("need to obtain drv we want to build"); + addWaitee(worker.makeGoal(DerivedPath::fromSingle(*drvReq))); + co_await Suspend{}; + } + + trace("outer load and build derivation"); + + if (nrFailed != 0) { + co_return amDone(ecFailed, Error("cannot build missing derivation '%s'", drvReq->to_string(worker.store))); + } + + StorePath drvPath = resolveDerivedPath(worker.store, *drvReq); + /* Build this step! */ + concreteDrvGoal = worker.makeDerivationGoal(drvPath, wantedOutputs, buildMode); + { + auto g = upcast_goal(concreteDrvGoal); + /* We will finish with it ourselves, as if we were the derivational goal. */ + g->preserveException = true; + } + optDrvPath = std::move(drvPath); + addWaitee(upcast_goal(concreteDrvGoal)); + co_await Suspend{}; + + trace("outer build done"); + + buildResult = upcast_goal(concreteDrvGoal) + ->getBuildResult(DerivedPath::Built{ + .drvPath = drvReq, + .outputs = wantedOutputs, + }); + + auto g = upcast_goal(concreteDrvGoal); + co_return amDone(g->exitCode, g->ex); +} + +} diff --git a/src/libstore/build/derivation-creation-and-realisation-goal.hh b/src/libstore/build/derivation-creation-and-realisation-goal.hh new file mode 100644 index 00000000000..9960484ac95 --- /dev/null +++ b/src/libstore/build/derivation-creation-and-realisation-goal.hh @@ -0,0 +1,89 @@ +#pragma once + +#include "parsed-derivations.hh" +#include "user-lock.hh" +#include "store-api.hh" +#include "pathlocks.hh" +#include "goal.hh" + +namespace nix { + +struct DerivationGoal; + +/** + * This goal type is essentially the serial composition (like function + * composition) of a goal for getting a derivation, and then a + * `DerivationGoal` using the newly-obtained derivation. + * + * In the (currently experimental) general inductive case of derivations + * that are themselves build outputs, that first goal will be *another* + * `DerivationCreationAndRealisationGoal`. In the (much more common) base-case + * where the derivation has no provence and is just referred to by + * (content-addressed) store path, that first goal is a + * `SubstitutionGoal`. + * + * If we already have the derivation (e.g. if the evaluator has created + * the derivation locally and then instructured the store to build it), + * we can skip the first goal entirely as a small optimization. + */ +struct DerivationCreationAndRealisationGoal : public Goal +{ + /** + * How to obtain a store path of the derivation to build. + */ + ref drvReq; + + /** + * The path of the derivation, once obtained. + **/ + std::optional optDrvPath; + + /** + * The goal for the corresponding concrete derivation. + **/ + std::shared_ptr concreteDrvGoal; + + /** + * The specific outputs that we need to build. + */ + OutputsSpec wantedOutputs; + + /** + * The final output paths of the build. + * + * - For input-addressed derivations, always the precomputed paths + * + * - For content-addressed derivations, calcuated from whatever the + * hash ends up being. (Note that fixed outputs derivations that + * produce the "wrong" output still install that data under its + * true content-address.) + */ + OutputPathMap finalOutputs; + + BuildMode buildMode; + + DerivationCreationAndRealisationGoal( + ref drvReq, + const OutputsSpec & wantedOutputs, + Worker & worker, + BuildMode buildMode = bmNormal); + virtual ~DerivationCreationAndRealisationGoal(); + + void timedOut(Error && ex) override; + + std::string key() override; + + /** + * Add wanted outputs to an already existing derivation goal. + */ + void addWantedOutputs(const OutputsSpec & outputs); + + Co init() override; + + JobCategory jobCategory() const override + { + return JobCategory::Administration; + }; +}; + +} diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 0d16f09750b..fee9c65be80 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -137,21 +137,8 @@ Goal::Co DerivationGoal::init() { trace("init"); if (useDerivation) { - /* The first thing to do is to make sure that the derivation - exists. If it doesn't, it may be created through a - substitute. */ - - if (buildMode != bmNormal || !worker.evalStore.isValidPath(drvPath)) { - addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath))); - co_await Suspend{}; - } - trace("loading derivation"); - if (nrFailed != 0) { - co_return done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); - } - /* `drvPath' should already be a root, but let's be on the safe side: if the user forgot to make it a root, we wouldn't want things being garbage collected while we're busy. */ @@ -1549,23 +1536,24 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) if (!useDerivation || !drv) return; auto & fullDrv = *dynamic_cast(drv.get()); - auto * dg = dynamic_cast(&*waitee); - if (!dg) return; + std::optional info = tryGetConcreteDrvGoal(waitee); + if (!info) return; + const auto & [dg, drvReq] = *info; - auto * nodeP = fullDrv.inputDrvs.findSlot(DerivedPath::Opaque { .path = dg->drvPath }); + auto * nodeP = fullDrv.inputDrvs.findSlot(drvReq.get()); if (!nodeP) return; auto & outputs = nodeP->value; for (auto & outputName : outputs) { - auto buildResult = dg->getBuildResult(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(dg->drvPath), + auto buildResult = dg.get().getBuildResult(DerivedPath::Built { + .drvPath = makeConstantStorePathRef(dg.get().drvPath), .outputs = OutputsSpec::Names { outputName }, }); if (buildResult.success()) { auto i = buildResult.builtOutputs.find(outputName); if (i != buildResult.builtOutputs.end()) inputDrvOutputs.insert_or_assign( - { dg->drvPath, outputName }, + { dg.get().drvPath, outputName }, i->second.outPath); } } diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 4e9c1451901..53884081301 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -56,6 +56,10 @@ struct InitialOutput { /** * A goal for building some or all of the outputs of a derivation. + * + * The derivation must already be present, either in the store in a drv + * or in memory. If the derivation itself needs to be gotten first, a + * `DerivationCreationAndRealisationGoal` goal must be used instead. */ struct DerivationGoal : public Goal { diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 3bf22320e3a..a473daff914 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -1,6 +1,7 @@ #include "worker.hh" #include "substitution-goal.hh" #ifndef _WIN32 // TODO Enable building on Windows +# include "derivation-creation-and-realisation-goal.hh" # include "derivation-goal.hh" #endif #include "local-store.hh" @@ -29,8 +30,8 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod } if (i->exitCode != Goal::ecSuccess) { #ifndef _WIN32 // TODO Enable building on Windows - if (auto i2 = dynamic_cast(i.get())) - failed.insert(printStorePath(i2->drvPath)); + if (auto i2 = dynamic_cast(i.get())) + failed.insert(i2->drvReq->to_string(*this)); else #endif if (auto i2 = dynamic_cast(i.get())) diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc index 9a16da14555..c381e5b581f 100644 --- a/src/libstore/build/goal.cc +++ b/src/libstore/build/goal.cc @@ -175,7 +175,7 @@ Goal::Done Goal::amDone(ExitCode result, std::optional ex) exitCode = result; if (ex) { - if (!waiters.empty()) + if (!preserveException && !waiters.empty()) logError(ex->info()); else this->ex = std::move(*ex); diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh index 1dd7ed52537..2db1098b736 100644 --- a/src/libstore/build/goal.hh +++ b/src/libstore/build/goal.hh @@ -50,6 +50,16 @@ enum struct JobCategory { * A substitution an arbitrary store object; it will use network resources. */ Substitution, + /** + * A goal that does no "real" work by itself, and just exists to depend on + * other goals which *do* do real work. These goals therefore are not + * limited. + * + * These goals cannot infinitely create themselves, so there is no risk of + * a "fork bomb" type situation (which would be a problem even though the + * goal do no real work) either. + */ + Administration, }; struct Goal : public std::enable_shared_from_this @@ -373,6 +383,17 @@ public: */ BuildResult getBuildResult(const DerivedPath &) const; + /** + * Hack to say that this goal should not log `ex`, but instead keep + * it around. Set by a waitee which sees itself as the designated + * continuation of this goal, responsible for reporting its + * successes or failures. + * + * @todo this is yet another not-nice hack in the goal system that + * we ought to get rid of. See #11927 + */ + bool preserveException = false; + /** * Exception containing an error message, if any. */ diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index dbe86f43f6a..6510bbe9edd 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -5,6 +5,7 @@ #include "drv-output-substitution-goal.hh" #include "derivation-goal.hh" #ifndef _WIN32 // TODO Enable building on Windows +# include "derivation-creation-and-realisation-goal.hh" # include "local-derivation-goal.hh" # include "hook-instance.hh" #endif @@ -43,6 +44,24 @@ Worker::~Worker() } +std::shared_ptr Worker::makeDerivationCreationAndRealisationGoal( + ref drvReq, + const OutputsSpec & wantedOutputs, + BuildMode buildMode) +{ + std::weak_ptr & goal_weak = outerDerivationGoals.ensureSlot(*drvReq).value; + std::shared_ptr goal = goal_weak.lock(); + if (!goal) { + goal = std::make_shared(drvReq, wantedOutputs, *this, buildMode); + goal_weak = goal; + wakeUp(goal); + } else { + goal->addWantedOutputs(wantedOutputs); + } + return goal; +} + + std::shared_ptr Worker::makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, @@ -120,10 +139,7 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) { return std::visit(overloaded { [&](const DerivedPath::Built & bfd) -> GoalPtr { - if (auto bop = std::get_if(&*bfd.drvPath)) - return makeDerivationGoal(bop->path, bfd.outputs, buildMode); - else - throw UnimplementedError("Building dynamic derivations in one shot is not yet implemented."); + return makeDerivationCreationAndRealisationGoal(bfd.drvPath, bfd.outputs, buildMode); }, [&](const DerivedPath::Opaque & bo) -> GoalPtr { return makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair); @@ -132,24 +148,46 @@ GoalPtr Worker::makeGoal(const DerivedPath & req, BuildMode buildMode) } +template +static void cullMap(std::map & goalMap, F f) +{ + for (auto i = goalMap.begin(); i != goalMap.end();) + if (!f(i->second)) + i = goalMap.erase(i); + else ++i; +} + + template static void removeGoal(std::shared_ptr goal, std::map> & goalMap) { /* !!! inefficient */ - for (auto i = goalMap.begin(); - i != goalMap.end(); ) - if (i->second.lock() == goal) { - auto j = i; ++j; - goalMap.erase(i); - i = j; - } - else ++i; + cullMap(goalMap, [&](const std::weak_ptr & gp) -> bool { + return gp.lock() != goal; + }); +} + +template +static void removeGoal(std::shared_ptr goal, std::map>::ChildNode> & goalMap); + +template +static void removeGoal(std::shared_ptr goal, std::map>::ChildNode> & goalMap) +{ + /* !!! inefficient */ + cullMap(goalMap, [&](DerivedPathMap>::ChildNode & node) -> bool { + if (node.value.lock() == goal) + node.value.reset(); + removeGoal(goal, node.childMap); + return !node.value.expired() || !node.childMap.empty(); + }); } void Worker::removeGoal(GoalPtr goal) { - if (auto drvGoal = std::dynamic_pointer_cast(goal)) + if (auto drvGoal = std::dynamic_pointer_cast(goal)) + nix::removeGoal(drvGoal, outerDerivationGoals.map); + else if (auto drvGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(drvGoal, derivationGoals); else if (auto subGoal = std::dynamic_pointer_cast(goal)) @@ -215,6 +253,9 @@ void Worker::childStarted(GoalPtr goal, const std::set 0); nrLocalBuilds--; break; + case JobCategory::Administration: + /* Intentionally not limited, see docs */ + break; default: unreachable(); } @@ -290,9 +334,9 @@ void Worker::run(const Goals & _topGoals) for (auto & i : _topGoals) { topGoals.insert(i); - if (auto goal = dynamic_cast(i.get())) { + if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Built { - .drvPath = makeConstantStorePathRef(goal->drvPath), + .drvPath = goal->drvReq, .outputs = goal->wantedOutputs, }); } else @@ -552,4 +596,22 @@ GoalPtr upcast_goal(std::shared_ptr subGoal) return subGoal; } +GoalPtr upcast_goal(std::shared_ptr subGoal) +{ + return subGoal; +} + +std::optional, std::reference_wrapper>> tryGetConcreteDrvGoal(GoalPtr waitee) +{ + auto * odg = dynamic_cast(&*waitee); + if (!odg) return std::nullopt; + /* If we failed to obtain the concrete drv, we won't have created + the concrete derivation goal. */ + if (!odg->concreteDrvGoal) return std::nullopt; + return {{ + std::cref(*odg->concreteDrvGoal), + std::cref(*odg->drvReq), + }}; +} + } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index f5e61720807..efd518f9995 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "store-api.hh" +#include "derived-path-map.hh" #include "goal.hh" #include "realisation.hh" #include "muxable-pipe.hh" @@ -13,6 +14,7 @@ namespace nix { /* Forward definition. */ +struct DerivationCreationAndRealisationGoal; struct DerivationGoal; struct PathSubstitutionGoal; class DrvOutputSubstitutionGoal; @@ -31,9 +33,25 @@ class DrvOutputSubstitutionGoal; */ GoalPtr upcast_goal(std::shared_ptr subGoal); GoalPtr upcast_goal(std::shared_ptr subGoal); +GoalPtr upcast_goal(std::shared_ptr subGoal); typedef std::chrono::time_point steady_time_point; +/** + * The current implementation of impure derivations has + * `DerivationGoal`s accumulate realisations from their waitees. + * Unfortunately, `DerivationGoal`s don't directly depend on other + * goals, but instead depend on `DerivationCreationAndRealisationGoal`s. + * + * We try not to share any of the details of any goal type with any + * other, for sake of modularity and quicker rebuilds. This means we + * cannot "just" downcast and fish out the field. So as an escape hatch, + * we have made the function, written in `worker.cc` where all the goal + * types are visible, and use it instead. + */ + +std::optional, std::reference_wrapper>> tryGetConcreteDrvGoal(GoalPtr waitee); + /** * A mapping used to remember for each child process to what goal it * belongs, and comm channels for receiving log data and output @@ -103,6 +121,9 @@ private: * Maps used to prevent multiple instantiations of a goal for the * same derivation / path. */ + + DerivedPathMap> outerDerivationGoals; + std::map> derivationGoals; std::map> substitutionGoals; std::map> drvOutputSubstitutionGoals; @@ -196,6 +217,9 @@ public: * @ref DerivationGoal "derivation goal" */ private: + std::shared_ptr makeDerivationCreationAndRealisationGoal( + ref drvPath, + const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal); std::shared_ptr makeDerivationGoalCommon( const StorePath & drvPath, const OutputsSpec & wantedOutputs, std::function()> mkDrvGoal); diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index c97d52773eb..0095a9d7814 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -52,6 +52,7 @@ typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const Single // instantiations +#include "derivation-creation-and-realisation-goal.hh" namespace nix { template<> @@ -68,4 +69,7 @@ std::strong_ordering DerivedPathMap>::ChildNode::operator template struct DerivedPathMap>::ChildNode; template struct DerivedPathMap>; +template struct DerivedPathMap>; + + }; diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh index bd60fe88710..61e0b5463e1 100644 --- a/src/libstore/derived-path-map.hh +++ b/src/libstore/derived-path-map.hh @@ -21,8 +21,11 @@ namespace nix { * * @param V A type to instantiate for each output. It should probably * should be an "optional" type so not every interior node has to have a - * value. `* const Something` or `std::optional` would be - * good choices for "optional" types. + * value. For example, the scheduler uses + * `DerivedPathMap>` to + * remember which goals correspond to which outputs. `* const Something` + * or `std::optional` would also be good choices for + * "optional" types. */ template struct DerivedPathMap { diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 79d91249722..d42b010614a 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -183,6 +183,7 @@ sources = files( 'binary-cache-store.cc', 'build-result.cc', 'build/derivation-goal.cc', + 'build/derivation-creation-and-realisation-goal.cc', 'build/drv-output-substitution-goal.cc', 'build/entry-points.cc', 'build/goal.cc', @@ -255,6 +256,7 @@ headers = [config_h] + files( 'binary-cache-store.hh', 'build-result.hh', 'build/derivation-goal.hh', + 'build/derivation-creation-and-realisation-goal.hh', 'build/drv-output-substitution-goal.hh', 'build/goal.hh', 'build/substitution-goal.hh', diff --git a/tests/functional/dyn-drv/build-built-drv.sh b/tests/functional/dyn-drv/build-built-drv.sh index 647be945716..fcb25a34b45 100644 --- a/tests/functional/dyn-drv/build-built-drv.sh +++ b/tests/functional/dyn-drv/build-built-drv.sh @@ -18,4 +18,9 @@ clearStore drvDep=$(nix-instantiate ./text-hashed-output.nix -A producingDrv) -expectStderr 1 nix build "${drvDep}^out^out" --no-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" +# Store layer needs bugfix +requireDaemonNewerThan "2.27pre20250205" + +out2=$(nix build "${drvDep}^out^out" --no-link) + +test $out1 == $out2 diff --git a/tests/functional/dyn-drv/dep-built-drv.sh b/tests/functional/dyn-drv/dep-built-drv.sh index 4f6e9b080fa..9d470099a0f 100644 --- a/tests/functional/dyn-drv/dep-built-drv.sh +++ b/tests/functional/dyn-drv/dep-built-drv.sh @@ -4,8 +4,11 @@ source common.sh out1=$(nix-build ./text-hashed-output.nix -A hello --no-out-link) +# Store layer needs bugfix +requireDaemonNewerThan "2.27pre20250205" + clearStore -expectStderr 1 nix-build ./text-hashed-output.nix -A wrapper --no-out-link | grepQuiet "Building dynamic derivations in one shot is not yet implemented" +out2=$(nix-build ./text-hashed-output.nix -A wrapper --no-out-link) -# diff -r $out1 $out2 +diff -r $out1 $out2 diff --git a/tests/functional/dyn-drv/failing-outer.sh b/tests/functional/dyn-drv/failing-outer.sh new file mode 100644 index 00000000000..d888ea876e6 --- /dev/null +++ b/tests/functional/dyn-drv/failing-outer.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +source common.sh + +# Store layer needs bugfix +requireDaemonNewerThan "2.27pre20250205" + +expected=100 +if [[ -v NIX_DAEMON_PACKAGE ]]; then expected=1; fi # work around the daemon not returning a 100 status correctly + +expectStderr "$expected" nix-build ./text-hashed-output.nix -A failingWrapper --no-out-link \ + | grepQuiet "build of '.*use-dynamic-drv-in-non-dynamic-drv-wrong.drv' failed" diff --git a/tests/functional/dyn-drv/meson.build b/tests/functional/dyn-drv/meson.build index 5b60a46980b..4ad343aee8e 100644 --- a/tests/functional/dyn-drv/meson.build +++ b/tests/functional/dyn-drv/meson.build @@ -12,6 +12,7 @@ suites += { 'recursive-mod-json.sh', 'build-built-drv.sh', 'eval-outputOf.sh', + 'failing-outer.sh', 'dep-built-drv.sh', 'old-daemon-error-hack.sh', ], diff --git a/tests/functional/dyn-drv/text-hashed-output.nix b/tests/functional/dyn-drv/text-hashed-output.nix index 99203b51849..a604a7cbf2c 100644 --- a/tests/functional/dyn-drv/text-hashed-output.nix +++ b/tests/functional/dyn-drv/text-hashed-output.nix @@ -13,6 +13,7 @@ rec { echo "Hello World" > $out/hello ''; }; + producingDrv = mkDerivation { name = "hello.drv"; buildCommand = '' @@ -23,6 +24,7 @@ rec { outputHashMode = "text"; outputHashAlgo = "sha256"; }; + wrapper = mkDerivation { name = "use-dynamic-drv-in-non-dynamic-drv"; buildCommand = '' @@ -30,4 +32,12 @@ rec { cp -r ${builtins.outputOf producingDrv.outPath "out"} $out ''; }; + + failingWrapper = mkDerivation { + name = "use-dynamic-drv-in-non-dynamic-drv-wrong"; + buildCommand = '' + echo "Fail at copying the output of the dynamic derivation" + fail ${builtins.outputOf producingDrv.outPath "out"} $out + ''; + }; } From 820df19e3ee2dc9e39a27ec51656e82c23e92ab4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 5 Feb 2025 17:22:56 -0500 Subject: [PATCH 2/2] More interesting dynamic derivations example Co-authored-by: Samuel Ainsworth --- tests/functional/dyn-drv/dep-built-drv-2.sh | 15 ++++ tests/functional/dyn-drv/meson.build | 1 + tests/functional/dyn-drv/non-trivial.nix | 77 +++++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 tests/functional/dyn-drv/dep-built-drv-2.sh create mode 100644 tests/functional/dyn-drv/non-trivial.nix diff --git a/tests/functional/dyn-drv/dep-built-drv-2.sh b/tests/functional/dyn-drv/dep-built-drv-2.sh new file mode 100644 index 00000000000..24628e85e52 --- /dev/null +++ b/tests/functional/dyn-drv/dep-built-drv-2.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +source common.sh + +# Store layer needs bugfix +requireDaemonNewerThan "2.27pre20250205" + +TODO_NixOS # can't enable a sandbox feature easily + +enableFeatures 'recursive-nix' +restartDaemon + +export NIX_BIN_DIR=$(dirname $(type -p nix)) + +nix build -L --file ./non-trivial.nix diff --git a/tests/functional/dyn-drv/meson.build b/tests/functional/dyn-drv/meson.build index 4ad343aee8e..07145000d83 100644 --- a/tests/functional/dyn-drv/meson.build +++ b/tests/functional/dyn-drv/meson.build @@ -15,6 +15,7 @@ suites += { 'failing-outer.sh', 'dep-built-drv.sh', 'old-daemon-error-hack.sh', + 'dep-built-drv-2.sh', ], 'workdir': meson.current_source_dir(), } diff --git a/tests/functional/dyn-drv/non-trivial.nix b/tests/functional/dyn-drv/non-trivial.nix new file mode 100644 index 00000000000..5cfafbb62f5 --- /dev/null +++ b/tests/functional/dyn-drv/non-trivial.nix @@ -0,0 +1,77 @@ +with import ./config.nix; + +builtins.outputOf + (mkDerivation { + name = "make-derivations.drv"; + + requiredSystemFeatures = [ "recursive-nix" ]; + + buildCommand = '' + set -e + set -u + + PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH + + export NIX_CONFIG='extra-experimental-features = nix-command ca-derivations dynamic-derivations' + + declare -A deps=( + [a]="" + [b]="a" + [c]="a" + [d]="b c" + [e]="b c d" + ) + + # Cannot just literally include this, or Nix will think it is the + # *outer* derivation that's trying to refer to itself, and + # substitute the string too soon. + placeholder=$(nix eval --raw --expr 'builtins.placeholder "out"') + + declare -A drvs=() + for word in a b c d e; do + inputDrvs="" + for dep in ''${deps[$word]}; do + if [[ "$inputDrvs" != "" ]]; then + inputDrvs+="," + fi + read -r -d "" line <> \"\$out\""], + "builder": "${shell}", + "env": { + "out": "$placeholder", + "$word": "hello, from $word!", + "PATH": ${builtins.toJSON path} + }, + "inputDrvs": { + $inputDrvs + }, + "inputSrcs": [], + "name": "build-$word", + "outputs": { + "out": { + "method": "nar", + "hashAlgo": "sha256" + } + }, + "system": "${system}" + } + EOF + drvs[$word]="$(echo "$json" | nix derivation add)" + done + cp "''${drvs[e]}" $out + ''; + + __contentAddressed = true; + outputHashMode = "text"; + outputHashAlgo = "sha256"; + }).outPath + "out"