Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Content-addressing derivations with shallow traces #12464

Draft
wants to merge 4 commits into
base: master
Choose a base branch
from
+1,315 −1,027
Draft

Content-addressing derivations with shallow traces #12464

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c985252
Revert "Revert "Adapt scheduler to work with dynamic derivations""
Ericson2314 Nov 20, 2023
b802581
Merge remote-tracking branch 'upstream/master' into HEAD
Ericson2314 Feb 13, 2025
1c6e766
Don't do store operations in `*DerivedPath::toJSON`
Ericson2314 Feb 13, 2025
40fc7b7
Revert "Use the hash modulo in the derivation outputs"
Ericson2314 Feb 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions maintainers/flake-module.nix
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,8 +257,6 @@
''^src/libstore/pathlocks\.hh$''
''^src/libstore/profiles\.cc$''
''^src/libstore/profiles\.hh$''
''^src/libstore/realisation\.cc$''
''^src/libstore/realisation\.hh$''
''^src/libstore/remote-fs-accessor\.cc$''
''^src/libstore/remote-fs-accessor\.hh$''
''^src/libstore/remote-store-connection\.hh$''
Expand Down
6 changes: 2 additions & 4 deletions src/build-remote/build-remote.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -345,21 +345,19 @@ static int main_build_remote(int argc, char * * argv)
}


auto outputHashes = staticOutputHashes(*store, drv);
std::set<Realisation> missingRealisations;
StorePathSet missingPaths;
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) {
for (auto & outputName : wantedOutputs) {
auto thisOutputHash = outputHashes.at(outputName);
auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
auto thisOutputId = DrvOutput{ *drvPath, outputName };
if (!store->queryRealisation(thisOutputId)) {
debug("missing output %s", outputName);
assert(optResult);
auto & result = *optResult;
auto i = result.builtOutputs.find(outputName);
assert(i != result.builtOutputs.end());
auto & newRealisation = i->second;
missingRealisations.insert(newRealisation);
missingRealisations.insert({newRealisation, thisOutputId});
missingPaths.insert(newRealisation.outPath);
}
}
Expand Down
23 changes: 9 additions & 14 deletions src/libcmd/built-path.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,21 +116,16 @@ RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const
overloaded{
[&](const BuiltPath::Opaque & p) { res.insert(p.path); },
[&](const BuiltPath::Built & p) {
auto drvHashes =
staticOutputHashes(store, store.readDerivation(p.drvPath->outPath()));
for (auto& [outputName, outputPath] : p.outputs) {
if (experimentalFeatureSettings.isEnabled(
Xp::CaDerivations)) {
auto drvOutput = get(drvHashes, outputName);
if (!drvOutput)
throw Error(
"the derivation '%s' has unrealised output '%s' (derived-path.cc/toRealisedPaths)",
store.printStorePath(p.drvPath->outPath()), outputName);
auto thisRealisation = store.queryRealisation(
DrvOutput{*drvOutput, outputName});
assert(thisRealisation); // We’ve built it, so we must
// have the realisation
res.insert(*thisRealisation);
if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
DrvOutput key{
.drvPath = p.drvPath->outPath(),
.outputName = outputName,
};
auto thisRealisation = store.queryRealisation(key);
// We’ve built it, so we must have the realisation.
assert(thisRealisation);
res.insert(Realisation{*thisRealisation, key});
} else {
res.insert(outputPath);
}
Expand Down
26 changes: 1 addition & 25 deletions src/libexpr/primops.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1522,31 +1522,7 @@ static void derivationStrictInternal(
DerivationOutput::Deferred { });
}

auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
switch (hashModulo.kind) {
case DrvHash::Kind::Regular:
for (auto & i : outputs) {
auto h = get(hashModulo.hashes, i);
if (!h)
state.error<AssertionError>(
"derivation produced no hash for output '%s'",
i
).atPos(v).debugThrow();
auto outPath = state.store->makeOutputPath(i, *h, drvName);
drv.env[i] = state.store->printStorePath(outPath);
drv.outputs.insert_or_assign(
i,
DerivationOutput::InputAddressed {
.path = std::move(outPath),
});
}
break;
;
case DrvHash::Kind::Deferred:
for (auto & i : outputs) {
drv.outputs.insert_or_assign(i, DerivationOutput::Deferred {});
}
}
resolveInputAddressed(*state.store, drv);
}

/* Write the resulting term into the Nix store directory. */
Expand Down
45 changes: 0 additions & 45 deletions src/libstore-tests/common-protocol.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,51 +96,6 @@ CHARACTERIZATION_TEST(
},
}))

CHARACTERIZATION_TEST(
drvOutput,
"drv-output",
(std::tuple<DrvOutput, DrvOutput> {
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "quux",
},
}))

CHARACTERIZATION_TEST(
realisation,
"realisation",
(std::tuple<Realisation, Realisation> {
Realisation {
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
.signatures = { "asdf", "qwer" },
},
Realisation {
.id = {
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
.signatures = { "asdf", "qwer" },
.dependentRealisations = {
{
DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "quux",
},
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
},
},
},
}))

CHARACTERIZATION_TEST(
vector,
"vector",
Expand Down
61 changes: 23 additions & 38 deletions src/libstore-tests/serve-protocol.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,51 +73,44 @@ VERSIONED_CHARACTERIZATION_TEST(
VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
drvOutput,
"drv-output",
defaultVersion,
"drv-output-2.8",
2 << 8 | 8,
(std::tuple<DrvOutput, DrvOutput> {
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.drvPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
.outputName = "baz",
},
DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.drvPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
.outputName = "quux",
},
}))

#endif

VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
unkeyedRealisation,
"unkeyed-realisation-2.8",
2 << 8 | 8,
(UnkeyedRealisation {
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
.signatures = { "asdf", "qwer" },
}))

VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
realisation,
"realisation",
defaultVersion,
(std::tuple<Realisation, Realisation> {
Realisation {
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
"realisation-2.8",
2 << 8 | 8,
(Realisation {
UnkeyedRealisation {
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
.signatures = { "asdf", "qwer" },
},
Realisation {
.id = {
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
.signatures = { "asdf", "qwer" },
.dependentRealisations = {
{
DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "quux",
},
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
},
},
DrvOutput {
.drvPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
.outputName = "baz",
},
}))

Expand Down Expand Up @@ -176,8 +169,8 @@ VERSIONED_CHARACTERIZATION_TEST(
VERSIONED_CHARACTERIZATION_TEST(
ServeProtoTest,
buildResult_2_6,
"build-result-2.6",
2 << 8 | 6,
"build-result-2.8",
2 << 8 | 8,
({
using namespace std::literals::chrono_literals;
std::tuple<BuildResult, BuildResult, BuildResult> t {
Expand All @@ -200,20 +193,12 @@ VERSIONED_CHARACTERIZATION_TEST(
{
"foo",
{
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "foo",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
},
},
{
"bar",
{
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "bar",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" },
},
},
Expand Down
73 changes: 21 additions & 52 deletions src/libstore-tests/worker-protocol.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,49 +125,42 @@ VERSIONED_CHARACTERIZATION_TEST(
VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
drvOutput,
"drv-output",
defaultVersion,
"drv-output-1.39",
1 << 8 | 39,
(std::tuple<DrvOutput, DrvOutput> {
{
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.drvPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
.outputName = "baz",
},
DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.drvPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
.outputName = "quux",
},
}))

VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
unkeyedRealisation,
"unkeyed-realisation-1.39",
1 << 8 | 39,
(UnkeyedRealisation {
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
.signatures = { "asdf", "qwer" },
}))

VERSIONED_CHARACTERIZATION_TEST(
WorkerProtoTest,
realisation,
"realisation",
defaultVersion,
(std::tuple<Realisation, Realisation> {
Realisation {
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
"realisation-1.39",
1 << 8 | 39,
(Realisation {
UnkeyedRealisation {
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
.signatures = { "asdf", "qwer" },
},
Realisation {
.id = {
.drvHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="),
.outputName = "baz",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
.signatures = { "asdf", "qwer" },
.dependentRealisations = {
{
DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "quux",
},
StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
},
},
DrvOutput {
.drvPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" },
.outputName = "baz",
},
}))

Expand Down Expand Up @@ -216,20 +209,12 @@ VERSIONED_CHARACTERIZATION_TEST(
{
"foo",
{
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "foo",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
},
},
{
"bar",
{
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "bar",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" },
},
},
Expand Down Expand Up @@ -266,20 +251,12 @@ VERSIONED_CHARACTERIZATION_TEST(
{
"foo",
{
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "foo",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
},
},
{
"bar",
{
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "bar",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" },
},
},
Expand Down Expand Up @@ -318,20 +295,12 @@ VERSIONED_CHARACTERIZATION_TEST(
{
"foo",
{
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "foo",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo" },
},
},
{
"bar",
{
.id = DrvOutput {
.drvHash = Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="),
.outputName = "bar",
},
.outPath = StorePath { "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar" },
},
},
Expand Down
Loading
Loading