Skip to content

Commit

Permalink
addMultipleToStore(): Move pathsToCopy
Browse files Browse the repository at this point in the history
This allows RemoteStore::addMultipleToStore() to free the Source
objects early (and in particular the associated sinkToSource()
buffers). This should fix NixOS#7359. For example, memory consumption of

  nix copy --derivation --to ssh-ng://localhost?remote-store=/tmp/nix --derivation --no-check-sigs \
    /nix/store/4p9xmfgnvclqpii8pxqcwcvl9bxqy2xf-nixos-system-...drv

went from 353 MB to 74 MB.
  • Loading branch information
edolstra committed Jan 20, 2025
1 parent bcb92a5 commit cc838e8
Show file tree
Hide file tree
Showing 4 changed files with 9 additions and 6 deletions.
7 changes: 5 additions & 2 deletions src/libstore/remote-store.cc
Original file line number Diff line number Diff line change
Expand Up @@ -534,21 +534,24 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source,


void RemoteStore::addMultipleToStore(
PathsSource & pathsToCopy,
PathsSource && pathsToCopy,
Activity & act,
RepairFlag repair,
CheckSigsFlag checkSigs)
{
auto source = sinkToSource([&](Sink & sink) {
sink << pathsToCopy.size();
for (auto & [pathInfo, pathSource] : pathsToCopy) {
std::reverse(pathsToCopy.begin(), pathsToCopy.end());
while (!pathsToCopy.empty()) {
auto & [pathInfo, pathSource] = pathsToCopy.back();
WorkerProto::Serialise<ValidPathInfo>::write(*this,
WorkerProto::WriteConn {
.to = sink,
.version = 16,
},
pathInfo);
pathSource->drainInto(sink);
pathsToCopy.pop_back();
}
});

Expand Down
2 changes: 1 addition & 1 deletion src/libstore/remote-store.hh
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ public:
CheckSigsFlag checkSigs) override;

void addMultipleToStore(
PathsSource & pathsToCopy,
PathsSource && pathsToCopy,
Activity & act,
RepairFlag repair,
CheckSigsFlag checkSigs) override;
Expand Down
4 changes: 2 additions & 2 deletions src/libstore/store-api.cc
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ StorePath Store::addToStore(
}

void Store::addMultipleToStore(
PathsSource & pathsToCopy,
PathsSource && pathsToCopy,
Activity & act,
RepairFlag repair,
CheckSigsFlag checkSigs)
Expand Down Expand Up @@ -1138,7 +1138,7 @@ std::map<StorePath, StorePath> copyPaths(
pathsToCopy.push_back(std::pair{infoForDst, std::move(source)});
}

dstStore.addMultipleToStore(pathsToCopy, act, repair, checkSigs);
dstStore.addMultipleToStore(std::move(pathsToCopy), act, repair, checkSigs);

return pathsMap;
}
Expand Down
2 changes: 1 addition & 1 deletion src/libstore/store-api.hh
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,7 @@ public:
CheckSigsFlag checkSigs = CheckSigs);

virtual void addMultipleToStore(
PathsSource & pathsToCopy,
PathsSource && pathsToCopy,
Activity & act,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs);
Expand Down

0 comments on commit cc838e8

Please sign in to comment.