From f90e7714454945a41a68ace35d54cd99d66445a7 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 15 Jul 2024 11:10:42 -0700 Subject: [PATCH 01/80] feat: Refactor unix domain socket store config Following what is outlined in #10766 refactor the uds-remote-store such that the member variables (state) don't live in the store itself but in the config object. Additionally, the config object includes a new necessary constructor that takes a scheme & authority. Minor: * code formatting * cleanup of getting default path * added some comments --- src/libstore/store-api.cc | 7 ++++- src/libstore/uds-remote-store.cc | 41 ++++++++------------------- src/libstore/uds-remote-store.hh | 48 ++++++++++++++++++++++++-------- 3 files changed, 54 insertions(+), 42 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 05c4e1c5e8f..f2925c7c187 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1289,7 +1289,12 @@ ref openStore(StoreReference && storeURI) if (access(stateDir.c_str(), R_OK | W_OK) == 0) return std::make_shared(params); else if (pathExists(settings.nixDaemonSocketFile)) - return std::make_shared(params); + // TODO(fzakaria): A bit gross that we now pass empty string + // but this is knowing that empty string will later default to the same + // nixDaemonSocketFile. Why don't we just wire it all through? + // I believe there are cases where it will live reload so we want to + // continue to account for that. + return std::make_shared("unix", "", params); #if __linux__ else if (!pathExists(stateDir) && params.empty() diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 499f7696712..d0ff5f575d2 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -2,19 +2,17 @@ #include "unix-domain-socket.hh" #include "worker-protocol.hh" -#include #include #include -#include #include #include #ifdef _WIN32 -# include -# include +# include +# include #else -# include -# include +# include +# include #endif namespace nix { @@ -22,51 +20,36 @@ namespace nix { std::string UDSRemoteStoreConfig::doc() { return - #include "uds-remote-store.md" +#include "uds-remote-store.md" ; } - -UDSRemoteStore::UDSRemoteStore(const Params & params) +UDSRemoteStore::UDSRemoteStore(std::string_view scheme, std::string_view authority, const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) , RemoteStoreConfig(params) - , UDSRemoteStoreConfig(params) + , UDSRemoteStoreConfig(scheme, authority, params) , Store(params) , LocalFSStore(params) , RemoteStore(params) { } - -UDSRemoteStore::UDSRemoteStore( - std::string_view scheme, - PathView socket_path, - const Params & params) - : UDSRemoteStore(params) +std::string UDSRemoteStore::getPathOrDefault() const { - if (!socket_path.empty()) - path.emplace(socket_path); + return path.value_or(settings.nixDaemonSocketFile); } - std::string UDSRemoteStore::getUri() { - if (path) { - return std::string("unix://") + *path; - } else { - // unix:// with no path also works. Change what we return? - return "daemon"; - } + return std::format("{}://{}", UNIX_SCHEME, getPathOrDefault()); } - void UDSRemoteStore::Connection::closeWrite() { shutdown(toSocket(fd.get()), SHUT_WR); } - ref UDSRemoteStore::openConnection() { auto conn = make_ref(); @@ -74,7 +57,7 @@ ref UDSRemoteStore::openConnection() /* Connect to a daemon that does the privileged work for us. */ conn->fd = createUnixDomainSocket(); - nix::connect(toSocket(conn->fd.get()), path ? *path : settings.nixDaemonSocketFile); + nix::connect(toSocket(conn->fd.get()), getPathOrDefault()); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); @@ -84,7 +67,6 @@ ref UDSRemoteStore::openConnection() return conn; } - void UDSRemoteStore::addIndirectRoot(const Path & path) { auto conn(getConnection()); @@ -93,7 +75,6 @@ void UDSRemoteStore::addIndirectRoot(const Path & path) readInt(conn->from); } - static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 6f0494bb63b..ba4748edd60 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -7,42 +7,67 @@ namespace nix { +const std::string UNIX_SCHEME = "unix"; + struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig { - UDSRemoteStoreConfig(const Params & params) + + // TODO(fzakaria): Delete this constructor once moved over to the factory pattern + // outlined in https://github.com/NixOS/nix/issues/10766 + using LocalFSStoreConfig::LocalFSStoreConfig; + using RemoteStoreConfig::RemoteStoreConfig; + + UDSRemoteStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) , RemoteStoreConfig(params) { + assert(scheme == UNIX_SCHEME && "Scheme must be 'unix'"); + if (!authority.empty()) { + path.emplace(authority); + } } - const std::string name() override { return "Local Daemon Store"; } + const std::string name() override + { + return "Local Daemon Store"; + } std::string doc() override; + + // The path to the unix-domain sock + // The default *could be* settings.nixDaemonSocketFile but that + // won't pick up live changes unfortunately. This optional handling is instead + // handled on opening of the connection + std::optional path; }; -class UDSRemoteStore : public virtual UDSRemoteStoreConfig - , public virtual IndirectRootStore - , public virtual RemoteStore +class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual IndirectRootStore, public virtual RemoteStore { public: - UDSRemoteStore(const Params & params); UDSRemoteStore( std::string_view scheme, - PathView path, + // authority is the socket path + std::string_view authority, const Params & params); std::string getUri() override; static std::set uriSchemes() - { return {"unix"}; } + { + return {UNIX_SCHEME}; + } ref getFSAccessor(bool requireValidPath = true) override - { return LocalFSStore::getFSAccessor(requireValidPath); } + { + return LocalFSStore::getFSAccessor(requireValidPath); + } void narFromPath(const StorePath & path, Sink & sink) override - { LocalFSStore::narFromPath(path, sink); } + { + LocalFSStore::narFromPath(path, sink); + } /** * Implementation of `IndirectRootStore::addIndirectRoot()` which @@ -63,7 +88,8 @@ private: }; ref openConnection() override; - std::optional path; + + std::string getPathOrDefault() const; }; } From 1f722674ddd6be43573004507d5611d79816a2c1 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 15 Jul 2024 11:42:08 -0700 Subject: [PATCH 02/80] uds-remote-store is now clang-formatted --- maintainers/flake-module.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 46b3e136324..d6bb3912266 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -249,8 +249,6 @@ ''^src/libstore/unix/pathlocks\.cc$'' ''^src/libstore/posix-fs-canonicalise\.cc$'' ''^src/libstore/posix-fs-canonicalise\.hh$'' - ''^src/libstore/uds-remote-store\.cc$'' - ''^src/libstore/uds-remote-store\.hh$'' ''^src/libstore/windows/build\.cc$'' ''^src/libstore/worker-protocol-impl\.hh$'' ''^src/libstore/worker-protocol\.cc$'' From 660d8688e196d0ad3624b30dbf3585b72cd26e3a Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 15 Jul 2024 13:00:27 -0700 Subject: [PATCH 03/80] Removed clang-format changes --- maintainers/flake-module.nix | 2 ++ src/libstore/uds-remote-store.cc | 16 +++++++++++----- src/libstore/uds-remote-store.hh | 21 +++++++-------------- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index d6bb3912266..46b3e136324 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -249,6 +249,8 @@ ''^src/libstore/unix/pathlocks\.cc$'' ''^src/libstore/posix-fs-canonicalise\.cc$'' ''^src/libstore/posix-fs-canonicalise\.hh$'' + ''^src/libstore/uds-remote-store\.cc$'' + ''^src/libstore/uds-remote-store\.hh$'' ''^src/libstore/windows/build\.cc$'' ''^src/libstore/worker-protocol-impl\.hh$'' ''^src/libstore/worker-protocol\.cc$'' diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index d0ff5f575d2..ac2d10ecb74 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -6,13 +6,14 @@ #include #include #include +#include #ifdef _WIN32 -# include -# include +# include +# include #else -# include -# include +# include +# include #endif namespace nix { @@ -20,10 +21,11 @@ namespace nix { std::string UDSRemoteStoreConfig::doc() { return -#include "uds-remote-store.md" + #include "uds-remote-store.md" ; } + UDSRemoteStore::UDSRemoteStore(std::string_view scheme, std::string_view authority, const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) @@ -45,11 +47,13 @@ std::string UDSRemoteStore::getUri() return std::format("{}://{}", UNIX_SCHEME, getPathOrDefault()); } + void UDSRemoteStore::Connection::closeWrite() { shutdown(toSocket(fd.get()), SHUT_WR); } + ref UDSRemoteStore::openConnection() { auto conn = make_ref(); @@ -67,6 +71,7 @@ ref UDSRemoteStore::openConnection() return conn; } + void UDSRemoteStore::addIndirectRoot(const Path & path) { auto conn(getConnection()); @@ -75,6 +80,7 @@ void UDSRemoteStore::addIndirectRoot(const Path & path) readInt(conn->from); } + static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index ba4748edd60..61cb87ce4e6 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -28,10 +28,7 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon } } - const std::string name() override - { - return "Local Daemon Store"; - } + const std::string name() override { return "Local Daemon Store"; } std::string doc() override; @@ -42,7 +39,9 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon std::optional path; }; -class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual IndirectRootStore, public virtual RemoteStore +class UDSRemoteStore : public virtual UDSRemoteStoreConfig + , public virtual IndirectRootStore + , public virtual RemoteStore { public: @@ -55,19 +54,13 @@ public: std::string getUri() override; static std::set uriSchemes() - { - return {UNIX_SCHEME}; - } + { return {UNIX_SCHEME}; } ref getFSAccessor(bool requireValidPath = true) override - { - return LocalFSStore::getFSAccessor(requireValidPath); - } + { return LocalFSStore::getFSAccessor(requireValidPath); } void narFromPath(const StorePath & path, Sink & sink) override - { - LocalFSStore::narFromPath(path, sink); - } + { LocalFSStore::narFromPath(path, sink); } /** * Implementation of `IndirectRootStore::addIndirectRoot()` which From 6585c6ed0c3a468a3a0358f1ad7be6c27ff69e95 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 15 Jul 2024 13:02:17 -0700 Subject: [PATCH 04/80] Reference public variable --- src/libstore/store-api.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f2925c7c187..16ae7df9feb 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1294,7 +1294,7 @@ ref openStore(StoreReference && storeURI) // nixDaemonSocketFile. Why don't we just wire it all through? // I believe there are cases where it will live reload so we want to // continue to account for that. - return std::make_shared("unix", "", params); + return std::make_shared(UNIX_SCHEME, "", params); #if __linux__ else if (!pathExists(stateDir) && params.empty() From 10c5c65c4534a214c644a94a87369050a5c2bc77 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 15 Jul 2024 13:03:48 -0700 Subject: [PATCH 05/80] Update src/libstore/uds-remote-store.hh Co-authored-by: John Ericson --- src/libstore/uds-remote-store.hh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 61cb87ce4e6..fafeec1e053 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -32,10 +32,13 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon std::string doc() override; - // The path to the unix-domain sock - // The default *could be* settings.nixDaemonSocketFile but that - // won't pick up live changes unfortunately. This optional handling is instead - // handled on opening of the connection + /** + * The path to the unix domain socket. + * + * The default *could be* settings.nixDaemonSocketFile but that + * won't pick up live changes unfortunately. This optional handling is instead + * handled on opening of the connection. + */ std::optional path; }; From 030d91df053ed81a026d14c7d450906791822a02 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 15 Jul 2024 13:08:05 -0700 Subject: [PATCH 06/80] Move constant to within Config object --- src/libstore/store-api.cc | 2 +- src/libstore/uds-remote-store.hh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 16ae7df9feb..a0c83fa8b4f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1294,7 +1294,7 @@ ref openStore(StoreReference && storeURI) // nixDaemonSocketFile. Why don't we just wire it all through? // I believe there are cases where it will live reload so we want to // continue to account for that. - return std::make_shared(UNIX_SCHEME, "", params); + return std::make_shared(UDSRemoteStoreConfig::UNIX_SCHEME, "", params); #if __linux__ else if (!pathExists(stateDir) && params.empty() diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index fafeec1e053..00cc2cb2f48 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -7,11 +7,11 @@ namespace nix { -const std::string UNIX_SCHEME = "unix"; - struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig { + static constexpr char const * UNIX_SCHEME = "unix"; + // TODO(fzakaria): Delete this constructor once moved over to the factory pattern // outlined in https://github.com/NixOS/nix/issues/10766 using LocalFSStoreConfig::LocalFSStoreConfig; From 71699057f6dac7196b6a75bb0dcf130798b757e5 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 15 Jul 2024 13:12:04 -0700 Subject: [PATCH 07/80] Lowercase constexpr --- src/libstore/store-api.cc | 2 +- src/libstore/uds-remote-store.cc | 2 +- src/libstore/uds-remote-store.hh | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index a0c83fa8b4f..02879ad80f4 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1294,7 +1294,7 @@ ref openStore(StoreReference && storeURI) // nixDaemonSocketFile. Why don't we just wire it all through? // I believe there are cases where it will live reload so we want to // continue to account for that. - return std::make_shared(UDSRemoteStoreConfig::UNIX_SCHEME, "", params); + return std::make_shared(UDSRemoteStoreConfig::scheme, "", params); #if __linux__ else if (!pathExists(stateDir) && params.empty() diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index ac2d10ecb74..842bc0fb2d9 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -44,7 +44,7 @@ std::string UDSRemoteStore::getPathOrDefault() const std::string UDSRemoteStore::getUri() { - return std::format("{}://{}", UNIX_SCHEME, getPathOrDefault()); + return std::format("{}://{}", scheme, getPathOrDefault()); } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 00cc2cb2f48..4ea959aac7d 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -10,7 +10,7 @@ namespace nix { struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig { - static constexpr char const * UNIX_SCHEME = "unix"; + static constexpr char const * scheme = "unix"; // TODO(fzakaria): Delete this constructor once moved over to the factory pattern // outlined in https://github.com/NixOS/nix/issues/10766 @@ -22,7 +22,7 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon , LocalFSStoreConfig(params) , RemoteStoreConfig(params) { - assert(scheme == UNIX_SCHEME && "Scheme must be 'unix'"); + assert(scheme == UDSRemoteStoreConfig::scheme && "Scheme must be 'unix'"); if (!authority.empty()) { path.emplace(authority); } @@ -57,7 +57,7 @@ public: std::string getUri() override; static std::set uriSchemes() - { return {UNIX_SCHEME}; } + { return {scheme}; } ref getFSAccessor(bool requireValidPath = true) override { return LocalFSStore::getFSAccessor(requireValidPath); } From 4af50928dda400547bfce9af31187cbaeb8990b5 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 15 Jul 2024 14:15:03 -0700 Subject: [PATCH 08/80] Update src/libstore/store-api.cc Revert back to basic constructor in store-api.cc Co-authored-by: John Ericson --- src/libstore/store-api.cc | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 02879ad80f4..05c4e1c5e8f 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1289,12 +1289,7 @@ ref openStore(StoreReference && storeURI) if (access(stateDir.c_str(), R_OK | W_OK) == 0) return std::make_shared(params); else if (pathExists(settings.nixDaemonSocketFile)) - // TODO(fzakaria): A bit gross that we now pass empty string - // but this is knowing that empty string will later default to the same - // nixDaemonSocketFile. Why don't we just wire it all through? - // I believe there are cases where it will live reload so we want to - // continue to account for that. - return std::make_shared(UDSRemoteStoreConfig::scheme, "", params); + return std::make_shared(params); #if __linux__ else if (!pathExists(stateDir) && params.empty() From dd74b72b97ab55c24323726ad7c31469b795cfd3 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 15 Jul 2024 14:23:27 -0700 Subject: [PATCH 09/80] Revert back returning daemon for URI if path empty --- src/libstore/uds-remote-store.cc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 842bc0fb2d9..8388a38cfef 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -6,7 +6,6 @@ #include #include #include -#include #ifdef _WIN32 # include @@ -44,7 +43,15 @@ std::string UDSRemoteStore::getPathOrDefault() const std::string UDSRemoteStore::getUri() { - return std::format("{}://{}", scheme, getPathOrDefault()); + if (path) { + return std::string(scheme) + "://" + *path; + } else { + // FIXME: Not clear why we return daemon here and not default to + // settings.nixDaemonSocketFile + // + // unix:// with no path also works. Change what we return? + return "daemon"; + } } From 01fbb4b8007bdf15346ec9959c62815e7d03705f Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 15 Jul 2024 14:32:54 -0700 Subject: [PATCH 10/80] Added unit test --- src/libstore/uds-remote-store.hh | 11 +++++++++++ tests/unit/libstore/uds-remote-store.cc | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 tests/unit/libstore/uds-remote-store.cc diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 4ea959aac7d..63a8ed8abff 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -48,6 +48,17 @@ class UDSRemoteStore : public virtual UDSRemoteStoreConfig { public: + /* + \deprecated This is the old API to construct the store. + A bit gross that we now pass empty string but this is knowing + that empty string will later default to the same nixDaemonSocketFile. + Why don't we just wire it all through? + I believe there are cases where it will live reload so we want to + continue to account for that. + */ + UDSRemoteStore(const Params & params) + : nix::UDSRemoteStore(scheme, "", params) {} + UDSRemoteStore( std::string_view scheme, // authority is the socket path diff --git a/tests/unit/libstore/uds-remote-store.cc b/tests/unit/libstore/uds-remote-store.cc new file mode 100644 index 00000000000..32a690b14b4 --- /dev/null +++ b/tests/unit/libstore/uds-remote-store.cc @@ -0,0 +1,18 @@ +#include + +#include "uds-remote-store.hh" + +namespace nix { + +TEST(UDSRemoteStore, constructConfig) +{ + UDSRemoteStoreConfig config { + "unix", + "/tmp/socket", + {} + }; + + EXPECT_EQ(*config.path, "/tmp/socket"); +} + +} \ No newline at end of file From 6eb3303c98985fe40040bf66d87466f8003ce31a Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 15 Jul 2024 14:35:52 -0700 Subject: [PATCH 11/80] Added to meson --- tests/unit/libstore/meson.build | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/libstore/meson.build b/tests/unit/libstore/meson.build index 90e7d3047c7..8e846904685 100644 --- a/tests/unit/libstore/meson.build +++ b/tests/unit/libstore/meson.build @@ -69,6 +69,7 @@ sources = files( 'serve-protocol.cc', 'store-reference.cc', 'worker-protocol.cc', + 'uds-remote-store.cc', ) include_dirs = [include_directories('.')] From d3275e8e2ae3ffd5fb138733bf16574bdea33cd6 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 15 Jul 2024 18:13:34 -0700 Subject: [PATCH 12/80] Changed to protected and added another test --- src/libstore/uds-remote-store.hh | 10 ++++++++-- tests/unit/libstore/uds-remote-store.cc | 7 ++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 63a8ed8abff..c59c179d483 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -10,7 +10,10 @@ namespace nix { struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig { - static constexpr char const * scheme = "unix"; + protected: + static constexpr char const * scheme = "unix"; + + public: // TODO(fzakaria): Delete this constructor once moved over to the factory pattern // outlined in https://github.com/NixOS/nix/issues/10766 @@ -22,7 +25,10 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon , LocalFSStoreConfig(params) , RemoteStoreConfig(params) { - assert(scheme == UDSRemoteStoreConfig::scheme && "Scheme must be 'unix'"); + if (scheme != UDSRemoteStoreConfig::scheme) { + throw UsageError("Scheme must be 'unix'"); + } + if (!authority.empty()) { path.emplace(authority); } diff --git a/tests/unit/libstore/uds-remote-store.cc b/tests/unit/libstore/uds-remote-store.cc index 32a690b14b4..597a78c7653 100644 --- a/tests/unit/libstore/uds-remote-store.cc +++ b/tests/unit/libstore/uds-remote-store.cc @@ -15,4 +15,9 @@ TEST(UDSRemoteStore, constructConfig) EXPECT_EQ(*config.path, "/tmp/socket"); } -} \ No newline at end of file +TEST(UDSRemoteStore, constructConfigWrongScheme) +{ + EXPECT_THROW(UDSRemoteStoreConfig("http", "/tmp/socket", {}), UsageError); +} + +} // namespace nix \ No newline at end of file From 0087d7d65b13a003c10ef0fe0c3f9af72d775f8b Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Mon, 15 Jul 2024 18:16:54 -0700 Subject: [PATCH 13/80] Clang formatted test file --- tests/unit/libstore/uds-remote-store.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/unit/libstore/uds-remote-store.cc b/tests/unit/libstore/uds-remote-store.cc index 597a78c7653..5a0be184778 100644 --- a/tests/unit/libstore/uds-remote-store.cc +++ b/tests/unit/libstore/uds-remote-store.cc @@ -6,11 +6,7 @@ namespace nix { TEST(UDSRemoteStore, constructConfig) { - UDSRemoteStoreConfig config { - "unix", - "/tmp/socket", - {} - }; + UDSRemoteStoreConfig config{"unix", "/tmp/socket", {}}; EXPECT_EQ(*config.path, "/tmp/socket"); } From e733f6d37ed6c356f3a3d125eaff69c46ced34d7 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 Jul 2024 23:11:03 -0400 Subject: [PATCH 14/80] Move `UdsRemoteStore{,Config}` defs out of headers, and other cleanups Hopefully this fixes the macOS link error. It's also good for compilation time. --- src/libstore/uds-remote-store.cc | 28 +++++++++++++++ src/libstore/uds-remote-store.hh | 45 +++++++++---------------- tests/unit/libstore/uds-remote-store.cc | 2 +- 3 files changed, 45 insertions(+), 30 deletions(-) diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 8388a38cfef..9399fc810be 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -17,6 +17,24 @@ namespace nix { +UDSRemoteStoreConfig::UDSRemoteStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , RemoteStoreConfig(params) +{ + if (scheme != UDSRemoteStoreConfig::scheme) { + throw UsageError("Scheme must be 'unix'"); + } + + if (!authority.empty()) { + path.emplace(authority); + } +} + + std::string UDSRemoteStoreConfig::doc() { return @@ -25,6 +43,15 @@ std::string UDSRemoteStoreConfig::doc() } +// A bit gross that we now pass empty string but this is knowing that +// empty string will later default to the same nixDaemonSocketFile. Why +// don't we just wire it all through? I believe there are cases where it +// will live reload so we want to continue to account for that. +UDSRemoteStore::UDSRemoteStore(const Params & params) + : UDSRemoteStore(scheme, "", params) +{} + + UDSRemoteStore::UDSRemoteStore(std::string_view scheme, std::string_view authority, const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) @@ -36,6 +63,7 @@ UDSRemoteStore::UDSRemoteStore(std::string_view scheme, std::string_view authori { } + std::string UDSRemoteStore::getPathOrDefault() const { return path.value_or(settings.nixDaemonSocketFile); diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index c59c179d483..2cf40edd6b0 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -9,30 +9,18 @@ namespace nix { struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig { - - protected: - static constexpr char const * scheme = "unix"; - - public: - // TODO(fzakaria): Delete this constructor once moved over to the factory pattern // outlined in https://github.com/NixOS/nix/issues/10766 using LocalFSStoreConfig::LocalFSStoreConfig; using RemoteStoreConfig::RemoteStoreConfig; - UDSRemoteStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RemoteStoreConfig(params) - { - if (scheme != UDSRemoteStoreConfig::scheme) { - throw UsageError("Scheme must be 'unix'"); - } - - if (!authority.empty()) { - path.emplace(authority); - } - } + /** + * @param authority is the socket path. + */ + UDSRemoteStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params); const std::string name() override { return "Local Daemon Store"; } @@ -46,6 +34,9 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon * handled on opening of the connection. */ std::optional path; + +protected: + static constexpr char const * scheme = "unix"; }; class UDSRemoteStore : public virtual UDSRemoteStoreConfig @@ -54,20 +45,16 @@ class UDSRemoteStore : public virtual UDSRemoteStoreConfig { public: - /* - \deprecated This is the old API to construct the store. - A bit gross that we now pass empty string but this is knowing - that empty string will later default to the same nixDaemonSocketFile. - Why don't we just wire it all through? - I believe there are cases where it will live reload so we want to - continue to account for that. + /** + * @deprecated This is the old API to construct the store. */ - UDSRemoteStore(const Params & params) - : nix::UDSRemoteStore(scheme, "", params) {} + UDSRemoteStore(const Params & params); + /** + * @param authority is the socket path. + */ UDSRemoteStore( std::string_view scheme, - // authority is the socket path std::string_view authority, const Params & params); diff --git a/tests/unit/libstore/uds-remote-store.cc b/tests/unit/libstore/uds-remote-store.cc index 5a0be184778..b9811a5e073 100644 --- a/tests/unit/libstore/uds-remote-store.cc +++ b/tests/unit/libstore/uds-remote-store.cc @@ -16,4 +16,4 @@ TEST(UDSRemoteStore, constructConfigWrongScheme) EXPECT_THROW(UDSRemoteStoreConfig("http", "/tmp/socket", {}), UsageError); } -} // namespace nix \ No newline at end of file +} // namespace nix From e435f3e23caa7e2b9941f49500d5c154c514ddf0 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 Jul 2024 23:44:06 -0400 Subject: [PATCH 15/80] Add two more headers to try to provide missing template instantiations --- src/libstore/uds-remote-store.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 9399fc810be..abefdcb75d4 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,6 +1,8 @@ #include "uds-remote-store.hh" #include "unix-domain-socket.hh" #include "worker-protocol.hh" +#include "args.hh" +#include "config-impl.hh" #include #include From ed2ccdca21d0032d61ae1772956327eac8a97cd3 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Tue, 16 Jul 2024 05:47:50 -0700 Subject: [PATCH 16/80] Add globals.hh to uds-remote-store.cc --- src/libstore/uds-remote-store.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index abefdcb75d4..a6199eaa7e4 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,8 +1,7 @@ #include "uds-remote-store.hh" #include "unix-domain-socket.hh" #include "worker-protocol.hh" -#include "args.hh" -#include "config-impl.hh" +#include "globals.hh" #include #include From e9c0636a11dceae3182dd1b03f499042b8aca768 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 16 Jul 2024 16:15:44 -0400 Subject: [PATCH 17/80] Fix linking issue, convert many more --- src/libstore/dummy-store.cc | 20 ++-- src/libstore/http-binary-cache-store.cc | 50 ++++----- src/libstore/http-binary-cache-store.hh | 21 ++++ src/libstore/local-binary-cache-store.cc | 36 +++---- src/libstore/local-binary-cache-store.hh | 21 ++++ src/libstore/local-fs-store.cc | 14 +++ src/libstore/local-fs-store.hh | 9 ++ src/libstore/local-overlay-store.cc | 6 +- src/libstore/local-overlay-store.hh | 21 ++-- src/libstore/local-store.cc | 33 +++--- src/libstore/local-store.hh | 5 + src/libstore/meson.build | 2 + src/libstore/s3-binary-cache-store.cc | 100 +++++------------- src/libstore/s3-binary-cache-store.hh | 79 ++++++++++++++ src/libstore/ssh-store.cc | 55 ++++------ src/libstore/ssh-store.hh | 23 ++++ src/libstore/uds-remote-store.cc | 29 ++--- src/libstore/uds-remote-store.hh | 9 +- .../unit/libstore/http-binary-cache-store.cc | 21 ++++ .../unit/libstore/local-binary-cache-store.cc | 14 +++ tests/unit/libstore/local-overlay-store.cc | 30 ++++++ tests/unit/libstore/local-store.cc | 36 +++++++ tests/unit/libstore/meson.build | 7 +- tests/unit/libstore/s3-binary-cache-store.cc | 18 ++++ tests/unit/libstore/ssh-store.cc | 27 ++++- tests/unit/libstore/uds-remote-store.cc | 2 +- 26 files changed, 474 insertions(+), 214 deletions(-) create mode 100644 src/libstore/http-binary-cache-store.hh create mode 100644 src/libstore/local-binary-cache-store.hh create mode 100644 tests/unit/libstore/http-binary-cache-store.cc create mode 100644 tests/unit/libstore/local-binary-cache-store.cc create mode 100644 tests/unit/libstore/local-overlay-store.cc create mode 100644 tests/unit/libstore/local-store.cc create mode 100644 tests/unit/libstore/s3-binary-cache-store.cc diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 17ebaace6a8..af0dd909214 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -6,6 +6,13 @@ namespace nix { struct DummyStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; + DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) + : StoreConfig(params) + { + if (!authority.empty()) + throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); + } + const std::string name() override { return "Dummy Store"; } std::string doc() override @@ -19,18 +26,15 @@ struct DummyStoreConfig : virtual StoreConfig { struct DummyStore : public virtual DummyStoreConfig, public virtual Store { DummyStore(std::string_view scheme, std::string_view authority, const Params & params) - : DummyStore(params) - { - if (!authority.empty()) - throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); - } - - DummyStore(const Params & params) : StoreConfig(params) - , DummyStoreConfig(params) + , DummyStoreConfig(scheme, authority, params) , Store(params) { } + DummyStore(const Params & params) + : DummyStore("dummy", "", params) + { } + std::string getUri() override { return *uriSchemes().begin(); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 3328caef9c4..49ec26b9f93 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -1,4 +1,4 @@ -#include "binary-cache-store.hh" +#include "http-binary-cache-store.hh" #include "filetransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" @@ -8,26 +8,37 @@ namespace nix { MakeError(UploadToHTTP, Error); -struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig + +HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( + std::string_view scheme, + std::string_view _cacheUri, + const Params & params) + : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , cacheUri( + std::string { scheme } + + "://" + + (!_cacheUri.empty() + ? _cacheUri + : throw UsageError("`%s` Store requires a non-empty authority in Store URL", scheme))) { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + while (!cacheUri.empty() && cacheUri.back() == '/') + cacheUri.pop_back(); +} - const std::string name() override { return "HTTP Binary Cache Store"; } - std::string doc() override - { - return - #include "http-binary-cache-store.md" - ; - } -}; +std::string HttpBinaryCacheStoreConfig::doc() +{ + return + #include "http-binary-cache-store.md" + ; +} + class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore { private: - Path cacheUri; - struct State { bool enabled = true; @@ -40,23 +51,14 @@ class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public v HttpBinaryCacheStore( std::string_view scheme, - PathView _cacheUri, + PathView cacheUri, const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) - , HttpBinaryCacheStoreConfig(params) + , HttpBinaryCacheStoreConfig(scheme, cacheUri, params) , Store(params) , BinaryCacheStore(params) - , cacheUri( - std::string { scheme } - + "://" - + (!_cacheUri.empty() - ? _cacheUri - : throw UsageError("`%s` Store requires a non-empty authority in Store URL", scheme))) { - while (!cacheUri.empty() && cacheUri.back() == '/') - cacheUri.pop_back(); - diskCache = getNarInfoDiskCache(); } diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh new file mode 100644 index 00000000000..230e52b3373 --- /dev/null +++ b/src/libstore/http-binary-cache-store.hh @@ -0,0 +1,21 @@ +#include "binary-cache-store.hh" + +namespace nix { + +struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + HttpBinaryCacheStoreConfig(std::string_view scheme, std::string_view _cacheUri, const Params & params); + + Path cacheUri; + + const std::string name() override + { + return "HTTP Binary Cache Store"; + } + + std::string doc() override; +}; + +} diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index aa2efdb8fb6..106663132c1 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -1,4 +1,4 @@ -#include "binary-cache-store.hh" +#include "local-binary-cache-store.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" #include "signals.hh" @@ -7,28 +7,27 @@ namespace nix { -struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig( + std::string_view scheme, + PathView binaryCacheDir, + const Params & params) + : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , binaryCacheDir(binaryCacheDir) { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - - const std::string name() override { return "Local Binary Cache Store"; } +} - std::string doc() override - { - return - #include "local-binary-cache-store.md" - ; - } -}; -class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore +std::string LocalBinaryCacheStoreConfig::doc() { -private: - - Path binaryCacheDir; + return + #include "local-binary-cache-store.md" + ; +} -public: +struct LocalBinaryCacheStore : virtual LocalBinaryCacheStoreConfig, virtual BinaryCacheStore +{ /** * @param binaryCacheDir `file://` is a short-hand for `file:///` * for now. @@ -39,10 +38,9 @@ class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) - , LocalBinaryCacheStoreConfig(params) + , LocalBinaryCacheStoreConfig(scheme, binaryCacheDir, params) , Store(params) , BinaryCacheStore(params) - , binaryCacheDir(binaryCacheDir) { } diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh new file mode 100644 index 00000000000..7701eb38b99 --- /dev/null +++ b/src/libstore/local-binary-cache-store.hh @@ -0,0 +1,21 @@ +#include "binary-cache-store.hh" + +namespace nix { + +struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + LocalBinaryCacheStoreConfig(std::string_view scheme, PathView binaryCacheDir, const Params & params); + + Path binaryCacheDir; + + const std::string name() override + { + return "Local Binary Cache Store"; + } + + std::string doc() override; +}; + +} diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 843c0d28881..5449b20eb3b 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -8,6 +8,20 @@ namespace nix { +LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params) + : StoreConfig(params) + // Default `?root` from `rootDir` if non set + // FIXME don't duplicate description once we don't have root setting + , rootDir{ + this, + !rootDir.empty() && params.count("root") == 0 + ? (std::optional{rootDir}) + : std::nullopt, + "root", + "Directory prefixed to all other paths."} +{ +} + LocalFSStore::LocalFSStore(const Params & params) : Store(params) { diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 8fb08120012..9bb569f0b25 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -11,6 +11,15 @@ struct LocalFSStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; + /** + * Used to override the `root` settings. Can't be done via modifying + * `params` reliably because this parameter is unused except for + * passing to base class constructors. + * + * @todo Make this less error-prone with new store settings system. + */ + LocalFSStoreConfig(PathView path, const Params & params); + const OptionalPathSetting rootDir{this, std::nullopt, "root", "Directory prefixed to all other paths."}; diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 598415db837..ec2c5f4e936 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -18,11 +18,11 @@ Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { return upperLayer + "/" + path.to_string(); } -LocalOverlayStore::LocalOverlayStore(const Params & params) +LocalOverlayStore::LocalOverlayStore(std::string_view scheme, PathView path, const Params & params) : StoreConfig(params) - , LocalFSStoreConfig(params) + , LocalFSStoreConfig(path, params) , LocalStoreConfig(params) - , LocalOverlayStoreConfig(params) + , LocalOverlayStoreConfig(scheme, path, params) , Store(params) , LocalFSStore(params) , LocalStore(params) diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 35a30101391..a3f15e484f0 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -8,11 +8,16 @@ namespace nix { struct LocalOverlayStoreConfig : virtual LocalStoreConfig { LocalOverlayStoreConfig(const StringMap & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , LocalStoreConfig(params) + : LocalOverlayStoreConfig("local-overlay", "", params) { } + LocalOverlayStoreConfig(std::string_view scheme, PathView path, const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(path, params) + , LocalStoreConfig(scheme, path, params) + { + } + const Setting lowerStoreUri{(StoreConfig*) this, "", "lower-store", R"( [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) @@ -90,15 +95,13 @@ class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual ref lowerStore; public: - LocalOverlayStore(const Params & params); - - LocalOverlayStore(std::string_view scheme, PathView path, const Params & params) - : LocalOverlayStore(params) + LocalOverlayStore(const Params & params) + : LocalOverlayStore("local-overlay", "", params) { - if (!path.empty()) - throw UsageError("local-overlay:// store url doesn't support path part, only scheme and query params"); } + LocalOverlayStore(std::string_view scheme, PathView path, const Params & params); + static std::set uriSchemes() { return { "local-overlay" }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 82b70ff2135..819cee34532 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -56,6 +56,15 @@ namespace nix { +LocalStoreConfig::LocalStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(authority, params) +{ +} + std::string LocalStoreConfig::doc() { return @@ -183,10 +192,13 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) } } -LocalStore::LocalStore(const Params & params) +LocalStore::LocalStore( + std::string_view scheme, + PathView path, + const Params & params) : StoreConfig(params) - , LocalFSStoreConfig(params) - , LocalStoreConfig(params) + , LocalFSStoreConfig(path, params) + , LocalStoreConfig(scheme, path, params) , Store(params) , LocalFSStore(params) , dbDir(stateDir + "/db") @@ -465,19 +477,8 @@ LocalStore::LocalStore(const Params & params) } -LocalStore::LocalStore( - std::string_view scheme, - PathView path, - const Params & _params) - : LocalStore([&]{ - // Default `?root` from `path` if non set - if (!path.empty() && _params.count("root") == 0) { - auto params = _params; - params.insert_or_assign("root", std::string { path }); - return params; - } - return _params; - }()) +LocalStore::LocalStore(const Params & params) + : LocalStore("local", "", params) { } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index b0a0def9aae..026729bf24c 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -38,6 +38,11 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; + LocalStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params); + Setting requireSigs{this, settings.requireSigs, "require-sigs", diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 297bf71870c..29ee95b75bc 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -243,10 +243,12 @@ headers = [config_h] + files( 'filetransfer.hh', 'gc-store.hh', 'globals.hh', + 'http-binary-cache-store.hh', 'indirect-root-store.hh', 'keys.hh', 'legacy-ssh-store.hh', 'length-prefixed-protocol-helper.hh', + 'local-binary-cache-store.hh', 'local-fs-store.hh', 'local-overlay-store.hh', 'local-store.hh', diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 27013c0f17a..d865684d7aa 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -1,5 +1,7 @@ #if ENABLE_S3 +#include + #include "s3.hh" #include "s3-binary-cache-store.hh" #include "nar-info.hh" @@ -190,76 +192,31 @@ S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) , BinaryCacheStore(params) { } -struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig + +S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( + std::string_view uriScheme, + std::string_view bucketName, + const Params & params) + : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , bucketName(bucketName) { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - - const Setting profile{this, "", "profile", - R"( - The name of the AWS configuration profile to use. By default - Nix will use the `default` profile. - )"}; - - const Setting region{this, Aws::Region::US_EAST_1, "region", - R"( - The region of the S3 bucket. If your bucket is not in - `us–east-1`, you should always explicitly specify the region - parameter. - )"}; - - const Setting scheme{this, "", "scheme", - R"( - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - )"}; - - const Setting endpoint{this, "", "endpoint", - R"( - The URL of the endpoint of an S3-compatible service such as MinIO. - Do not specify this setting if you're using Amazon S3. - - > **Note** - > - > This endpoint must support HTTPS and will use path-based - > addressing instead of virtual host based addressing. - )"}; - - const Setting narinfoCompression{this, "", "narinfo-compression", - "Compression method for `.narinfo` files."}; - - const Setting lsCompression{this, "", "ls-compression", - "Compression method for `.ls` files."}; - - const Setting logCompression{this, "", "log-compression", - R"( - Compression method for `log/*` files. It is recommended to - use a compression method supported by most web browsers - (e.g. `brotli`). - )"}; - - const Setting multipartUpload{ - this, false, "multipart-upload", - "Whether to use multi-part uploads."}; - - const Setting bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", - "Size (in bytes) of each part in multi-part uploads."}; - - const std::string name() override { return "S3 Binary Cache Store"; } - - std::string doc() override - { - return - #include "s3-binary-cache-store.md" - ; - } -}; + // Don't want to use use AWS SDK in header, so we check the default + // here. TODO do this better after we overhaul the store settings + // system. + assert(std::string{defaultRegion} == std::string{Aws::Region::US_EAST_1}); + + if (bucketName.empty()) + throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); +} + +std::string S3BinaryCacheStoreConfig::doc() +{ + return + #include "s3-binary-cache-store.md" + ; +} + struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore { @@ -275,15 +232,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) - , S3BinaryCacheStoreConfig(params) + , S3BinaryCacheStoreConfig(uriScheme, bucketName, params) , Store(params) , BinaryCacheStore(params) , S3BinaryCacheStore(params) - , bucketName(bucketName) , s3Helper(profile, region, scheme, endpoint) { - if (bucketName.empty()) - throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); diskCache = getNarInfoDiskCache(); } diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index c62ea5147f3..723c8c54e61 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -7,6 +7,85 @@ namespace nix { +struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + std::string bucketName; + + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + S3BinaryCacheStoreConfig( + std::string_view uriScheme, + std::string_view bucketName, + const Params & params); + + const Setting profile{this, "", "profile", + R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )"}; + +protected: + + constexpr static const char * defaultRegion = "us-east-1"; + +public: + + const Setting region{this, defaultRegion, "region", + R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )"}; + + const Setting scheme{this, "", "scheme", + R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )"}; + + const Setting endpoint{this, "", "endpoint", + R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )"}; + + const Setting narinfoCompression{this, "", "narinfo-compression", + "Compression method for `.narinfo` files."}; + + const Setting lsCompression{this, "", "ls-compression", + "Compression method for `.ls` files."}; + + const Setting logCompression{this, "", "log-compression", + R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )"}; + + const Setting multipartUpload{ + this, false, "multipart-upload", + "Whether to use multi-part uploads."}; + + const Setting bufferSize{ + this, 5 * 1024 * 1024, "buffer-size", + "Size (in bytes) of each part in multi-part uploads."}; + + const std::string name() override { return "S3 Binary Cache Store"; } + + std::string doc() override; +}; + class S3BinaryCacheStore : public virtual BinaryCacheStore { protected: diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index b21c22d7e08..d63995bf3bc 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -89,43 +89,32 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore }; }; -struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig -{ - using SSHStoreConfig::SSHStoreConfig; - using LocalFSStoreConfig::LocalFSStoreConfig; - - MountedSSHStoreConfig(StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) - { - } - MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) - { - } +MountedSSHStoreConfig::MountedSSHStoreConfig(StringMap params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) +{ +} - const std::string name() override { return "Experimental SSH Store with filesystem mounted"; } +MountedSSHStoreConfig::MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(scheme, host, params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) +{ +} - std::string doc() override - { - return - #include "mounted-ssh-store.md" - ; - } +std::string MountedSSHStoreConfig::doc() +{ + return + #include "mounted-ssh-store.md" + ; +} - std::optional experimentalFeature() const override - { - return ExperimentalFeature::MountedSSHStore; - } -}; /** * The mounted ssh store assumes that filesystems on the remote host are diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh index 6ef2219a2e9..feb57ccba80 100644 --- a/src/libstore/ssh-store.hh +++ b/src/libstore/ssh-store.hh @@ -3,6 +3,7 @@ #include "common-ssh-store-config.hh" #include "store-api.hh" +#include "local-fs-store.hh" #include "remote-store.hh" namespace nix { @@ -25,4 +26,26 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig std::string doc() override; }; +struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig +{ + using LocalFSStoreConfig::LocalFSStoreConfig; + using SSHStoreConfig::SSHStoreConfig; + + MountedSSHStoreConfig(StringMap params); + + MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params); + + const std::string name() override + { + return "Experimental SSH Store with filesystem mounted"; + } + + std::string doc() override; + + std::optional experimentalFeature() const override + { + return ExperimentalFeature::MountedSSHStore; + } +}; + } diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index a6199eaa7e4..3c445eb1318 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,7 +1,6 @@ #include "uds-remote-store.hh" #include "unix-domain-socket.hh" #include "worker-protocol.hh" -#include "globals.hh" #include #include @@ -25,14 +24,11 @@ UDSRemoteStoreConfig::UDSRemoteStoreConfig( : StoreConfig(params) , LocalFSStoreConfig(params) , RemoteStoreConfig(params) + , path{authority.empty() ? settings.nixDaemonSocketFile : authority} { if (scheme != UDSRemoteStoreConfig::scheme) { throw UsageError("Scheme must be 'unix'"); } - - if (!authority.empty()) { - path.emplace(authority); - } } @@ -65,22 +61,15 @@ UDSRemoteStore::UDSRemoteStore(std::string_view scheme, std::string_view authori } -std::string UDSRemoteStore::getPathOrDefault() const -{ - return path.value_or(settings.nixDaemonSocketFile); -} - std::string UDSRemoteStore::getUri() { - if (path) { - return std::string(scheme) + "://" + *path; - } else { - // FIXME: Not clear why we return daemon here and not default to - // settings.nixDaemonSocketFile - // - // unix:// with no path also works. Change what we return? - return "daemon"; - } + return path == settings.nixDaemonSocketFile + ? // FIXME: Not clear why we return daemon here and not default + // to settings.nixDaemonSocketFile + // + // unix:// with no path also works. Change what we return? + "daemon" + : std::string(scheme) + "://" + path; } @@ -97,7 +86,7 @@ ref UDSRemoteStore::openConnection() /* Connect to a daemon that does the privileged work for us. */ conn->fd = createUnixDomainSocket(); - nix::connect(toSocket(conn->fd.get()), getPathOrDefault()); + nix::connect(toSocket(conn->fd.get()), path); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 2cf40edd6b0..0e9542eabba 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -29,11 +29,10 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon /** * The path to the unix domain socket. * - * The default *could be* settings.nixDaemonSocketFile but that - * won't pick up live changes unfortunately. This optional handling is instead - * handled on opening of the connection. + * The default is `settings.nixDaemonSocketFile`, but we don't write + * that below, instead putting in the constructor. */ - std::optional path; + Path path; protected: static constexpr char const * scheme = "unix"; @@ -88,8 +87,6 @@ private: }; ref openConnection() override; - - std::string getPathOrDefault() const; }; } diff --git a/tests/unit/libstore/http-binary-cache-store.cc b/tests/unit/libstore/http-binary-cache-store.cc new file mode 100644 index 00000000000..1e415f6251a --- /dev/null +++ b/tests/unit/libstore/http-binary-cache-store.cc @@ -0,0 +1,21 @@ +#include + +#include "http-binary-cache-store.hh" + +namespace nix { + +TEST(HttpBinaryCacheStore, constructConfig) +{ + HttpBinaryCacheStoreConfig config{"http", "foo.bar.baz", {}}; + + EXPECT_EQ(config.cacheUri, "http://foo.bar.baz"); +} + +TEST(HttpBinaryCacheStore, constructConfigNoTrailingSlash) +{ + HttpBinaryCacheStoreConfig config{"https", "foo.bar.baz/a/b/", {}}; + + EXPECT_EQ(config.cacheUri, "https://foo.bar.baz/a/b"); +} + +} // namespace nix diff --git a/tests/unit/libstore/local-binary-cache-store.cc b/tests/unit/libstore/local-binary-cache-store.cc new file mode 100644 index 00000000000..2e840228dad --- /dev/null +++ b/tests/unit/libstore/local-binary-cache-store.cc @@ -0,0 +1,14 @@ +#include + +#include "local-binary-cache-store.hh" + +namespace nix { + +TEST(LocalBinaryCacheStore, constructConfig) +{ + LocalBinaryCacheStoreConfig config{"local", "/foo/bar/baz", {}}; + + EXPECT_EQ(config.binaryCacheDir, "/foo/bar/baz"); +} + +} // namespace nix diff --git a/tests/unit/libstore/local-overlay-store.cc b/tests/unit/libstore/local-overlay-store.cc new file mode 100644 index 00000000000..4fe7e607f47 --- /dev/null +++ b/tests/unit/libstore/local-overlay-store.cc @@ -0,0 +1,30 @@ +#include + +#include "local-overlay-store.hh" + +namespace nix { + +TEST(LocalOverlayStore, constructConfig_rootQueryParam) +{ + LocalOverlayStoreConfig config{ + "local-overlay", + "", + { + { + "root", + "/foo/bar", + }, + }, + }; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +TEST(LocalOverlayStore, constructConfig_rootPath) +{ + LocalOverlayStoreConfig config{"local-overlay", "/foo/bar", {}}; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +} // namespace nix diff --git a/tests/unit/libstore/local-store.cc b/tests/unit/libstore/local-store.cc new file mode 100644 index 00000000000..e85cf0b83f0 --- /dev/null +++ b/tests/unit/libstore/local-store.cc @@ -0,0 +1,36 @@ +#include + +#include "local-store.hh" + +// Needed for template specialisations. This is not good! When we +// overhaul how store configs work, this should be fixed. +#include "args.hh" +#include "config-impl.hh" +#include "abstract-setting-to-json.hh" + +namespace nix { + +TEST(LocalStore, constructConfig_rootQueryParam) +{ + LocalStoreConfig config{ + "local", + "", + { + { + "root", + "/foo/bar", + }, + }, + }; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +TEST(LocalStore, constructConfig_rootPath) +{ + LocalStoreConfig config{"local", "/foo/bar", {}}; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +} // namespace nix diff --git a/tests/unit/libstore/meson.build b/tests/unit/libstore/meson.build index 60e1cb2fc99..8534ba8c52d 100644 --- a/tests/unit/libstore/meson.build +++ b/tests/unit/libstore/meson.build @@ -58,7 +58,11 @@ sources = files( 'derivation.cc', 'derived-path.cc', 'downstream-placeholder.cc', + 'http-binary-cache-store.cc', 'legacy-ssh-store.cc', + 'local-binary-cache-store.cc', + 'local-overlay-store.cc', + 'local-store.cc', 'machines.cc', 'nar-info-disk-cache.cc', 'nar-info.cc', @@ -67,11 +71,12 @@ sources = files( 'path-info.cc', 'path.cc', 'references.cc', + 's3-binary-cache-store.cc', 'serve-protocol.cc', 'ssh-store.cc', 'store-reference.cc', - 'worker-protocol.cc', 'uds-remote-store.cc', + 'worker-protocol.cc', ) include_dirs = [include_directories('.')] diff --git a/tests/unit/libstore/s3-binary-cache-store.cc b/tests/unit/libstore/s3-binary-cache-store.cc new file mode 100644 index 00000000000..1708a5f12f1 --- /dev/null +++ b/tests/unit/libstore/s3-binary-cache-store.cc @@ -0,0 +1,18 @@ +#if ENABLE_S3 + +#include + +#include "s3-binary-cache-store.hh" + +namespace nix { + +TEST(S3BinaryCacheStore, constructConfig) +{ + S3BinaryCacheStoreConfig config{"s3", "foobar", {}}; + + EXPECT_EQ(config.bucketName, "foobar"); +} + +} // namespace nix + +#endif diff --git a/tests/unit/libstore/ssh-store.cc b/tests/unit/libstore/ssh-store.cc index 7010ad157dd..a1fcef6b863 100644 --- a/tests/unit/libstore/ssh-store.cc +++ b/tests/unit/libstore/ssh-store.cc @@ -15,7 +15,9 @@ TEST(SSHStore, constructConfig) // TODO #11106, no more split on space "foo bar", }, - }}; + }, + }; + EXPECT_EQ( config.remoteProgram.get(), (Strings{ @@ -23,4 +25,27 @@ TEST(SSHStore, constructConfig) "bar", })); } + +TEST(MountedSSHStore, constructConfig) +{ + MountedSSHStoreConfig config{ + "mounted-ssh", + "localhost", + StoreConfig::Params{ + { + "remote-program", + // TODO #11106, no more split on space + "foo bar", + }, + }, + }; + + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); +} + } diff --git a/tests/unit/libstore/uds-remote-store.cc b/tests/unit/libstore/uds-remote-store.cc index b9811a5e073..4bd3bc18f5f 100644 --- a/tests/unit/libstore/uds-remote-store.cc +++ b/tests/unit/libstore/uds-remote-store.cc @@ -8,7 +8,7 @@ TEST(UDSRemoteStore, constructConfig) { UDSRemoteStoreConfig config{"unix", "/tmp/socket", {}}; - EXPECT_EQ(*config.path, "/tmp/socket"); + EXPECT_EQ(config.path, "/tmp/socket"); } TEST(UDSRemoteStore, constructConfigWrongScheme) From f23d1683855fd0c83d3c1d7b11b9b51a6995cc94 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 16 Jul 2024 16:26:30 -0400 Subject: [PATCH 18/80] `make format` --- src/libstore/s3-binary-cache-store.hh | 49 ++++++++++++-------- tests/unit/libstore/s3-binary-cache-store.cc | 4 +- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 723c8c54e61..bca5196cde0 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -13,12 +13,12 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - S3BinaryCacheStoreConfig( - std::string_view uriScheme, - std::string_view bucketName, - const Params & params); + S3BinaryCacheStoreConfig(std::string_view uriScheme, std::string_view bucketName, const Params & params); - const Setting profile{this, "", "profile", + const Setting profile{ + this, + "", + "profile", R"( The name of the AWS configuration profile to use. By default Nix will use the `default` profile. @@ -30,14 +30,20 @@ protected: public: - const Setting region{this, defaultRegion, "region", + const Setting region{ + this, + defaultRegion, + "region", R"( The region of the S3 bucket. If your bucket is not in `us–east-1`, you should always explicitly specify the region parameter. )"}; - const Setting scheme{this, "", "scheme", + const Setting scheme{ + this, + "", + "scheme", R"( The scheme used for S3 requests, `https` (default) or `http`. This option allows you to disable HTTPS for binary caches which don't @@ -49,7 +55,10 @@ public: > information. )"}; - const Setting endpoint{this, "", "endpoint", + const Setting endpoint{ + this, + "", + "endpoint", R"( The URL of the endpoint of an S3-compatible service such as MinIO. Do not specify this setting if you're using Amazon S3. @@ -60,28 +69,30 @@ public: > addressing instead of virtual host based addressing. )"}; - const Setting narinfoCompression{this, "", "narinfo-compression", - "Compression method for `.narinfo` files."}; + const Setting narinfoCompression{ + this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; - const Setting lsCompression{this, "", "ls-compression", - "Compression method for `.ls` files."}; + const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; - const Setting logCompression{this, "", "log-compression", + const Setting logCompression{ + this, + "", + "log-compression", R"( Compression method for `log/*` files. It is recommended to use a compression method supported by most web browsers (e.g. `brotli`). )"}; - const Setting multipartUpload{ - this, false, "multipart-upload", - "Whether to use multi-part uploads."}; + const Setting multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."}; const Setting bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", - "Size (in bytes) of each part in multi-part uploads."}; + this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; - const std::string name() override { return "S3 Binary Cache Store"; } + const std::string name() override + { + return "S3 Binary Cache Store"; + } std::string doc() override; }; diff --git a/tests/unit/libstore/s3-binary-cache-store.cc b/tests/unit/libstore/s3-binary-cache-store.cc index 1708a5f12f1..7aa5f2f2c06 100644 --- a/tests/unit/libstore/s3-binary-cache-store.cc +++ b/tests/unit/libstore/s3-binary-cache-store.cc @@ -1,8 +1,8 @@ #if ENABLE_S3 -#include +# include -#include "s3-binary-cache-store.hh" +# include "s3-binary-cache-store.hh" namespace nix { From c4c148fd034379a81293a17a32af7986927a60d4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 16 Jul 2024 18:34:32 -0400 Subject: [PATCH 19/80] Declare some template specializations in the header Not having these causes some issues with the new unit tests. --- src/libstore/globals.hh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index dfe25f31726..50cdf9ffe9d 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -16,6 +16,11 @@ namespace nix { typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode; +template<> SandboxMode BaseSetting::parse(const std::string & str) const; +template<> struct BaseSetting::trait; +template<> std::string BaseSetting::to_string() const; +template<> void BaseSetting::convertToArg(Args & args, const std::string & category); + struct MaxBuildJobsSetting : public BaseSetting { MaxBuildJobsSetting(Config * options, From 4e79b4eab887ec03005f22f2b82113b5da4a2120 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 15 Jul 2024 23:26:39 -0400 Subject: [PATCH 20/80] Move `uriSchemes` to `*StoreConfig` It is a property of the configuration of a store --- how a store URL is parsed into a store config, not a store itself. Progress towards #10766 --- src/libstore/dummy-store.cc | 8 ++++---- src/libstore/http-binary-cache-store.cc | 8 -------- src/libstore/http-binary-cache-store.hh | 8 ++++++++ src/libstore/legacy-ssh-store.hh | 4 ++-- src/libstore/local-binary-cache-store.cc | 4 +--- src/libstore/local-binary-cache-store.hh | 2 ++ src/libstore/local-overlay-store.hh | 10 +++++----- src/libstore/local-store.hh | 6 +++--- src/libstore/s3-binary-cache-store.cc | 3 --- src/libstore/s3-binary-cache-store.hh | 2 ++ src/libstore/ssh-store.cc | 7 ------- src/libstore/ssh-store.hh | 10 ++++++++++ src/libstore/store-api.hh | 6 +++++- src/libstore/uds-remote-store.hh | 7 ++++--- 14 files changed, 46 insertions(+), 39 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index af0dd909214..c1e871e9384 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -21,6 +21,10 @@ struct DummyStoreConfig : virtual StoreConfig { #include "dummy-store.md" ; } + + static std::set uriSchemes() { + return {"dummy"}; + } }; struct DummyStore : public virtual DummyStoreConfig, public virtual Store @@ -54,10 +58,6 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store return Trusted; } - static std::set uriSchemes() { - return {"dummy"}; - } - std::optional queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 49ec26b9f93..b15ef4e4cba 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -83,14 +83,6 @@ class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public v } } - static std::set uriSchemes() - { - static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; - auto ret = std::set({"http", "https"}); - if (forceHttp) ret.insert("file"); - return ret; - } - protected: void maybeDisable() diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh index 230e52b3373..879cab064cd 100644 --- a/src/libstore/http-binary-cache-store.hh +++ b/src/libstore/http-binary-cache-store.hh @@ -15,6 +15,14 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig return "HTTP Binary Cache Store"; } + static std::set uriSchemes() + { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; + auto ret = std::set({"http", "https"}); + if (forceHttp) ret.insert("file"); + return ret; + } + std::string doc() override; }; diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index f2665189828..b541455b4e5 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -26,6 +26,8 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig const std::string name() override { return "SSH Store"; } + static std::set uriSchemes() { return {"ssh"}; } + std::string doc() override; }; @@ -46,8 +48,6 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor SSHMaster master; - static std::set uriSchemes() { return {"ssh"}; } - LegacySSHStore( std::string_view scheme, std::string_view host, diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 106663132c1..dcc6affe4a1 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -51,8 +51,6 @@ struct LocalBinaryCacheStore : virtual LocalBinaryCacheStoreConfig, virtual Bina return "file://" + binaryCacheDir; } - static std::set uriSchemes(); - protected: bool fileExists(const std::string & path) override; @@ -121,7 +119,7 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path) return pathExists(binaryCacheDir + "/" + path); } -std::set LocalBinaryCacheStore::uriSchemes() +std::set LocalBinaryCacheStoreConfig::uriSchemes() { if (getEnv("_NIX_FORCE_HTTP") == "1") return {}; diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh index 7701eb38b99..997e8ecbb51 100644 --- a/src/libstore/local-binary-cache-store.hh +++ b/src/libstore/local-binary-cache-store.hh @@ -15,6 +15,8 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig return "Local Binary Cache Store"; } + static std::set uriSchemes(); + std::string doc() override; }; diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index a3f15e484f0..63628abed50 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -63,6 +63,11 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig return ExperimentalFeature::LocalOverlayStore; } + static std::set uriSchemes() + { + return { "local-overlay" }; + } + std::string doc() override; protected: @@ -102,11 +107,6 @@ public: LocalOverlayStore(std::string_view scheme, PathView path, const Params & params); - static std::set uriSchemes() - { - return { "local-overlay" }; - } - std::string getUri() override { return "local-overlay://"; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 026729bf24c..a03cfc03b30 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -67,6 +67,9 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig const std::string name() override { return "Local Store"; } + static std::set uriSchemes() + { return {"local"}; } + std::string doc() override; }; @@ -149,9 +152,6 @@ public: ~LocalStore(); - static std::set uriSchemes() - { return {"local"}; } - /** * Implementations of abstract store API methods. */ diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index d865684d7aa..1a0ec11112e 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -475,9 +475,6 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual { return std::nullopt; } - - static std::set uriSchemes() { return {"s3"}; } - }; static RegisterStoreImplementation regS3BinaryCacheStore; diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index bca5196cde0..0fa6965c67b 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -94,6 +94,8 @@ public: return "S3 Binary Cache Store"; } + static std::set uriSchemes() { return {"s3"}; } + std::string doc() override; }; diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index d63995bf3bc..954a9746774 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -47,8 +47,6 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore { } - static std::set uriSchemes() { return {"ssh-ng"}; } - std::string getUri() override { return *uriSchemes().begin() + "://" + host; @@ -154,11 +152,6 @@ class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSH }; } - static std::set uriSchemes() - { - return {"mounted-ssh-ng"}; - } - std::string getUri() override { return *uriSchemes().begin() + "://" + host; diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh index feb57ccba80..29a2a8b2c2d 100644 --- a/src/libstore/ssh-store.hh +++ b/src/libstore/ssh-store.hh @@ -23,6 +23,11 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig return "Experimental SSH Store"; } + static std::set uriSchemes() + { + return {"ssh-ng"}; + } + std::string doc() override; }; @@ -40,6 +45,11 @@ struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfi return "Experimental SSH Store with filesystem mounted"; } + static std::set uriSchemes() + { + return {"mounted-ssh-ng"}; + } + std::string doc() override; std::optional experimentalFeature() const override diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 749d7ea098d..7d5f533c5a3 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -216,6 +216,10 @@ public: virtual ~Store() { } + /** + * @todo move to `StoreConfig` one we store enough information in + * those to recover the scheme and authority in all cases. + */ virtual std::string getUri() = 0; /** @@ -897,7 +901,7 @@ struct Implementations { if (!registered) registered = new std::vector(); StoreFactory factory{ - .uriSchemes = T::uriSchemes(), + .uriSchemes = TConfig::uriSchemes(), .create = ([](auto scheme, auto uri, auto & params) -> std::shared_ptr diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 0e9542eabba..a8e57166416 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -36,6 +36,10 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon protected: static constexpr char const * scheme = "unix"; + +public: + static std::set uriSchemes() + { return {scheme}; } }; class UDSRemoteStore : public virtual UDSRemoteStoreConfig @@ -59,9 +63,6 @@ public: std::string getUri() override; - static std::set uriSchemes() - { return {scheme}; } - ref getFSAccessor(bool requireValidPath = true) override { return LocalFSStore::getFSAccessor(requireValidPath); } From 1e25c2253d7a72d01d9790a4c5c64e79a6673026 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 16 Jul 2024 19:12:43 -0400 Subject: [PATCH 21/80] Revert "Declare some template specializations in the header" This reverts commit c4c148fd034379a81293a17a32af7986927a60d4. --- src/libstore/globals.hh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 50cdf9ffe9d..dfe25f31726 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -16,11 +16,6 @@ namespace nix { typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode; -template<> SandboxMode BaseSetting::parse(const std::string & str) const; -template<> struct BaseSetting::trait; -template<> std::string BaseSetting::to_string() const; -template<> void BaseSetting::convertToArg(Args & args, const std::string & category); - struct MaxBuildJobsSetting : public BaseSetting { MaxBuildJobsSetting(Config * options, From 414d6ada5072a177697af7813f4fcc2a0749cc0a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 16 Jul 2024 20:13:37 -0400 Subject: [PATCH 22/80] `make format` --- src/libstore/http-binary-cache-store.hh | 3 ++- src/libstore/s3-binary-cache-store.hh | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh index 879cab064cd..d2fc43210a2 100644 --- a/src/libstore/http-binary-cache-store.hh +++ b/src/libstore/http-binary-cache-store.hh @@ -19,7 +19,8 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; auto ret = std::set({"http", "https"}); - if (forceHttp) ret.insert("file"); + if (forceHttp) + ret.insert("file"); return ret; } diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 0fa6965c67b..7d303a115f4 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -94,7 +94,10 @@ public: return "S3 Binary Cache Store"; } - static std::set uriSchemes() { return {"s3"}; } + static std::set uriSchemes() + { + return {"s3"}; + } std::string doc() override; }; From 3b0edea4082b88aa100ead806f35c420e722271e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 16 Jul 2024 20:13:13 -0400 Subject: [PATCH 23/80] WIP Start getting store settings off config.hh --- src/libstore/config-parse.hh | 14 +++++++++++ src/libstore/path.cc | 2 +- src/libstore/store-api.hh | 2 -- src/libstore/store-dir-config.cc | 41 ++++++++++++++++++++++++++++++++ src/libstore/store-dir-config.hh | 28 ++++++++++++---------- src/libutil/config-abstract.hh | 32 +++++++++++++++++++++++++ 6 files changed, 103 insertions(+), 16 deletions(-) create mode 100644 src/libstore/config-parse.hh create mode 100644 src/libstore/store-dir-config.cc create mode 100644 src/libutil/config-abstract.hh diff --git a/src/libstore/config-parse.hh b/src/libstore/config-parse.hh new file mode 100644 index 00000000000..e0cdaaeaccd --- /dev/null +++ b/src/libstore/config-parse.hh @@ -0,0 +1,14 @@ +#pragma once + +/** + * Look up the setting's name in a map, falling back on the default if + * it does not exist. + */ +#define CONFIG_ROW(FIELD) \ + .FIELD = { \ + .value = ({ \ + auto p = get(params, descriptions.FIELD.name); \ + p ? *p : defaults.FIELD.value; \ + }) \ + } + diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 3e9d054778c..f2fa91cb2e2 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -89,7 +89,7 @@ StorePath StoreDirConfig::parseStorePath(std::string_view path) const canonPath(std::string(path)) #endif ; - if (dirOf(p) != storeDir) + if (dirOf(p) != storeDir.get()) throw BadStorePath("path '%s' is not in the Nix store", p); return StorePath(baseNameOf(p)); } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7d5f533c5a3..a4d2138f66d 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -99,8 +99,6 @@ typedef std::map> StorePathCAMap; struct StoreConfig : public StoreDirConfig { - using Params = StoreReference::Params; - using StoreDirConfig::StoreDirConfig; StoreConfig() = delete; diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc new file mode 100644 index 00000000000..775f678be4a --- /dev/null +++ b/src/libstore/store-dir-config.cc @@ -0,0 +1,41 @@ +#include "store-dir-config.hh" +#include "config-parse.hh" +#include "util.hh" + +namespace nix { + +const StoreDirConfigT storeDirConfigDefaults = { + .storeDir = + { + .value = settings.nixStore, + }, +}; + +const StoreDirConfigT storeDirConfigDescriptions = { + .storeDir = + { + .name = "store", + .description = R"( + Logical location of the Nix store, usually + `/nix/store`. Note that you can only copy store paths + between stores if they have the same `store` setting. + )", + }, +}; + +StoreDirConfigT parseStoreDirConfig(const StoreReference::Params & params) +{ + constexpr auto & defaults = storeDirConfigDefaults; + constexpr auto & descriptions = storeDirConfigDescriptions; + + return { + CONFIG_ROW(storeDir), + }; +} + +StoreDirConfig::StoreDirConfig(const StoreReference::Params & params) + : StoreDirConfigT{parseStoreDirConfig(params)} +{ +} + +} diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index 64c0dd8b70b..e2f61a72995 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -3,8 +3,9 @@ #include "path.hh" #include "hash.hh" #include "content-address.hh" +#include "store-reference.hh" +#include "config-abstract.hh" #include "globals.hh" -#include "config.hh" #include #include @@ -18,22 +19,23 @@ struct SourcePath; MakeError(BadStorePath, Error); MakeError(BadStorePathName, BadStorePath); -struct StoreDirConfig : public Config +template class F> +struct StoreDirConfigT { - using Config::Config; + F storeDir; +}; - StoreDirConfig() = delete; +extern const StoreDirConfigT storeDirConfigDefaults; - virtual ~StoreDirConfig() = default; +extern const StoreDirConfigT storeDirConfigDescriptions; + +StoreDirConfigT parseStoreDirConfig(const StoreReference::Params &); - const PathSetting storeDir_{this, settings.nixStore, - "store", - R"( - Logical location of the Nix store, usually - `/nix/store`. Note that you can only copy store paths - between stores if they have the same `store` setting. - )"}; - const Path storeDir = storeDir_; +struct StoreDirConfig : StoreDirConfigT +{ + StoreDirConfig(const StoreReference::Params & params); + + virtual ~StoreDirConfig() = default; // pure methods diff --git a/src/libutil/config-abstract.hh b/src/libutil/config-abstract.hh new file mode 100644 index 00000000000..081478dc195 --- /dev/null +++ b/src/libutil/config-abstract.hh @@ -0,0 +1,32 @@ +#pragma once +///@type + +namespace nix { + +template +struct JustValue +{ + T value; + + operator const T &() const + { + return value; + } + operator T &() + { + return value; + } + const T & get() const + { + return value; + } +}; + +template +struct SettingInfo +{ + std::string name; + std::string description; +}; + +} From d095fa32395b133907b65df4647bfd48787cd010 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 17 Jul 2024 14:51:05 -0400 Subject: [PATCH 24/80] WIP --- src/libstore/config-parse-impl.hh | 24 +++++ src/libstore/config-parse.hh | 29 +++-- src/libstore/path.cc | 2 +- src/libstore/store-api.cc | 168 ++++++++++++----------------- src/libstore/store-api.hh | 128 ++++------------------ src/libstore/store-dir-config.cc | 19 ++-- src/libstore/store-dir-config.hh | 14 +-- src/libstore/store-open.hh | 38 +++++++ src/libstore/store-reference.hh | 3 +- src/libstore/store-registration.cc | 95 ++++++++++++++++ src/libstore/store-registration.hh | 55 ++++++++++ src/libutil/config-abstract.hh | 9 +- 12 files changed, 340 insertions(+), 244 deletions(-) create mode 100644 src/libstore/config-parse-impl.hh create mode 100644 src/libstore/store-open.hh create mode 100644 src/libstore/store-registration.cc create mode 100644 src/libstore/store-registration.hh diff --git a/src/libstore/config-parse-impl.hh b/src/libstore/config-parse-impl.hh new file mode 100644 index 00000000000..ebb65305ae3 --- /dev/null +++ b/src/libstore/config-parse-impl.hh @@ -0,0 +1,24 @@ +#pragma once +//@file + +#include + +#include "config-parse.hh" +#include "util.hh" + +namespace nix::config { + +template +std::optional> SettingInfo::parseConfig(const nlohmann::json::object_t & map) const +{ + auto * p = get(map, name); + return p ? (JustValue{.value = p->get()}) : (std::optional>{}); +} + +/** + * Look up the setting's name in a map, falling back on the default if + * it does not exist. + */ +#define CONFIG_ROW(FIELD) .FIELD = descriptions.FIELD.parseConfig(params).value_or(defaults.FIELD) + +} diff --git a/src/libstore/config-parse.hh b/src/libstore/config-parse.hh index e0cdaaeaccd..5d17f9f8d6a 100644 --- a/src/libstore/config-parse.hh +++ b/src/libstore/config-parse.hh @@ -1,14 +1,21 @@ #pragma once +//@file -/** - * Look up the setting's name in a map, falling back on the default if - * it does not exist. - */ -#define CONFIG_ROW(FIELD) \ - .FIELD = { \ - .value = ({ \ - auto p = get(params, descriptions.FIELD.name); \ - p ? *p : defaults.FIELD.value; \ - }) \ - } +#include +#include "config-abstract.hh" +#include "util.hh" + +namespace nix::config { + +template +struct SettingInfo +{ + std::string name; + std::string description; + bool documentDefault = true; + + std::optional> parseConfig(const nlohmann::json::object_t & map) const; +}; + +} diff --git a/src/libstore/path.cc b/src/libstore/path.cc index f2fa91cb2e2..3e9d054778c 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -89,7 +89,7 @@ StorePath StoreDirConfig::parseStorePath(std::string_view path) const canonPath(std::string(path)) #endif ; - if (dirOf(p) != storeDir.get()) + if (dirOf(p) != storeDir) throw BadStorePath("path '%s' is not in the Nix store", p); return StorePath(baseNameOf(p)); } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 2c4dee518b4..bf47026ed47 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -5,6 +5,7 @@ #include "realisation.hh" #include "derivations.hh" #include "store-api.hh" +#include "store-open.hh" #include "util.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" @@ -18,6 +19,7 @@ #include "worker-protocol.hh" #include "signals.hh" #include "users.hh" +#include "config-parse-impl.hh" #include #include @@ -28,7 +30,6 @@ using json = nlohmann::json; namespace nix { - bool StoreDirConfig::isInStore(PathView path) const { return isInDir(path, storeDir); @@ -188,6 +189,70 @@ std::pair StoreDirConfig::computeStorePath( } +const StoreConfigT storeConfigDefaults = { + .pathInfoCacheSize = { .value = 65536 }, + .isTrusted = { .value = false }, + .priority = { .value = 0 }, + .wantMassQuery = { .value = false }, + .systemFeatures = { .value = StoreConfig::getDefaultSystemFeatures() }, +}; + +const StoreConfigT storeConfigDescriptions = { + .pathInfoCacheSize = { + .name = "path-info-cache-size", + .description = "Size of the in-memory store path metadata cache.", + }, + .isTrusted = { + .name = "trusted", + .description = R"( + Whether paths from this store can be used as substitutes + even if they are not signed by a key listed in the + [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) + setting. + )", + }, + .priority = { + .name = "priority", + .description = R"( + Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + A lower value means a higher priority. + )", + }, + .wantMassQuery = { + .name = "want-mass-query", + .description = R"( + Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + )", + }, + + .systemFeatures = { + .name = "system-features", + .description = R"( + Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. + + Example: `"kvm"` + )", + // The default value is CPU- and OS-specific, and thus + // unsuitable to be rendered in the documentation. + .documentDefault = false, + }, +}; + +StoreConfigT parseStoreConfig(const StoreReference::Params & params) +{ + constexpr auto & defaults = storeConfigDefaults; + constexpr auto & descriptions = storeConfigDescriptions; + + return { + CONFIG_ROW(pathInfoCacheSize), + CONFIG_ROW(isTrusted), + CONFIG_ROW(priority), + CONFIG_ROW(wantMassQuery), + CONFIG_ROW(systemFeatures), + }; +} + + StorePath Store::addToStore( std::string_view name, const SourcePath & path, @@ -432,7 +497,7 @@ StringSet StoreConfig::getDefaultSystemFeatures() return res; } -Store::Store(const Params & params) +Store::Store(const StoreReference::Params & params) : StoreConfig(params) , state({(size_t) pathInfoCacheSize}) { @@ -1267,102 +1332,3 @@ Derivation Store::readInvalidDerivation(const StorePath & drvPath) { return readDerivationCommon(*this, drvPath, false); } } - - -#include "local-store.hh" -#include "uds-remote-store.hh" - - -namespace nix { - -ref openStore(const std::string & uri, - const Store::Params & extraParams) -{ - return openStore(StoreReference::parse(uri, extraParams)); -} - -ref openStore(StoreReference && storeURI) -{ - auto & params = storeURI.params; - - auto store = std::visit(overloaded { - [&](const StoreReference::Auto &) -> std::shared_ptr { - auto stateDir = getOr(params, "state", settings.nixStateDir); - if (access(stateDir.c_str(), R_OK | W_OK) == 0) - return std::make_shared(params); - else if (pathExists(settings.nixDaemonSocketFile)) - return std::make_shared(params); - #if __linux__ - else if (!pathExists(stateDir) - && params.empty() - && !isRootUser() - && !getEnv("NIX_STORE_DIR").has_value() - && !getEnv("NIX_STATE_DIR").has_value()) - { - /* If /nix doesn't exist, there is no daemon socket, and - we're not root, then automatically set up a chroot - store in ~/.local/share/nix/root. */ - auto chrootStore = getDataDir() + "/nix/root"; - if (!pathExists(chrootStore)) { - try { - createDirs(chrootStore); - } catch (SystemError & e) { - return std::make_shared(params); - } - warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); - } else - debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); - return std::make_shared("local", chrootStore, params); - } - #endif - else - return std::make_shared(params); - }, - [&](const StoreReference::Specified & g) { - for (auto implem : *Implementations::registered) - if (implem.uriSchemes.count(g.scheme)) - return implem.create(g.scheme, g.authority, params); - - throw Error("don't know how to open Nix store with scheme '%s'", g.scheme); - }, - }, storeURI.variant); - - experimentalFeatureSettings.require(store->experimentalFeature()); - store->warnUnknownSettings(); - store->init(); - - return ref { store }; -} - -std::list> getDefaultSubstituters() -{ - static auto stores([]() { - std::list> stores; - - StringSet done; - - auto addStore = [&](const std::string & uri) { - if (!done.insert(uri).second) return; - try { - stores.push_back(openStore(uri)); - } catch (Error & e) { - logWarning(e.info()); - } - }; - - for (auto uri : settings.substituters.get()) - addStore(uri); - - stores.sort([](ref & a, ref & b) { - return a->priority < b->priority; - }); - - return stores; - } ()); - - return stores; -} - -std::vector * Implementations::registered = 0; - -} diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index a4d2138f66d..11d5d1c61f6 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -9,7 +9,6 @@ #include "lru-cache.hh" #include "sync.hh" #include "globals.hh" -#include "config.hh" #include "path-info.hh" #include "repair-flag.hh" #include "store-dir-config.hh" @@ -41,7 +40,7 @@ namespace nix { * 2. A class `Foo : virtual Store, virtual FooConfig` that contains the * implementation of the store. * - * This class is expected to have a constructor `Foo(const Params & params)` + * This class is expected to have a constructor `Foo(const StoreReference::Params & params)` * that calls `StoreConfig(params)` (otherwise you're gonna encounter an * `assertion failure` when trying to instantiate it). * @@ -97,7 +96,27 @@ struct KeyedBuildResult; typedef std::map> StorePathCAMap; -struct StoreConfig : public StoreDirConfig +template class F> +struct StoreConfigT +{ + F pathInfoCacheSize; + + F isTrusted; + + F priority; + + F wantMassQuery; + + F systemFeatures; +}; + +extern const StoreConfigT storeConfigDefaults; + +extern const StoreConfigT storeConfigDescriptions; + +StoreConfigT parseStoreConfig(const StoreReference::Params &); + +struct StoreConfig : StoreConfigT, StoreDirConfig { using StoreDirConfig::StoreDirConfig; @@ -128,39 +147,6 @@ struct StoreConfig : public StoreDirConfig { return std::nullopt; } - - const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", - "Size of the in-memory store path metadata cache."}; - - const Setting isTrusted{this, false, "trusted", - R"( - Whether paths from this store can be used as substitutes - even if they are not signed by a key listed in the - [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) - setting. - )"}; - - Setting priority{this, 0, "priority", - R"( - Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). - A lower value means a higher priority. - )"}; - - Setting wantMassQuery{this, false, "want-mass-query", - R"( - Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). - )"}; - - Setting systemFeatures{this, getDefaultSystemFeatures(), - "system-features", - R"( - Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. - - Example: `"kvm"` - )", - {}, - // Don't document the machine-specific default value - false}; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig @@ -203,7 +189,7 @@ protected: std::shared_ptr diskCache; - Store(const Params & params); + Store(const StoreReference::Params & params); public: /** @@ -855,74 +841,6 @@ StorePath resolveDerivedPath(Store &, const SingleDerivedPath &, Store * evalSto OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); -/** - * @return a Store object to access the Nix store denoted by - * ‘uri’ (slight misnomer...). - */ -ref openStore(StoreReference && storeURI); - - -/** - * Opens the store at `uri`, where `uri` is in the format expected by `StoreReference::parse` - - */ -ref openStore(const std::string & uri = settings.storeUri.get(), - const Store::Params & extraParams = Store::Params()); - - -/** - * @return the default substituter stores, defined by the - * ‘substituters’ option and various legacy options. - */ -std::list> getDefaultSubstituters(); - -struct StoreFactory -{ - std::set uriSchemes; - /** - * The `authorityPath` parameter is `/`, or really - * whatever comes after `://` and before `?`. - */ - std::function ( - std::string_view scheme, - std::string_view authorityPath, - const Store::Params & params)> create; - std::function ()> getConfig; -}; - -struct Implementations -{ - static std::vector * registered; - - template - static void add() - { - if (!registered) registered = new std::vector(); - StoreFactory factory{ - .uriSchemes = TConfig::uriSchemes(), - .create = - ([](auto scheme, auto uri, auto & params) - -> std::shared_ptr - { return std::make_shared(scheme, uri, params); }), - .getConfig = - ([]() - -> std::shared_ptr - { return std::make_shared(StringMap({})); }) - }; - registered->push_back(factory); - } -}; - -template -struct RegisterStoreImplementation -{ - RegisterStoreImplementation() - { - Implementations::add(); - } -}; - - /** * Display a set of paths in human-readable form (i.e., between quotes * and separated by commas). diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc index 775f678be4a..960f2b88266 100644 --- a/src/libstore/store-dir-config.cc +++ b/src/libstore/store-dir-config.cc @@ -1,18 +1,15 @@ #include "store-dir-config.hh" -#include "config-parse.hh" +#include "config-parse-impl.hh" #include "util.hh" namespace nix { -const StoreDirConfigT storeDirConfigDefaults = { - .storeDir = - { - .value = settings.nixStore, - }, +const StoreDirConfigT storeDirConfigDefaults = { + ._storeDir = {.value = settings.nixStore}, }; -const StoreDirConfigT storeDirConfigDescriptions = { - .storeDir = +const StoreDirConfigT storeDirConfigDescriptions = { + ._storeDir = { .name = "store", .description = R"( @@ -23,18 +20,18 @@ const StoreDirConfigT storeDirConfigDescriptions = { }, }; -StoreDirConfigT parseStoreDirConfig(const StoreReference::Params & params) +StoreDirConfigT parseStoreDirConfig(const StoreReference::Params & params) { constexpr auto & defaults = storeDirConfigDefaults; constexpr auto & descriptions = storeDirConfigDescriptions; return { - CONFIG_ROW(storeDir), + CONFIG_ROW(_storeDir), }; } StoreDirConfig::StoreDirConfig(const StoreReference::Params & params) - : StoreDirConfigT{parseStoreDirConfig(params)} + : StoreDirConfigT{parseStoreDirConfig(params)} { } diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index e2f61a72995..a2717534501 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -4,7 +4,7 @@ #include "hash.hh" #include "content-address.hh" #include "store-reference.hh" -#include "config-abstract.hh" +#include "config-parse.hh" #include "globals.hh" #include @@ -22,21 +22,23 @@ MakeError(BadStorePathName, BadStorePath); template class F> struct StoreDirConfigT { - F storeDir; + F _storeDir; }; -extern const StoreDirConfigT storeDirConfigDefaults; +extern const StoreDirConfigT storeDirConfigDefaults; -extern const StoreDirConfigT storeDirConfigDescriptions; +extern const StoreDirConfigT storeDirConfigDescriptions; -StoreDirConfigT parseStoreDirConfig(const StoreReference::Params &); +StoreDirConfigT parseStoreDirConfig(const StoreReference::Params &); -struct StoreDirConfig : StoreDirConfigT +struct StoreDirConfig : StoreDirConfigT { StoreDirConfig(const StoreReference::Params & params); virtual ~StoreDirConfig() = default; + const Path & storeDir = _storeDir.value; + // pure methods StorePath parseStorePath(std::string_view path) const; diff --git a/src/libstore/store-open.hh b/src/libstore/store-open.hh new file mode 100644 index 00000000000..ab6f3cb7739 --- /dev/null +++ b/src/libstore/store-open.hh @@ -0,0 +1,38 @@ +#pragma once +/** + * @file + * + * For opening a store described by an `StoreReference`, which is an "untyped" + * notion which needs to be decoded against a collection of specific + * implementations. + * + * For consumers of the store registration machinery defined in + * `store-registration.hh`. Not needed by store implementation definitions, or + * usages of a given `Store` which will be passed in. + */ + +#include "store-api.hh" + +namespace nix { + +/** + * @return a Store object to access the Nix store denoted by + * ‘uri’ (slight misnomer...). + */ +ref openStore(StoreReference && storeURI); + +/** + * Opens the store at `uri`, where `uri` is in the format expected by `StoreReference::parse` + + */ +ref openStore( + const std::string & uri = settings.storeUri.get(), + const StoreReference::Params & extraParams = StoreReference::Params()); + +/** + * @return the default substituter stores, defined by the + * ‘substituters’ option and various legacy options. + */ +std::list> getDefaultSubstituters(); + +} diff --git a/src/libstore/store-reference.hh b/src/libstore/store-reference.hh index 459cea9c20e..c1066296afe 100644 --- a/src/libstore/store-reference.hh +++ b/src/libstore/store-reference.hh @@ -2,6 +2,7 @@ ///@file #include +#include #include "types.hh" @@ -41,7 +42,7 @@ namespace nix { */ struct StoreReference { - using Params = std::map; + using Params = nlohmann::json::object_t; /** * Special store reference `""` or `"auto"` diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc new file mode 100644 index 00000000000..85ece9623cf --- /dev/null +++ b/src/libstore/store-registration.cc @@ -0,0 +1,95 @@ +#include "store-registration.hh" +#include "store-open.hh" +#include "local-store.hh" +#include "uds-remote-store.hh" + +namespace nix { + +ref openStore(const std::string & uri, const Store::Params & extraParams) +{ + return openStore(StoreReference::parse(uri, extraParams)); +} + +ref openStore(StoreReference && storeURI) +{ + auto & params = storeURI.params; + + auto store = std::visit( + overloaded{ + [&](const StoreReference::Auto &) -> std::shared_ptr { + auto stateDir = getOr(params, "state", settings.nixStateDir); + if (access(stateDir.c_str(), R_OK | W_OK) == 0) + return std::make_shared(params); + else if (pathExists(settings.nixDaemonSocketFile)) + return std::make_shared(params); +#if __linux__ + else if ( + !pathExists(stateDir) && params.empty() && !isRootUser() && !getEnv("NIX_STORE_DIR").has_value() + && !getEnv("NIX_STATE_DIR").has_value()) { + /* If /nix doesn't exist, there is no daemon socket, and + we're not root, then automatically set up a chroot + store in ~/.local/share/nix/root. */ + auto chrootStore = getDataDir() + "/nix/root"; + if (!pathExists(chrootStore)) { + try { + createDirs(chrootStore); + } catch (SystemError & e) { + return std::make_shared(params); + } + warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); + } else + debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); + return std::make_shared("local", chrootStore, params); + } +#endif + else + return std::make_shared(params); + }, + [&](const StoreReference::Specified & g) { + for (auto implem : *Implementations::registered) + if (implem.uriSchemes.count(g.scheme)) + return implem.create(g.scheme, g.authority, params); + + throw Error("don't know how to open Nix store with scheme '%s'", g.scheme); + }, + }, + storeURI.variant); + + experimentalFeatureSettings.require(store->experimentalFeature()); + store->warnUnknownSettings(); + store->init(); + + return ref{store}; +} + +std::vector * Implementations::registered = 0; + +std::list> getDefaultSubstituters() +{ + static auto stores([]() { + std::list> stores; + + StringSet done; + + auto addStore = [&](const std::string & uri) { + if (!done.insert(uri).second) + return; + try { + stores.push_back(openStore(uri)); + } catch (Error & e) { + logWarning(e.info()); + } + }; + + for (auto uri : settings.substituters.get()) + addStore(uri); + + stores.sort([](ref & a, ref & b) { return a->priority < b->priority; }); + + return stores; + }()); + + return stores; +} + +} diff --git a/src/libstore/store-registration.hh b/src/libstore/store-registration.hh new file mode 100644 index 00000000000..29cee23a3fb --- /dev/null +++ b/src/libstore/store-registration.hh @@ -0,0 +1,55 @@ +#pragma once +/** + * @file + * + * Infrastructure for "registering" store implementations. Used by the + * store implementation definitions themselves but not by consumers of + * those implementations. + */ + +#include "store-api.hh" + +namespace nix { + +struct StoreFactory +{ + std::set uriSchemes; + /** + * The `authorityPath` parameter is `/`, or really + * whatever comes after `://` and before `?`. + */ + std::function( + std::string_view scheme, std::string_view authorityPath, const Store::Params & params)> + create; + std::function()> getConfig; +}; + +struct Implementations +{ + static std::vector * registered; + + template + static void add() + { + if (!registered) + registered = new std::vector(); + StoreFactory factory{ + .uriSchemes = TConfig::uriSchemes(), + .create = ([](auto scheme, auto uri, auto & params) -> std::shared_ptr { + return std::make_shared(scheme, uri, params); + }), + .getConfig = ([]() -> std::shared_ptr { return std::make_shared(StringMap({})); })}; + registered->push_back(factory); + } +}; + +template +struct RegisterStoreImplementation +{ + RegisterStoreImplementation() + { + Implementations::add(); + } +}; + +} diff --git a/src/libutil/config-abstract.hh b/src/libutil/config-abstract.hh index 081478dc195..68f20605e06 100644 --- a/src/libutil/config-abstract.hh +++ b/src/libutil/config-abstract.hh @@ -1,7 +1,7 @@ #pragma once ///@type -namespace nix { +namespace nix::config { template struct JustValue @@ -22,11 +22,4 @@ struct JustValue } }; -template -struct SettingInfo -{ - std::string name; - std::string description; -}; - } From 7b135d5e68707e1ba22d1d79ca702e9538cdedd6 Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Wed, 17 Jul 2024 12:18:13 -0700 Subject: [PATCH 25/80] Comment out tests --- tests/unit/libstore/local-store.cc | 4 ++++ tests/unit/libstore/ssh-store.cc | 4 ++++ tests/unit/libstore/uds-remote-store.cc | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/tests/unit/libstore/local-store.cc b/tests/unit/libstore/local-store.cc index e85cf0b83f0..2ab9f43cc19 100644 --- a/tests/unit/libstore/local-store.cc +++ b/tests/unit/libstore/local-store.cc @@ -1,3 +1,6 @@ +//FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 #include #include "local-store.hh" @@ -34,3 +37,4 @@ TEST(LocalStore, constructConfig_rootPath) } } // namespace nix +#endif \ No newline at end of file diff --git a/tests/unit/libstore/ssh-store.cc b/tests/unit/libstore/ssh-store.cc index a1fcef6b863..f0559c1fea2 100644 --- a/tests/unit/libstore/ssh-store.cc +++ b/tests/unit/libstore/ssh-store.cc @@ -1,3 +1,6 @@ +//FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 #include #include "ssh-store.hh" @@ -49,3 +52,4 @@ TEST(MountedSSHStore, constructConfig) } } +#endif \ No newline at end of file diff --git a/tests/unit/libstore/uds-remote-store.cc b/tests/unit/libstore/uds-remote-store.cc index 4bd3bc18f5f..d5f495428cd 100644 --- a/tests/unit/libstore/uds-remote-store.cc +++ b/tests/unit/libstore/uds-remote-store.cc @@ -1,3 +1,6 @@ +//FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 #include #include "uds-remote-store.hh" @@ -17,3 +20,4 @@ TEST(UDSRemoteStore, constructConfigWrongScheme) } } // namespace nix +#endif \ No newline at end of file From 596a682a87b98b079c729b84b1efc3925c9570fd Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 17 Jul 2024 16:18:29 -0400 Subject: [PATCH 26/80] Dummy store compiles --- src/libstore/dummy-store.cc | 35 ++++++++++++++++++++++-------- src/libstore/store-api.cc | 10 ++++----- src/libstore/store-api.hh | 22 ++++++++++++++++--- src/libstore/store-registration.hh | 23 ++++++++++---------- 4 files changed, 62 insertions(+), 28 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index c1e871e9384..4d0bbec2975 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -1,12 +1,22 @@ #include "store-api.hh" +#include "store-registration.hh" #include "callback.hh" namespace nix { +struct DummyStoreConfigDescription : virtual Store::Config::Description +{ + DummyStoreConfigDescription() : StoreConfigDescription{Store::Config::schema} {} +}; + struct DummyStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) + using Description = DummyStoreConfigDescription; + + static const Description schema; + + DummyStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params) : StoreConfig(params) { if (!authority.empty()) @@ -25,18 +35,20 @@ struct DummyStoreConfig : virtual StoreConfig { static std::set uriSchemes() { return {"dummy"}; } + + std::shared_ptr open() override; }; +decltype(DummyStoreConfig::schema) DummyStoreConfig::schema{}; + struct DummyStore : public virtual DummyStoreConfig, public virtual Store { - DummyStore(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - , DummyStoreConfig(scheme, authority, params) - , Store(params) - { } + using Config = DummyStoreConfig; - DummyStore(const Params & params) - : DummyStore("dummy", "", params) + DummyStore(const DummyStoreConfig & config) + : StoreConfig(config) + , DummyStoreConfig(config) + , Store(config) { } std::string getUri() override @@ -86,6 +98,11 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store { unsupported("getFSAccessor"); } }; -static RegisterStoreImplementation regDummyStore; +std::shared_ptr DummyStoreConfig::open() +{ + return std::make_shared(*this); +} + +static RegisterStoreImplementation regDummyStore; } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index bf47026ed47..381bb3ed6c4 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -197,7 +197,7 @@ const StoreConfigT storeConfigDefaults = { .systemFeatures = { .value = StoreConfig::getDefaultSystemFeatures() }, }; -const StoreConfigT storeConfigDescriptions = { +decltype(StoreConfig::schema) StoreConfig::schema = {{ .pathInfoCacheSize = { .name = "path-info-cache-size", .description = "Size of the in-memory store path metadata cache.", @@ -236,12 +236,12 @@ const StoreConfigT storeConfigDescriptions = { // unsuitable to be rendered in the documentation. .documentDefault = false, }, -}; +}}; StoreConfigT parseStoreConfig(const StoreReference::Params & params) { constexpr auto & defaults = storeConfigDefaults; - constexpr auto & descriptions = storeConfigDescriptions; + constexpr auto & descriptions = StoreConfig::schema; return { CONFIG_ROW(pathInfoCacheSize), @@ -497,8 +497,8 @@ StringSet StoreConfig::getDefaultSystemFeatures() return res; } -Store::Store(const StoreReference::Params & params) - : StoreConfig(params) +Store::Store(const Store::Config & config) + : Store::Config(config) , state({(size_t) pathInfoCacheSize}) { assertLibStoreInitialized(); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 11d5d1c61f6..924f037af24 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -112,16 +112,22 @@ struct StoreConfigT extern const StoreConfigT storeConfigDefaults; -extern const StoreConfigT storeConfigDescriptions; - StoreConfigT parseStoreConfig(const StoreReference::Params &); +struct StoreConfigDescription : StoreConfigT, StoreDirConfigT +{ +}; + struct StoreConfig : StoreConfigT, StoreDirConfig { using StoreDirConfig::StoreDirConfig; StoreConfig() = delete; + using Description = StoreConfigDescription; + + static const Description schema; + static StringSet getDefaultSystemFeatures(); virtual ~StoreConfig() { } @@ -147,10 +153,20 @@ struct StoreConfig : StoreConfigT, StoreDirConfig { return std::nullopt; } + + /** + * Open a store of the type corresponding to this configuration + * type. + */ + virtual std::shared_ptr open() = 0; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig { +public: + + using Config = StoreConfig; + protected: struct PathInfoCacheValue { @@ -189,7 +205,7 @@ protected: std::shared_ptr diskCache; - Store(const StoreReference::Params & params); + Store(const Store::Config & config); public: /** diff --git a/src/libstore/store-registration.hh b/src/libstore/store-registration.hh index 29cee23a3fb..9bc1869eae1 100644 --- a/src/libstore/store-registration.hh +++ b/src/libstore/store-registration.hh @@ -18,37 +18,38 @@ struct StoreFactory * The `authorityPath` parameter is `/`, or really * whatever comes after `://` and before `?`. */ - std::function( - std::string_view scheme, std::string_view authorityPath, const Store::Params & params)> - create; - std::function()> getConfig; + std::function( + std::string_view scheme, std::string_view authorityPath, const StoreReference::Params & params)> + parseConfig; + const StoreConfigDescription & configSchema; }; struct Implementations { static std::vector * registered; - template + template static void add() { if (!registered) registered = new std::vector(); StoreFactory factory{ - .uriSchemes = TConfig::uriSchemes(), - .create = ([](auto scheme, auto uri, auto & params) -> std::shared_ptr { - return std::make_shared(scheme, uri, params); + .uriSchemes = T::Config::uriSchemes(), + .parseConfig = ([](auto scheme, auto uri, auto & params) -> std::shared_ptr { + return std::make_shared(scheme, uri, params); }), - .getConfig = ([]() -> std::shared_ptr { return std::make_shared(StringMap({})); })}; + .configSchema = T::Config::schema, + }; registered->push_back(factory); } }; -template +template struct RegisterStoreImplementation { RegisterStoreImplementation() { - Implementations::add(); + Implementations::add(); } }; From 34f876f350f6cb366580dd8755bbcaec866d8535 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Wed, 17 Jul 2024 16:19:57 -0400 Subject: [PATCH 27/80] Format, CI hopefully happy now --- tests/unit/libstore/local-store.cc | 14 +++++++------- tests/unit/libstore/ssh-store.cc | 8 ++++---- tests/unit/libstore/uds-remote-store.cc | 8 ++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/unit/libstore/local-store.cc b/tests/unit/libstore/local-store.cc index 2ab9f43cc19..abc3ea7963f 100644 --- a/tests/unit/libstore/local-store.cc +++ b/tests/unit/libstore/local-store.cc @@ -1,15 +1,15 @@ -//FIXME: Odd failures for templates that are causing the PR to break +// FIXME: Odd failures for templates that are causing the PR to break // for now with discussion with @Ericson2314 to comment out. #if 0 -#include +# include -#include "local-store.hh" +# include "local-store.hh" // Needed for template specialisations. This is not good! When we // overhaul how store configs work, this should be fixed. -#include "args.hh" -#include "config-impl.hh" -#include "abstract-setting-to-json.hh" +# include "args.hh" +# include "config-impl.hh" +# include "abstract-setting-to-json.hh" namespace nix { @@ -37,4 +37,4 @@ TEST(LocalStore, constructConfig_rootPath) } } // namespace nix -#endif \ No newline at end of file +#endif diff --git a/tests/unit/libstore/ssh-store.cc b/tests/unit/libstore/ssh-store.cc index f0559c1fea2..b853a5f1fb9 100644 --- a/tests/unit/libstore/ssh-store.cc +++ b/tests/unit/libstore/ssh-store.cc @@ -1,9 +1,9 @@ -//FIXME: Odd failures for templates that are causing the PR to break +// FIXME: Odd failures for templates that are causing the PR to break // for now with discussion with @Ericson2314 to comment out. #if 0 -#include +# include -#include "ssh-store.hh" +# include "ssh-store.hh" namespace nix { @@ -52,4 +52,4 @@ TEST(MountedSSHStore, constructConfig) } } -#endif \ No newline at end of file +#endif diff --git a/tests/unit/libstore/uds-remote-store.cc b/tests/unit/libstore/uds-remote-store.cc index d5f495428cd..5ccb208714f 100644 --- a/tests/unit/libstore/uds-remote-store.cc +++ b/tests/unit/libstore/uds-remote-store.cc @@ -1,9 +1,9 @@ -//FIXME: Odd failures for templates that are causing the PR to break +// FIXME: Odd failures for templates that are causing the PR to break // for now with discussion with @Ericson2314 to comment out. #if 0 -#include +# include -#include "uds-remote-store.hh" +# include "uds-remote-store.hh" namespace nix { @@ -20,4 +20,4 @@ TEST(UDSRemoteStore, constructConfigWrongScheme) } } // namespace nix -#endif \ No newline at end of file +#endif From 4909f9a52b67e960954a4d48a46c5a4b9e80664a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 01:02:12 -0400 Subject: [PATCH 28/80] Get `LocalFSStore` building --- src/libstore/dummy-store.cc | 24 ++++--- src/libstore/local-fs-store.cc | 90 +++++++++++++++++++++---- src/libstore/local-fs-store.hh | 62 ++++++++++------- src/libstore/store-api.cc | 103 +++++++++++++++-------------- src/libstore/store-api.hh | 24 +++---- src/libstore/store-dir-config.cc | 22 ++---- src/libstore/store-dir-config.hh | 12 ++-- src/libstore/store-registration.hh | 4 +- src/libutil/config-abstract.hh | 6 ++ 9 files changed, 215 insertions(+), 132 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 4d0bbec2975..f7509bcdeaf 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -4,20 +4,20 @@ namespace nix { -struct DummyStoreConfigDescription : virtual Store::Config::Description -{ - DummyStoreConfigDescription() : StoreConfigDescription{Store::Config::schema} {} -}; - struct DummyStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - using Description = DummyStoreConfigDescription; + struct Descriptions : virtual Store::Config::Descriptions + { + Descriptions() + : StoreConfig::Descriptions{Store::Config::descriptions} + {} + }; - static const Description schema; + static const Descriptions descriptions; DummyStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params) - : StoreConfig(params) + : StoreConfig{params} { if (!authority.empty()) throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); @@ -39,13 +39,15 @@ struct DummyStoreConfig : virtual StoreConfig { std::shared_ptr open() override; }; -decltype(DummyStoreConfig::schema) DummyStoreConfig::schema{}; + +const DummyStoreConfig::Descriptions DummyStoreConfig::descriptions{}; + struct DummyStore : public virtual DummyStoreConfig, public virtual Store { using Config = DummyStoreConfig; - DummyStore(const DummyStoreConfig & config) + DummyStore(const Config & config) : StoreConfig(config) , DummyStoreConfig(config) , Store(config) @@ -98,7 +100,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store { unsupported("getFSAccessor"); } }; -std::shared_ptr DummyStoreConfig::open() +std::shared_ptr DummyStore::Config::open() { return std::make_shared(*this); } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 5449b20eb3b..fbaf6c3cb1c 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -5,25 +5,87 @@ #include "globals.hh" #include "compression.hh" #include "derivations.hh" +#include "config-parse-impl.hh" namespace nix { -LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params) +LocalFSStore::Config::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , LocalFSStoreConfigT{ + .rootDir = { + .name = "root", + .description = "Directory prefixed to all other paths.", + }, + .stateDir = { + .name = "state", + .description = "Directory where Nix will store state.", + }, + .logDir = { + .name = "log", + .description = "directory where Nix will store log files.", + }, + .realStoreDir{ + .name = "real", + .description = "Physical path of the Nix store.", + }, + } +{} + +const LocalFSStore::Config::Descriptions LocalFSStore::Config::descriptions{}; + +LocalFSStoreConfigT LocalFSStoreConfig::defaults( + const Store::Config & storeConfig, + const std::optional rootDir) +{ + return { + .rootDir = {.value = std::nullopt }, + .stateDir = {.value = rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir }, + .logDir = {.value = rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir }, + .realStoreDir = {.value = rootDir ? *rootDir + "/nix/store" : storeConfig.storeDir }, + }; +} + +/** + * @param rootDir Fallback if not in `params` + */ +auto localFSStoreConfig( + const Store::Config & storeConfig, + const std::optional _rootDir, + const StoreReference::Params & params) +{ + const auto & descriptions = LocalFSStore::Config::descriptions; + + auto rootDir = descriptions.rootDir.parseConfig(params) + .value_or(config::JustValue{.value = std::move(_rootDir)}); + + auto defaults = LocalFSStore::Config::defaults(storeConfig, rootDir.value); + + return LocalFSStoreConfigT{ + CONFIG_ROW(rootDir), + CONFIG_ROW(stateDir), + CONFIG_ROW(logDir), + CONFIG_ROW(realStoreDir), + }; +} + +LocalFSStore::Config::LocalFSStoreConfig(const StoreReference::Params & params) + : StoreConfig{params} + , LocalFSStoreConfigT{localFSStoreConfig(*this, std::nullopt, params)} +{ +} + +LocalFSStore::Config::LocalFSStoreConfig(PathView rootDir, const StoreReference::Params & params) : StoreConfig(params) - // Default `?root` from `rootDir` if non set - // FIXME don't duplicate description once we don't have root setting - , rootDir{ - this, - !rootDir.empty() && params.count("root") == 0 - ? (std::optional{rootDir}) - : std::nullopt, - "root", - "Directory prefixed to all other paths."} + , LocalFSStoreConfigT{localFSStoreConfig( + *this, + // Default `?root` from `rootDir` if non set + !rootDir.empty() ? (std::optional{rootDir}) : std::nullopt, + params)} { } -LocalFSStore::LocalFSStore(const Params & params) - : Store(params) +LocalFSStore::LocalFSStore(const Config & config) + : Store(config) { } @@ -97,8 +159,8 @@ std::optional LocalFSStore::getBuildLogExact(const StorePath & path Path logPath = j == 0 - ? fmt("%s/%s/%s/%s", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)) - : fmt("%s/%s/%s", logDir, drvsLogDir, baseName); + ? fmt("%s/%s/%s/%s", logDir.get(), drvsLogDir, baseName.substr(0, 2), baseName.substr(2)) + : fmt("%s/%s/%s", logDir.get(), drvsLogDir, baseName); Path logBz2Path = logPath + ".bz2"; if (pathExists(logPath)) diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 9bb569f0b25..f36cd56af27 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -7,9 +7,23 @@ namespace nix { -struct LocalFSStoreConfig : virtual StoreConfig +template class F> +struct LocalFSStoreConfigT { - using StoreConfig::StoreConfig; + F> rootDir; + + F stateDir; + + F logDir; + + F realStoreDir; +}; + +struct LocalFSStoreConfig : + virtual Store::Config, + LocalFSStoreConfigT +{ + LocalFSStoreConfig(const StoreReference::Params &); /** * Used to override the `root` settings. Can't be done via modifying @@ -18,38 +32,38 @@ struct LocalFSStoreConfig : virtual StoreConfig * * @todo Make this less error-prone with new store settings system. */ - LocalFSStoreConfig(PathView path, const Params & params); + LocalFSStoreConfig(PathView path, const StoreReference::Params & params); - const OptionalPathSetting rootDir{this, std::nullopt, - "root", - "Directory prefixed to all other paths."}; - - const PathSetting stateDir{this, - rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir, - "state", - "Directory where Nix will store state."}; + struct Descriptions : + virtual Store::Config::Descriptions, + LocalFSStoreConfigT + { + Descriptions(); + }; - const PathSetting logDir{this, - rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir, - "log", - "directory where Nix will store log files."}; + static const Descriptions descriptions; - const PathSetting realStoreDir{this, - rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", - "Physical path of the Nix store."}; + /** + * The other defaults depend on the choice of `storeDir` and `rootDir` + */ + static LocalFSStoreConfigT defaults( + const Store::Config &, + const std::optional rootDir); }; -class LocalFSStore : public virtual LocalFSStoreConfig, - public virtual Store, - public virtual GcStore, - public virtual LogStore +struct LocalFSStore : + virtual LocalFSStoreConfig, + virtual Store, + virtual GcStore, + virtual LogStore { -public: + using Config = LocalFSStoreConfig; + inline static std::string operationName = "Local Filesystem Store"; const static std::string drvsLogDir; - LocalFSStore(const Params & params); + LocalFSStore(const Config & params); void narFromPath(const StorePath & path, Sink & sink) override; ref getFSAccessor(bool requireValidPath = true) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 381bb3ed6c4..95aa103e46e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -189,7 +189,56 @@ std::pair StoreDirConfig::computeStorePath( } -const StoreConfigT storeConfigDefaults = { +StoreConfig::Descriptions::Descriptions() + : StoreDirConfig::Descriptions{StoreDirConfig::descriptions} + , StoreConfigT{ + .pathInfoCacheSize = { + .name = "path-info-cache-size", + .description = "Size of the in-memory store path metadata cache.", + }, + .isTrusted = { + .name = "trusted", + .description = R"( + Whether paths from this store can be used as substitutes + even if they are not signed by a key listed in the + [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) + setting. + )", + }, + .priority = { + .name = "priority", + .description = R"( + Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + A lower value means a higher priority. + )", + }, + .wantMassQuery = { + .name = "want-mass-query", + .description = R"( + Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + )", + }, + + .systemFeatures = { + .name = "system-features", + .description = R"( + Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. + + Example: `"kvm"` + )", + // The default value is CPU- and OS-specific, and thus + // unsuitable to be rendered in the documentation. + .documentDefault = false, + }, + } +{ +} + + +const StoreConfig::Descriptions StoreConfig::descriptions{}; + + +decltype(StoreConfig::defaults) StoreConfig::defaults = { .pathInfoCacheSize = { .value = 65536 }, .isTrusted = { .value = false }, .priority = { .value = 0 }, @@ -197,59 +246,17 @@ const StoreConfigT storeConfigDefaults = { .systemFeatures = { .value = StoreConfig::getDefaultSystemFeatures() }, }; -decltype(StoreConfig::schema) StoreConfig::schema = {{ - .pathInfoCacheSize = { - .name = "path-info-cache-size", - .description = "Size of the in-memory store path metadata cache.", - }, - .isTrusted = { - .name = "trusted", - .description = R"( - Whether paths from this store can be used as substitutes - even if they are not signed by a key listed in the - [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) - setting. - )", - }, - .priority = { - .name = "priority", - .description = R"( - Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). - A lower value means a higher priority. - )", - }, - .wantMassQuery = { - .name = "want-mass-query", - .description = R"( - Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). - )", - }, - - .systemFeatures = { - .name = "system-features", - .description = R"( - Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. - - Example: `"kvm"` - )", - // The default value is CPU- and OS-specific, and thus - // unsuitable to be rendered in the documentation. - .documentDefault = false, - }, -}}; - -StoreConfigT parseStoreConfig(const StoreReference::Params & params) -{ - constexpr auto & defaults = storeConfigDefaults; - constexpr auto & descriptions = StoreConfig::schema; - return { +StoreConfig::StoreConfig(const StoreReference::Params & params) + : StoreDirConfig{params} + , StoreConfigT{ CONFIG_ROW(pathInfoCacheSize), CONFIG_ROW(isTrusted), CONFIG_ROW(priority), CONFIG_ROW(wantMassQuery), CONFIG_ROW(systemFeatures), - }; + } +{ } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 924f037af24..296b81be6a0 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -110,23 +110,23 @@ struct StoreConfigT F systemFeatures; }; -extern const StoreConfigT storeConfigDefaults; - StoreConfigT parseStoreConfig(const StoreReference::Params &); - -struct StoreConfigDescription : StoreConfigT, StoreDirConfigT +struct StoreConfig : + StoreDirConfig, + StoreConfigT { -}; + StoreConfig(const StoreReference::Params &); -struct StoreConfig : StoreConfigT, StoreDirConfig -{ - using StoreDirConfig::StoreDirConfig; - - StoreConfig() = delete; + struct Descriptions : + StoreDirConfig::Descriptions, + StoreConfigT + { + Descriptions(); + }; - using Description = StoreConfigDescription; + static const StoreConfigT defaults; - static const Description schema; + static const Descriptions descriptions; static StringSet getDefaultSystemFeatures(); diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc index 960f2b88266..a51962d7fa9 100644 --- a/src/libstore/store-dir-config.cc +++ b/src/libstore/store-dir-config.cc @@ -4,11 +4,7 @@ namespace nix { -const StoreDirConfigT storeDirConfigDefaults = { - ._storeDir = {.value = settings.nixStore}, -}; - -const StoreDirConfigT storeDirConfigDescriptions = { +const StoreDirConfigT StoreDirConfig::descriptions = { ._storeDir = { .name = "store", @@ -20,18 +16,14 @@ const StoreDirConfigT storeDirConfigDescriptions = { }, }; -StoreDirConfigT parseStoreDirConfig(const StoreReference::Params & params) -{ - constexpr auto & defaults = storeDirConfigDefaults; - constexpr auto & descriptions = storeDirConfigDescriptions; - - return { - CONFIG_ROW(_storeDir), - }; -} +const StoreDirConfigT StoreDirConfig::defaults = { + ._storeDir = {.value = settings.nixStore}, +}; StoreDirConfig::StoreDirConfig(const StoreReference::Params & params) - : StoreDirConfigT{parseStoreDirConfig(params)} + : StoreDirConfigT{ + CONFIG_ROW(_storeDir), + } { } diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index a2717534501..5359c75b043 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -25,16 +25,16 @@ struct StoreDirConfigT F _storeDir; }; -extern const StoreDirConfigT storeDirConfigDefaults; - -extern const StoreDirConfigT storeDirConfigDescriptions; - -StoreDirConfigT parseStoreDirConfig(const StoreReference::Params &); - struct StoreDirConfig : StoreDirConfigT { StoreDirConfig(const StoreReference::Params & params); + static const StoreDirConfigT defaults; + + using Descriptions = StoreDirConfigT; + + static const Descriptions descriptions; + virtual ~StoreDirConfig() = default; const Path & storeDir = _storeDir.value; diff --git a/src/libstore/store-registration.hh b/src/libstore/store-registration.hh index 9bc1869eae1..8d57f4a587a 100644 --- a/src/libstore/store-registration.hh +++ b/src/libstore/store-registration.hh @@ -21,7 +21,7 @@ struct StoreFactory std::function( std::string_view scheme, std::string_view authorityPath, const StoreReference::Params & params)> parseConfig; - const StoreConfigDescription & configSchema; + const Store::Config::Descriptions & configDescriptions; }; struct Implementations @@ -38,7 +38,7 @@ struct Implementations .parseConfig = ([](auto scheme, auto uri, auto & params) -> std::shared_ptr { return std::make_shared(scheme, uri, params); }), - .configSchema = T::Config::schema, + .configDescriptions = T::Config::descriptions, }; registered->push_back(factory); } diff --git a/src/libutil/config-abstract.hh b/src/libutil/config-abstract.hh index 68f20605e06..4737a29cc7c 100644 --- a/src/libutil/config-abstract.hh +++ b/src/libutil/config-abstract.hh @@ -22,4 +22,10 @@ struct JustValue } }; +template +auto operator <<(auto str, const JustValue & opt) +{ + return str << opt.get(); +} + } From 307440b80c6c290758861dd45780b8ed9a781e10 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 02:54:14 -0400 Subject: [PATCH 29/80] Convert `LocalStoreConfig` to new system --- src/libstore-c/nix_api_store.cc | 1 + src/libstore/binary-cache-store.hh | 6 +- src/libstore/dummy-store.cc | 6 +- src/libstore/local-fs-store.cc | 3 +- src/libstore/local-fs-store.hh | 22 +++--- src/libstore/local-store.cc | 79 ++++++++++++++----- src/libstore/local-store.hh | 69 ++++++++-------- src/libstore/machines.hh | 2 + src/libstore/store-api.cc | 12 +-- src/libstore/store-api.hh | 13 +-- src/libstore/store-dir-config.hh | 4 +- src/libutil/config-abstract.hh | 2 +- src/nix-channel/nix-channel.cc | 1 + tests/unit/libstore-support/tests/libstore.hh | 1 + 14 files changed, 135 insertions(+), 86 deletions(-) diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index 79841ca49a1..ba4a6e6e5eb 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -5,6 +5,7 @@ #include "path.hh" #include "store-api.hh" +#include "store-open.hh" #include "build-result.hh" #include "globals.hh" diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 695bc925277..234c39caf3a 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -96,7 +96,11 @@ public: public: - virtual void init() override; + /** + * Perform any necessary effectful operation to make the store up and + * running + */ + virtual void init(); private: diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index f7509bcdeaf..c6dd71ba669 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -36,7 +36,7 @@ struct DummyStoreConfig : virtual StoreConfig { return {"dummy"}; } - std::shared_ptr open() override; + std::shared_ptr openStore() override; }; @@ -50,7 +50,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store DummyStore(const Config & config) : StoreConfig(config) , DummyStoreConfig(config) - , Store(config) + , Store{static_cast(*this)} { } std::string getUri() override @@ -100,7 +100,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store { unsupported("getFSAccessor"); } }; -std::shared_ptr DummyStore::Config::open() +std::shared_ptr DummyStore::Config::openStore() { return std::make_shared(*this); } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index fbaf6c3cb1c..a8b23996159 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -85,7 +85,8 @@ LocalFSStore::Config::LocalFSStoreConfig(PathView rootDir, const StoreReference: } LocalFSStore::LocalFSStore(const Config & config) - : Store(config) + : LocalFSStore::Config{config} + , Store{static_cast(*this)} { } diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index a400faec05a..654dd210f40 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -23,17 +23,6 @@ struct LocalFSStoreConfig : virtual Store::Config, LocalFSStoreConfigT { - LocalFSStoreConfig(const StoreReference::Params &); - - /** - * Used to override the `root` settings. Can't be done via modifying - * `params` reliably because this parameter is unused except for - * passing to base class constructors. - * - * @todo Make this less error-prone with new store settings system. - */ - LocalFSStoreConfig(PathView path, const StoreReference::Params & params); - struct Descriptions : virtual Store::Config::Descriptions, LocalFSStoreConfigT @@ -49,6 +38,17 @@ struct LocalFSStoreConfig : static LocalFSStoreConfigT defaults( const Store::Config &, const std::optional rootDir); + + LocalFSStoreConfig(const StoreReference::Params &); + + /** + * Used to override the `root` settings. Can't be done via modifying + * `params` reliably because this parameter is unused except for + * passing to base class constructors. + * + * @todo Make this less error-prone with new store settings system. + */ + LocalFSStoreConfig(PathView path, const StoreReference::Params & params); }; struct LocalFSStore : diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 819cee34532..2dfc78ebda3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -17,6 +17,9 @@ #include "posix-source-accessor.hh" #include "keys.hh" #include "users.hh" +#include "config-parse-impl.hh" +#include "store-open.hh" +#include "store-registration.hh" #include #include @@ -56,12 +59,50 @@ namespace nix { -LocalStoreConfig::LocalStoreConfig( +LocalStore::Config::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , LocalFSStore::Config::Descriptions{LocalFSStore::Config::descriptions} + , LocalStoreConfigT{ + .requireSigs = { + .name = "require-sigs", + .description = "Whether store paths copied into this store should have a trusted signature.", + }, + .readOnly = { + .name = "read-only", + .description = R"( + Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. + + Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. + + Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. + + > **Warning** + > Do not use this unless the filesystem is read-only. + > + > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. + > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. + )", + }, + } +{} + +const LocalStore::Config::Descriptions LocalStore::Config::descriptions{}; + +decltype(LocalStore::Config::defaults) LocalStore::Config::defaults = { + .requireSigs = {.value = settings.requireSigs }, + .readOnly = {.value = false }, +}; + +LocalStore::Config::LocalStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(authority, params) + const StoreReference::Params & params) + : Store::Config(params) + , LocalFSStore::Config(authority, params) + , LocalStoreConfigT{ + CONFIG_ROW(requireSigs), + CONFIG_ROW(readOnly), + } { } @@ -72,6 +113,11 @@ std::string LocalStoreConfig::doc() ; } +std::shared_ptr LocalStore::Config::openStore() +{ + return std::make_shared(*this); +} + struct LocalStore::State::Stmts { /* Some precompiled SQLite statements. */ SQLiteStmt RegisterValidPath; @@ -192,15 +238,12 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) } } -LocalStore::LocalStore( - std::string_view scheme, - PathView path, - const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(path, params) - , LocalStoreConfig(scheme, path, params) - , Store(params) - , LocalFSStore(params) +LocalStore::LocalStore(const Config & config) + : Store::Config{config} + , LocalFSStore::Config{config} + , LocalStore::Config{config} + , Store{static_cast(*this)} + , LocalFSStore{static_cast(*this)} , dbDir(stateDir + "/db") , linksDir(realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") @@ -477,12 +520,6 @@ LocalStore::LocalStore( } -LocalStore::LocalStore(const Params & params) - : LocalStore("local", "", params) -{ -} - - AutoCloseFD LocalStore::openGCLock() { Path fnGCLock = stateDir + "/gc.lock"; @@ -1512,7 +1549,7 @@ LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair database and the filesystem) in the loop below, in order to catch invalid states. */ - for (auto & i : std::filesystem::directory_iterator{realStoreDir.to_string()}) { + for (auto & i : std::filesystem::directory_iterator{realStoreDir.get()}) { checkInterrupt(); try { storePathsInStoreDir.insert({i.path().filename().string()}); @@ -1802,6 +1839,6 @@ std::optional LocalStore::getVersion() return nixVersion; } -static RegisterStoreImplementation regLocalStore; +static RegisterStoreImplementation regLocalStore; } // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a03cfc03b30..13e26922c0a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -34,36 +34,40 @@ struct OptimiseStats uint64_t bytesFreed = 0; }; -struct LocalStoreConfig : virtual LocalFSStoreConfig +template class F> +struct LocalStoreConfigT { - using LocalFSStoreConfig::LocalFSStoreConfig; + const F requireSigs; - LocalStoreConfig( - std::string_view scheme, - std::string_view authority, - const Params & params); + const F readOnly; +}; - Setting requireSigs{this, - settings.requireSigs, - "require-sigs", - "Whether store paths copied into this store should have a trusted signature."}; +struct LocalStoreConfig : + virtual Store::Config, + virtual LocalFSStore::Config, + LocalStoreConfigT +{ + struct Descriptions : + virtual Store::Config::Descriptions, + virtual LocalFSStore::Config::Descriptions, + LocalStoreConfigT + { + Descriptions(); + }; - Setting readOnly{this, - false, - "read-only", - R"( - Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. + static const Descriptions descriptions; - Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. + /** + * The other defaults depend on the choice of `storeDir` and `rootDir` + */ + static LocalStoreConfigT defaults; - Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. + LocalStoreConfig(const StoreReference::Params &); - > **Warning** - > Do not use this unless the filesystem is read-only. - > - > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. - > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. - )"}; + LocalStoreConfig( + std::string_view scheme, + std::string_view authority, + const StoreReference::Params & params); const std::string name() override { return "Local Store"; } @@ -71,12 +75,19 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig { return {"local"}; } std::string doc() override; + + std::shared_ptr openStore() override; }; -class LocalStore : public virtual LocalStoreConfig - , public virtual IndirectRootStore - , public virtual GcStore +class LocalStore : + public virtual LocalStoreConfig, + public virtual IndirectRootStore, + public virtual GcStore { +public: + + using Config = LocalStoreConfig; + private: /** @@ -144,11 +155,7 @@ public: * Initialise the local store, upgrading the schema if * necessary. */ - LocalStore(const Params & params); - LocalStore( - std::string_view scheme, - PathView path, - const Params & params); + LocalStore(const Config & params); ~LocalStore(); diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index 983652d5f8b..671de208468 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + #include "ref.hh" #include "store-reference.hh" diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 95aa103e46e..c4bb9d594b0 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -189,7 +189,7 @@ std::pair StoreDirConfig::computeStorePath( } -StoreConfig::Descriptions::Descriptions() +Store::Config::Descriptions::Descriptions() : StoreDirConfig::Descriptions{StoreDirConfig::descriptions} , StoreConfigT{ .pathInfoCacheSize = { @@ -235,19 +235,19 @@ StoreConfig::Descriptions::Descriptions() } -const StoreConfig::Descriptions StoreConfig::descriptions{}; +const Store::Config::Descriptions Store::Config::descriptions{}; -decltype(StoreConfig::defaults) StoreConfig::defaults = { +decltype(Store::Config::defaults) Store::Config::defaults = { .pathInfoCacheSize = { .value = 65536 }, .isTrusted = { .value = false }, .priority = { .value = 0 }, .wantMassQuery = { .value = false }, - .systemFeatures = { .value = StoreConfig::getDefaultSystemFeatures() }, + .systemFeatures = { .value = Store::Config::getDefaultSystemFeatures() }, }; -StoreConfig::StoreConfig(const StoreReference::Params & params) +Store::Config::StoreConfig(const StoreReference::Params & params) : StoreDirConfig{params} , StoreConfigT{ CONFIG_ROW(pathInfoCacheSize), @@ -491,7 +491,7 @@ ValidPathInfo Store::addToStoreSlow( return info; } -StringSet StoreConfig::getDefaultSystemFeatures() +StringSet Store::Config::getDefaultSystemFeatures() { auto res = settings.systemFeatures.get(); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 9d114ff40c7..09352e601ba 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -115,8 +115,6 @@ struct StoreConfig : StoreDirConfig, StoreConfigT { - StoreConfig(const StoreReference::Params &); - struct Descriptions : StoreDirConfig::Descriptions, StoreConfigT @@ -128,10 +126,12 @@ struct StoreConfig : static const Descriptions descriptions; - static StringSet getDefaultSystemFeatures(); + StoreConfig(const StoreReference::Params &); virtual ~StoreConfig() { } + static StringSet getDefaultSystemFeatures(); + /** * The name of this type of store. */ @@ -158,7 +158,7 @@ struct StoreConfig : * Open a store of the type corresponding to this configuration * type. */ - virtual std::shared_ptr open() = 0; + virtual std::shared_ptr openStore() = 0; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig @@ -208,11 +208,6 @@ protected: Store(const Store::Config & config); public: - /** - * Perform any necessary effectful operation to make the store up and - * running - */ - virtual void init() {}; virtual ~Store() { } diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index 4f90c5b5fc5..19391322907 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -27,14 +27,14 @@ struct StoreDirConfigT struct StoreDirConfig : StoreDirConfigT { - StoreDirConfig(const StoreReference::Params & params); - static const StoreDirConfigT defaults; using Descriptions = StoreDirConfigT; static const Descriptions descriptions; + StoreDirConfig(const StoreReference::Params & params); + virtual ~StoreDirConfig() = default; const Path & storeDir = _storeDir.value; diff --git a/src/libutil/config-abstract.hh b/src/libutil/config-abstract.hh index 4737a29cc7c..7d546b33332 100644 --- a/src/libutil/config-abstract.hh +++ b/src/libutil/config-abstract.hh @@ -23,7 +23,7 @@ struct JustValue }; template -auto operator <<(auto str, const JustValue & opt) +auto && operator <<(auto && str, const JustValue & opt) { return str << opt.get(); } diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 9f7f557b59d..d8d0d518b78 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -3,6 +3,7 @@ #include "globals.hh" #include "filetransfer.hh" #include "store-api.hh" +#include "store-open.hh" #include "legacy.hh" #include "eval-settings.hh" // for defexpr #include "users.hh" diff --git a/tests/unit/libstore-support/tests/libstore.hh b/tests/unit/libstore-support/tests/libstore.hh index 84be52c230b..5710a12d9b6 100644 --- a/tests/unit/libstore-support/tests/libstore.hh +++ b/tests/unit/libstore-support/tests/libstore.hh @@ -5,6 +5,7 @@ #include #include "store-api.hh" +#include "store-open.hh" namespace nix { From de1ec5a63444f90fea58bef4d6da10ca85830fcc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 03:15:26 -0400 Subject: [PATCH 30/80] A bunch of small fixes --- src/build-remote/build-remote.cc | 2 +- src/libcmd/command.cc | 2 +- src/libexpr/primops/fetchClosure.cc | 2 +- src/libstore-c/nix_api_store.cc | 2 +- src/libstore/build/drv-output-substitution-goal.cc | 1 + src/libstore/profiles.hh | 2 +- src/libstore/remote-store.hh | 7 ++++--- src/libstore/store-reference.cc | 2 ++ src/libutil/config-abstract.hh | 2 +- src/nix-build/nix-build.cc | 2 +- src/nix-channel/nix-channel.cc | 1 - src/nix-collect-garbage/nix-collect-garbage.cc | 2 +- src/nix-copy-closure/nix-copy-closure.cc | 2 +- src/nix-env/nix-env.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix-store/nix-store.cc | 1 + src/nix/flake.cc | 2 +- src/nix/log.cc | 2 +- src/nix/main.cc | 3 ++- src/nix/make-content-addressed.cc | 2 +- src/nix/prefetch.cc | 2 +- src/nix/repl.cc | 1 + src/nix/sigs.cc | 2 +- src/nix/unix/daemon.cc | 3 ++- src/nix/verify.cc | 2 +- tests/unit/libstore/local-overlay-store.cc | 8 ++------ tests/unit/libstore/local-store.cc | 14 ++------------ tests/unit/libstore/ssh-store.cc | 8 ++------ tests/unit/libstore/uds-remote-store.cc | 8 ++------ 29 files changed, 38 insertions(+), 53 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index a0a404e575a..1824e20024e 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -16,7 +16,7 @@ #include "globals.hh" #include "serialise.hh" #include "build-result.hh" -#include "store-api.hh" +#include "store-open.hh" #include "strings.hh" #include "derivations.hh" #include "local-store.hh" diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 67fef190920..349fb37d704 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -2,7 +2,7 @@ #include "command.hh" #include "markdown.hh" -#include "store-api.hh" +#include "store-open.hh" #include "local-fs-store.hh" #include "derivations.hh" #include "nixexpr.hh" diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index fc5bb31454f..7f59cac2f70 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -1,5 +1,5 @@ #include "primops.hh" -#include "store-api.hh" +#include "store-open.hh" #include "realisation.hh" #include "make-content-addressed.hh" #include "url.hh" diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index ba4a6e6e5eb..4961937cf12 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -43,7 +43,7 @@ Store * nix_store_open(nix_c_context * context, const char * uri, const char *** if (!params) return new Store{nix::openStore(uri_str)}; - nix::Store::Params params_map; + nix::StoreReference::Params params_map; for (size_t i = 0; params[i] != nullptr; i++) { params_map[params[i][0]] = params[i][1]; } diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index 02284d93c1a..4488efdcb59 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -3,6 +3,7 @@ #include "worker.hh" #include "substitution-goal.hh" #include "callback.hh" +#include "store-open.hh" namespace nix { diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index b10a72330b4..63aa9972c9b 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -84,7 +84,7 @@ typedef std::list Generations; */ std::pair> findGenerations(Path profile); -class LocalFSStore; +struct LocalFSStore; /** * Create a new generation of the given profile diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 4e18962683d..f19aa0fc57b 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -35,14 +35,15 @@ struct RemoteStoreConfig : virtual StoreConfig * \todo RemoteStore is a misnomer - should be something like * DaemonStore. */ -class RemoteStore : public virtual RemoteStoreConfig, +struct RemoteStore : + public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore, public virtual LogStore { -public: + using Config = RemoteStoreConfig; - RemoteStore(const Params & params); + RemoteStore(const Config & config); /* Implementations of abstract store API methods. */ diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc index b4968dfadbd..5e5d30fc117 100644 --- a/src/libstore/store-reference.cc +++ b/src/libstore/store-reference.cc @@ -1,5 +1,7 @@ #include +#include + #include "error.hh" #include "url.hh" #include "store-reference.hh" diff --git a/src/libutil/config-abstract.hh b/src/libutil/config-abstract.hh index 7d546b33332..10c98f0a73f 100644 --- a/src/libutil/config-abstract.hh +++ b/src/libutil/config-abstract.hh @@ -23,7 +23,7 @@ struct JustValue }; template -auto && operator <<(auto && str, const JustValue & opt) +auto && operator<<(auto && str, const JustValue & opt) { return str << opt.get(); } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 0ce987d8a5c..9e5dfd1a04c 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -11,7 +11,7 @@ #include "current-process.hh" #include "parsed-derivations.hh" -#include "store-api.hh" +#include "store-open.hh" #include "local-fs-store.hh" #include "globals.hh" #include "realisation.hh" diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index d8d0d518b78..028a7f36f24 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -2,7 +2,6 @@ #include "shared.hh" #include "globals.hh" #include "filetransfer.hh" -#include "store-api.hh" #include "store-open.hh" #include "legacy.hh" #include "eval-settings.hh" // for defexpr diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 91209c97898..9830b1058a3 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -1,6 +1,6 @@ #include "file-system.hh" #include "signals.hh" -#include "store-api.hh" +#include "store-open.hh" #include "store-cast.hh" #include "gc-store.hh" #include "profiles.hh" diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index b64af758fcb..d910adcf876 100644 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -1,6 +1,6 @@ #include "shared.hh" #include "realisation.hh" -#include "store-api.hh" +#include "store-open.hh" #include "legacy.hh" using namespace nix; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 5e170c99d37..056e9025800 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -9,7 +9,7 @@ #include "profiles.hh" #include "path-with-outputs.hh" #include "shared.hh" -#include "store-api.hh" +#include "store-open.hh" #include "local-fs-store.hh" #include "user-env.hh" #include "value-to-json.hh" diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index c4854951124..5474ed0b7cb 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -8,7 +8,7 @@ #include "signals.hh" #include "value-to-xml.hh" #include "value-to-json.hh" -#include "store-api.hh" +#include "store-open.hh" #include "local-fs-store.hh" #include "common-eval-args.hh" #include "legacy.hh" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index f073074e851..40d20b37690 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -2,6 +2,7 @@ #include "derivations.hh" #include "dotgraph.hh" #include "globals.hh" +#include "store-open.hh" #include "store-cast.hh" #include "local-fs-store.hh" #include "log-store.hh" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 3f9f8f99b06..26e553bde07 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -8,7 +8,7 @@ #include "flake/flake.hh" #include "get-drvs.hh" #include "signals.hh" -#include "store-api.hh" +#include "store-open.hh" #include "derivations.hh" #include "outputs-spec.hh" #include "attr-path.hh" diff --git a/src/nix/log.cc b/src/nix/log.cc index 7f590c708f6..efa4a584ccc 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "common-args.hh" #include "shared.hh" -#include "store-api.hh" +#include "store-open.hh" #include "log-store.hh" #include "progress-bar.hh" diff --git a/src/nix/main.cc b/src/nix/main.cc index 00ad6fe2c97..0d839425afb 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -8,7 +8,8 @@ #include "globals.hh" #include "legacy.hh" #include "shared.hh" -#include "store-api.hh" +#include "store-open.hh" +#include "store-registration.hh" #include "filetransfer.hh" #include "finally.hh" #include "loggers.hh" diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc index d9c988a9f5d..0c3efd7f499 100644 --- a/src/nix/make-content-addressed.cc +++ b/src/nix/make-content-addressed.cc @@ -1,5 +1,5 @@ #include "command.hh" -#include "store-api.hh" +#include "store-open.hh" #include "make-content-addressed.hh" #include "common-args.hh" diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index db7d9e4efe6..fa087430f86 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "common-args.hh" #include "shared.hh" -#include "store-api.hh" +#include "store-open.hh" #include "filetransfer.hh" #include "finally.hh" #include "progress-bar.hh" diff --git a/src/nix/repl.cc b/src/nix/repl.cc index a2f3e033e72..8f74149042d 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,6 +1,7 @@ #include "eval.hh" #include "eval-settings.hh" #include "globals.hh" +#include "store-open.hh" #include "command.hh" #include "installable-value.hh" #include "repl.hh" diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 1e277cbbee7..5a8693ab6c4 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -1,7 +1,7 @@ #include "signals.hh" #include "command.hh" #include "shared.hh" -#include "store-api.hh" +#include "store-open.hh" #include "thread-pool.hh" #include "progress-bar.hh" diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 4a7997b1fb0..8b618c19e2b 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -7,6 +7,7 @@ #include "local-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" +#include "store-open.hh" #include "serialise.hh" #include "archive.hh" #include "globals.hh" @@ -239,7 +240,7 @@ static PeerInfo getPeerInfo(int remote) */ static ref openUncachedStore() { - Store::Params params; // FIXME: get params from somewhere + StoreReference::Params params; // FIXME: get params from somewhere // Disable caching since the client already does that. params["path-info-cache-size"] = "0"; return openStore(settings.storeUri, params); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 124a05bed2c..3e2253fdf04 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -1,6 +1,6 @@ #include "command.hh" #include "shared.hh" -#include "store-api.hh" +#include "store-open.hh" #include "thread-pool.hh" #include "signals.hh" #include "keys.hh" diff --git a/tests/unit/libstore/local-overlay-store.cc b/tests/unit/libstore/local-overlay-store.cc index b34ca92375e..4fe7e607f47 100644 --- a/tests/unit/libstore/local-overlay-store.cc +++ b/tests/unit/libstore/local-overlay-store.cc @@ -1,9 +1,6 @@ -// FIXME: Odd failures for templates that are causing the PR to break -// for now with discussion with @Ericson2314 to comment out. -#if 0 -# include +#include -# include "local-overlay-store.hh" +#include "local-overlay-store.hh" namespace nix { @@ -31,4 +28,3 @@ TEST(LocalOverlayStore, constructConfig_rootPath) } } // namespace nix -#endif diff --git a/tests/unit/libstore/local-store.cc b/tests/unit/libstore/local-store.cc index abc3ea7963f..e22f54bd45d 100644 --- a/tests/unit/libstore/local-store.cc +++ b/tests/unit/libstore/local-store.cc @@ -1,15 +1,6 @@ -// FIXME: Odd failures for templates that are causing the PR to break -// for now with discussion with @Ericson2314 to comment out. -#if 0 -# include +#include -# include "local-store.hh" - -// Needed for template specialisations. This is not good! When we -// overhaul how store configs work, this should be fixed. -# include "args.hh" -# include "config-impl.hh" -# include "abstract-setting-to-json.hh" +#include "local-store.hh" namespace nix { @@ -37,4 +28,3 @@ TEST(LocalStore, constructConfig_rootPath) } } // namespace nix -#endif diff --git a/tests/unit/libstore/ssh-store.cc b/tests/unit/libstore/ssh-store.cc index b853a5f1fb9..a1fcef6b863 100644 --- a/tests/unit/libstore/ssh-store.cc +++ b/tests/unit/libstore/ssh-store.cc @@ -1,9 +1,6 @@ -// FIXME: Odd failures for templates that are causing the PR to break -// for now with discussion with @Ericson2314 to comment out. -#if 0 -# include +#include -# include "ssh-store.hh" +#include "ssh-store.hh" namespace nix { @@ -52,4 +49,3 @@ TEST(MountedSSHStore, constructConfig) } } -#endif diff --git a/tests/unit/libstore/uds-remote-store.cc b/tests/unit/libstore/uds-remote-store.cc index 5ccb208714f..4bd3bc18f5f 100644 --- a/tests/unit/libstore/uds-remote-store.cc +++ b/tests/unit/libstore/uds-remote-store.cc @@ -1,9 +1,6 @@ -// FIXME: Odd failures for templates that are causing the PR to break -// for now with discussion with @Ericson2314 to comment out. -#if 0 -# include +#include -# include "uds-remote-store.hh" +#include "uds-remote-store.hh" namespace nix { @@ -20,4 +17,3 @@ TEST(UDSRemoteStore, constructConfigWrongScheme) } } // namespace nix -#endif From bb7bf73682fee2b57e0d08f7b728358c1b1ecc42 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 12:01:10 -0400 Subject: [PATCH 31/80] More progress converting --- src/libcmd/common-eval-args.cc | 2 +- src/libcmd/repl.cc | 4 +- src/libstore/binary-cache-store.cc | 72 +++++++++++++++++++++++-- src/libstore/binary-cache-store.hh | 60 ++++++++++----------- src/libstore/build/substitution-goal.cc | 1 + src/libstore/common-ssh-store-config.cc | 49 ++++++++++++++++- src/libstore/common-ssh-store-config.hh | 43 +++++++++------ src/libstore/local-fs-store.cc | 10 ++-- src/libstore/local-fs-store.hh | 3 -- src/libstore/local-store.cc | 4 +- src/libstore/local-store.hh | 1 - src/libstore/remote-fs-accessor.hh | 2 +- src/libstore/remote-store.cc | 18 +++++++ src/libstore/remote-store.hh | 30 ++++++++--- src/libstore/store-api.cc | 10 ++-- src/libstore/store-api.hh | 4 -- src/libstore/store-dir-config.cc | 2 +- src/libstore/uds-remote-store.hh | 28 ++++------ src/libutil/config-abstract.hh | 12 +++++ 19 files changed, 251 insertions(+), 104 deletions(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index decadd751cc..fc75f047f4b 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -9,7 +9,7 @@ #include "registry.hh" #include "flake/flakeref.hh" #include "flake/settings.hh" -#include "store-api.hh" +#include "store-open.hh" #include "command.hh" #include "tarball.hh" #include "fetch-to-store.hh" diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index b5d0816dd2c..4dd41ca663d 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -12,7 +12,7 @@ #include "eval-settings.hh" #include "attr-path.hh" #include "signals.hh" -#include "store-api.hh" +#include "store-open.hh" #include "log-store.hh" #include "common-eval-args.hh" #include "get-drvs.hh" @@ -635,7 +635,7 @@ ProcessLineResult NixRepl::processLine(std::string line) // When missing, trigger the normal exception // e.g. :doc builtins.foo // behaves like - // nix-repl> builtins.foo + // nix-repl> builtins.foo // error: attribute 'foo' missing evalString(arg, v); assert(false); diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index e8c8892b337..4adc599007e 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -13,6 +13,7 @@ #include "callback.hh" #include "signals.hh" #include "archive.hh" +#include "config-parse-impl.hh" #include #include @@ -24,9 +25,74 @@ namespace nix { -BinaryCacheStore::BinaryCacheStore(const Params & params) - : BinaryCacheStoreConfig(params) - , Store(params) +BinaryCacheStore::Config::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , BinaryCacheStoreConfigT{ + .compression = { + .name = "compression", + .description = "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`).", + }, + .writeNARListing = { + .name = "write-nar-listing", + .description = "Whether to write a JSON file that lists the files in each NAR.", + }, + .writeDebugInfo = { + .name = "index-debug-info", + .description = R"( + Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to + fetch debug info on demand + )", + }, + .secretKeyFile{ + .name = "secret-key", + .description = "Path to the secret key used to sign the binary cache.", + }, + .localNarCache{ + .name = "local-nar-cache", + .description = "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`.", + }, + .parallelCompression{ + .name = "parallel-compression", + .description = "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`.", + }, + .compressionLevel{ + .name = "compression-level", + .description = R"( + The *preset level* to be used when compressing NARs. + The meaning and accepted values depend on the compression method selected. + `-1` specifies that the default compression level should be used. + )", + }, + } +{} + +const BinaryCacheStore::Config::Descriptions BinaryCacheStore::Config::descriptions{}; + +decltype(BinaryCacheStore::Config::defaults) BinaryCacheStore::Config::defaults = { + .compression = {"xz"}, + .writeNARListing{false}, + .writeDebugInfo{false}, + .secretKeyFile{""}, + .localNarCache{""}, + .parallelCompression{false}, + .compressionLevel{-1}, +}; + +BinaryCacheStore::Config::BinaryCacheStoreConfig(const StoreReference::Params & params) + : StoreConfig{params} + , BinaryCacheStoreConfigT{ + CONFIG_ROW(compression), + CONFIG_ROW(secretKeyFile), + CONFIG_ROW(localNarCache), + CONFIG_ROW(parallelCompression), + CONFIG_ROW(compressionLevel), + } +{ +} + +BinaryCacheStore::BinaryCacheStore(const Config & config) + : Config{config} + , Store{static_cast(*this)} { if (secretKeyFile != "") signer = std::make_unique( diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 234c39caf3a..27d7a845ab7 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -13,48 +13,46 @@ namespace nix { struct NarInfo; -struct BinaryCacheStoreConfig : virtual StoreConfig +template class F> +struct BinaryCacheStoreConfigT { - using StoreConfig::StoreConfig; - - const Setting compression{this, "xz", "compression", - "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; - - const Setting writeNARListing{this, false, "write-nar-listing", - "Whether to write a JSON file that lists the files in each NAR."}; - - const Setting writeDebugInfo{this, false, "index-debug-info", - R"( - Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to - fetch debug info on demand - )"}; + const F compression; + const F writeNARListing; + const F writeDebugInfo; + const F secretKeyFile; + const F localNarCache; + const F parallelCompression; + const F compressionLevel; +}; - const Setting secretKeyFile{this, "", "secret-key", - "Path to the secret key used to sign the binary cache."}; +struct BinaryCacheStoreConfig : + virtual Store::Config, + BinaryCacheStoreConfigT +{ + struct Descriptions : + virtual Store::Config::Descriptions, + BinaryCacheStoreConfigT + { + Descriptions(); + }; - const Setting localNarCache{this, "", "local-nar-cache", - "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; + static const Descriptions descriptions; - const Setting parallelCompression{this, false, "parallel-compression", - "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; + static BinaryCacheStoreConfigT defaults; - const Setting compressionLevel{this, -1, "compression-level", - R"( - The *preset level* to be used when compressing NARs. - The meaning and accepted values depend on the compression method selected. - `-1` specifies that the default compression level should be used. - )"}; + BinaryCacheStoreConfig(const StoreReference::Params &); }; - /** * @note subclasses must implement at least one of the two * virtual getFile() methods. */ -class BinaryCacheStore : public virtual BinaryCacheStoreConfig, - public virtual Store, - public virtual LogStore +struct BinaryCacheStore : + virtual BinaryCacheStoreConfig, + virtual Store, + virtual LogStore { + using Config = BinaryCacheStoreConfig; private: std::unique_ptr signer; @@ -64,7 +62,7 @@ protected: // The prefix under which realisation infos will be stored const std::string realisationsPrefix = "realisations"; - BinaryCacheStore(const Params & params); + BinaryCacheStore(const Config &); public: diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 7deeb47487d..32189ca7ae5 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -1,4 +1,5 @@ #include "worker.hh" +#include "store-open.hh" #include "substitution-goal.hh" #include "nar-info.hh" #include "finally.hh" diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index 05332b9bb5c..e1448b4b4eb 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -2,9 +2,45 @@ #include "common-ssh-store-config.hh" #include "ssh.hh" +#include "config-parse-impl.hh" namespace nix { +CommonSSHStoreConfig::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , CommonSSHStoreConfigT{ + .sshKey{ + .name = "ssh-key", + .description = "Path to the SSH private key used to authenticate to the remote machine.", + }, + .sshPublicHostKey = { + .name = "base64-ssh-public-host-key", + .description = "The public host key of the remote machine.", + }, + .compress = { + .name = "compress", + .description = "Whether to enable SSH compression.", + }, + .remoteStore{ + .name = "remote-store", + .description = R"( + [Store URL](@docroot@/store/types/index.md#store-url-format) + to be used on the remote machine. The default is `auto` + (i.e. use the Nix daemon or `/nix/store` directly). + )", + }, + } +{} + +const CommonSSHStoreConfig::Descriptions CommonSSHStoreConfig::descriptions{}; + +decltype(CommonSSHStoreConfig::defaults) CommonSSHStoreConfig::defaults = { + .sshKey{""}, + .sshPublicHostKey{""}, + .compress{false}, + .remoteStore{""}, +}; + static std::string extractConnStr(std::string_view scheme, std::string_view _connStr) { if (_connStr.empty()) @@ -22,8 +58,17 @@ static std::string extractConnStr(std::string_view scheme, std::string_view _con return connStr; } -CommonSSHStoreConfig::CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params) - : StoreConfig(params) +CommonSSHStoreConfig::CommonSSHStoreConfig( + std::string_view scheme, + std::string_view host, + const StoreReference::Params & params) + : Store::Config(params) + , CommonSSHStoreConfigT{ + CONFIG_ROW(sshKey), + CONFIG_ROW(sshPublicHostKey), + CONFIG_ROW(compress), + CONFIG_ROW(remoteStore), + } , host(extractConnStr(scheme, host)) { } diff --git a/src/libstore/common-ssh-store-config.hh b/src/libstore/common-ssh-store-config.hh index 5deb6f4c9e9..4def51da0e5 100644 --- a/src/libstore/common-ssh-store-config.hh +++ b/src/libstore/common-ssh-store-config.hh @@ -7,27 +7,38 @@ namespace nix { class SSHMaster; -struct CommonSSHStoreConfig : virtual StoreConfig +template class F> +struct CommonSSHStoreConfigT { - using StoreConfig::StoreConfig; - - CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params); + const F sshKey; + const F sshPublicHostKey; + const F compress; + const F remoteStore; +}; - const Setting sshKey{this, "", "ssh-key", - "Path to the SSH private key used to authenticate to the remote machine."}; +struct CommonSSHStoreConfig : + virtual Store::Config, + CommonSSHStoreConfigT +{ + struct Descriptions : + virtual Store::Config::Descriptions, + CommonSSHStoreConfigT + { + Descriptions(); + }; - const Setting sshPublicHostKey{this, "", "base64-ssh-public-host-key", - "The public host key of the remote machine."}; + static const Descriptions descriptions; - const Setting compress{this, false, "compress", - "Whether to enable SSH compression."}; + static CommonSSHStoreConfigT defaults; - const Setting remoteStore{this, "", "remote-store", - R"( - [Store URL](@docroot@/store/types/index.md#store-url-format) - to be used on the remote machine. The default is `auto` - (i.e. use the Nix daemon or `/nix/store` directly). - )"}; + /** + * @param scheme Note this isn't stored by this mix-in class, but + * just used for better error messages. + */ + CommonSSHStoreConfig( + std::string_view scheme, + std::string_view host, + const StoreReference::Params & params); /** * The `parseURL` function supports both IPv6 URIs as defined in diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index a8b23996159..12035c1efe7 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -33,15 +33,15 @@ LocalFSStore::Config::Descriptions::Descriptions() const LocalFSStore::Config::Descriptions LocalFSStore::Config::descriptions{}; -LocalFSStoreConfigT LocalFSStoreConfig::defaults( +LocalFSStoreConfigT LocalFSStore::Config::defaults( const Store::Config & storeConfig, const std::optional rootDir) { return { - .rootDir = {.value = std::nullopt }, - .stateDir = {.value = rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir }, - .logDir = {.value = rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir }, - .realStoreDir = {.value = rootDir ? *rootDir + "/nix/store" : storeConfig.storeDir }, + .rootDir = {std::nullopt}, + .stateDir = {rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir}, + .logDir = {rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir}, + .realStoreDir = {rootDir ? *rootDir + "/nix/store" : storeConfig.storeDir}, }; } diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 654dd210f40..051e4494012 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -11,11 +11,8 @@ template class F> struct LocalFSStoreConfigT { const F> rootDir; - const F stateDir; - const F logDir; - const F realStoreDir; }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2dfc78ebda3..aff70589d18 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -89,8 +89,8 @@ LocalStore::Config::Descriptions::Descriptions() const LocalStore::Config::Descriptions LocalStore::Config::descriptions{}; decltype(LocalStore::Config::defaults) LocalStore::Config::defaults = { - .requireSigs = {.value = settings.requireSigs }, - .readOnly = {.value = false }, + .requireSigs = {settings.requireSigs}, + .readOnly = {false}, }; LocalStore::Config::LocalStoreConfig( diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 13e26922c0a..06e402f80f9 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -38,7 +38,6 @@ template class F> struct LocalStoreConfigT { const F requireSigs; - const F readOnly; }; diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index d09762a53c4..6946074affa 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -19,7 +19,7 @@ class RemoteFSAccessor : public SourceAccessor std::pair, CanonPath> fetch(const CanonPath & path); - friend class BinaryCacheStore; + friend struct BinaryCacheStore; Path makeCacheFile(std::string_view hashPart, const std::string & ext); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 6e8931ca2e4..30d1523b550 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -23,6 +23,24 @@ namespace nix { +#if 0 + const Setting maxConnections{this, 1, "max-connections", + "Maximum number of concurrent connections to the Nix daemon."}; + + const Setting maxConnectionAge{this, + std::numeric_limits::max(), + "max-connection-age", + "Maximum age of a connection before it is closed."}; + + const Setting maxConnections{this, 1, "max-connections", + "Maximum number of concurrent connections to the Nix daemon."}; + + const Setting maxConnectionAge{this, + std::numeric_limits::max(), + "max-connection-age", + "Maximum age of a connection before it is closed."}; +#endif + /* TODO: Separate these store types into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) : RemoteStoreConfig(params) diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index f19aa0fc57b..437c4aee734 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -18,17 +18,31 @@ struct FdSink; struct FdSource; template class Pool; -struct RemoteStoreConfig : virtual StoreConfig +template class F> +struct RemoteStoreConfigT { - using StoreConfig::StoreConfig; + const F maxConnections; - const Setting maxConnections{this, 1, "max-connections", - "Maximum number of concurrent connections to the Nix daemon."}; + const F maxConnectionAge; +}; + +struct RemoteStoreConfig : + virtual Store::Config, + RemoteStoreConfigT +{ + struct Descriptions : + virtual Store::Config::Descriptions, + RemoteStoreConfigT + { + Descriptions(); + }; - const Setting maxConnectionAge{this, - std::numeric_limits::max(), - "max-connection-age", - "Maximum age of a connection before it is closed."}; + static const Descriptions descriptions; + + /** + * The other defaults depend on the choice of `storeDir` and `rootDir` + */ + static RemoteStoreConfigT defaults; }; /** diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c4bb9d594b0..b67faf38a5e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -239,11 +239,11 @@ const Store::Config::Descriptions Store::Config::descriptions{}; decltype(Store::Config::defaults) Store::Config::defaults = { - .pathInfoCacheSize = { .value = 65536 }, - .isTrusted = { .value = false }, - .priority = { .value = 0 }, - .wantMassQuery = { .value = false }, - .systemFeatures = { .value = Store::Config::getDefaultSystemFeatures() }, + .pathInfoCacheSize = {65536}, + .isTrusted = {false}, + .priority = {0}, + .wantMassQuery = {false}, + .systemFeatures = {Store::Config::getDefaultSystemFeatures()}, }; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 09352e601ba..7e9a5fa53f5 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -100,13 +100,9 @@ template class F> struct StoreConfigT { const F pathInfoCacheSize; - const F isTrusted; - F priority; - F wantMassQuery; - F systemFeatures; }; diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc index a51962d7fa9..026a6cd9837 100644 --- a/src/libstore/store-dir-config.cc +++ b/src/libstore/store-dir-config.cc @@ -17,7 +17,7 @@ const StoreDirConfigT StoreDirConfig::descriptions = { }; const StoreDirConfigT StoreDirConfig::defaults = { - ._storeDir = {.value = settings.nixStore}, + ._storeDir = {settings.nixStore}, }; StoreDirConfig::StoreDirConfig(const StoreReference::Params & params) diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index a8e57166416..8d19ad5f5bc 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -7,7 +7,9 @@ namespace nix { -struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig +struct UDSRemoteStoreConfig : + virtual LocalFSStoreConfig, + virtual RemoteStoreConfig { // TODO(fzakaria): Delete this constructor once moved over to the factory pattern // outlined in https://github.com/NixOS/nix/issues/10766 @@ -34,7 +36,7 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon */ Path path; -protected: +private: static constexpr char const * scheme = "unix"; public: @@ -42,24 +44,12 @@ public: { return {scheme}; } }; -class UDSRemoteStore : public virtual UDSRemoteStoreConfig - , public virtual IndirectRootStore - , public virtual RemoteStore +struct UDSRemoteStore : + virtual UDSRemoteStoreConfig, + virtual IndirectRootStore, + virtual RemoteStore { -public: - - /** - * @deprecated This is the old API to construct the store. - */ - UDSRemoteStore(const Params & params); - - /** - * @param authority is the socket path. - */ - UDSRemoteStore( - std::string_view scheme, - std::string_view authority, - const Params & params); + UDSRemoteStore(const UDSRemoteStoreConfig &); std::string getUri() override; diff --git a/src/libutil/config-abstract.hh b/src/libutil/config-abstract.hh index 10c98f0a73f..fcde0696b98 100644 --- a/src/libutil/config-abstract.hh +++ b/src/libutil/config-abstract.hh @@ -12,14 +12,26 @@ struct JustValue { return value; } + operator T &() { return value; } + const T & get() const { return value; } + + bool operator==(auto && v2) const + { + return value == v2; + } + + bool operator!=(auto && v2) const + { + return value != v2; + } }; template From ca1e89513141503450b3ee9fb371060f9cc92d9e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 12:31:46 -0400 Subject: [PATCH 32/80] WIP --- src/libstore/common-ssh-store-config.cc | 43 ++++----- src/libstore/dummy-store.cc | 4 +- src/libstore/filetransfer.cc | 2 +- src/libstore/http-binary-cache-store.cc | 32 ++++--- src/libstore/http-binary-cache-store.hh | 7 +- src/libstore/local-binary-cache-store.cc | 30 +++---- src/libstore/local-binary-cache-store.hh | 11 ++- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 5 +- src/libstore/machines.cc | 2 +- src/libstore/misc.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 76 ++++++++++++++-- src/libstore/s3-binary-cache-store.hh | 109 +++++++---------------- src/libstore/store-api.hh | 2 +- 14 files changed, 175 insertions(+), 152 deletions(-) diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index e1448b4b4eb..a2d7e2ebad6 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -9,28 +9,29 @@ namespace nix { CommonSSHStoreConfig::Descriptions::Descriptions() : Store::Config::Descriptions{Store::Config::descriptions} , CommonSSHStoreConfigT{ - .sshKey{ - .name = "ssh-key", - .description = "Path to the SSH private key used to authenticate to the remote machine.", - }, - .sshPublicHostKey = { - .name = "base64-ssh-public-host-key", - .description = "The public host key of the remote machine.", - }, - .compress = { - .name = "compress", - .description = "Whether to enable SSH compression.", - }, - .remoteStore{ - .name = "remote-store", - .description = R"( - [Store URL](@docroot@/store/types/index.md#store-url-format) - to be used on the remote machine. The default is `auto` - (i.e. use the Nix daemon or `/nix/store` directly). + .sshKey{ + .name = "ssh-key", + .description = "Path to the SSH private key used to authenticate to the remote machine.", + }, + .sshPublicHostKey{ + .name = "base64-ssh-public-host-key", + .description = "The public host key of the remote machine.", + }, + .compress{ + .name = "compress", + .description = "Whether to enable SSH compression.", + }, + .remoteStore{ + .name = "remote-store", + .description = R"( + [Store URL](@docroot@/store/types/index.md#store-url-format) + to be used on the remote machine. The default is `auto` + (i.e. use the Nix daemon or `/nix/store` directly). )", - }, - } -{} + }, + } +{ +} const CommonSSHStoreConfig::Descriptions CommonSSHStoreConfig::descriptions{}; diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index c6dd71ba669..968c2d3fbac 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -36,7 +36,7 @@ struct DummyStoreConfig : virtual StoreConfig { return {"dummy"}; } - std::shared_ptr openStore() override; + std::shared_ptr openStore() const override; }; @@ -100,7 +100,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store { unsupported("getFSAccessor"); } }; -std::shared_ptr DummyStore::Config::openStore() +std::shared_ptr DummyStore::Config::openStore() const { return std::make_shared(*this); } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index cbbb0fe7a34..ce296668798 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -715,7 +715,7 @@ struct curlFileTransfer : public FileTransfer } #if ENABLE_S3 - std::tuple parseS3Uri(std::string uri) + std::tuple parseS3Uri(std::string uri) { auto [path, params] = splitUriAndParams(uri); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index b15ef4e4cba..0a663ec036d 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -3,6 +3,7 @@ #include "globals.hh" #include "nar-info-disk-cache.hh" #include "callback.hh" +#include "store-registration.hh" namespace nix { @@ -12,7 +13,7 @@ MakeError(UploadToHTTP, Error); HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( std::string_view scheme, std::string_view _cacheUri, - const Params & params) + const StoreReference::Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) , cacheUri( @@ -35,10 +36,10 @@ std::string HttpBinaryCacheStoreConfig::doc() } -class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore +class HttpBinaryCacheStore : + public virtual HttpBinaryCacheStoreConfig, + public virtual BinaryCacheStore { -private: - struct State { bool enabled = true; @@ -49,15 +50,14 @@ class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public v public: - HttpBinaryCacheStore( - std::string_view scheme, - PathView cacheUri, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , HttpBinaryCacheStoreConfig(scheme, cacheUri, params) - , Store(params) - , BinaryCacheStore(params) + using Config = HttpBinaryCacheStoreConfig; + + HttpBinaryCacheStore(const Config & config) + : Store::Config{config} + , BinaryCacheStore::Config{config} + , HttpBinaryCacheStore::Config{config} + , Store{static_cast(*this)} + , BinaryCacheStore{static_cast(*this)} { diskCache = getNarInfoDiskCache(); } @@ -207,6 +207,10 @@ class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public v } }; -static RegisterStoreImplementation regHttpBinaryCacheStore; +std::shared_ptr HttpBinaryCacheStore::Config::openStore() const { + return std::make_shared(*this); +} + +static RegisterStoreImplementation regHttpBinaryCacheStore; } diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh index d2fc43210a2..d15c6e3d0db 100644 --- a/src/libstore/http-binary-cache-store.hh +++ b/src/libstore/http-binary-cache-store.hh @@ -4,9 +4,8 @@ namespace nix { struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - - HttpBinaryCacheStoreConfig(std::string_view scheme, std::string_view _cacheUri, const Params & params); + HttpBinaryCacheStoreConfig( + std::string_view scheme, std::string_view cacheUri, const StoreReference::Params & params); Path cacheUri; @@ -25,6 +24,8 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig } std::string doc() override; + + std::shared_ptr openStore() const override; }; } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index dcc6affe4a1..8fb163e3064 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -2,6 +2,7 @@ #include "globals.hh" #include "nar-info-disk-cache.hh" #include "signals.hh" +#include "store-registration.hh" #include @@ -10,7 +11,7 @@ namespace nix { LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig( std::string_view scheme, PathView binaryCacheDir, - const Params & params) + const StoreReference::Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) , binaryCacheDir(binaryCacheDir) @@ -26,21 +27,18 @@ std::string LocalBinaryCacheStoreConfig::doc() } -struct LocalBinaryCacheStore : virtual LocalBinaryCacheStoreConfig, virtual BinaryCacheStore +struct LocalBinaryCacheStore : + virtual LocalBinaryCacheStoreConfig, + virtual BinaryCacheStore { - /** - * @param binaryCacheDir `file://` is a short-hand for `file:///` - * for now. - */ - LocalBinaryCacheStore( - std::string_view scheme, - PathView binaryCacheDir, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , LocalBinaryCacheStoreConfig(scheme, binaryCacheDir, params) - , Store(params) - , BinaryCacheStore(params) + using Config = LocalBinaryCacheStoreConfig; + + LocalBinaryCacheStore(const Config & config) + : Store::Config{config} + , BinaryCacheStore::Config{config} + , LocalBinaryCacheStore::Config{config} + , Store{static_cast(*this)} + , BinaryCacheStore{static_cast(*this)} { } @@ -127,6 +125,6 @@ std::set LocalBinaryCacheStoreConfig::uriSchemes() return {"file"}; } -static RegisterStoreImplementation regLocalBinaryCacheStore; +static RegisterStoreImplementation regLocalBinaryCacheStore; } diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh index 997e8ecbb51..01f7969eb61 100644 --- a/src/libstore/local-binary-cache-store.hh +++ b/src/libstore/local-binary-cache-store.hh @@ -4,9 +4,12 @@ namespace nix { struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - - LocalBinaryCacheStoreConfig(std::string_view scheme, PathView binaryCacheDir, const Params & params); + /** + * @param binaryCacheDir `file://` is a short-hand for `file:///` + * for now. + */ + LocalBinaryCacheStoreConfig( + std::string_view scheme, PathView binaryCacheDir, const StoreReference::Params & params); Path binaryCacheDir; @@ -18,6 +21,8 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig static std::set uriSchemes(); std::string doc() override; + + std::shared_ptr openStore() const override; }; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index aff70589d18..d3677fdeb39 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -113,7 +113,7 @@ std::string LocalStoreConfig::doc() ; } -std::shared_ptr LocalStore::Config::openStore() +std::shared_ptr LocalStore::Config::openStore() const { return std::make_shared(*this); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 06e402f80f9..45cb04b7159 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -56,9 +56,6 @@ struct LocalStoreConfig : static const Descriptions descriptions; - /** - * The other defaults depend on the choice of `storeDir` and `rootDir` - */ static LocalStoreConfigT defaults; LocalStoreConfig(const StoreReference::Params &); @@ -75,7 +72,7 @@ struct LocalStoreConfig : std::string doc() override; - std::shared_ptr openStore() override; + std::shared_ptr openStore() const override; }; class LocalStore : diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 256cf918892..9e4145e0ac5 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -1,6 +1,6 @@ #include "machines.hh" #include "globals.hh" -#include "store-api.hh" +#include "store-open.hh" #include diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index bcc02206bc9..31762f34d2c 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -3,7 +3,7 @@ #include "derivations.hh" #include "parsed-derivations.hh" #include "globals.hh" -#include "store-api.hh" +#include "store-open.hh" #include "thread-pool.hh" #include "realisation.hh" #include "topo-sort.hh" diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 1a0ec11112e..abd2727194b 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -1,7 +1,5 @@ #if ENABLE_S3 -#include - #include "s3.hh" #include "s3-binary-cache-store.hh" #include "nar-info.hh" @@ -187,6 +185,75 @@ S3Helper::FileTransferResult S3Helper::getObject( return res; } + const Setting profile{ + this, + "", + "profile", + R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )"}; + + const Setting region{ + this, + Aws::Region::US_EAST_1, + "region", + R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )"}; + + const Setting scheme{ + this, + "", + "scheme", + R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )"}; + + const Setting endpoint{ + this, + "", + "endpoint", + R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )"}; + + const Setting narinfoCompression{ + this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; + + const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; + + const Setting logCompression{ + this, + "", + "log-compression", + R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )"}; + + const Setting multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."}; + + const Setting bufferSize{ + this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; + + S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) : BinaryCacheStoreConfig(params) , BinaryCacheStore(params) @@ -201,11 +268,6 @@ S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( , BinaryCacheStoreConfig(params) , bucketName(bucketName) { - // Don't want to use use AWS SDK in header, so we check the default - // here. TODO do this better after we overhaul the store settings - // system. - assert(std::string{defaultRegion} == std::string{Aws::Region::US_EAST_1}); - if (bucketName.empty()) throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); } diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 7d303a115f4..c28e9320378 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -7,87 +7,40 @@ namespace nix { -struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +template class F> +struct S3BinaryCacheStoreConfigT { - std::string bucketName; - - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - - S3BinaryCacheStoreConfig(std::string_view uriScheme, std::string_view bucketName, const Params & params); + const F profile; + const F region; + const F scheme; + const F endpoint; + const F narinfoCompression; + const F lsCompression; + const F logCompression; + const F multipartUpload; + const F bufferSize; +}; - const Setting profile{ - this, - "", - "profile", - R"( - The name of the AWS configuration profile to use. By default - Nix will use the `default` profile. - )"}; +struct S3BinaryCacheStoreConfig : + virtual BinaryCacheStoreConfig, + S3BinaryCacheStoreConfigT +{ + struct Descriptions : + virtual Store::Config::Descriptions, + virtual BinaryCacheStore::Config::Descriptions, + S3BinaryCacheStoreConfigT + { + Descriptions(); + }; -protected: + static const Descriptions descriptions; - constexpr static const char * defaultRegion = "us-east-1"; + static S3BinaryCacheStoreConfigT defaults; -public: + S3BinaryCacheStoreConfig( + std::string_view uriScheme, std::string_view bucketName, const StoreReference::Params & params); - const Setting region{ - this, - defaultRegion, - "region", - R"( - The region of the S3 bucket. If your bucket is not in - `us–east-1`, you should always explicitly specify the region - parameter. - )"}; - - const Setting scheme{ - this, - "", - "scheme", - R"( - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - )"}; - - const Setting endpoint{ - this, - "", - "endpoint", - R"( - The URL of the endpoint of an S3-compatible service such as MinIO. - Do not specify this setting if you're using Amazon S3. - - > **Note** - > - > This endpoint must support HTTPS and will use path-based - > addressing instead of virtual host based addressing. - )"}; - - const Setting narinfoCompression{ - this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; - - const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; - - const Setting logCompression{ - this, - "", - "log-compression", - R"( - Compression method for `log/*` files. It is recommended to - use a compression method supported by most web browsers - (e.g. `brotli`). - )"}; - - const Setting multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."}; - - const Setting bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; + std::string bucketName; const std::string name() override { @@ -102,11 +55,13 @@ public: std::string doc() override; }; -class S3BinaryCacheStore : public virtual BinaryCacheStore +struct S3BinaryCacheStore : virtual BinaryCacheStore { + using Config = S3BinaryCacheStoreConfig; + protected: - S3BinaryCacheStore(const Params & params); + S3BinaryCacheStore(const Config &); public: diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7e9a5fa53f5..6571fd37c10 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -154,7 +154,7 @@ struct StoreConfig : * Open a store of the type corresponding to this configuration * type. */ - virtual std::shared_ptr openStore() = 0; + virtual std::shared_ptr openStore() const = 0; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig From 901b34fb71ff224a3dde41d0cf24a428511f5246 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 13:22:29 -0400 Subject: [PATCH 33/80] WIP --- src/libstore/dummy-store.cc | 6 +- src/libstore/http-binary-cache-store.cc | 5 +- src/libstore/http-binary-cache-store.hh | 2 +- src/libstore/local-binary-cache-store.hh | 2 +- src/libstore/local-store.cc | 4 +- src/libstore/local-store.hh | 2 +- src/libstore/s3-binary-cache-store.cc | 216 ++++++++++-------- src/libstore/s3-binary-cache-store.hh | 2 + src/libstore/store-api.hh | 2 +- .../unix/build/local-derivation-goal.cc | 63 +++-- 10 files changed, 180 insertions(+), 124 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 968c2d3fbac..3d2931cd9ba 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -36,7 +36,7 @@ struct DummyStoreConfig : virtual StoreConfig { return {"dummy"}; } - std::shared_ptr openStore() const override; + ref openStore() const override; }; @@ -100,9 +100,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store { unsupported("getFSAccessor"); } }; -std::shared_ptr DummyStore::Config::openStore() const +ref DummyStore::Config::openStore() const { - return std::make_shared(*this); + return make_ref(*this); } static RegisterStoreImplementation regDummyStore; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 0a663ec036d..accd84899f1 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -207,8 +207,9 @@ class HttpBinaryCacheStore : } }; -std::shared_ptr HttpBinaryCacheStore::Config::openStore() const { - return std::make_shared(*this); +ref HttpBinaryCacheStore::Config::openStore() const +{ + return make_ref(*this); } static RegisterStoreImplementation regHttpBinaryCacheStore; diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh index d15c6e3d0db..397bd0592a6 100644 --- a/src/libstore/http-binary-cache-store.hh +++ b/src/libstore/http-binary-cache-store.hh @@ -25,7 +25,7 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig std::string doc() override; - std::shared_ptr openStore() const override; + ref openStore() const override; }; } diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh index 01f7969eb61..647cd83d099 100644 --- a/src/libstore/local-binary-cache-store.hh +++ b/src/libstore/local-binary-cache-store.hh @@ -22,7 +22,7 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig std::string doc() override; - std::shared_ptr openStore() const override; + ref openStore() const override; }; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index d3677fdeb39..2a930051d4f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -113,9 +113,9 @@ std::string LocalStoreConfig::doc() ; } -std::shared_ptr LocalStore::Config::openStore() const +ref LocalStore::Config::openStore() const { - return std::make_shared(*this); + return make_ref(*this); } struct LocalStore::State::Stmts { diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 45cb04b7159..3d5aa1d3dbd 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -72,7 +72,7 @@ struct LocalStoreConfig : std::string doc() override; - std::shared_ptr openStore() const override; + ref openStore() const override; }; class LocalStore : diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index abd2727194b..6b9a809af96 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -7,6 +7,8 @@ #include "globals.hh" #include "compression.hh" #include "filetransfer.hh" +#include "config-parse-impl.hh" +#include "store-registration.hh" #include #include @@ -185,93 +187,119 @@ S3Helper::FileTransferResult S3Helper::getObject( return res; } - const Setting profile{ - this, - "", - "profile", - R"( - The name of the AWS configuration profile to use. By default - Nix will use the `default` profile. - )"}; - - const Setting region{ - this, - Aws::Region::US_EAST_1, - "region", - R"( - The region of the S3 bucket. If your bucket is not in - `us–east-1`, you should always explicitly specify the region - parameter. - )"}; - - const Setting scheme{ - this, - "", - "scheme", - R"( - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - )"}; - - const Setting endpoint{ - this, - "", - "endpoint", - R"( - The URL of the endpoint of an S3-compatible service such as MinIO. - Do not specify this setting if you're using Amazon S3. - - > **Note** - > - > This endpoint must support HTTPS and will use path-based - > addressing instead of virtual host based addressing. - )"}; - - const Setting narinfoCompression{ - this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; - - const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; - - const Setting logCompression{ - this, - "", - "log-compression", - R"( - Compression method for `log/*` files. It is recommended to - use a compression method supported by most web browsers - (e.g. `brotli`). - )"}; - - const Setting multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."}; - - const Setting bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; - - -S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) - : BinaryCacheStoreConfig(params) - , BinaryCacheStore(params) -{ } +S3BinaryCacheStore::Config::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , BinaryCacheStore::Config::Descriptions{BinaryCacheStore::Config::descriptions} + , S3BinaryCacheStoreConfigT{ + .profile{ + .name = "profile", + .description = R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )", + }, + .region{ + .name = "region", + .description = R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )", + }, + .scheme{ + .name = "scheme", + .description = R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )", + }, + .endpoint{ + .name = "endpoint", + .description = R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )", + }, + .narinfoCompression{ + .name = "narinfo-compression", + .description = "Compression method for `.narinfo` files.", + }, + .lsCompression{ + .name = "ls-compression", + .description = "Compression method for `.ls` files.", + }, + .logCompression{ + .name = "log-compression", + .description = R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )", + }, + .multipartUpload{ + .name = "multipart-upload", + .description = "Whether to use multi-part uploads.", + }, + .bufferSize{ + .name = "buffer-size", + .description = "Size (in bytes) of each part in multi-part uploads.", + }, + } +{} + +const S3BinaryCacheStore::Config::Descriptions S3BinaryCacheStore::Config::descriptions{}; + +decltype(S3BinaryCacheStore::Config::defaults) S3BinaryCacheStore::Config::defaults = { + .profile{""}, + .region{Aws::Region::US_EAST_1}, + .scheme{""}, + .endpoint{""}, + .narinfoCompression{""}, + .lsCompression{""}, + .logCompression{""}, + .multipartUpload{false}, + .bufferSize{5 * 1024 * 1024}, +}; -S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( - std::string_view uriScheme, - std::string_view bucketName, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , bucketName(bucketName) +S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig( + std::string_view scheme, + std::string_view authority, + const StoreReference::Params & params) + : Store::Config(params) + , BinaryCacheStore::Config(params) + , S3BinaryCacheStoreConfigT{ + CONFIG_ROW(profile), + CONFIG_ROW(region), + CONFIG_ROW(scheme), + CONFIG_ROW(endpoint), + CONFIG_ROW(narinfoCompression), + CONFIG_ROW(lsCompression), + CONFIG_ROW(logCompression), + CONFIG_ROW(multipartUpload), + CONFIG_ROW(bufferSize), + } { if (bucketName.empty()) - throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); + throw UsageError("`%s` store requires a bucket name in its Store URI", scheme); } + +S3BinaryCacheStore::S3BinaryCacheStore(const Config & config) + : BinaryCacheStore(config) +{ } + std::string S3BinaryCacheStoreConfig::doc() { return @@ -282,22 +310,17 @@ std::string S3BinaryCacheStoreConfig::doc() struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore { - std::string bucketName; - Stats stats; S3Helper s3Helper; - S3BinaryCacheStoreImpl( - std::string_view uriScheme, - std::string_view bucketName, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , S3BinaryCacheStoreConfig(uriScheme, bucketName, params) - , Store(params) - , BinaryCacheStore(params) - , S3BinaryCacheStore(params) + S3BinaryCacheStoreImpl(const Config & config) + : Store::Config{config} + , BinaryCacheStore::Config{config} + , S3BinaryCacheStore::Config{config} + , Store{static_cast(*this)} + , BinaryCacheStore{static_cast(*this)} + , S3BinaryCacheStore{static_cast(*this)} , s3Helper(profile, region, scheme, endpoint) { diskCache = getNarInfoDiskCache(); @@ -539,7 +562,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual } }; -static RegisterStoreImplementation regS3BinaryCacheStore; +ref S3BinaryCacheStoreImpl::Config::openStore() const +{ + return make_ref(*this); +} + +static RegisterStoreImplementation regS3BinaryCacheStore; } diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index c28e9320378..bff26f6b3d0 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -53,6 +53,8 @@ struct S3BinaryCacheStoreConfig : } std::string doc() override; + + ref openStore() const override; }; struct S3BinaryCacheStore : virtual BinaryCacheStore diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 6571fd37c10..56fef0f1a7b 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -154,7 +154,7 @@ struct StoreConfig : * Open a store of the type corresponding to this configuration * type. */ - virtual std::shared_ptr openStore() const = 0; + virtual ref openStore() const = 0; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index f968bbc5b7f..681ac032150 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1254,29 +1254,46 @@ bool LocalDerivationGoal::isAllowed(const DerivedPath & req) return this->isAllowed(pathPartOfReq(req)); } - struct RestrictedStoreConfig : virtual LocalFSStoreConfig { - using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } -}; + const std::string name() override { return "Restricted Store"; } + + ref openStore() const override; -/* A wrapper around LocalStore that only allows building/querying of - paths that are in the input closures of the build or were added via - recursive Nix calls. */ -struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual IndirectRootStore, public virtual GcStore -{ ref next; LocalDerivationGoal & goal; - RestrictedStore(const Params & params, ref next, LocalDerivationGoal & goal) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RestrictedStoreConfig(params) - , Store(params) - , LocalFSStore(params) - , next(next), goal(goal) + RestrictedStoreConfig( + const StoreReference::Params & params, + ref next, + LocalDerivationGoal & goal) + : Store::Config{params} + , LocalFSStore::Config{params} + , next{next} + , goal{goal} + { + } +}; + +/** + * A wrapper around LocalStore that only allows building/querying of + * paths that are in the input closures of the build or were added via + * recursive Nix calls. + */ +struct RestrictedStore : + virtual RestrictedStoreConfig, + virtual IndirectRootStore, + virtual GcStore +{ + using Config = RestrictedStoreConfig; + + RestrictedStore(const RestrictedStoreConfig & config) + : Store::Config(config) + , LocalFSStore::Config(config) + , RestrictedStore::Config(config) + , Store(static_cast(*this)) + , LocalFSStore(static_cast(*this)) { } Path getRealStoreDir() override @@ -1480,20 +1497,28 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In }; +ref RestrictedStore::Config::openStore() const +{ + return make_ref(*this); +} + + void LocalDerivationGoal::startDaemon() { experimentalFeatureSettings.require(Xp::RecursiveNix); - Store::Params params; + StoreReference::Params params; params["path-info-cache-size"] = "0"; params["store"] = worker.store.storeDir; if (auto & optRoot = getLocalStore().rootDir.get()) params["root"] = *optRoot; params["state"] = "/no-such-path"; params["log"] = "/no-such-path"; - auto store = make_ref(params, + auto store = RestrictedStore::Config{ + params, ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), - *this); + *this, + }.openStore(); addedPaths.clear(); From 6e5d7fc5d88b5fab7788c84b8dd85e4d9de5419a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 15:23:24 -0400 Subject: [PATCH 34/80] Convert ssh:// to new system --- src/libstore/legacy-ssh-store.cc | 58 +++++++++++++++++++++------ src/libstore/legacy-ssh-store.hh | 51 +++++++++++++---------- src/libstore/s3-binary-cache-store.hh | 11 ++--- 3 files changed, 80 insertions(+), 40 deletions(-) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index eac360a4f7a..c27c7bc563e 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -12,18 +12,55 @@ #include "ssh.hh" #include "derivations.hh" #include "callback.hh" +#include "config-parse-impl.hh" +#include "store-registration.hh" namespace nix { -LegacySSHStoreConfig::LegacySSHStoreConfig( +LegacySSHStore::Config::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , CommonSSHStoreConfig::Descriptions{CommonSSHStoreConfig::descriptions} + , LegacySSHStoreConfigT{ + .remoteProgram{ + .name = "remote-program", + .description = "Path to the `nix-store` executable on the remote machine.", + }, + .maxConnections{ + .name = "max-connections", + .description = "Maximum number of concurrent SSH connections.", + }, + } +{} + + +const LegacySSHStore::Config::Descriptions LegacySSHStore::Config::descriptions{}; + + +decltype(LegacySSHStore::Config::defaults) LegacySSHStore::Config::defaults = { + .remoteProgram = {{"nix-store"}}, + .maxConnections = {1}, +}; + + +LegacySSHStore::Config::LegacySSHStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params) - : StoreConfig(params) + const StoreReference::Params & params) + : Store::Config(params) , CommonSSHStoreConfig(scheme, authority, params) + , LegacySSHStoreConfigT{ + CONFIG_ROW(remoteProgram), + CONFIG_ROW(maxConnections), + } { +#ifndef _WIN32 + if (auto * p = get(params, "log-fd")) { + logFD = p->get(); + } +#endif } + std::string LegacySSHStoreConfig::doc() { return @@ -38,14 +75,11 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection bool good = true; }; -LegacySSHStore::LegacySSHStore( - std::string_view scheme, - std::string_view host, - const Params & params) - : StoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , LegacySSHStoreConfig(scheme, host, params) - , Store(params) +LegacySSHStore::LegacySSHStore(const Config & config) + : Store::Config(config) + , CommonSSHStoreConfig(config) + , LegacySSHStore::Config(config) + , Store(static_cast(*this)) , connections(make_ref>( std::max(1, (int) maxConnections), [this]() { return openConnection(); }, @@ -317,6 +351,6 @@ std::optional isTrustedClient() } -static RegisterStoreImplementation regLegacySSHStore; +static RegisterStoreImplementation regLegacySSHStore; } diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index b541455b4e5..0a58c803126 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -9,38 +9,50 @@ namespace nix { -struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig +template class F> +struct LegacySSHStoreConfigT { - using CommonSSHStoreConfig::CommonSSHStoreConfig; + const F remoteProgram; + const F maxConnections; +}; + +struct LegacySSHStoreConfig : + virtual CommonSSHStoreConfig, + LegacySSHStoreConfigT +{ + struct Descriptions : + virtual CommonSSHStoreConfig::Descriptions, + LegacySSHStoreConfigT + { + Descriptions(); + }; + + static const Descriptions descriptions; + + static LegacySSHStoreConfigT defaults; + /** + * Hack for getting remote build log output. Intentionally not a + * documented user-visible setting. + */ + Descriptor logFD = INVALID_DESCRIPTOR; LegacySSHStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params); - - const Setting remoteProgram{this, {"nix-store"}, "remote-program", - "Path to the `nix-store` executable on the remote machine."}; - - const Setting maxConnections{this, 1, "max-connections", - "Maximum number of concurrent SSH connections."}; + const StoreReference::Params & params); const std::string name() override { return "SSH Store"; } static std::set uriSchemes() { return {"ssh"}; } std::string doc() override; + + ref openStore() const override; }; struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store { -#ifndef _WIN32 - // Hack for getting remote build log output. - // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in - // the documentation - const Setting logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"}; -#else - Descriptor logFD = INVALID_DESCRIPTOR; -#endif + using Config = LegacySSHStoreConfig; struct Connection; @@ -48,10 +60,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor SSHMaster master; - LegacySSHStore( - std::string_view scheme, - std::string_view host, - const Params & params); + LegacySSHStore(const Config &); ref openConnection(); diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index bff26f6b3d0..df7197113ae 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -21,14 +21,11 @@ struct S3BinaryCacheStoreConfigT const F bufferSize; }; -struct S3BinaryCacheStoreConfig : - virtual BinaryCacheStoreConfig, - S3BinaryCacheStoreConfigT +struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig, S3BinaryCacheStoreConfigT { - struct Descriptions : - virtual Store::Config::Descriptions, - virtual BinaryCacheStore::Config::Descriptions, - S3BinaryCacheStoreConfigT + struct Descriptions : virtual Store::Config::Descriptions, + virtual BinaryCacheStore::Config::Descriptions, + S3BinaryCacheStoreConfigT { Descriptions(); }; From 74620b97d9687806b6daae80f08883c586d8caf5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 15:54:42 -0400 Subject: [PATCH 35/80] WIP --- src/libstore/legacy-ssh-store.cc | 12 ++-- src/libstore/legacy-ssh-store.hh | 1 + src/libstore/remote-store.cc | 53 +++++++++----- src/libstore/remote-store.hh | 3 +- src/libstore/ssh-store.cc | 118 ++++++++++++++++++------------- src/libstore/ssh-store.hh | 39 +++++++--- 6 files changed, 139 insertions(+), 87 deletions(-) diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index c27c7bc563e..10301b79e66 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -46,8 +46,8 @@ LegacySSHStore::Config::LegacySSHStoreConfig( std::string_view scheme, std::string_view authority, const StoreReference::Params & params) - : Store::Config(params) - , CommonSSHStoreConfig(scheme, authority, params) + : Store::Config{params} + , CommonSSHStoreConfig{scheme, authority, params} , LegacySSHStoreConfigT{ CONFIG_ROW(remoteProgram), CONFIG_ROW(maxConnections), @@ -76,10 +76,10 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection }; LegacySSHStore::LegacySSHStore(const Config & config) - : Store::Config(config) - , CommonSSHStoreConfig(config) - , LegacySSHStore::Config(config) - , Store(static_cast(*this)) + : Store::Config{config} + , CommonSSHStoreConfig{config} + , LegacySSHStore::Config{config} + , Store{static_cast(*this)} , connections(make_ref>( std::max(1, (int) maxConnections), [this]() { return openConnection(); }, diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index 0a58c803126..9d2234faaf7 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -30,6 +30,7 @@ struct LegacySSHStoreConfig : static const Descriptions descriptions; static LegacySSHStoreConfigT defaults; + /** * Hack for getting remote build log output. Intentionally not a * documented user-visible setting. diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 30d1523b550..4caabdf9f3c 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -18,33 +18,50 @@ #include "callback.hh" #include "filetransfer.hh" #include "signals.hh" +#include "config-parse-impl.hh" #include namespace nix { -#if 0 - const Setting maxConnections{this, 1, "max-connections", - "Maximum number of concurrent connections to the Nix daemon."}; +RemoteStore::Config::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , RemoteStoreConfigT{ + .maxConnections{ + .name = "max-connections", + .description = "Maximum number of concurrent connections to the Nix daemon.", + }, + .maxConnectionAge{ + .name = "max-connection-age", + .description = "Maximum age of a connection before it is closed.", + }, + } +{} + + +const RemoteStore::Config::Descriptions RemoteStore::Config::descriptions{}; + - const Setting maxConnectionAge{this, - std::numeric_limits::max(), - "max-connection-age", - "Maximum age of a connection before it is closed."}; +decltype(RemoteStore::Config::defaults) RemoteStore::Config::defaults = { + .maxConnections = {1}, + .maxConnectionAge = {std::numeric_limits::max()}, +}; - const Setting maxConnections{this, 1, "max-connections", - "Maximum number of concurrent connections to the Nix daemon."}; - const Setting maxConnectionAge{this, - std::numeric_limits::max(), - "max-connection-age", - "Maximum age of a connection before it is closed."}; -#endif +RemoteStore::Config::RemoteStoreConfig(const StoreReference::Params & params) + : Store::Config(params) + , RemoteStoreConfigT{ + CONFIG_ROW(maxConnections), + CONFIG_ROW(maxConnectionAge), + } +{ +} + /* TODO: Separate these store types into different files, give them better names */ -RemoteStore::RemoteStore(const Params & params) - : RemoteStoreConfig(params) - , Store(params) +RemoteStore::RemoteStore(const Config & config) + : RemoteStore::Config(config) + , Store(static_cast(*this)) , connections(make_ref>( std::max(1, (int) maxConnections), [this]() { @@ -134,7 +151,7 @@ void RemoteStore::setOptions(Connection & conn) << settings.useSubstitutes; if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { - std::map overrides; + std::map overrides; settings.getSettings(overrides, true); // libstore settings fileTransferSettings.getSettings(overrides, true); overrides.erase(settings.keepFailed.name); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 437c4aee734..126f060a02b 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -22,7 +22,6 @@ template class F> struct RemoteStoreConfigT { const F maxConnections; - const F maxConnectionAge; }; @@ -43,6 +42,8 @@ struct RemoteStoreConfig : * The other defaults depend on the choice of `storeDir` and `rootDir` */ static RemoteStoreConfigT defaults; + + RemoteStoreConfig(const StoreReference::Params &); }; /** diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 954a9746774..eb97360160f 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -7,19 +7,45 @@ #include "worker-protocol-impl.hh" #include "pool.hh" #include "ssh.hh" +#include "config-parse-impl.hh" +#include "store-registration.hh" namespace nix { +SSHStoreConfig::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , CommonSSHStoreConfig::Descriptions{CommonSSHStoreConfig::descriptions} + , SSHStoreConfigT{ + .remoteProgram{ + .name = "remote-program", + .description = "Path to the `nix-daemon` executable on the remote machine.", + }, + } +{} + + +const SSHStoreConfig::Descriptions SSHStoreConfig::descriptions{}; + + +decltype(SSHStoreConfig::defaults) SSHStoreConfig::defaults = { + .remoteProgram = {{"nix-daemon"}}, +}; + + SSHStoreConfig::SSHStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, authority, params) + const StoreReference::Params & params) + : Store::Config{params} + , RemoteStore::Config{params} + , CommonSSHStoreConfig{scheme, authority, params} + , SSHStoreConfigT{ + CONFIG_ROW(remoteProgram), + } { } + std::string SSHStoreConfig::doc() { return @@ -27,20 +53,19 @@ std::string SSHStoreConfig::doc() ; } -class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore +struct SSHStore : + public virtual SSHStoreConfig, + public virtual RemoteStore { -public: - - SSHStore( - std::string_view scheme, - std::string_view host, - const Params & params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(scheme, host, params) - , Store(params) - , RemoteStore(params) + using Config = SSHStoreConfig; + + SSHStore(const Config & config) + : Store::Config{config} + , RemoteStore::Config{config} + , CommonSSHStoreConfig{config} + , SSHStore::Config{config} + , Store{static_cast(*this)} + , RemoteStore{static_cast(*this)} , master(createSSHMaster( // Use SSH master only if using more than 1 connection. connections->capacity() > 1)) @@ -88,21 +113,15 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore }; -MountedSSHStoreConfig::MountedSSHStoreConfig(StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) -{ -} - -MountedSSHStoreConfig::MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) +MountedSSHStoreConfig::MountedSSHStoreConfig( + std::string_view scheme, + std::string_view host, + const StoreReference::Params & params) + : Store::Config{params} + , RemoteStore::Config{params} + , CommonSSHStoreConfig{scheme, host, params} + , SSHStoreConfig{scheme, host, params} + , LocalFSStoreConfig{params} { } @@ -128,24 +147,21 @@ std::string MountedSSHStoreConfig::doc() * The difference lies in how they manage GC roots. See addPermRoot * below for details. */ -class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSHStore, public virtual LocalFSStore +struct MountedSSHStore : virtual MountedSSHStoreConfig, virtual SSHStore, virtual LocalFSStore { -public: - - MountedSSHStore( - std::string_view scheme, - std::string_view host, - const Params & params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) - , MountedSSHStoreConfig(params) - , Store(params) - , RemoteStore(params) - , SSHStore(scheme, host, params) - , LocalFSStore(params) + using Config = MountedSSHStoreConfig; + + MountedSSHStore(const Config & config) + : Store::Config{config} + , RemoteStore::Config{config} + , CommonSSHStoreConfig{config} + , SSHStore::Config{config} + , LocalFSStore::Config{config} + , MountedSSHStore::Config{config} + , Store{static_cast(*this)} + , RemoteStore{static_cast(*this)} + , SSHStore{static_cast(*this)} + , LocalFSStore{static_cast(*this)} { extraRemoteProgramArgs = { "--process-ops", @@ -215,7 +231,7 @@ ref SSHStore::openConnection() return conn; } -static RegisterStoreImplementation regSSHStore; -static RegisterStoreImplementation regMountedSSHStore; +static RegisterStoreImplementation regSSHStore; +static RegisterStoreImplementation regMountedSSHStore; } diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh index 29a2a8b2c2d..c7ed62d7577 100644 --- a/src/libstore/ssh-store.hh +++ b/src/libstore/ssh-store.hh @@ -8,15 +8,26 @@ namespace nix { -struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig +template class F> +struct SSHStoreConfigT { - using CommonSSHStoreConfig::CommonSSHStoreConfig; - using RemoteStoreConfig::RemoteStoreConfig; + const F remoteProgram; +}; + +struct SSHStoreConfig : virtual RemoteStoreConfig, + virtual CommonSSHStoreConfig, + SSHStoreConfigT +{ + struct Descriptions : virtual CommonSSHStoreConfig::Descriptions, SSHStoreConfigT + { + Descriptions(); + }; - SSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); + static const Descriptions descriptions; - const Setting remoteProgram{ - this, {"nix-daemon"}, "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; + static SSHStoreConfigT defaults; + + SSHStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params); const std::string name() override { @@ -29,16 +40,20 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig } std::string doc() override; + + ref openStore() const override; }; -struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig +struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStore::Config { - using LocalFSStoreConfig::LocalFSStoreConfig; - using SSHStoreConfig::SSHStoreConfig; + struct Descriptions : virtual SSHStoreConfig::Descriptions, virtual LocalFSStore::Config::Descriptions + { + Descriptions(); + }; - MountedSSHStoreConfig(StringMap params); + static const Descriptions descriptions; - MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params); + MountedSSHStoreConfig(std::string_view scheme, std::string_view host, const StoreReference::Params & params); const std::string name() override { @@ -56,6 +71,8 @@ struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfi { return ExperimentalFeature::MountedSSHStore; } + + ref openStore() const override; }; } From 469ac00e91d3dc8ba07fdc989dba3c8a50bb309e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 16:06:12 -0400 Subject: [PATCH 36/80] Convert `UDSRemoteStore` --- src/libstore/uds-remote-store.cc | 34 ++++++++++++-------------------- src/libstore/uds-remote-store.hh | 30 ++++++++++++++++------------ 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 3c445eb1318..6f6dd70f6e2 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,6 +1,7 @@ #include "uds-remote-store.hh" #include "unix-domain-socket.hh" #include "worker-protocol.hh" +#include "store-registration.hh" #include #include @@ -20,13 +21,13 @@ namespace nix { UDSRemoteStoreConfig::UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params) + const StoreReference::Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) , RemoteStoreConfig(params) , path{authority.empty() ? settings.nixDaemonSocketFile : authority} { - if (scheme != UDSRemoteStoreConfig::scheme) { + if (uriSchemes().count(std::string{scheme}) == 0) { throw UsageError("Scheme must be 'unix'"); } } @@ -40,23 +41,14 @@ std::string UDSRemoteStoreConfig::doc() } -// A bit gross that we now pass empty string but this is knowing that -// empty string will later default to the same nixDaemonSocketFile. Why -// don't we just wire it all through? I believe there are cases where it -// will live reload so we want to continue to account for that. -UDSRemoteStore::UDSRemoteStore(const Params & params) - : UDSRemoteStore(scheme, "", params) -{} - - -UDSRemoteStore::UDSRemoteStore(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RemoteStoreConfig(params) - , UDSRemoteStoreConfig(scheme, authority, params) - , Store(params) - , LocalFSStore(params) - , RemoteStore(params) +UDSRemoteStore::UDSRemoteStore(const Config & config) + : Store::Config{config} + , LocalFSStore::Config{config} + , RemoteStore::Config{config} + , UDSRemoteStore::Config{config} + , Store(static_cast(*this)) + , LocalFSStore(static_cast(*this)) + , RemoteStore(static_cast(*this)) { } @@ -69,7 +61,7 @@ std::string UDSRemoteStore::getUri() // // unix:// with no path also works. Change what we return? "daemon" - : std::string(scheme) + "://" + path; + : std::string(*uriSchemes().begin()) + "://" + path; } @@ -106,6 +98,6 @@ void UDSRemoteStore::addIndirectRoot(const Path & path) } -static RegisterStoreImplementation regUDSRemoteStore; +static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 8d19ad5f5bc..69fbf6f60a1 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -8,13 +8,17 @@ namespace nix { struct UDSRemoteStoreConfig : - virtual LocalFSStoreConfig, - virtual RemoteStoreConfig + virtual LocalFSStore::Config, + virtual RemoteStore::Config { - // TODO(fzakaria): Delete this constructor once moved over to the factory pattern - // outlined in https://github.com/NixOS/nix/issues/10766 - using LocalFSStoreConfig::LocalFSStoreConfig; - using RemoteStoreConfig::RemoteStoreConfig; + struct Descriptions : + virtual LocalFSStore::Config::Descriptions, + virtual RemoteStore::Config::Descriptions + { + Descriptions(); + }; + + static const Descriptions descriptions; /** * @param authority is the socket path. @@ -22,7 +26,7 @@ struct UDSRemoteStoreConfig : UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params); + const StoreReference::Params & params); const std::string name() override { return "Local Daemon Store"; } @@ -36,12 +40,10 @@ struct UDSRemoteStoreConfig : */ Path path; -private: - static constexpr char const * scheme = "unix"; - -public: static std::set uriSchemes() - { return {scheme}; } + { return {"unix"}; } + + ref openStore() const override; }; struct UDSRemoteStore : @@ -49,7 +51,9 @@ struct UDSRemoteStore : virtual IndirectRootStore, virtual RemoteStore { - UDSRemoteStore(const UDSRemoteStoreConfig &); + using Config = UDSRemoteStoreConfig; + + UDSRemoteStore(const Config &); std::string getUri() override; From 2fcb196c8e34b00ec18abe8306db05990b374c1f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 16:38:31 -0400 Subject: [PATCH 37/80] Fix `StoreReference` --- src/libstore/store-reference.cc | 37 +++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc index 5e5d30fc117..40a2867ed31 100644 --- a/src/libstore/store-reference.cc +++ b/src/libstore/store-reference.cc @@ -35,20 +35,49 @@ std::string StoreReference::render() const }, variant); + StringMap params2; + for (auto & [k, v] : params) { + auto * p = v.get_ptr(); + // if it is a JSON string, just use that + + // FIXME: Ensure the literal string isn't itself valid JSON. If + // it is, we still need to dump to escape it. + params2.insert_or_assign(k, p ? *p : v.dump()); + } + if (!params.empty()) { res += "?"; - res += encodeQuery(params); + res += encodeQuery(params2); } return res; } +static StoreReference::Params decodeParamsJson(StringMap paramsRaw) +{ + StoreReference::Params params; + for (auto && [k, v] : std::move(paramsRaw)) { + nlohmann::json j; + try { + j = nlohmann::json::parse(v); + } catch (nlohmann::json::exception &) { + // if its not valid JSON, keep the literal string + j = std::move(v); + } + params.insert_or_assign(std::move(k), std::move(j)); + } + return params; +} + StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams) { auto params = extraParams; try { auto parsedUri = parseURL(uri); - params.insert(parsedUri.query.begin(), parsedUri.query.end()); + { + auto params2 = decodeParamsJson(std::move(parsedUri.query)); + params.insert(params2.begin(), params2.end()); + } auto baseURI = parsedUri.authority.value_or("") + parsedUri.path; @@ -106,13 +135,13 @@ StoreReference StoreReference::parse(const std::string & uri, const StoreReferen std::pair splitUriAndParams(const std::string & uri_) { auto uri(uri_); - StoreReference::Params params; + StringMap params; auto q = uri.find('?'); if (q != std::string::npos) { params = decodeQuery(uri.substr(q + 1)); uri = uri_.substr(0, q); } - return {uri, params}; + return {uri, decodeParamsJson(std::move(params))}; } } From 103c90b96714955a66732b3bbeb55feeac91e517 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 17:14:26 -0400 Subject: [PATCH 38/80] More progress --- src/libstore/local-overlay-store.cc | 73 ++++++++++++++++++---- src/libstore/local-overlay-store.hh | 96 +++++++++++------------------ 2 files changed, 98 insertions(+), 71 deletions(-) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index ec2c5f4e936..19eaa04c126 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -1,12 +1,63 @@ +#include + #include "local-overlay-store.hh" #include "callback.hh" #include "realisation.hh" #include "processes.hh" #include "url.hh" -#include +#include "store-open.hh" +#include "store-registration.hh" namespace nix { +#if 0 + const Setting lowerStoreUri{(StoreConfig*) this, "", "lower-store", + R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). + + Must be a store with a store dir on the file system. + Must be used as OverlayFS lower layer for this store's store dir. + )"}; + + const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer", + R"( + Directory containing the OverlayFS upper layer for this store's store dir. + )"}; + + Setting checkMount{(StoreConfig*) this, true, "check-mount", + R"( + Check that the overlay filesystem is correctly mounted. + + Nix does not manage the overlayfs mount point itself, but the correct + functioning of the overlay store does depend on this mount point being set up + correctly. Rather than just assume this is the case, check that the lowerdir + and upperdir options are what we expect them to be. This check is on by + default, but can be disabled if needed. + )"}; + + const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook", + R"( + Script or other executable to run when overlay filesystem needs remounting. + + This is occasionally necessary when deleting a store path that exists in both upper and lower layers. + In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly + is the only way to perform the deletion without creating a "whiteout". + However this causes the OverlayFS kernel data structures to get out-of-sync, + and can lead to 'stale file handle' errors; remounting solves the problem. + + The store directory is passed as an argument to the invoked executable. + )"}; + + : Store::Config{config} + , LocalFSStore::Configcpath, params) + , LocalStoreConfig(params) + , LocalOverlayStoreConfig(scheme, path, params) + , Store(params) + , LocalFSStore(params) + , LocalStore(params) +#endif + std::string LocalOverlayStoreConfig::doc() { return @@ -18,15 +69,15 @@ Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { return upperLayer + "/" + path.to_string(); } -LocalOverlayStore::LocalOverlayStore(std::string_view scheme, PathView path, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(path, params) - , LocalStoreConfig(params) - , LocalOverlayStoreConfig(scheme, path, params) - , Store(params) - , LocalFSStore(params) - , LocalStore(params) - , lowerStore(openStore(percentDecode(lowerStoreUri.get())).dynamic_pointer_cast()) +LocalOverlayStore::LocalOverlayStore(const Config & config) + : Store::Config{config} + , LocalFSStore::Config{config} + , LocalStore::Config{config} + , LocalOverlayStore::Config{config} + , Store{static_cast(*this)} + , LocalFSStore{static_cast(*this)} + , LocalStore{static_cast(*this)} + , lowerStore(nix::openStore(lowerStoreUri.get()).dynamic_pointer_cast()) { if (checkMount.get()) { std::smatch match; @@ -287,6 +338,6 @@ void LocalOverlayStore::remountIfNecessary() } -static RegisterStoreImplementation regLocalOverlayStore; +static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 63628abed50..927d006027f 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -2,59 +2,39 @@ namespace nix { + +template class F> +struct LocalOverlayStoreConfigT +{ + const F lowerStoreUri; + const F upperLayer; + const F checkMount; + const F remountHook; +}; + /** * Configuration for `LocalOverlayStore`. */ -struct LocalOverlayStoreConfig : virtual LocalStoreConfig +struct LocalOverlayStoreConfig : + virtual LocalStoreConfig, + LocalOverlayStoreConfigT { - LocalOverlayStoreConfig(const StringMap & params) - : LocalOverlayStoreConfig("local-overlay", "", params) - { } - - LocalOverlayStoreConfig(std::string_view scheme, PathView path, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(path, params) - , LocalStoreConfig(scheme, path, params) + struct Descriptions : + virtual Store::Config::Descriptions, + virtual LocalStore::Config::Descriptions, + LocalOverlayStoreConfigT { - } - - const Setting lowerStoreUri{(StoreConfig*) this, "", "lower-store", - R"( - [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) - for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). - - Must be a store with a store dir on the file system. - Must be used as OverlayFS lower layer for this store's store dir. - )"}; + Descriptions(); + }; - const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer", - R"( - Directory containing the OverlayFS upper layer for this store's store dir. - )"}; + static const Descriptions descriptions; - Setting checkMount{(StoreConfig*) this, true, "check-mount", - R"( - Check that the overlay filesystem is correctly mounted. + static LocalStoreConfigT defaults; - Nix does not manage the overlayfs mount point itself, but the correct - functioning of the overlay store does depend on this mount point being set up - correctly. Rather than just assume this is the case, check that the lowerdir - and upperdir options are what we expect them to be. This check is on by - default, but can be disabled if needed. - )"}; - - const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook", - R"( - Script or other executable to run when overlay filesystem needs remounting. - - This is occasionally necessary when deleting a store path that exists in both upper and lower layers. - In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly - is the only way to perform the deletion without creating a "whiteout". - However this causes the OverlayFS kernel data structures to get out-of-sync, - and can lead to 'stale file handle' errors; remounting solves the problem. - - The store directory is passed as an argument to the invoked executable. - )"}; + LocalOverlayStoreConfig( + std::string_view scheme, + PathView path, + const StoreReference::Params & params); const std::string name() override { return "Experimental Local Overlay Store"; } @@ -88,8 +68,18 @@ protected: * Documentation on overridden methods states how they differ from their * `LocalStore` counterparts. */ -class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual LocalStore +struct LocalOverlayStore : virtual LocalOverlayStoreConfig, virtual LocalStore { + using Config = LocalOverlayStoreConfig; + + LocalOverlayStore(const Config &); + + std::string getUri() override + { + return "local-overlay://"; + } + +private: /** * The store beneath us. * @@ -99,20 +89,6 @@ class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual */ ref lowerStore; -public: - LocalOverlayStore(const Params & params) - : LocalOverlayStore("local-overlay", "", params) - { - } - - LocalOverlayStore(std::string_view scheme, PathView path, const Params & params); - - std::string getUri() override - { - return "local-overlay://"; - } - -private: /** * First copy up any lower store realisation with the same key, so we * merge rather than mask it. From 3a149b521dee082a085d3afc077590bb88143d37 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 17:32:31 -0400 Subject: [PATCH 39/80] Getting close! --- src/libstore/store-open.hh | 9 ++++- src/libstore/store-registration.cc | 40 ++++++++++++------- src/libstore/store-registration.hh | 6 +-- .../functional/test-libstoreconsumer/main.cc | 2 +- tests/unit/libstore/ssh-store.cc | 4 +- 5 files changed, 38 insertions(+), 23 deletions(-) diff --git a/src/libstore/store-open.hh b/src/libstore/store-open.hh index ab6f3cb7739..c074b846bcc 100644 --- a/src/libstore/store-open.hh +++ b/src/libstore/store-open.hh @@ -15,6 +15,11 @@ namespace nix { +/** + * @return The store config denoted `uri` (slight misnomer...). + */ +ref resolveStoreConfig(StoreReference && storeURI); + /** * @return a Store object to access the Nix store denoted by * ‘uri’ (slight misnomer...). @@ -22,8 +27,8 @@ namespace nix { ref openStore(StoreReference && storeURI); /** - * Opens the store at `uri`, where `uri` is in the format expected by `StoreReference::parse` - + * Opens the store at `uri`, where `uri` is in the format expected by + * `StoreReference::parse` */ ref openStore( const std::string & uri = settings.storeUri.get(), diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc index 85ece9623cf..3b23debe9b8 100644 --- a/src/libstore/store-registration.cc +++ b/src/libstore/store-registration.cc @@ -2,26 +2,40 @@ #include "store-open.hh" #include "local-store.hh" #include "uds-remote-store.hh" +#include "json-utils.hh" namespace nix { -ref openStore(const std::string & uri, const Store::Params & extraParams) +ref openStore(const std::string & uri, const StoreReference::Params & extraParams) { return openStore(StoreReference::parse(uri, extraParams)); } ref openStore(StoreReference && storeURI) +{ + auto store = resolveStoreConfig(std::move(storeURI))->openStore(); + + experimentalFeatureSettings.require(store->experimentalFeature()); +#if 0 // FIXME + store->warnUnknownSettings(); + store->init(); +#endif + + return store; +} + +ref resolveStoreConfig(StoreReference && storeURI) { auto & params = storeURI.params; - auto store = std::visit( + auto storeConfig = std::visit( overloaded{ - [&](const StoreReference::Auto &) -> std::shared_ptr { - auto stateDir = getOr(params, "state", settings.nixStateDir); + [&](const StoreReference::Auto &) -> ref { + auto stateDir = getString(getOr(params, "state", settings.nixStateDir)); if (access(stateDir.c_str(), R_OK | W_OK) == 0) - return std::make_shared(params); + return make_ref(params); else if (pathExists(settings.nixDaemonSocketFile)) - return std::make_shared(params); + return make_ref("unix", "", params); #if __linux__ else if ( !pathExists(stateDir) && params.empty() && !isRootUser() && !getEnv("NIX_STORE_DIR").has_value() @@ -34,32 +48,28 @@ ref openStore(StoreReference && storeURI) try { createDirs(chrootStore); } catch (SystemError & e) { - return std::make_shared(params); + return make_ref(params); } warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); } else debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); - return std::make_shared("local", chrootStore, params); + return make_ref("local", chrootStore, params); } #endif else - return std::make_shared(params); + return make_ref(params); }, [&](const StoreReference::Specified & g) { for (auto implem : *Implementations::registered) if (implem.uriSchemes.count(g.scheme)) - return implem.create(g.scheme, g.authority, params); + return implem.parseConfig(g.scheme, g.authority, params); throw Error("don't know how to open Nix store with scheme '%s'", g.scheme); }, }, storeURI.variant); - experimentalFeatureSettings.require(store->experimentalFeature()); - store->warnUnknownSettings(); - store->init(); - - return ref{store}; + return storeConfig; } std::vector * Implementations::registered = 0; diff --git a/src/libstore/store-registration.hh b/src/libstore/store-registration.hh index 8d57f4a587a..dd61a45f1ae 100644 --- a/src/libstore/store-registration.hh +++ b/src/libstore/store-registration.hh @@ -18,7 +18,7 @@ struct StoreFactory * The `authorityPath` parameter is `/`, or really * whatever comes after `://` and before `?`. */ - std::function( + std::function( std::string_view scheme, std::string_view authorityPath, const StoreReference::Params & params)> parseConfig; const Store::Config::Descriptions & configDescriptions; @@ -35,8 +35,8 @@ struct Implementations registered = new std::vector(); StoreFactory factory{ .uriSchemes = T::Config::uriSchemes(), - .parseConfig = ([](auto scheme, auto uri, auto & params) -> std::shared_ptr { - return std::make_shared(scheme, uri, params); + .parseConfig = ([](auto scheme, auto uri, auto & params) -> ref { + return make_ref(scheme, uri, params); }), .configDescriptions = T::Config::descriptions, }; diff --git a/tests/functional/test-libstoreconsumer/main.cc b/tests/functional/test-libstoreconsumer/main.cc index c61489af69a..00763038e9f 100644 --- a/tests/functional/test-libstoreconsumer/main.cc +++ b/tests/functional/test-libstoreconsumer/main.cc @@ -1,5 +1,5 @@ #include "globals.hh" -#include "store-api.hh" +#include "store-open.hh" #include "build-result.hh" #include diff --git a/tests/unit/libstore/ssh-store.cc b/tests/unit/libstore/ssh-store.cc index a1fcef6b863..dad20025873 100644 --- a/tests/unit/libstore/ssh-store.cc +++ b/tests/unit/libstore/ssh-store.cc @@ -9,7 +9,7 @@ TEST(SSHStore, constructConfig) SSHStoreConfig config{ "ssh", "localhost", - StoreConfig::Params{ + StoreReference::Params{ { "remote-program", // TODO #11106, no more split on space @@ -31,7 +31,7 @@ TEST(MountedSSHStore, constructConfig) MountedSSHStoreConfig config{ "mounted-ssh", "localhost", - StoreConfig::Params{ + StoreReference::Params{ { "remote-program", // TODO #11106, no more split on space From 68ae4dc3fd9c2fa4b2e8f766e9b9b178599b703c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 18:57:41 -0400 Subject: [PATCH 40/80] It builds, debugging --- src/libstore/binary-cache-store.cc | 8 +- src/libstore/http-binary-cache-store.cc | 6 +- src/libstore/legacy-ssh-store.cc | 7 +- src/libstore/legacy-ssh-store.hh | 5 +- src/libstore/local-binary-cache-store.cc | 4 + src/libstore/local-overlay-store.cc | 124 ++++++++++++++--------- src/libstore/local-overlay-store.hh | 2 +- src/libstore/local-store.hh | 4 +- src/libstore/s3-binary-cache-store.cc | 7 +- src/libstore/ssh-store.cc | 21 ++++ src/libstore/store-api.cc | 2 + src/libstore/store-api.hh | 16 ++- src/libstore/store-reference.cc | 10 +- src/libstore/store-registration.cc | 2 +- src/libstore/uds-remote-store.cc | 15 +++ src/libstore/uds-remote-store.hh | 4 + tests/unit/libstore/legacy-ssh-store.cc | 2 +- 17 files changed, 174 insertions(+), 65 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 4adc599007e..08367595842 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -101,6 +101,8 @@ BinaryCacheStore::BinaryCacheStore(const Config & config) StringSink sink; sink << narVersionMagic1; narMagic = sink.s; + + init(); } void BinaryCacheStore::init() @@ -121,9 +123,11 @@ void BinaryCacheStore::init() throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'", getUri(), value, storeDir); } else if (name == "WantMassQuery") { - wantMassQuery.setDefault(value == "1"); + if (defaultWantMassQuery) + wantMassQuery.value = value == "1"; } else if (name == "Priority") { - priority.setDefault(std::stoi(value)); + if (defaultPriority) + priority.value = std::stoi(value); } } } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index accd84899f1..8d9a9d96335 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -71,8 +71,10 @@ class HttpBinaryCacheStore : { // FIXME: do this lazily? if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) { - wantMassQuery.setDefault(cacheInfo->wantMassQuery); - priority.setDefault(cacheInfo->priority); + if (defaultWantMassQuery) + wantMassQuery.value = cacheInfo->wantMassQuery; + if (defaultPriority) + priority.value = cacheInfo->priority; } else { try { BinaryCacheStore::init(); diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 10301b79e66..d95a155f5c4 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -345,12 +345,17 @@ unsigned int LegacySSHStore::getProtocol() * The legacy ssh protocol doesn't support checking for trusted-user. * Try using ssh-ng:// instead if you want to know. */ -std::optional isTrustedClient() +std::optional LegacySSHStore::isTrustedClient() { return std::nullopt; } +ref LegacySSHStore::Config::openStore() const { + return make_ref(*this); +} + + static RegisterStoreImplementation regLegacySSHStore; } diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index 9d2234faaf7..779798390d8 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -137,10 +137,7 @@ public: * The legacy ssh protocol doesn't support checking for trusted-user. * Try using ssh-ng:// instead if you want to know. */ - std::optional isTrustedClient() override - { - return std::nullopt; - } + std::optional isTrustedClient() override; void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept override diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 8fb163e3064..c00a34d2bce 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -125,6 +125,10 @@ std::set LocalBinaryCacheStoreConfig::uriSchemes() return {"file"}; } +ref LocalBinaryCacheStoreConfig::openStore() const { + return make_ref(*this); +} + static RegisterStoreImplementation regLocalBinaryCacheStore; } diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 19eaa04c126..ade81b20a9f 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -7,56 +7,86 @@ #include "url.hh" #include "store-open.hh" #include "store-registration.hh" +#include "config-parse-impl.hh" namespace nix { -#if 0 - const Setting lowerStoreUri{(StoreConfig*) this, "", "lower-store", - R"( - [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) - for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). - - Must be a store with a store dir on the file system. - Must be used as OverlayFS lower layer for this store's store dir. - )"}; - - const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer", - R"( - Directory containing the OverlayFS upper layer for this store's store dir. - )"}; - - Setting checkMount{(StoreConfig*) this, true, "check-mount", - R"( - Check that the overlay filesystem is correctly mounted. - - Nix does not manage the overlayfs mount point itself, but the correct - functioning of the overlay store does depend on this mount point being set up - correctly. Rather than just assume this is the case, check that the lowerdir - and upperdir options are what we expect them to be. This check is on by - default, but can be disabled if needed. - )"}; - - const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook", - R"( - Script or other executable to run when overlay filesystem needs remounting. - - This is occasionally necessary when deleting a store path that exists in both upper and lower layers. - In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly - is the only way to perform the deletion without creating a "whiteout". - However this causes the OverlayFS kernel data structures to get out-of-sync, - and can lead to 'stale file handle' errors; remounting solves the problem. - - The store directory is passed as an argument to the invoked executable. - )"}; - - : Store::Config{config} - , LocalFSStore::Configcpath, params) - , LocalStoreConfig(params) - , LocalOverlayStoreConfig(scheme, path, params) - , Store(params) - , LocalFSStore(params) - , LocalStore(params) -#endif +LocalOverlayStore::Config::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , LocalFSStore::Config::Descriptions{LocalFSStore::Config::descriptions} + , LocalStore::Config::Descriptions{LocalStore::Config::descriptions} + , LocalOverlayStoreConfigT{ + .lowerStoreUri{ + .name = "lower-store", + .description = R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). + + Must be a store with a store dir on the file system. + Must be used as OverlayFS lower layer for this store's store dir. + )", + }, + .upperLayer{ + .name = "upper-layer", + .description = R"( + Directory containing the OverlayFS upper layer for this store's store dir. + )", + }, + .checkMount{ + .name = "check-mount", + .description = R"( + Check that the overlay filesystem is correctly mounted. + + Nix does not manage the overlayfs mount point itself, but the correct + functioning of the overlay store does depend on this mount point being set up + correctly. Rather than just assume this is the case, check that the lowerdir + and upperdir options are what we expect them to be. This check is on by + default, but can be disabled if needed. + )", + }, + .remountHook{ + .name = "remount-hook", + .description = R"( + Script or other executable to run when overlay filesystem needs remounting. + + This is occasionally necessary when deleting a store path that exists in both upper and lower layers. + In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly + is the only way to perform the deletion without creating a "whiteout". + However this causes the OverlayFS kernel data structures to get out-of-sync, + and can lead to 'stale file handle' errors; remounting solves the problem. + + The store directory is passed as an argument to the invoked executable. + )", + }, + } +{} + + +const LocalOverlayStore::Config::Descriptions LocalOverlayStore::Config::descriptions{}; + + +decltype(LocalOverlayStore::Config::defaults) LocalOverlayStore::Config::defaults = { + .lowerStoreUri{""}, + .upperLayer{""}, + .checkMount{true}, + .remountHook{""}, +}; + +LocalOverlayStore::Config::LocalOverlayStoreConfig( + std::string_view scheme, + std::string_view authority, + const StoreReference::Params & params) + : Store::Config(params) + , LocalFSStore::Config(authority, params) + , LocalStore::Config(scheme, authority, params) + , LocalOverlayStoreConfigT{ + CONFIG_ROW(lowerStoreUri), + CONFIG_ROW(upperLayer), + CONFIG_ROW(checkMount), + CONFIG_ROW(remountHook), + } +{ +} std::string LocalOverlayStoreConfig::doc() { diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 927d006027f..4cf898b8fc7 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -29,7 +29,7 @@ struct LocalOverlayStoreConfig : static const Descriptions descriptions; - static LocalStoreConfigT defaults; + static LocalOverlayStoreConfigT defaults; LocalOverlayStoreConfig( std::string_view scheme, diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 3d5aa1d3dbd..ca483df1584 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -58,7 +58,9 @@ struct LocalStoreConfig : static LocalStoreConfigT defaults; - LocalStoreConfig(const StoreReference::Params &); + LocalStoreConfig(const StoreReference::Params & params) + : LocalStoreConfig{"local", "", params} + {} LocalStoreConfig( std::string_view scheme, diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 6b9a809af96..7f8ce3a7bdd 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -290,6 +290,7 @@ S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig( CONFIG_ROW(multipartUpload), CONFIG_ROW(bufferSize), } + , bucketName{authority} { if (bucketName.empty()) throw UsageError("`%s` store requires a bucket name in its Store URI", scheme); @@ -334,8 +335,10 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual void init() override { if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) { - wantMassQuery.setDefault(cacheInfo->wantMassQuery); - priority.setDefault(cacheInfo->priority); + if (defaultWantMassQuery) + wantMassQuery.value = cacheInfo->wantMassQuery; + if (defaultPriority) + priority.value = cacheInfo->priority; } else { BinaryCacheStore::init(); diskCache->createCache(getUri(), storeDir, wantMassQuery, priority); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index eb97360160f..0e1af43a131 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -113,6 +113,21 @@ struct SSHStore : }; +ref SSHStore::Config::openStore() const { + return make_ref(*this); +} + + +MountedSSHStoreConfig::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , CommonSSHStoreConfig::Descriptions{CommonSSHStoreConfig::descriptions} + , SSHStore::Config::Descriptions{SSHStoreConfig::descriptions} +{} + + +const MountedSSHStoreConfig::Descriptions MountedSSHStoreConfig::descriptions{}; + + MountedSSHStoreConfig::MountedSSHStoreConfig( std::string_view scheme, std::string_view host, @@ -214,6 +229,12 @@ struct MountedSSHStore : virtual MountedSSHStoreConfig, virtual SSHStore, virtua } }; + +ref MountedSSHStore::Config::openStore() const { + return make_ref(*this); +} + + ref SSHStore::openConnection() { auto conn = make_ref(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index b67faf38a5e..f51e7c27cf9 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -256,6 +256,8 @@ Store::Config::StoreConfig(const StoreReference::Params & params) CONFIG_ROW(wantMassQuery), CONFIG_ROW(systemFeatures), } + , defaultPriority{params.count(descriptions.priority.name) == 0} + , defaultWantMassQuery{params.count(descriptions.wantMassQuery.name) == 0} { } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 56fef0f1a7b..e2151ef5b42 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -103,7 +103,7 @@ struct StoreConfigT const F isTrusted; F priority; F wantMassQuery; - F systemFeatures; + const F systemFeatures; }; StoreConfigT parseStoreConfig(const StoreReference::Params &); @@ -155,6 +155,20 @@ struct StoreConfig : * type. */ virtual ref openStore() const = 0; + +protected: + + /** + * So caches themselves (not our config) can update default + * settings, but aren't allowed to update settings specied by the + * client (i.e. us). + */ + bool defaultPriority; + + /** + * @see defaultPriority + */ + bool defaultWantMassQuery; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc index 40a2867ed31..7be513f4fcc 100644 --- a/src/libstore/store-reference.cc +++ b/src/libstore/store-reference.cc @@ -61,8 +61,14 @@ static StoreReference::Params decodeParamsJson(StringMap paramsRaw) try { j = nlohmann::json::parse(v); } catch (nlohmann::json::exception &) { - // if its not valid JSON, keep the literal string - j = std::move(v); + // if its not valid JSON... + if (k == "remoteProgram" || k == "systemFeatures") { + // Back compat hack! Split and take that array + j = tokenizeString>(v); + } else { + // ...keep the literal string. + j = std::move(v); + } } params.insert_or_assign(std::move(k), std::move(j)); } diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc index 3b23debe9b8..1f3edfc71c1 100644 --- a/src/libstore/store-registration.cc +++ b/src/libstore/store-registration.cc @@ -35,7 +35,7 @@ ref resolveStoreConfig(StoreReference && storeURI) if (access(stateDir.c_str(), R_OK | W_OK) == 0) return make_ref(params); else if (pathExists(settings.nixDaemonSocketFile)) - return make_ref("unix", "", params); + return make_ref(params); #if __linux__ else if ( !pathExists(stateDir) && params.empty() && !isRootUser() && !getEnv("NIX_STORE_DIR").has_value() diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 6f6dd70f6e2..8604961fbe5 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -18,6 +18,16 @@ namespace nix { +UDSRemoteStoreConfig::Descriptions::Descriptions() + : Store::Config::Descriptions{Store::Config::descriptions} + , LocalFSStore::Config::Descriptions{LocalFSStore::Config::descriptions} + , RemoteStore::Config::Descriptions{RemoteStore::Config::descriptions} +{} + + +const UDSRemoteStoreConfig::Descriptions UDSRemoteStoreConfig::descriptions{}; + + UDSRemoteStoreConfig::UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, @@ -98,6 +108,11 @@ void UDSRemoteStore::addIndirectRoot(const Path & path) } +ref UDSRemoteStore::Config::openStore() const { + return make_ref(*this); +} + + static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 69fbf6f60a1..4af4eac624e 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -20,6 +20,10 @@ struct UDSRemoteStoreConfig : static const Descriptions descriptions; + UDSRemoteStoreConfig(const StoreReference::Params & params) + : UDSRemoteStoreConfig{"unix", "", params} + {} + /** * @param authority is the socket path. */ diff --git a/tests/unit/libstore/legacy-ssh-store.cc b/tests/unit/libstore/legacy-ssh-store.cc index eb31a240804..4d0510cc656 100644 --- a/tests/unit/libstore/legacy-ssh-store.cc +++ b/tests/unit/libstore/legacy-ssh-store.cc @@ -9,7 +9,7 @@ TEST(LegacySSHStore, constructConfig) LegacySSHStoreConfig config{ "ssh", "localhost", - StoreConfig::Params{ + StoreReference::Params{ { "remote-program", // TODO #11106, no more split on space From 387767b9e944653c8c1439b4c25c78b02202842e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 19:07:45 -0400 Subject: [PATCH 41/80] Fix some bugs --- src/libstore/local-fs-store.cc | 2 +- src/libstore/ssh-store.hh | 4 +--- tests/unit/libstore/legacy-ssh-store.cc | 6 ++++-- tests/unit/libstore/ssh-store.cc | 12 ++++++++---- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 12035c1efe7..f30abe88d22 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -38,7 +38,7 @@ LocalFSStoreConfigT LocalFSStore::Config::defaults( const std::optional rootDir) { return { - .rootDir = {std::nullopt}, + .rootDir = {rootDir}, .stateDir = {rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir}, .logDir = {rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir}, .realStoreDir = {rootDir ? *rootDir + "/nix/store" : storeConfig.storeDir}, diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh index c7ed62d7577..6aa9c9cdf09 100644 --- a/src/libstore/ssh-store.hh +++ b/src/libstore/ssh-store.hh @@ -14,9 +14,7 @@ struct SSHStoreConfigT const F remoteProgram; }; -struct SSHStoreConfig : virtual RemoteStoreConfig, - virtual CommonSSHStoreConfig, - SSHStoreConfigT +struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig, SSHStoreConfigT { struct Descriptions : virtual CommonSSHStoreConfig::Descriptions, SSHStoreConfigT { diff --git a/tests/unit/libstore/legacy-ssh-store.cc b/tests/unit/libstore/legacy-ssh-store.cc index 4d0510cc656..ba877d10318 100644 --- a/tests/unit/libstore/legacy-ssh-store.cc +++ b/tests/unit/libstore/legacy-ssh-store.cc @@ -12,8 +12,10 @@ TEST(LegacySSHStore, constructConfig) StoreReference::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, }, }}; EXPECT_EQ( diff --git a/tests/unit/libstore/ssh-store.cc b/tests/unit/libstore/ssh-store.cc index dad20025873..0e2651f2e9b 100644 --- a/tests/unit/libstore/ssh-store.cc +++ b/tests/unit/libstore/ssh-store.cc @@ -12,8 +12,10 @@ TEST(SSHStore, constructConfig) StoreReference::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, }, }, }; @@ -34,8 +36,10 @@ TEST(MountedSSHStore, constructConfig) StoreReference::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, }, }, }; From b53a4a0188b338c809f8f95b50f408657b607f29 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 21:57:35 -0400 Subject: [PATCH 42/80] More tests --- src/libstore/dummy-store.cc | 48 +++++++++--------------------- tests/unit/libstore/dummy-store.cc | 20 +++++++++++++ 2 files changed, 34 insertions(+), 34 deletions(-) create mode 100644 tests/unit/libstore/dummy-store.cc diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 3d2931cd9ba..e8934ec9a19 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -1,43 +1,23 @@ -#include "store-api.hh" +#include "dummy-store.hh" #include "store-registration.hh" #include "callback.hh" namespace nix { -struct DummyStoreConfig : virtual StoreConfig { - using StoreConfig::StoreConfig; - - struct Descriptions : virtual Store::Config::Descriptions - { - Descriptions() - : StoreConfig::Descriptions{Store::Config::descriptions} - {} - }; - - static const Descriptions descriptions; - - DummyStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params) - : StoreConfig{params} - { - if (!authority.empty()) - throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); - } - - const std::string name() override { return "Dummy Store"; } - - std::string doc() override - { - return - #include "dummy-store.md" - ; - } - - static std::set uriSchemes() { - return {"dummy"}; - } +DummyStoreConfig::DummyStoreConfig( + std::string_view scheme, std::string_view authority, const StoreReference::Params & params) + : StoreConfig{params} +{ + if (!authority.empty()) + throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); +} - ref openStore() const override; -}; +std::string DummyStoreConfig::doc() +{ + return + #include "dummy-store.md" + ; +} const DummyStoreConfig::Descriptions DummyStoreConfig::descriptions{}; diff --git a/tests/unit/libstore/dummy-store.cc b/tests/unit/libstore/dummy-store.cc new file mode 100644 index 00000000000..df005da61ca --- /dev/null +++ b/tests/unit/libstore/dummy-store.cc @@ -0,0 +1,20 @@ +#include + +#include "dummy-store.hh" +#include "globals.hh" + +namespace nix { + +TEST(DummyStore, constructConfig) +{ + DummyStoreConfig config{"dummy", "", {}}; + + EXPECT_EQ(config.storeDir, settings.nixStore); +} + +TEST(DummyStore, constructConfigNoAuthority) +{ + EXPECT_THROW(DummyStoreConfig("dummy", "not-allowed", {}), UsageError); +} + +} // namespace nix From 34f9b13ec2282e134b8aa1286b934c83a8631a86 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 22:50:36 -0400 Subject: [PATCH 43/80] Inline defaults This helps avoid reading globals before they are initialized --- src/libstore/binary-cache-store.cc | 22 ++++++------------ src/libstore/binary-cache-store.hh | 2 -- src/libstore/common-ssh-store-config.cc | 15 ++++--------- src/libstore/common-ssh-store-config.hh | 2 -- src/libstore/config-parse-impl.hh | 6 ++--- src/libstore/config-parse.hh | 2 +- src/libstore/legacy-ssh-store.cc | 10 ++------- src/libstore/legacy-ssh-store.hh | 2 -- src/libstore/local-fs-store.cc | 22 ++++-------------- src/libstore/local-fs-store.hh | 7 ------ src/libstore/local-overlay-store.cc | 18 ++++++--------- src/libstore/local-overlay-store.hh | 2 -- src/libstore/local-store.cc | 9 ++------ src/libstore/local-store.hh | 2 -- src/libstore/remote-store.cc | 10 ++------- src/libstore/remote-store.hh | 5 ----- src/libstore/s3-binary-cache-store.cc | 30 ++++++++----------------- src/libstore/s3-binary-cache-store.hh | 2 -- src/libstore/ssh-store.cc | 7 +----- src/libstore/ssh-store.hh | 2 -- src/libstore/store-api.cc | 19 +++++----------- src/libstore/store-api.hh | 2 -- src/libstore/store-dir-config.cc | 6 +---- src/libstore/store-dir-config.hh | 2 -- 24 files changed, 48 insertions(+), 158 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 08367595842..1bd9363a272 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -68,24 +68,16 @@ BinaryCacheStore::Config::Descriptions::Descriptions() const BinaryCacheStore::Config::Descriptions BinaryCacheStore::Config::descriptions{}; -decltype(BinaryCacheStore::Config::defaults) BinaryCacheStore::Config::defaults = { - .compression = {"xz"}, - .writeNARListing{false}, - .writeDebugInfo{false}, - .secretKeyFile{""}, - .localNarCache{""}, - .parallelCompression{false}, - .compressionLevel{-1}, -}; - BinaryCacheStore::Config::BinaryCacheStoreConfig(const StoreReference::Params & params) : StoreConfig{params} , BinaryCacheStoreConfigT{ - CONFIG_ROW(compression), - CONFIG_ROW(secretKeyFile), - CONFIG_ROW(localNarCache), - CONFIG_ROW(parallelCompression), - CONFIG_ROW(compressionLevel), + CONFIG_ROW(compression, "xz"), + CONFIG_ROW(writeNARListing, false), + CONFIG_ROW(writeDebugInfo, false), + CONFIG_ROW(secretKeyFile, ""), + CONFIG_ROW(localNarCache, ""), + CONFIG_ROW(parallelCompression, false), + CONFIG_ROW(compressionLevel, -1), } { } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 27d7a845ab7..877a5c3343c 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -38,8 +38,6 @@ struct BinaryCacheStoreConfig : static const Descriptions descriptions; - static BinaryCacheStoreConfigT defaults; - BinaryCacheStoreConfig(const StoreReference::Params &); }; diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index a2d7e2ebad6..113ba6b8b4e 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -35,13 +35,6 @@ CommonSSHStoreConfig::Descriptions::Descriptions() const CommonSSHStoreConfig::Descriptions CommonSSHStoreConfig::descriptions{}; -decltype(CommonSSHStoreConfig::defaults) CommonSSHStoreConfig::defaults = { - .sshKey{""}, - .sshPublicHostKey{""}, - .compress{false}, - .remoteStore{""}, -}; - static std::string extractConnStr(std::string_view scheme, std::string_view _connStr) { if (_connStr.empty()) @@ -65,10 +58,10 @@ CommonSSHStoreConfig::CommonSSHStoreConfig( const StoreReference::Params & params) : Store::Config(params) , CommonSSHStoreConfigT{ - CONFIG_ROW(sshKey), - CONFIG_ROW(sshPublicHostKey), - CONFIG_ROW(compress), - CONFIG_ROW(remoteStore), + CONFIG_ROW(sshKey, ""), + CONFIG_ROW(sshPublicHostKey, ""), + CONFIG_ROW(compress, false), + CONFIG_ROW(remoteStore, ""), } , host(extractConnStr(scheme, host)) { diff --git a/src/libstore/common-ssh-store-config.hh b/src/libstore/common-ssh-store-config.hh index 4def51da0e5..e7bb16367ff 100644 --- a/src/libstore/common-ssh-store-config.hh +++ b/src/libstore/common-ssh-store-config.hh @@ -29,8 +29,6 @@ struct CommonSSHStoreConfig : static const Descriptions descriptions; - static CommonSSHStoreConfigT defaults; - /** * @param scheme Note this isn't stored by this mix-in class, but * just used for better error messages. diff --git a/src/libstore/config-parse-impl.hh b/src/libstore/config-parse-impl.hh index ebb65305ae3..ed640019dbe 100644 --- a/src/libstore/config-parse-impl.hh +++ b/src/libstore/config-parse-impl.hh @@ -9,16 +9,16 @@ namespace nix::config { template -std::optional> SettingInfo::parseConfig(const nlohmann::json::object_t & map) const +std::optional SettingInfo::parseConfig(const nlohmann::json::object_t & map) const { auto * p = get(map, name); - return p ? (JustValue{.value = p->get()}) : (std::optional>{}); + return p ? p->get() : (std::optional{}); } /** * Look up the setting's name in a map, falling back on the default if * it does not exist. */ -#define CONFIG_ROW(FIELD) .FIELD = descriptions.FIELD.parseConfig(params).value_or(defaults.FIELD) +#define CONFIG_ROW(FIELD, DEFAULT) .FIELD = {.value = descriptions.FIELD.parseConfig(params).value_or(DEFAULT)} } diff --git a/src/libstore/config-parse.hh b/src/libstore/config-parse.hh index 5d17f9f8d6a..e016caf15d9 100644 --- a/src/libstore/config-parse.hh +++ b/src/libstore/config-parse.hh @@ -15,7 +15,7 @@ struct SettingInfo std::string description; bool documentDefault = true; - std::optional> parseConfig(const nlohmann::json::object_t & map) const; + std::optional parseConfig(const nlohmann::json::object_t & map) const; }; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index d95a155f5c4..0fc5181ff6d 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -36,12 +36,6 @@ LegacySSHStore::Config::Descriptions::Descriptions() const LegacySSHStore::Config::Descriptions LegacySSHStore::Config::descriptions{}; -decltype(LegacySSHStore::Config::defaults) LegacySSHStore::Config::defaults = { - .remoteProgram = {{"nix-store"}}, - .maxConnections = {1}, -}; - - LegacySSHStore::Config::LegacySSHStoreConfig( std::string_view scheme, std::string_view authority, @@ -49,8 +43,8 @@ LegacySSHStore::Config::LegacySSHStoreConfig( : Store::Config{params} , CommonSSHStoreConfig{scheme, authority, params} , LegacySSHStoreConfigT{ - CONFIG_ROW(remoteProgram), - CONFIG_ROW(maxConnections), + CONFIG_ROW(remoteProgram, Strings{"nix-store"}), + CONFIG_ROW(maxConnections, 1), } { #ifndef _WIN32 diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index 779798390d8..12ca9f12514 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -29,8 +29,6 @@ struct LegacySSHStoreConfig : static const Descriptions descriptions; - static LegacySSHStoreConfigT defaults; - /** * Hack for getting remote build log output. Intentionally not a * documented user-visible setting. diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index f30abe88d22..bedc0aca94e 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -33,18 +33,6 @@ LocalFSStore::Config::Descriptions::Descriptions() const LocalFSStore::Config::Descriptions LocalFSStore::Config::descriptions{}; -LocalFSStoreConfigT LocalFSStore::Config::defaults( - const Store::Config & storeConfig, - const std::optional rootDir) -{ - return { - .rootDir = {rootDir}, - .stateDir = {rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir}, - .logDir = {rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir}, - .realStoreDir = {rootDir ? *rootDir + "/nix/store" : storeConfig.storeDir}, - }; -} - /** * @param rootDir Fallback if not in `params` */ @@ -58,13 +46,11 @@ auto localFSStoreConfig( auto rootDir = descriptions.rootDir.parseConfig(params) .value_or(config::JustValue{.value = std::move(_rootDir)}); - auto defaults = LocalFSStore::Config::defaults(storeConfig, rootDir.value); - return LocalFSStoreConfigT{ - CONFIG_ROW(rootDir), - CONFIG_ROW(stateDir), - CONFIG_ROW(logDir), - CONFIG_ROW(realStoreDir), + .rootDir = std::move(rootDir), + CONFIG_ROW(stateDir, rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir), + CONFIG_ROW(logDir, rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir), + CONFIG_ROW(realStoreDir, rootDir ? *rootDir + "/nix/store" : storeConfig.storeDir), }; } diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 051e4494012..678e93ec0a4 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -29,13 +29,6 @@ struct LocalFSStoreConfig : static const Descriptions descriptions; - /** - * The other defaults depend on the choice of `storeDir` and `rootDir` - */ - static LocalFSStoreConfigT defaults( - const Store::Config &, - const std::optional rootDir); - LocalFSStoreConfig(const StoreReference::Params &); /** diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index ade81b20a9f..7a483304841 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -65,13 +65,6 @@ LocalOverlayStore::Config::Descriptions::Descriptions() const LocalOverlayStore::Config::Descriptions LocalOverlayStore::Config::descriptions{}; -decltype(LocalOverlayStore::Config::defaults) LocalOverlayStore::Config::defaults = { - .lowerStoreUri{""}, - .upperLayer{""}, - .checkMount{true}, - .remountHook{""}, -}; - LocalOverlayStore::Config::LocalOverlayStoreConfig( std::string_view scheme, std::string_view authority, @@ -80,14 +73,15 @@ LocalOverlayStore::Config::LocalOverlayStoreConfig( , LocalFSStore::Config(authority, params) , LocalStore::Config(scheme, authority, params) , LocalOverlayStoreConfigT{ - CONFIG_ROW(lowerStoreUri), - CONFIG_ROW(upperLayer), - CONFIG_ROW(checkMount), - CONFIG_ROW(remountHook), + CONFIG_ROW(lowerStoreUri, ""), + CONFIG_ROW(upperLayer, ""), + CONFIG_ROW(checkMount, true), + CONFIG_ROW(remountHook, ""), } { } + std::string LocalOverlayStoreConfig::doc() { return @@ -95,10 +89,12 @@ std::string LocalOverlayStoreConfig::doc() ; } + Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { return upperLayer + "/" + path.to_string(); } + LocalOverlayStore::LocalOverlayStore(const Config & config) : Store::Config{config} , LocalFSStore::Config{config} diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 4cf898b8fc7..3d16c0c4695 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -29,8 +29,6 @@ struct LocalOverlayStoreConfig : static const Descriptions descriptions; - static LocalOverlayStoreConfigT defaults; - LocalOverlayStoreConfig( std::string_view scheme, PathView path, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2a930051d4f..9100879c923 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -88,11 +88,6 @@ LocalStore::Config::Descriptions::Descriptions() const LocalStore::Config::Descriptions LocalStore::Config::descriptions{}; -decltype(LocalStore::Config::defaults) LocalStore::Config::defaults = { - .requireSigs = {settings.requireSigs}, - .readOnly = {false}, -}; - LocalStore::Config::LocalStoreConfig( std::string_view scheme, std::string_view authority, @@ -100,8 +95,8 @@ LocalStore::Config::LocalStoreConfig( : Store::Config(params) , LocalFSStore::Config(authority, params) , LocalStoreConfigT{ - CONFIG_ROW(requireSigs), - CONFIG_ROW(readOnly), + CONFIG_ROW(requireSigs, settings.requireSigs), + CONFIG_ROW(readOnly, false), } { } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ca483df1584..55a2e504ff9 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -56,8 +56,6 @@ struct LocalStoreConfig : static const Descriptions descriptions; - static LocalStoreConfigT defaults; - LocalStoreConfig(const StoreReference::Params & params) : LocalStoreConfig{"local", "", params} {} diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 4caabdf9f3c..b2bb6f7d6a6 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -42,17 +42,11 @@ RemoteStore::Config::Descriptions::Descriptions() const RemoteStore::Config::Descriptions RemoteStore::Config::descriptions{}; -decltype(RemoteStore::Config::defaults) RemoteStore::Config::defaults = { - .maxConnections = {1}, - .maxConnectionAge = {std::numeric_limits::max()}, -}; - - RemoteStore::Config::RemoteStoreConfig(const StoreReference::Params & params) : Store::Config(params) , RemoteStoreConfigT{ - CONFIG_ROW(maxConnections), - CONFIG_ROW(maxConnectionAge), + CONFIG_ROW(maxConnections, 1), + CONFIG_ROW(maxConnectionAge, std::numeric_limits::max()), } { } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 126f060a02b..d4a63de1f13 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -38,11 +38,6 @@ struct RemoteStoreConfig : static const Descriptions descriptions; - /** - * The other defaults depend on the choice of `storeDir` and `rootDir` - */ - static RemoteStoreConfigT defaults; - RemoteStoreConfig(const StoreReference::Params &); }; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 7f8ce3a7bdd..7f0fcc671d1 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -261,18 +261,6 @@ S3BinaryCacheStore::Config::Descriptions::Descriptions() const S3BinaryCacheStore::Config::Descriptions S3BinaryCacheStore::Config::descriptions{}; -decltype(S3BinaryCacheStore::Config::defaults) S3BinaryCacheStore::Config::defaults = { - .profile{""}, - .region{Aws::Region::US_EAST_1}, - .scheme{""}, - .endpoint{""}, - .narinfoCompression{""}, - .lsCompression{""}, - .logCompression{""}, - .multipartUpload{false}, - .bufferSize{5 * 1024 * 1024}, -}; - S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig( std::string_view scheme, std::string_view authority, @@ -280,15 +268,15 @@ S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig( : Store::Config(params) , BinaryCacheStore::Config(params) , S3BinaryCacheStoreConfigT{ - CONFIG_ROW(profile), - CONFIG_ROW(region), - CONFIG_ROW(scheme), - CONFIG_ROW(endpoint), - CONFIG_ROW(narinfoCompression), - CONFIG_ROW(lsCompression), - CONFIG_ROW(logCompression), - CONFIG_ROW(multipartUpload), - CONFIG_ROW(bufferSize), + CONFIG_ROW(profile, ""), + CONFIG_ROW(region, Aws::Region::US_EAST_1), + CONFIG_ROW(scheme, ""), + CONFIG_ROW(endpoint, ""), + CONFIG_ROW(narinfoCompression, ""), + CONFIG_ROW(lsCompression, ""), + CONFIG_ROW(logCompression, ""), + CONFIG_ROW(multipartUpload, false), + CONFIG_ROW(bufferSize, 5 * 1024 * 1024), } , bucketName{authority} { diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index df7197113ae..2097bb5b427 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -32,8 +32,6 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig, S3BinaryCacheS static const Descriptions descriptions; - static S3BinaryCacheStoreConfigT defaults; - S3BinaryCacheStoreConfig( std::string_view uriScheme, std::string_view bucketName, const StoreReference::Params & params); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 0e1af43a131..3b04fec8227 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -27,11 +27,6 @@ SSHStoreConfig::Descriptions::Descriptions() const SSHStoreConfig::Descriptions SSHStoreConfig::descriptions{}; -decltype(SSHStoreConfig::defaults) SSHStoreConfig::defaults = { - .remoteProgram = {{"nix-daemon"}}, -}; - - SSHStoreConfig::SSHStoreConfig( std::string_view scheme, std::string_view authority, @@ -40,7 +35,7 @@ SSHStoreConfig::SSHStoreConfig( , RemoteStore::Config{params} , CommonSSHStoreConfig{scheme, authority, params} , SSHStoreConfigT{ - CONFIG_ROW(remoteProgram), + CONFIG_ROW(remoteProgram, Strings{"nix-daemon"}), } { } diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh index 6aa9c9cdf09..7f664fd2d6e 100644 --- a/src/libstore/ssh-store.hh +++ b/src/libstore/ssh-store.hh @@ -23,8 +23,6 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig, static const Descriptions descriptions; - static SSHStoreConfigT defaults; - SSHStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params); const std::string name() override diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f51e7c27cf9..78931a5d097 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -238,23 +238,14 @@ Store::Config::Descriptions::Descriptions() const Store::Config::Descriptions Store::Config::descriptions{}; -decltype(Store::Config::defaults) Store::Config::defaults = { - .pathInfoCacheSize = {65536}, - .isTrusted = {false}, - .priority = {0}, - .wantMassQuery = {false}, - .systemFeatures = {Store::Config::getDefaultSystemFeatures()}, -}; - - Store::Config::StoreConfig(const StoreReference::Params & params) : StoreDirConfig{params} , StoreConfigT{ - CONFIG_ROW(pathInfoCacheSize), - CONFIG_ROW(isTrusted), - CONFIG_ROW(priority), - CONFIG_ROW(wantMassQuery), - CONFIG_ROW(systemFeatures), + CONFIG_ROW(pathInfoCacheSize, 65536), + CONFIG_ROW(isTrusted, false), + CONFIG_ROW(priority, 0), + CONFIG_ROW(wantMassQuery, false), + CONFIG_ROW(systemFeatures, getDefaultSystemFeatures()), } , defaultPriority{params.count(descriptions.priority.name) == 0} , defaultWantMassQuery{params.count(descriptions.wantMassQuery.name) == 0} diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e2151ef5b42..f744c441cdb 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -118,8 +118,6 @@ struct StoreConfig : Descriptions(); }; - static const StoreConfigT defaults; - static const Descriptions descriptions; StoreConfig(const StoreReference::Params &); diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc index 026a6cd9837..2f4873ec3c4 100644 --- a/src/libstore/store-dir-config.cc +++ b/src/libstore/store-dir-config.cc @@ -16,13 +16,9 @@ const StoreDirConfigT StoreDirConfig::descriptions = { }, }; -const StoreDirConfigT StoreDirConfig::defaults = { - ._storeDir = {settings.nixStore}, -}; - StoreDirConfig::StoreDirConfig(const StoreReference::Params & params) : StoreDirConfigT{ - CONFIG_ROW(_storeDir), + CONFIG_ROW(_storeDir, settings.nixStore), } { } diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index 19391322907..d4a4288f58e 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -27,8 +27,6 @@ struct StoreDirConfigT struct StoreDirConfig : StoreDirConfigT { - static const StoreDirConfigT defaults; - using Descriptions = StoreDirConfigT; static const Descriptions descriptions; From ae9377617c0b7fecd12926e9a44b590f4df0ed20 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 23:29:30 -0400 Subject: [PATCH 44/80] Avoid potentially problematic move --- src/libstore/local-fs-store.cc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index bedc0aca94e..47f24f27c1b 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -43,11 +43,10 @@ auto localFSStoreConfig( { const auto & descriptions = LocalFSStore::Config::descriptions; - auto rootDir = descriptions.rootDir.parseConfig(params) - .value_or(config::JustValue{.value = std::move(_rootDir)}); + auto rootDir = descriptions.rootDir.parseConfig(params).value_or(std::move(_rootDir)); return LocalFSStoreConfigT{ - .rootDir = std::move(rootDir), + .rootDir = {.value = rootDir}, CONFIG_ROW(stateDir, rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir), CONFIG_ROW(logDir, rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir), CONFIG_ROW(realStoreDir, rootDir ? *rootDir + "/nix/store" : storeConfig.storeDir), From 40955fa5bf37cf97d5a7b8c73b628cc0aa1edf42 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Thu, 18 Jul 2024 23:32:00 -0400 Subject: [PATCH 45/80] WIP hack so it builds --- src/nix/main.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/main.cc b/src/nix/main.cc index 0d839425afb..524c4626350 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -223,12 +223,14 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs auto stores = nlohmann::json::object(); for (auto & implem : *Implementations::registered) { +#if 0 auto storeConfig = implem.getConfig(); auto storeName = storeConfig->name(); auto & j = stores[storeName]; j["doc"] = storeConfig->doc(); j["settings"] = storeConfig->toJSON(); j["experimentalFeature"] = storeConfig->experimentalFeature(); +#endif } res["stores"] = std::move(stores); res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo(); From 1c5fac5534cd560da65034ef43299d2a204d4847 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 19 Jul 2024 10:37:11 -0400 Subject: [PATCH 46/80] Adjust registration infra, fix Meson build --- src/libstore/dummy-store.cc | 2 +- src/libstore/http-binary-cache-store.cc | 2 +- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-binary-cache-store.cc | 2 +- src/libstore/local-overlay-store.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/meson.build | 4 ++++ src/libstore/s3-binary-cache-store.cc | 2 +- src/libstore/ssh-store.cc | 4 ++-- src/libstore/store-registration.hh | 12 ++++++------ src/libstore/uds-remote-store.cc | 2 +- src/perl/lib/Nix/Store.xs | 2 +- tests/unit/libstore/meson.build | 1 + 13 files changed, 22 insertions(+), 17 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index e8934ec9a19..2f3269b6830 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -85,6 +85,6 @@ ref DummyStore::Config::openStore() const return make_ref(*this); } -static RegisterStoreImplementation regDummyStore; +static RegisterStoreImplementation regDummyStore; } diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 8d9a9d96335..3cdd7e8874d 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -214,6 +214,6 @@ ref HttpBinaryCacheStore::Config::openStore() const return make_ref(*this); } -static RegisterStoreImplementation regHttpBinaryCacheStore; +static RegisterStoreImplementation regHttpBinaryCacheStore; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 0fc5181ff6d..ea49b6c8ac6 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -350,6 +350,6 @@ ref LegacySSHStore::Config::openStore() const { } -static RegisterStoreImplementation regLegacySSHStore; +static RegisterStoreImplementation regLegacySSHStore; } diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index c00a34d2bce..0997cfd77c1 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -129,6 +129,6 @@ ref LocalBinaryCacheStoreConfig::openStore() const { return make_ref(*this); } -static RegisterStoreImplementation regLocalBinaryCacheStore; +static RegisterStoreImplementation regLocalBinaryCacheStore; } diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 7a483304841..a1bf594cdb4 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -364,6 +364,6 @@ void LocalOverlayStore::remountIfNecessary() } -static RegisterStoreImplementation regLocalOverlayStore; +static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9100879c923..66bdc51c756 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1834,6 +1834,6 @@ std::optional LocalStore::getVersion() return nixVersion; } -static RegisterStoreImplementation regLocalStore; +static RegisterStoreImplementation regLocalStore; } // namespace nix diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 29ee95b75bc..a9c92545a4d 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -210,6 +210,8 @@ sources = files( 'ssh-store.cc', 'ssh.cc', 'store-api.cc', + 'store-dir-config.cc', + 'store-registration.cc', 'store-reference.cc', 'uds-remote-store.cc', 'worker-protocol-connection.cc', @@ -282,6 +284,8 @@ headers = [config_h] + files( 'sqlite.hh', 'ssh.hh', 'store-api.hh', + 'store-open.hh', + 'store-registration.hh', 'store-cast.hh', 'store-dir-config.hh', 'store-reference.hh', diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 7f0fcc671d1..19f2a69c419 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -558,7 +558,7 @@ ref S3BinaryCacheStoreImpl::Config::openStore() const return make_ref(*this); } -static RegisterStoreImplementation regS3BinaryCacheStore; +static RegisterStoreImplementation regS3BinaryCacheStore; } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 3b04fec8227..a83a11b4fdd 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -247,7 +247,7 @@ ref SSHStore::openConnection() return conn; } -static RegisterStoreImplementation regSSHStore; -static RegisterStoreImplementation regMountedSSHStore; +static RegisterStoreImplementation regSSHStore; +static RegisterStoreImplementation regMountedSSHStore; } diff --git a/src/libstore/store-registration.hh b/src/libstore/store-registration.hh index dd61a45f1ae..20ca0c93e48 100644 --- a/src/libstore/store-registration.hh +++ b/src/libstore/store-registration.hh @@ -28,28 +28,28 @@ struct Implementations { static std::vector * registered; - template + template static void add() { if (!registered) registered = new std::vector(); StoreFactory factory{ - .uriSchemes = T::Config::uriSchemes(), + .uriSchemes = TConfig::uriSchemes(), .parseConfig = ([](auto scheme, auto uri, auto & params) -> ref { - return make_ref(scheme, uri, params); + return make_ref(scheme, uri, params); }), - .configDescriptions = T::Config::descriptions, + .configDescriptions = TConfig::descriptions, }; registered->push_back(factory); } }; -template +template struct RegisterStoreImplementation { RegisterStoreImplementation() { - Implementations::add(); + Implementations::add(); } }; diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 8604961fbe5..0cc9140edb7 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -113,6 +113,6 @@ ref UDSRemoteStore::Config::openStore() const { } -static RegisterStoreImplementation regUDSRemoteStore; +static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index f951437c899..d88e2648ebc 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -12,7 +12,7 @@ #include "derivations.hh" #include "realisation.hh" #include "globals.hh" -#include "store-api.hh" +#include "store-open.hh" #include "posix-source-accessor.hh" #include diff --git a/tests/unit/libstore/meson.build b/tests/unit/libstore/meson.build index 8534ba8c52d..5d5382a6da0 100644 --- a/tests/unit/libstore/meson.build +++ b/tests/unit/libstore/meson.build @@ -57,6 +57,7 @@ sources = files( 'derivation-advanced-attrs.cc', 'derivation.cc', 'derived-path.cc', + 'dummy-store.cc', 'downstream-placeholder.cc', 'http-binary-cache-store.cc', 'legacy-ssh-store.cc', From a6c6852afc6ee5f1b929f5e1a7737edf5f51f546 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 14:50:28 -0400 Subject: [PATCH 47/80] WIP Store doesn't inherit from config --- src/libstore/common-ssh-store-config.cc | 40 +++--- src/libstore/common-ssh-store-config.hh | 2 - src/libstore/config-parse-impl.hh | 43 +++++- src/libstore/config-parse.hh | 82 ++++++++++- src/libstore/legacy-ssh-store.hh | 2 + src/libstore/local-fs-store.cc | 129 +++++++++++------- src/libstore/local-fs-store.hh | 35 +++-- src/libstore/local-store.cc | 149 ++++++++++---------- src/libstore/local-store.hh | 24 ++-- src/libstore/meson.build | 1 + src/libstore/path.cc | 12 +- src/libstore/ssh-store.hh | 4 +- src/libstore/store-api.cc | 173 +++++++++++++++--------- src/libstore/store-api.hh | 70 ++++++---- src/libstore/store-dir-config.cc | 37 +++-- src/libstore/store-dir-config.hh | 49 +++++-- src/libstore/store-registration.hh | 4 +- src/libutil/config-abstract.hh | 8 ++ src/libutil/config.hh | 3 +- 19 files changed, 561 insertions(+), 306 deletions(-) diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index 113ba6b8b4e..ab328cdd3e2 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -7,29 +7,28 @@ namespace nix { CommonSSHStoreConfig::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , CommonSSHStoreConfigT{ - .sshKey{ - .name = "ssh-key", - .description = "Path to the SSH private key used to authenticate to the remote machine.", - }, - .sshPublicHostKey{ - .name = "base64-ssh-public-host-key", - .description = "The public host key of the remote machine.", - }, - .compress{ - .name = "compress", - .description = "Whether to enable SSH compression.", - }, - .remoteStore{ - .name = "remote-store", - .description = R"( + : CommonSSHStoreConfigT{ + .sshKey{ + .name = "ssh-key", + .description = "Path to the SSH private key used to authenticate to the remote machine.", + }, + .sshPublicHostKey{ + .name = "base64-ssh-public-host-key", + .description = "The public host key of the remote machine.", + }, + .compress{ + .name = "compress", + .description = "Whether to enable SSH compression.", + }, + .remoteStore{ + .name = "remote-store", + .description = R"( [Store URL](@docroot@/store/types/index.md#store-url-format) to be used on the remote machine. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). )", - }, - } + }, + } { } @@ -56,8 +55,7 @@ CommonSSHStoreConfig::CommonSSHStoreConfig( std::string_view scheme, std::string_view host, const StoreReference::Params & params) - : Store::Config(params) - , CommonSSHStoreConfigT{ + : CommonSSHStoreConfigT{ CONFIG_ROW(sshKey, ""), CONFIG_ROW(sshPublicHostKey, ""), CONFIG_ROW(compress, false), diff --git a/src/libstore/common-ssh-store-config.hh b/src/libstore/common-ssh-store-config.hh index e7bb16367ff..d9361e259a3 100644 --- a/src/libstore/common-ssh-store-config.hh +++ b/src/libstore/common-ssh-store-config.hh @@ -17,11 +17,9 @@ struct CommonSSHStoreConfigT }; struct CommonSSHStoreConfig : - virtual Store::Config, CommonSSHStoreConfigT { struct Descriptions : - virtual Store::Config::Descriptions, CommonSSHStoreConfigT { Descriptions(); diff --git a/src/libstore/config-parse-impl.hh b/src/libstore/config-parse-impl.hh index ed640019dbe..b73801793e8 100644 --- a/src/libstore/config-parse-impl.hh +++ b/src/libstore/config-parse-impl.hh @@ -9,16 +9,51 @@ namespace nix::config { template -std::optional SettingInfo::parseConfig(const nlohmann::json::object_t & map) const +OptValue SettingInfo::parseConfig(const nlohmann::json::object_t & map) const { - auto * p = get(map, name); - return p ? p->get() : (std::optional{}); + const nlohmann::json * p = get(map, name); + return {.optValue = p ? (std::optional{p->get()}) : std::nullopt}; +} + +template +std::pair SettingInfo::describe(const JustValue & def) const +{ + return { + name, + SettingDescription{ + .description = description, + .defaultValue = + documentDefault ? (std::optional{}) : (std::optional{nlohmann::json{def.value}}), + }, + }; } /** * Look up the setting's name in a map, falling back on the default if * it does not exist. */ -#define CONFIG_ROW(FIELD, DEFAULT) .FIELD = {.value = descriptions.FIELD.parseConfig(params).value_or(DEFAULT)} +#define CONFIG_ROW(FIELD) .FIELD = descriptions.FIELD.parseConfig(params) + +#define APPLY_ROW(FIELD) .FIELD = {.value = parsed.FIELD.optValue.value_or(std::move(defaults.FIELD))} + +#define DESC_ROW(FIELD) \ + { \ + descriptions.FIELD.describe(defaults.FIELD), \ + } + +#define MAKE_PARSE(CAPITAL, LOWER, FIELDS) \ + static CAPITAL##T LOWER##Parse(const StoreReference::Params & params) \ + { \ + constexpr auto & descriptions = LOWER##Descriptions; \ + return {FIELDS(CONFIG_ROW)}; \ + } + +#define MAKE_APPLY_PARSE(CAPITAL, LOWER, FIELDS) \ + static CAPITAL##T LOWER##ApplyParse(const StoreReference::Params & params) \ + { \ + auto defaults = LOWER##Defaults(); \ + auto parsed = LOWER##Parse(params); \ + return {FIELDS(APPLY_ROW)}; \ + } } diff --git a/src/libstore/config-parse.hh b/src/libstore/config-parse.hh index e016caf15d9..4b44e8197c8 100644 --- a/src/libstore/config-parse.hh +++ b/src/libstore/config-parse.hh @@ -4,18 +4,96 @@ #include #include "config-abstract.hh" -#include "util.hh" +#include "json-impls.hh" namespace nix::config { +struct SettingDescription; + +/** + * Typed version used as source of truth, and for operations like + * defaulting configurations. + */ template struct SettingInfo { + /** + * Name of the setting, used when parsing configuration maps + */ std::string name; + + /** + * Description of the setting. It is used just for documentation. + */ std::string description; + +#if 0 + /** + * Other names of the setting also used when parsing configuration + * maps. This is useful for back-compat, etc. + */ + std::set aliases; + + /** + * `ExperimentalFeature` that must be enabled if the setting is + * allowed to be used + */ + std::optional experimentalFeature; +#endif + + /** + * Whether to document the default value. (Some defaults are + * system-specific and should not be documented.) + */ bool documentDefault = true; - std::optional parseConfig(const nlohmann::json::object_t & map) const; + /** + * Describe the setting as a key-value pair (name -> other info). + * The default value will be rendered to JSON if it is to be + * documented. + */ + std::pair describe(const JustValue & def) const; + + OptValue parseConfig(const nlohmann::json::object_t & map) const; }; +/** + * Untyped version used for rendering docs. This is not the source of + * truth, it is generated from the typed one. + * + * @note No `name` field because this is intended to be used as the value type + * of a map + */ +struct SettingDescription +{ + /** + * @see SettingInfo::description + */ + std::string description; + +#if 0 + /** + * @see SettingInfo::aliases + */ + std::set aliases; + + /** + * @see SettingInfo::experimentalFeature + */ + std::optional experimentalFeature; +#endif + + /** + * Optional, for the `SettingInfo::documentDefault = false` case. + */ + std::optional defaultValue; +}; + +/** + * Map of setting names to descriptions of those settings. + */ +using SettingDescriptionMap = std::map; + } + +JSON_IMPL(config::SettingDescription) diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index 12ca9f12514..4da2d02dfa6 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -17,10 +17,12 @@ struct LegacySSHStoreConfigT }; struct LegacySSHStoreConfig : + virtual Store::Config, virtual CommonSSHStoreConfig, LegacySSHStoreConfigT { struct Descriptions : + virtual Store::Config::Descriptions, virtual CommonSSHStoreConfig::Descriptions, LegacySSHStoreConfigT { diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 47f24f27c1b..ce383e6cc6e 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -9,69 +9,104 @@ namespace nix { -LocalFSStore::Config::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , LocalFSStoreConfigT{ - .rootDir = { - .name = "root", - .description = "Directory prefixed to all other paths.", - }, - .stateDir = { - .name = "state", - .description = "Directory where Nix will store state.", - }, - .logDir = { - .name = "log", - .description = "directory where Nix will store log files.", - }, - .realStoreDir{ - .name = "real", - .description = "Physical path of the Nix store.", - }, - } -{} +static const LocalFSStoreConfigT localFSStoreConfigDescriptions = { + .rootDir = { + .name = "root", + .description = "Directory prefixed to all other paths.", + }, + .stateDir = { + .name = "state", + .description = "Directory where Nix will store state.", + }, + .logDir = { + .name = "log", + .description = "directory where Nix will store log files.", + }, + .realStoreDir{ + .name = "real", + .description = "Physical path of the Nix store.", + }, +}; + +#define LOCAL_FS_STORE_CONFIG_FIELDS(X) \ + X(rootDir), \ + X(stateDir), \ + X(logDir), \ + X(realStoreDir), -const LocalFSStore::Config::Descriptions LocalFSStore::Config::descriptions{}; +MAKE_PARSE(LocalFSStoreConfig, localFSStoreConfig, LOCAL_FS_STORE_CONFIG_FIELDS) /** * @param rootDir Fallback if not in `params` */ -auto localFSStoreConfig( +static LocalFSStoreConfigT localFSStoreConfigDefaults( + const Path & storeDir, + const std::optional & rootDir) +{ + return { + .rootDir = {std::nullopt}, + .stateDir = {rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir}, + .logDir = {rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir}, + .realStoreDir = {rootDir ? *rootDir + "/nix/store" : storeDir}, + }; +} + +static LocalFSStoreConfigT localFSStoreConfigApplyParse( + const Path & storeDir, + LocalFSStoreConfigT parsed) +{ + auto defaults = localFSStoreConfigDefaults( + storeDir, + parsed.rootDir.optValue.value_or(std::nullopt)); + return {LOCAL_FS_STORE_CONFIG_FIELDS(APPLY_ROW)}; +} + +LocalFSStore::Config::LocalFSStoreConfig( const Store::Config & storeConfig, - const std::optional _rootDir, const StoreReference::Params & params) + : LocalFSStoreConfigT{ + localFSStoreConfigApplyParse( + storeConfig.storeDir, + localFSStoreConfigParse(params))} + , storeConfig{storeConfig} { - const auto & descriptions = LocalFSStore::Config::descriptions; - - auto rootDir = descriptions.rootDir.parseConfig(params).value_or(std::move(_rootDir)); +} - return LocalFSStoreConfigT{ - .rootDir = {.value = rootDir}, - CONFIG_ROW(stateDir, rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir), - CONFIG_ROW(logDir, rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir), - CONFIG_ROW(realStoreDir, rootDir ? *rootDir + "/nix/store" : storeConfig.storeDir), - }; +static LocalFSStoreConfigT applyAuthority( + LocalFSStoreConfigT parsed, + PathView rootDir) +{ + if (!rootDir.empty()) + parsed.rootDir = {.optValue = {Path{rootDir}}}; + return parsed; } -LocalFSStore::Config::LocalFSStoreConfig(const StoreReference::Params & params) - : StoreConfig{params} - , LocalFSStoreConfigT{localFSStoreConfig(*this, std::nullopt, params)} +LocalFSStore::Config::LocalFSStoreConfig( + const Store::Config & storeConfig, + PathView rootDir, + const StoreReference::Params & params) + : LocalFSStoreConfigT{ + localFSStoreConfigApplyParse( + storeConfig.storeDir, + applyAuthority( + localFSStoreConfigParse(params), + rootDir))} + , storeConfig{storeConfig} { } -LocalFSStore::Config::LocalFSStoreConfig(PathView rootDir, const StoreReference::Params & params) - : StoreConfig(params) - , LocalFSStoreConfigT{localFSStoreConfig( - *this, - // Default `?root` from `rootDir` if non set - !rootDir.empty() ? (std::optional{rootDir}) : std::nullopt, - params)} +config::SettingDescriptionMap LocalFSStoreConfig::descriptions() { + constexpr auto & descriptions = localFSStoreConfigDescriptions; + auto defaults = localFSStoreConfigDefaults(settings.nixStore, std::nullopt); + return { + LOCAL_FS_STORE_CONFIG_FIELDS(DESC_ROW) + }; } LocalFSStore::LocalFSStore(const Config & config) - : LocalFSStore::Config{config} - , Store{static_cast(*this)} + : Store{static_cast(*this)} + , config{config} { } @@ -145,8 +180,8 @@ std::optional LocalFSStore::getBuildLogExact(const StorePath & path Path logPath = j == 0 - ? fmt("%s/%s/%s/%s", logDir.get(), drvsLogDir, baseName.substr(0, 2), baseName.substr(2)) - : fmt("%s/%s/%s", logDir.get(), drvsLogDir, baseName); + ? fmt("%s/%s/%s/%s", config.logDir.get(), drvsLogDir, baseName.substr(0, 2), baseName.substr(2)) + : fmt("%s/%s/%s", config.logDir.get(), drvsLogDir, baseName); Path logBz2Path = logPath + ".bz2"; if (pathExists(logPath)) diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 678e93ec0a4..27ef32ee94c 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -10,26 +10,21 @@ namespace nix { template class F> struct LocalFSStoreConfigT { - const F> rootDir; - const F stateDir; - const F logDir; - const F realStoreDir; + F> rootDir; + F stateDir; + F logDir; + F realStoreDir; }; -struct LocalFSStoreConfig : - virtual Store::Config, - LocalFSStoreConfigT +struct LocalFSStoreConfig : LocalFSStoreConfigT { - struct Descriptions : - virtual Store::Config::Descriptions, - LocalFSStoreConfigT - { - Descriptions(); - }; + const Store::Config & storeConfig; - static const Descriptions descriptions; + config::SettingDescriptionMap descriptions(); - LocalFSStoreConfig(const StoreReference::Params &); + LocalFSStoreConfig( + const Store::Config & storeConfig, + const StoreReference::Params &); /** * Used to override the `root` settings. Can't be done via modifying @@ -38,17 +33,21 @@ struct LocalFSStoreConfig : * * @todo Make this less error-prone with new store settings system. */ - LocalFSStoreConfig(PathView path, const StoreReference::Params & params); + LocalFSStoreConfig( + const Store::Config & storeConfig, + PathView path, + const StoreReference::Params & params); }; struct LocalFSStore : - virtual LocalFSStoreConfig, virtual Store, virtual GcStore, virtual LogStore { using Config = LocalFSStoreConfig; + const Config & config; + inline static std::string operationName = "Local Filesystem Store"; const static std::string drvsLogDir; @@ -74,7 +73,7 @@ struct LocalFSStore : */ virtual Path addPermRoot(const StorePath & storePath, const Path & gcRoot) = 0; - virtual Path getRealStoreDir() { return realStoreDir; } + virtual Path getRealStoreDir() { return config.realStoreDir; } Path toRealPath(const Path & storePath) override { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 66bdc51c756..b16d56ebb3d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -59,45 +59,52 @@ namespace nix { -LocalStore::Config::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , LocalFSStore::Config::Descriptions{LocalFSStore::Config::descriptions} - , LocalStoreConfigT{ - .requireSigs = { - .name = "require-sigs", - .description = "Whether store paths copied into this store should have a trusted signature.", - }, - .readOnly = { - .name = "read-only", - .description = R"( - Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. - - Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. - - Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. - - > **Warning** - > Do not use this unless the filesystem is read-only. - > - > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. - > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. - )", - }, - } -{} +static const LocalStoreConfigT localStoreConfigDescriptions = { + .requireSigs = { + .name = "require-sigs", + .description = "Whether store paths copied into this store should have a trusted signature.", + }, + .readOnly = { + .name = "read-only", + .description = R"( + Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. + + Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. + + Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. + + > **Warning** + > Do not use this unless the filesystem is read-only. + > + > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. + > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. + )", + }, +}; + +#define LOCAL_STORE_CONFIG_FIELDS(X) \ + X(requireSigs), \ + X(readOnly), + +MAKE_PARSE(LocalStoreConfig, localStoreConfig, LOCAL_STORE_CONFIG_FIELDS) + +static LocalStoreConfigT localStoreConfigDefaults() +{ + return { + .requireSigs = {settings.requireSigs}, + .readOnly = {false}, + }; +} -const LocalStore::Config::Descriptions LocalStore::Config::descriptions{}; +MAKE_APPLY_PARSE(LocalStoreConfig, localStoreConfig, LOCAL_STORE_CONFIG_FIELDS) LocalStore::Config::LocalStoreConfig( std::string_view scheme, std::string_view authority, const StoreReference::Params & params) : Store::Config(params) - , LocalFSStore::Config(authority, params) - , LocalStoreConfigT{ - CONFIG_ROW(requireSigs, settings.requireSigs), - CONFIG_ROW(readOnly, false), - } + , LocalFSStore::Config(*this, authority, params) + , LocalStoreConfigT{localStoreConfigApplyParse(params)} { } @@ -110,7 +117,7 @@ std::string LocalStoreConfig::doc() ref LocalStore::Config::openStore() const { - return make_ref(*this); + return make_ref(ref{shared_from_this()}); } struct LocalStore::State::Stmts { @@ -233,17 +240,15 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) } } -LocalStore::LocalStore(const Config & config) - : Store::Config{config} - , LocalFSStore::Config{config} - , LocalStore::Config{config} - , Store{static_cast(*this)} - , LocalFSStore{static_cast(*this)} - , dbDir(stateDir + "/db") - , linksDir(realStoreDir + "/.links") +LocalStore::LocalStore(ref config) + : Store{*config} + , LocalFSStore{*config} + , config{config} + , dbDir(config->stateDir + "/db") + , linksDir(config->realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") , schemaPath(dbDir + "/schema") - , tempRootsDir(stateDir + "/temproots") + , tempRootsDir(config->stateDir + "/temproots") , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) , locksHeld(tokenizeString(getEnv("NIX_HELD_LOCKS").value_or(""))) { @@ -251,18 +256,18 @@ LocalStore::LocalStore(const Config & config) state->stmts = std::make_unique(); /* Create missing state directories if they don't already exist. */ - createDirs(realStoreDir); - if (readOnly) { + createDirs(config->realStoreDir); + if (config->readOnly) { experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore); } else { makeStoreWritable(); } createDirs(linksDir); - Path profilesDir = stateDir + "/profiles"; + Path profilesDir = config->stateDir + "/profiles"; createDirs(profilesDir); createDirs(tempRootsDir); createDirs(dbDir); - Path gcRootsDir = stateDir + "/gcroots"; + Path gcRootsDir = config->stateDir + "/gcroots"; if (!pathExists(gcRootsDir)) { createDirs(gcRootsDir); createSymlink(profilesDir, gcRootsDir + "/profiles"); @@ -270,7 +275,7 @@ LocalStore::LocalStore(const Config & config) for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { createDirs(perUserDir); - if (!readOnly) { + if (!config->readOnly) { if (chmod(perUserDir.c_str(), 0755) == -1) throw SysError("could not set permissions on '%s' to 755", perUserDir); } @@ -285,16 +290,16 @@ LocalStore::LocalStore(const Config & config) struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); - else if (!readOnly) { + else if (!config->readOnly) { struct stat st; - if (stat(realStoreDir.get().c_str(), &st)) - throw SysError("getting attributes of path '%1%'", realStoreDir); + if (stat(config->realStoreDir.get().c_str(), &st)) + throw SysError("getting attributes of path '%1%'", config->realStoreDir); if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { - if (chown(realStoreDir.get().c_str(), 0, gr->gr_gid) == -1) - throw SysError("changing ownership of path '%1%'", realStoreDir); - if (chmod(realStoreDir.get().c_str(), perm) == -1) - throw SysError("changing permissions on path '%1%'", realStoreDir); + if (chown(config->realStoreDir.get().c_str(), 0, gr->gr_gid) == -1) + throw SysError("changing ownership of path '%1%'", config->realStoreDir); + if (chmod(config->realStoreDir.get().c_str(), perm) == -1) + throw SysError("changing permissions on path '%1%'", config->realStoreDir); } } } @@ -302,7 +307,7 @@ LocalStore::LocalStore(const Config & config) /* Ensure that the store and its parents are not symlinks. */ if (!settings.allowSymlinkedStore) { - Path path = realStoreDir; + Path path = config->realStoreDir; struct stat st; while (path != "/") { st = lstat(path); @@ -350,12 +355,12 @@ LocalStore::LocalStore(const Config & config) /* Acquire the big fat lock in shared mode to make sure that no schema upgrade is in progress. */ - if (!readOnly) { + if (!config->readOnly) { Path globalLockPath = dbDir + "/big-lock"; globalLock = openLockFile(globalLockPath.c_str(), true); } - if (!readOnly && !lockFile(globalLock.get(), ltRead, false)) { + if (!config->readOnly && !lockFile(globalLock.get(), ltRead, false)) { printInfo("waiting for the big Nix store lock..."); lockFile(globalLock.get(), ltRead, true); } @@ -363,7 +368,7 @@ LocalStore::LocalStore(const Config & config) /* Check the current database schema and if necessary do an upgrade. */ int curSchema = getSchema(); - if (readOnly && curSchema < nixSchemaVersion) { + if (config->readOnly && curSchema < nixSchemaVersion) { debug("current schema version: %d", curSchema); debug("supported schema version: %d", nixSchemaVersion); throw Error(curSchema == 0 ? @@ -435,7 +440,7 @@ LocalStore::LocalStore(const Config & config) else openDB(*state, false); if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { - if (!readOnly) { + if (!config->readOnly) { migrateCASchema(state->db, dbDir + "/ca-schema", globalLock); } else { throw Error("need to migrate to content-addressed schema, but this cannot be done in read-only mode"); @@ -517,7 +522,7 @@ LocalStore::LocalStore(const Config & config) AutoCloseFD LocalStore::openGCLock() { - Path fnGCLock = stateDir + "/gc.lock"; + Path fnGCLock = config->stateDir + "/gc.lock"; auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT #ifndef _WIN32 | O_CLOEXEC @@ -573,17 +578,17 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { - if (create && readOnly) { + if (create && config->readOnly) { throw Error("cannot create database while in read-only mode"); } - if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK))) + if (access(dbDir.c_str(), R_OK | (config->readOnly ? 0 : W_OK))) throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ std::string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - auto openMode = readOnly ? SQLiteOpenMode::Immutable + auto openMode = config->readOnly ? SQLiteOpenMode::Immutable : create ? SQLiteOpenMode::Normal : SQLiteOpenMode::NoCreate; state.db = SQLite(dbPath, openMode); @@ -660,12 +665,12 @@ void LocalStore::makeStoreWritable() if (!isRootUser()) return; /* Check if /nix/store is on a read-only mount. */ struct statvfs stat; - if (statvfs(realStoreDir.get().c_str(), &stat) != 0) + if (statvfs(config->realStoreDir.get().c_str(), &stat) != 0) throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { - if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) - throw SysError("remounting %1% writable", realStoreDir); + if (mount(0, config->realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) + throw SysError("remounting %1% writable", config->realStoreDir); } #endif } @@ -1005,7 +1010,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) for (auto & sub : getDefaultSubstituters()) { if (remaining.empty()) break; if (sub->storeDir != storeDir) continue; - if (!sub->wantMassQuery) continue; + if (!sub->resolvedSubstConfig.wantMassQuery) continue; auto valid = sub->queryValidPaths(remaining); @@ -1117,12 +1122,12 @@ const PublicKeys & LocalStore::getPublicKeys() bool LocalStore::pathInfoIsUntrusted(const ValidPathInfo & info) { - return requireSigs && !info.checkSignatures(*this, getPublicKeys()); + return config->requireSigs && !info.checkSignatures(*this, getPublicKeys()); } bool LocalStore::realisationIsUntrusted(const Realisation & realisation) { - return requireSigs && !realisation.checkSignatures(getPublicKeys()); + return config->requireSigs && !realisation.checkSignatures(getPublicKeys()); } void LocalStore::addToStore(const ValidPathInfo & info, Source & source, @@ -1403,7 +1408,7 @@ std::pair LocalStore::createTempDirInStore() /* There is a slight possibility that `tmpDir' gets deleted by the GC between createTempDir() and when we acquire a lock on it. We'll repeat until 'tmpDir' exists and we've locked it. */ - tmpDirFn = createTempDir(realStoreDir, "tmp"); + tmpDirFn = createTempDir(config->realStoreDir, "tmp"); tmpDirFd = openDirectory(tmpDirFn); if (!tmpDirFd) { continue; @@ -1544,7 +1549,7 @@ LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair database and the filesystem) in the loop below, in order to catch invalid states. */ - for (auto & i : std::filesystem::directory_iterator{realStoreDir.get()}) { + for (auto & i : std::filesystem::directory_iterator{config->realStoreDir.get()}) { checkInterrupt(); try { storePathsInStoreDir.insert({i.path().filename().string()}); @@ -1816,7 +1821,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) auto baseName = drvPath.to_string(); - auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)); + auto logPath = fmt("%s/%s/%s/%s.bz2", config->logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)); if (pathExists(logPath)) return; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 55a2e504ff9..e05d3cc3b85 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -37,24 +37,17 @@ struct OptimiseStats template class F> struct LocalStoreConfigT { - const F requireSigs; - const F readOnly; + F requireSigs; + F readOnly; }; struct LocalStoreConfig : - virtual Store::Config, - virtual LocalFSStore::Config, + std::enable_shared_from_this, + Store::Config, + LocalFSStore::Config, LocalStoreConfigT { - struct Descriptions : - virtual Store::Config::Descriptions, - virtual LocalFSStore::Config::Descriptions, - LocalStoreConfigT - { - Descriptions(); - }; - - static const Descriptions descriptions; + static config::SettingDescriptionMap descriptions(); LocalStoreConfig(const StoreReference::Params & params) : LocalStoreConfig{"local", "", params} @@ -76,7 +69,6 @@ struct LocalStoreConfig : }; class LocalStore : - public virtual LocalStoreConfig, public virtual IndirectRootStore, public virtual GcStore { @@ -84,6 +76,8 @@ public: using Config = LocalStoreConfig; + ref config; + private: /** @@ -151,7 +145,7 @@ public: * Initialise the local store, upgrading the schema if * necessary. */ - LocalStore(const Config & params); + LocalStore(ref params); ~LocalStore(); diff --git a/src/libstore/meson.build b/src/libstore/meson.build index a9c92545a4d..ef01704de9a 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -163,6 +163,7 @@ sources = files( 'builtins/unpack-channel.cc', 'common-protocol.cc', 'common-ssh-store-config.cc', + 'config-parse.cc', 'content-address.cc', 'daemon.cc', 'derivations.cc', diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 3e9d054778c..b0ec8db52c0 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -75,7 +75,7 @@ StorePath StorePath::random(std::string_view name) return StorePath(Hash::random(HashAlgorithm::SHA1), name); } -StorePath StoreDirConfig::parseStorePath(std::string_view path) const +StorePath MixStoreDirMethods::parseStorePath(std::string_view path) const { // On Windows, `/nix/store` is not a canonical path. More broadly it // is unclear whether this function should be using the native @@ -94,7 +94,7 @@ StorePath StoreDirConfig::parseStorePath(std::string_view path) const return StorePath(baseNameOf(p)); } -std::optional StoreDirConfig::maybeParseStorePath(std::string_view path) const +std::optional MixStoreDirMethods::maybeParseStorePath(std::string_view path) const { try { return parseStorePath(path); @@ -103,24 +103,24 @@ std::optional StoreDirConfig::maybeParseStorePath(std::string_view pa } } -bool StoreDirConfig::isStorePath(std::string_view path) const +bool MixStoreDirMethods::isStorePath(std::string_view path) const { return (bool) maybeParseStorePath(path); } -StorePathSet StoreDirConfig::parseStorePathSet(const PathSet & paths) const +StorePathSet MixStoreDirMethods::parseStorePathSet(const PathSet & paths) const { StorePathSet res; for (auto & i : paths) res.insert(parseStorePath(i)); return res; } -std::string StoreDirConfig::printStorePath(const StorePath & path) const +std::string MixStoreDirMethods::printStorePath(const StorePath & path) const { return (storeDir + "/").append(path.to_string()); } -PathSet StoreDirConfig::printStorePathSet(const StorePathSet & paths) const +PathSet MixStoreDirMethods::printStorePathSet(const StorePathSet & paths) const { PathSet res; for (auto & i : paths) res.insert(printStorePath(i)); diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh index 7f664fd2d6e..e934876c393 100644 --- a/src/libstore/ssh-store.hh +++ b/src/libstore/ssh-store.hh @@ -16,7 +16,9 @@ struct SSHStoreConfigT struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig, SSHStoreConfigT { - struct Descriptions : virtual CommonSSHStoreConfig::Descriptions, SSHStoreConfigT + struct Descriptions : virtual RemoteStoreConfig::Descriptions, + virtual CommonSSHStoreConfig::Descriptions, + SSHStoreConfigT { Descriptions(); }; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 78931a5d097..675f09a1231 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -30,13 +30,13 @@ using json = nlohmann::json; namespace nix { -bool StoreDirConfig::isInStore(PathView path) const +bool MixStoreDirMethods::isInStore(PathView path) const { return isInDir(path, storeDir); } -std::pair StoreDirConfig::toStorePath(PathView path) const +std::pair MixStoreDirMethods::toStorePath(PathView path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); @@ -78,7 +78,7 @@ to match. */ -StorePath StoreDirConfig::makeStorePath(std::string_view type, +StorePath MixStoreDirMethods::makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ @@ -89,14 +89,14 @@ StorePath StoreDirConfig::makeStorePath(std::string_view type, } -StorePath StoreDirConfig::makeStorePath(std::string_view type, +StorePath MixStoreDirMethods::makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const { return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name); } -StorePath StoreDirConfig::makeOutputPath(std::string_view id, +StorePath MixStoreDirMethods::makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const { return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id)); @@ -107,7 +107,7 @@ StorePath StoreDirConfig::makeOutputPath(std::string_view id, hacky, but we can't put them in, say, (per the grammar above) since that would be ambiguous. */ static std::string makeType( - const StoreDirConfig & store, + const MixStoreDirMethods & store, std::string && type, const StoreReferences & references) { @@ -120,7 +120,7 @@ static std::string makeType( } -StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const +StorePath MixStoreDirMethods::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { if (info.method == FileIngestionMethod::Git && info.hash.algo != HashAlgorithm::SHA1) throw Error("Git file ingestion must use SHA-1 hash"); @@ -142,7 +142,7 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed } -StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const +StorePath MixStoreDirMethods::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { @@ -163,7 +163,7 @@ StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const } -std::pair StoreDirConfig::computeStorePath( +std::pair MixStoreDirMethods::computeStorePath( std::string_view name, const SourcePath & path, ContentAddressMethod method, @@ -189,67 +189,111 @@ std::pair StoreDirConfig::computeStorePath( } -Store::Config::Descriptions::Descriptions() - : StoreDirConfig::Descriptions{StoreDirConfig::descriptions} - , StoreConfigT{ - .pathInfoCacheSize = { - .name = "path-info-cache-size", - .description = "Size of the in-memory store path metadata cache.", - }, - .isTrusted = { - .name = "trusted", - .description = R"( - Whether paths from this store can be used as substitutes - even if they are not signed by a key listed in the - [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) - setting. - )", - }, - .priority = { - .name = "priority", - .description = R"( - Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). - A lower value means a higher priority. - )", - }, - .wantMassQuery = { - .name = "want-mass-query", - .description = R"( - Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). - )", - }, - - .systemFeatures = { - .name = "system-features", - .description = R"( - Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. +static const StoreConfigT storeConfigDescriptions = { + .pathInfoCacheSize{ + .name = "path-info-cache-size", + .description = "Size of the in-memory store path metadata cache.", + }, + .isTrusted{ + .name = "trusted", + .description = R"( + Whether paths from this store can be used as substitutes + even if they are not signed by a key listed in the + [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) + setting. + )", + }, + .systemFeatures{ + .name = "system-features", + .description = R"( + Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. + + Example: `"kvm"` + )", + // The default value is CPU- and OS-specific, and thus + // unsuitable to be rendered in the documentation. + .documentDefault = false, + }, +}; + +static const SubstituterConfigT substituterConfigDescriptions = { + .priority{ + .name = "priority", + .description = R"( + Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + A lower value means a higher priority. + )", + }, + .wantMassQuery{ + .name = "want-mass-query", + .description = R"( + Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + )", + }, +}; + + +#define STORE_CONFIG_FIELDS(X) \ + X(pathInfoCacheSize), \ + X(isTrusted), \ + X(systemFeatures), + +#define SUBSTITUTER_CONFIG_FIELDS(X) \ + X(priority), \ + X(wantMassQuery), + + +MAKE_PARSE(StoreConfig, storeConfig, STORE_CONFIG_FIELDS) +MAKE_PARSE(SubstituterConfig, substituterConfig, SUBSTITUTER_CONFIG_FIELDS) + + +static StoreConfigT storeConfigDefaults() +{ + return { + .pathInfoCacheSize = {65536}, + .isTrusted = {false}, + .systemFeatures = {StoreConfig::getDefaultSystemFeatures()}, + }; +}; - Example: `"kvm"` - )", - // The default value is CPU- and OS-specific, and thus - // unsuitable to be rendered in the documentation. - .documentDefault = false, - }, - } +static SubstituterConfigT substituterConfigDefaults() { -} + return { + .priority = {0}, + .wantMassQuery = {false}, + }; +}; -const Store::Config::Descriptions Store::Config::descriptions{}; +MAKE_APPLY_PARSE(StoreConfig, storeConfig, STORE_CONFIG_FIELDS) Store::Config::StoreConfig(const StoreReference::Params & params) : StoreDirConfig{params} - , StoreConfigT{ - CONFIG_ROW(pathInfoCacheSize, 65536), - CONFIG_ROW(isTrusted, false), - CONFIG_ROW(priority, 0), - CONFIG_ROW(wantMassQuery, false), - CONFIG_ROW(systemFeatures, getDefaultSystemFeatures()), - } - , defaultPriority{params.count(descriptions.priority.name) == 0} - , defaultWantMassQuery{params.count(descriptions.wantMassQuery.name) == 0} + , StoreConfigT{storeConfigApplyParse(params)} + , SubstituterConfigT{substituterConfigParse(params)} +{ +} + + +config::SettingDescriptionMap StoreConfig::descriptions() { + auto ret = StoreDirConfig::descriptions(); + { + constexpr auto & descriptions = storeConfigDescriptions; + auto defaults = storeConfigDefaults(); + ret.merge(config::SettingDescriptionMap { + STORE_CONFIG_FIELDS(DESC_ROW) + }); + } + { + constexpr auto & descriptions = substituterConfigDescriptions; + auto defaults = substituterConfigDefaults(); + ret.merge(config::SettingDescriptionMap { + SUBSTITUTER_CONFIG_FIELDS(DESC_ROW) + }); + }; + return ret; } @@ -498,8 +542,9 @@ StringSet Store::Config::getDefaultSystemFeatures() } Store::Store(const Store::Config & config) - : Store::Config(config) - , state({(size_t) pathInfoCacheSize}) + : MixStoreDirMethods{config} + , config{config} + , state({(size_t) config.pathInfoCacheSize}) { assertLibStoreInitialized(); } @@ -1262,7 +1307,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre } -std::string StoreDirConfig::showPaths(const StorePathSet & paths) +std::string MixStoreDirMethods::showPaths(const StorePathSet & paths) { std::string s; for (auto & i : paths) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index f744c441cdb..206a35373db 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -99,26 +99,30 @@ typedef std::map> StorePathCAMap; template class F> struct StoreConfigT { - const F pathInfoCacheSize; - const F isTrusted; + F pathInfoCacheSize; + F isTrusted; + F systemFeatures; +}; + +template class F> +struct SubstituterConfigT +{ F priority; F wantMassQuery; - const F systemFeatures; }; -StoreConfigT parseStoreConfig(const StoreReference::Params &); +/** + * @note `config::OptValue` rather than `config::JustValue` is applied to + * `SubstitutorConfigT` because these are overrides. Caches themselves (not our + * config) can update default settings, but aren't allowed to update settings + * specified by the client (i.e. us). + */ struct StoreConfig : StoreDirConfig, - StoreConfigT + StoreConfigT, + SubstituterConfigT { - struct Descriptions : - StoreDirConfig::Descriptions, - StoreConfigT - { - Descriptions(); - }; - - static const Descriptions descriptions; + static config::SettingDescriptionMap descriptions(); StoreConfig(const StoreReference::Params &); @@ -153,27 +157,39 @@ struct StoreConfig : * type. */ virtual ref openStore() const = 0; +}; -protected: +/** + * A Store (client) + * + * This is an interface type allowing for create and read operations on + * a collection of store objects, and also building new store objects + * from `Derivation`s. See the manual for further details. + * + * "client" used is because this is just one view/actor onto an + * underlying resource, which could be an external process (daemon + * server), file system state, etc. + */ +class Store : public std::enable_shared_from_this, public MixStoreDirMethods +{ +public: + + using Config = StoreConfig; + + const Config & config; /** - * So caches themselves (not our config) can update default - * settings, but aren't allowed to update settings specied by the - * client (i.e. us). + * @note Avoid churn, since we used to inherit from `Config`. */ - bool defaultPriority; + operator const Config &() const { return config; } /** - * @see defaultPriority + * Resolved substituter configuration. This is intentionally mutable + * as store clients may do IO to ask the underlying store for their + * default setting values if the client config did not statically + * override them. */ - bool defaultWantMassQuery; -}; - -class Store : public std::enable_shared_from_this, public virtual StoreConfig -{ -public: - - using Config = StoreConfig; + SubstituterConfigT resolvedSubstConfig; protected: diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc index 2f4873ec3c4..ae22ba159f1 100644 --- a/src/libstore/store-dir-config.cc +++ b/src/libstore/store-dir-config.cc @@ -1,26 +1,45 @@ #include "store-dir-config.hh" #include "config-parse-impl.hh" #include "util.hh" +#include "globals.hh" namespace nix { -const StoreDirConfigT StoreDirConfig::descriptions = { - ._storeDir = - { - .name = "store", - .description = R"( +static const StoreDirConfigT storeDirConfigDescriptions = { + ._storeDir{ + .name = "store", + .description = R"( Logical location of the Nix store, usually `/nix/store`. Note that you can only copy store paths between stores if they have the same `store` setting. )", - }, + }, }; +#define STORE_DIR_CONFIG_FIELDS(X) X(_storeDir), + +MAKE_PARSE(StoreDirConfig, storeDirConfig, STORE_DIR_CONFIG_FIELDS) + +static StoreDirConfigT storeDirConfigDefaults() +{ + return { + ._storeDir = {settings.nixStore}, + }; +} + +MAKE_APPLY_PARSE(StoreDirConfig, storeDirConfig, STORE_DIR_CONFIG_FIELDS) + StoreDirConfig::StoreDirConfig(const StoreReference::Params & params) - : StoreDirConfigT{ - CONFIG_ROW(_storeDir, settings.nixStore), - } + : StoreDirConfigT{storeDirConfigApplyParse(params)} + , MixStoreDirMethods{_storeDir.value} +{ +} + +config::SettingDescriptionMap StoreDirConfig::descriptions() { + constexpr auto & descriptions = storeDirConfigDescriptions; + auto defaults = storeDirConfigDefaults(); + return {STORE_DIR_CONFIG_FIELDS(DESC_ROW)}; } } diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index d4a4288f58e..f247f917a50 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -5,7 +5,6 @@ #include "content-address.hh" #include "store-reference.hh" #include "config-parse.hh" -#include "globals.hh" #include #include @@ -19,25 +18,32 @@ struct SourcePath; MakeError(BadStorePath, Error); MakeError(BadStorePathName, BadStorePath); +/** + * Underlying store directory configuration type. + * + * Don't worry to much about the `F` parameter, it just some abstract + * nonsense for the "higher-kinded data" pattern. It is used in each + * settings record in order to ensure don't forgot to parse or document + * settings field. + */ template class F> struct StoreDirConfigT { - const F _storeDir; + F _storeDir; }; -struct StoreDirConfig : StoreDirConfigT +/** + * @todo This should just be part of `StoreDirConfig`. However, it would + * be a huge amount of churn if `Store` didn't have these methods + * anymore, forcing a bunch of code to go from `store.method(...)` to + * `store.config.method(...)`. + * + * So we instead pull out the methods into their own mix-in, so can put + * them directly on the Store too. + */ +struct MixStoreDirMethods { - using Descriptions = StoreDirConfigT; - - static const Descriptions descriptions; - - StoreDirConfig(const StoreReference::Params & params); - - virtual ~StoreDirConfig() = default; - - const Path & storeDir = _storeDir.value; - - // pure methods + const Path & storeDir; StorePath parseStorePath(std::string_view path) const; @@ -106,4 +112,19 @@ struct StoreDirConfig : StoreDirConfigT PathFilter & filter = defaultPathFilter) const; }; +/** + * Store directory configuration type. + * + * Combines the underlying `*T` type (with plain values for the fields) + * and the methods. + */ +struct StoreDirConfig : StoreDirConfigT, MixStoreDirMethods +{ + static config::SettingDescriptionMap descriptions(); + + StoreDirConfig(const StoreReference::Params & params); + + virtual ~StoreDirConfig() = default; +}; + } diff --git a/src/libstore/store-registration.hh b/src/libstore/store-registration.hh index 20ca0c93e48..289bdf341ff 100644 --- a/src/libstore/store-registration.hh +++ b/src/libstore/store-registration.hh @@ -21,7 +21,7 @@ struct StoreFactory std::function( std::string_view scheme, std::string_view authorityPath, const StoreReference::Params & params)> parseConfig; - const Store::Config::Descriptions & configDescriptions; + config::SettingDescriptionMap configDescriptions; }; struct Implementations @@ -38,7 +38,7 @@ struct Implementations .parseConfig = ([](auto scheme, auto uri, auto & params) -> ref { return make_ref(scheme, uri, params); }), - .configDescriptions = TConfig::descriptions, + .configDescriptions = TConfig::descriptions(), }; registered->push_back(factory); } diff --git a/src/libutil/config-abstract.hh b/src/libutil/config-abstract.hh index fcde0696b98..2c34ff50bdd 100644 --- a/src/libutil/config-abstract.hh +++ b/src/libutil/config-abstract.hh @@ -1,6 +1,8 @@ #pragma once ///@type +#include + namespace nix::config { template @@ -40,4 +42,10 @@ auto && operator<<(auto && str, const JustValue & opt) return str << opt.get(); } +template +struct OptValue +{ + std::optional optValue; +}; + } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 1952ba1b8d7..595538e9574 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -178,13 +178,12 @@ public: const std::string name; const std::string description; const std::set aliases; + std::optional experimentalFeature; int created = 123; bool overridden = false; - std::optional experimentalFeature; - protected: AbstractSetting( From 48aa3bb98e9dd513df8ee3e1e4e2dbd5ebb18ac8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 15:51:21 -0400 Subject: [PATCH 48/80] WIP --- src/libstore/local-fs-store.hh | 2 +- src/libstore/local-overlay-store.cc | 172 ++++++++++++++-------------- src/libstore/local-overlay-store.hh | 28 ++--- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 4 +- src/libstore/remote-store.hh | 20 +--- src/libstore/store-api.cc | 2 +- src/libstore/store-api.hh | 7 +- src/libstore/store-dir-config.hh | 2 +- src/libstore/store-registration.cc | 58 +++++++++- src/libstore/uds-remote-store.hh | 22 ++-- 11 files changed, 181 insertions(+), 138 deletions(-) diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 27ef32ee94c..8fbd98db99d 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -20,7 +20,7 @@ struct LocalFSStoreConfig : LocalFSStoreConfigT { const Store::Config & storeConfig; - config::SettingDescriptionMap descriptions(); + static config::SettingDescriptionMap descriptions(); LocalFSStoreConfig( const Store::Config & storeConfig, diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index a1bf594cdb4..5f3d9281814 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -5,84 +5,88 @@ #include "realisation.hh" #include "processes.hh" #include "url.hh" -#include "store-open.hh" +#include "store-api.hh" #include "store-registration.hh" #include "config-parse-impl.hh" namespace nix { -LocalOverlayStore::Config::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , LocalFSStore::Config::Descriptions{LocalFSStore::Config::descriptions} - , LocalStore::Config::Descriptions{LocalStore::Config::descriptions} - , LocalOverlayStoreConfigT{ - .lowerStoreUri{ - .name = "lower-store", - .description = R"( - [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) - for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). - - Must be a store with a store dir on the file system. - Must be used as OverlayFS lower layer for this store's store dir. - )", - }, - .upperLayer{ - .name = "upper-layer", - .description = R"( - Directory containing the OverlayFS upper layer for this store's store dir. - )", - }, - .checkMount{ - .name = "check-mount", - .description = R"( - Check that the overlay filesystem is correctly mounted. - - Nix does not manage the overlayfs mount point itself, but the correct - functioning of the overlay store does depend on this mount point being set up - correctly. Rather than just assume this is the case, check that the lowerdir - and upperdir options are what we expect them to be. This check is on by - default, but can be disabled if needed. - )", - }, - .remountHook{ - .name = "remount-hook", - .description = R"( - Script or other executable to run when overlay filesystem needs remounting. - - This is occasionally necessary when deleting a store path that exists in both upper and lower layers. - In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly - is the only way to perform the deletion without creating a "whiteout". - However this causes the OverlayFS kernel data structures to get out-of-sync, - and can lead to 'stale file handle' errors; remounting solves the problem. - - The store directory is passed as an argument to the invoked executable. - )", - }, - } -{} - - -const LocalOverlayStore::Config::Descriptions LocalOverlayStore::Config::descriptions{}; +static LocalOverlayStoreConfigT localOverlayStoreConfigDescriptions = { + .lowerStoreConfig{ + .name = "lower-store", + .description = R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). + + Must be a store with a store dir on the file system. + Must be used as OverlayFS lower layer for this store's store dir. + )", + }, + .upperLayer{ + .name = "upper-layer", + .description = R"( + Directory containing the OverlayFS upper layer for this store's store dir. + )", + }, + .checkMount{ + .name = "check-mount", + .description = R"( + Check that the overlay filesystem is correctly mounted. + + Nix does not manage the overlayfs mount point itself, but the correct + functioning of the overlay store does depend on this mount point being set up + correctly. Rather than just assume this is the case, check that the lowerdir + and upperdir options are what we expect them to be. This check is on by + default, but can be disabled if needed. + )", + }, + .remountHook{ + .name = "remount-hook", + .description = R"( + Script or other executable to run when overlay filesystem needs remounting. + + This is occasionally necessary when deleting a store path that exists in both upper and lower layers. + In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly + is the only way to perform the deletion without creating a "whiteout". + However this causes the OverlayFS kernel data structures to get out-of-sync, + and can lead to 'stale file handle' errors; remounting solves the problem. + + The store directory is passed as an argument to the invoked executable. + )", + }, +}; + +#define LOCAL_OVERLAY_STORE_CONFIG_FIELDS(X) \ + X(lowerStoreConfig), \ + X(upperLayer), \ + X(checkMount), \ + X(remountHook), + +MAKE_PARSE(LocalOverlayStoreConfig, localOverlayStoreConfig, LOCAL_OVERLAY_STORE_CONFIG_FIELDS) + +static LocalOverlayStoreConfigT localOverlayStoreConfigDefaults() +{ + return { + .lowerStoreConfig = {make_ref(StoreReference::Params{})}, + .upperLayer = {""}, + .checkMount = {true}, + .remountHook = {""}, + }; +} +MAKE_APPLY_PARSE(LocalOverlayStoreConfig, localOverlayStoreConfig, LOCAL_OVERLAY_STORE_CONFIG_FIELDS) LocalOverlayStore::Config::LocalOverlayStoreConfig( std::string_view scheme, std::string_view authority, const StoreReference::Params & params) - : Store::Config(params) - , LocalFSStore::Config(authority, params) - , LocalStore::Config(scheme, authority, params) - , LocalOverlayStoreConfigT{ - CONFIG_ROW(lowerStoreUri, ""), - CONFIG_ROW(upperLayer, ""), - CONFIG_ROW(checkMount, true), - CONFIG_ROW(remountHook, ""), - } + : LocalStore::Config(scheme, authority, params) + , LocalOverlayStoreConfigT{localOverlayStoreConfigApplyParse(params)} { } -std::string LocalOverlayStoreConfig::doc() +std::string LocalOverlayStoreConfig::doc() const { return #include "local-overlay-store.md" @@ -90,26 +94,24 @@ std::string LocalOverlayStoreConfig::doc() } -Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { +Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) const +{ return upperLayer + "/" + path.to_string(); } -LocalOverlayStore::LocalOverlayStore(const Config & config) - : Store::Config{config} - , LocalFSStore::Config{config} - , LocalStore::Config{config} - , LocalOverlayStore::Config{config} - , Store{static_cast(*this)} - , LocalFSStore{static_cast(*this)} - , LocalStore{static_cast(*this)} - , lowerStore(nix::openStore(lowerStoreUri.get()).dynamic_pointer_cast()) +LocalOverlayStore::LocalOverlayStore(ref config) + : Store{*config} + , LocalFSStore{*config} + , LocalStore{static_cast>(config)} + , config{config} + , lowerStore(config->lowerStoreConfig.value->openStore().dynamic_pointer_cast()) { - if (checkMount.get()) { + if (config->checkMount.get()) { std::smatch match; std::string mountInfo; auto mounts = readFile("/proc/self/mounts"); - auto regex = std::regex(R"((^|\n)overlay )" + realStoreDir.get() + R"( .*(\n|$))"); + auto regex = std::regex(R"((^|\n)overlay )" + config->realStoreDir.get() + R"( .*(\n|$))"); // Mount points can be stacked, so there might be multiple matching entries. // Loop until the last match, which will be the current state of the mount point. @@ -122,13 +124,13 @@ LocalOverlayStore::LocalOverlayStore(const Config & config) return std::regex_search(mountInfo, std::regex("\\b" + option + "=" + value + "( |,)")); }; - auto expectedLowerDir = lowerStore->realStoreDir.get(); - if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", upperLayer)) { + auto expectedLowerDir = lowerStore->config.realStoreDir.get(); + if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", config->upperLayer)) { debug("expected lowerdir: %s", expectedLowerDir); - debug("expected upperdir: %s", upperLayer); + debug("expected upperdir: %s", config->upperLayer); debug("actual mount: %s", mountInfo); throw Error("overlay filesystem '%s' mounted incorrectly", - realStoreDir.get()); + config->realStoreDir.get()); } } } @@ -278,14 +280,14 @@ void LocalOverlayStore::collectGarbage(const GCOptions & options, GCResults & re void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) { - auto mergedDir = realStoreDir.get() + "/"; + auto mergedDir = config->realStoreDir.get() + "/"; if (path.substr(0, mergedDir.length()) != mergedDir) { warn("local-overlay: unexpected gc path '%s' ", path); return; } StorePath storePath = {path.substr(mergedDir.length())}; - auto upperPath = toUpperPath(storePath); + auto upperPath = config->toUpperPath(storePath); if (pathExists(upperPath)) { debug("upper exists: %s", path); @@ -334,7 +336,7 @@ LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag StorePathSet done; auto existsInStoreDir = [&](const StorePath & storePath) { - return pathExists(realStoreDir.get() + "/" + storePath.to_string()); + return pathExists(config->realStoreDir.get() + "/" + storePath.to_string()); }; bool errors = false; @@ -354,10 +356,10 @@ void LocalOverlayStore::remountIfNecessary() { if (!_remountRequired) return; - if (remountHook.get().empty()) { - warn("'%s' needs remounting, set remount-hook to do this automatically", realStoreDir.get()); + if (config->remountHook.get().empty()) { + warn("'%s' needs remounting, set remount-hook to do this automatically", config->realStoreDir.get()); } else { - runProgram(remountHook, false, {realStoreDir}); + runProgram(config->remountHook, false, {config->realStoreDir}); } _remountRequired = false; diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 3d16c0c4695..112abd6be6b 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -6,7 +6,7 @@ namespace nix { template class F> struct LocalOverlayStoreConfigT { - const F lowerStoreUri; + const F> lowerStoreConfig; const F upperLayer; const F checkMount; const F remountHook; @@ -16,25 +16,17 @@ struct LocalOverlayStoreConfigT * Configuration for `LocalOverlayStore`. */ struct LocalOverlayStoreConfig : - virtual LocalStoreConfig, + LocalStoreConfig, LocalOverlayStoreConfigT { - struct Descriptions : - virtual Store::Config::Descriptions, - virtual LocalStore::Config::Descriptions, - LocalOverlayStoreConfigT - { - Descriptions(); - }; - - static const Descriptions descriptions; + static config::SettingDescriptionMap descriptions(); LocalOverlayStoreConfig( std::string_view scheme, PathView path, const StoreReference::Params & params); - const std::string name() override { return "Experimental Local Overlay Store"; } + const std::string name() const override { return "Experimental Local Overlay Store"; } std::optional experimentalFeature() const override { @@ -46,7 +38,7 @@ struct LocalOverlayStoreConfig : return { "local-overlay" }; } - std::string doc() override; + std::string doc() const override; protected: /** @@ -57,7 +49,9 @@ protected: * at that file path. It might be stored in the lower layer instead, * or it might not be part of this store at all. */ - Path toUpperPath(const StorePath & path); + Path toUpperPath(const StorePath & path) const; + + friend struct LocalOverlayStore; }; /** @@ -66,11 +60,13 @@ protected: * Documentation on overridden methods states how they differ from their * `LocalStore` counterparts. */ -struct LocalOverlayStore : virtual LocalOverlayStoreConfig, virtual LocalStore +struct LocalOverlayStore : virtual LocalStore { using Config = LocalOverlayStoreConfig; - LocalOverlayStore(const Config &); + ref config; + + LocalOverlayStore(ref); std::string getUri() override { diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b16d56ebb3d..2572ed9e75a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -108,7 +108,7 @@ LocalStore::Config::LocalStoreConfig( { } -std::string LocalStoreConfig::doc() +std::string LocalStoreConfig::doc() const { return #include "local-store.md" diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index e05d3cc3b85..fd72249abe0 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -58,12 +58,12 @@ struct LocalStoreConfig : std::string_view authority, const StoreReference::Params & params); - const std::string name() override { return "Local Store"; } + const std::string name() const override { return "Local Store"; } static std::set uriSchemes() { return {"local"}; } - std::string doc() override; + std::string doc() const override; ref openStore() const override; }; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index d4a63de1f13..b79f68b3c53 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -21,22 +21,13 @@ template class Pool; template class F> struct RemoteStoreConfigT { - const F maxConnections; - const F maxConnectionAge; + F maxConnections; + F maxConnectionAge; }; -struct RemoteStoreConfig : - virtual Store::Config, - RemoteStoreConfigT +struct RemoteStoreConfig : RemoteStoreConfigT { - struct Descriptions : - virtual Store::Config::Descriptions, - RemoteStoreConfigT - { - Descriptions(); - }; - - static const Descriptions descriptions; + static config::SettingDescriptionMap descriptions(); RemoteStoreConfig(const StoreReference::Params &); }; @@ -46,13 +37,14 @@ struct RemoteStoreConfig : * DaemonStore. */ struct RemoteStore : - public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore, public virtual LogStore { using Config = RemoteStoreConfig; + const Config & config; + RemoteStore(const Config & config); /* Implementations of abstract store API methods. */ diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 675f09a1231..1629135b751 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1307,7 +1307,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre } -std::string MixStoreDirMethods::showPaths(const StorePathSet & paths) +std::string MixStoreDirMethods::showPaths(const StorePathSet & paths) const { std::string s; for (auto & i : paths) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 206a35373db..87b8335430a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -133,12 +133,12 @@ struct StoreConfig : /** * The name of this type of store. */ - virtual const std::string name() = 0; + virtual const std::string name() const = 0; /** * Documentation for this type of store. */ - virtual std::string doc() + virtual std::string doc() const { return ""; } @@ -897,3 +897,6 @@ std::map drvOutputReferences( Store * evalStore = nullptr); } + +// Parses a Store URL, uses global state not pure so think about this +JSON_IMPL(ref) diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index f247f917a50..dae372147c9 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -64,7 +64,7 @@ struct MixStoreDirMethods * Display a set of paths in human-readable form (i.e., between quotes * and separated by commas). */ - std::string showPaths(const StorePathSet & paths); + std::string showPaths(const StorePathSet & paths) const; /** * @return true if ‘path’ is in the Nix store (but not the Nix diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc index 1f3edfc71c1..306416b2681 100644 --- a/src/libstore/store-registration.cc +++ b/src/libstore/store-registration.cc @@ -15,7 +15,6 @@ ref openStore(StoreReference && storeURI) { auto store = resolveStoreConfig(std::move(storeURI))->openStore(); - experimentalFeatureSettings.require(store->experimentalFeature()); #if 0 // FIXME store->warnUnknownSettings(); store->init(); @@ -69,6 +68,8 @@ ref resolveStoreConfig(StoreReference && storeURI) }, storeURI.variant); + experimentalFeatureSettings.require(storeConfig->experimentalFeature()); + return storeConfig; } @@ -94,7 +95,9 @@ std::list> getDefaultSubstituters() for (auto uri : settings.substituters.get()) addStore(uri); - stores.sort([](ref & a, ref & b) { return a->priority < b->priority; }); + stores.sort([](ref & a, ref & b) { + return a->resolvedSubstConfig.priority < b->resolvedSubstConfig.priority; + }); return stores; }()); @@ -103,3 +106,54 @@ std::list> getDefaultSubstituters() } } + +namespace nlohmann { + +using namespace nix::config; + +ref adl_serializer>::from_json(const json & json) +{ + StoreReference ref; + switch (json.type()) { + + case json::value_t::string: { + ref = StoreReference::parse(json.get_ref()); + break; + } + + case json::value_t::object: { + auto & obj = json.get_ref(); + ref = StoreReference { + .variant = StoreReference::Specified{ + .scheme = getString(valueAt(obj, "scheme")), + .authority = getString(valueAt(obj, "authority")), + }, + .params = obj, + }; + break; + } + + case json::value_t::null: + case json::value_t::number_unsigned: + case json::value_t::number_integer: + case json::value_t::number_float: + case json::value_t::boolean: + case json::value_t::array: + case json::value_t::binary: + case json::value_t::discarded: + default: + throw UsageError( + "Invalid JSON for Store configuration: is type '%s' but must be string or object", + json.type_name()); + }; + + return resolveStoreConfig(std::move(ref)); +} + +void adl_serializer>::to_json(json & obj, ref s) +{ + // TODO, for tests maybe + assert(false); +} + +} diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 4af4eac624e..a584bb90385 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -8,17 +8,12 @@ namespace nix { struct UDSRemoteStoreConfig : - virtual LocalFSStore::Config, - virtual RemoteStore::Config + std::enable_shared_from_this, + Store::Config, + LocalFSStore::Config, + RemoteStore::Config { - struct Descriptions : - virtual LocalFSStore::Config::Descriptions, - virtual RemoteStore::Config::Descriptions - { - Descriptions(); - }; - - static const Descriptions descriptions; + static config::SettingDescriptionMap descriptions(); UDSRemoteStoreConfig(const StoreReference::Params & params) : UDSRemoteStoreConfig{"unix", "", params} @@ -32,9 +27,9 @@ struct UDSRemoteStoreConfig : std::string_view authority, const StoreReference::Params & params); - const std::string name() override { return "Local Daemon Store"; } + const std::string name() const override { return "Local Daemon Store"; } - std::string doc() override; + std::string doc() const override; /** * The path to the unix domain socket. @@ -51,12 +46,13 @@ struct UDSRemoteStoreConfig : }; struct UDSRemoteStore : - virtual UDSRemoteStoreConfig, virtual IndirectRootStore, virtual RemoteStore { using Config = UDSRemoteStoreConfig; + ref config; + UDSRemoteStore(const Config &); std::string getUri() override; From 1344bcd0b2b8364447779ffb256ec1724fc47cd6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 22:07:48 -0400 Subject: [PATCH 49/80] WIP --- src/libstore/dummy-store.cc | 20 +++++++++----------- src/libstore/store-registration.cc | 14 +++++++------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 2f3269b6830..503ec346d13 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -12,7 +12,7 @@ DummyStoreConfig::DummyStoreConfig( throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); } -std::string DummyStoreConfig::doc() +std::string DummyStoreConfig::doc() const { return #include "dummy-store.md" @@ -20,22 +20,20 @@ std::string DummyStoreConfig::doc() } -const DummyStoreConfig::Descriptions DummyStoreConfig::descriptions{}; - - -struct DummyStore : public virtual DummyStoreConfig, public virtual Store +struct DummyStore : virtual Store { using Config = DummyStoreConfig; - DummyStore(const Config & config) - : StoreConfig(config) - , DummyStoreConfig(config) - , Store{static_cast(*this)} + ref config; + + DummyStore(ref config) + : Store{*config} + , config(config) { } std::string getUri() override { - return *uriSchemes().begin(); + return *Config::uriSchemes().begin() + "://"; } void queryPathInfoUncached(const StorePath & path, @@ -82,7 +80,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store ref DummyStore::Config::openStore() const { - return make_ref(*this); + return make_ref(ref{shared_from_this()}); } static RegisterStoreImplementation regDummyStore; diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc index 306416b2681..00bd522d600 100644 --- a/src/libstore/store-registration.cc +++ b/src/libstore/store-registration.cc @@ -123,11 +123,12 @@ ref adl_serializer>::from_json(const j case json::value_t::object: { auto & obj = json.get_ref(); - ref = StoreReference { - .variant = StoreReference::Specified{ - .scheme = getString(valueAt(obj, "scheme")), - .authority = getString(valueAt(obj, "authority")), - }, + ref = StoreReference{ + .variant = + StoreReference::Specified{ + .scheme = getString(valueAt(obj, "scheme")), + .authority = getString(valueAt(obj, "authority")), + }, .params = obj, }; break; @@ -143,8 +144,7 @@ ref adl_serializer>::from_json(const j case json::value_t::discarded: default: throw UsageError( - "Invalid JSON for Store configuration: is type '%s' but must be string or object", - json.type_name()); + "Invalid JSON for Store configuration: is type '%s' but must be string or object", json.type_name()); }; return resolveStoreConfig(std::move(ref)); From 5a1b10969dfbde502c6f6f906e9515a09446052e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 22:14:05 -0400 Subject: [PATCH 50/80] WIP --- src/libstore/uds-remote-store.cc | 35 ++++++++++---------------------- src/libstore/uds-remote-store.hh | 2 +- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 0cc9140edb7..5b3518c0d8f 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -18,22 +18,12 @@ namespace nix { -UDSRemoteStoreConfig::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , LocalFSStore::Config::Descriptions{LocalFSStore::Config::descriptions} - , RemoteStore::Config::Descriptions{RemoteStore::Config::descriptions} -{} - - -const UDSRemoteStoreConfig::Descriptions UDSRemoteStoreConfig::descriptions{}; - - UDSRemoteStoreConfig::UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, const StoreReference::Params & params) : StoreConfig(params) - , LocalFSStoreConfig(params) + , LocalFSStoreConfig(*this, params) , RemoteStoreConfig(params) , path{authority.empty() ? settings.nixDaemonSocketFile : authority} { @@ -43,7 +33,7 @@ UDSRemoteStoreConfig::UDSRemoteStoreConfig( } -std::string UDSRemoteStoreConfig::doc() +std::string UDSRemoteStoreConfig::doc() const { return #include "uds-remote-store.md" @@ -51,27 +41,24 @@ std::string UDSRemoteStoreConfig::doc() } -UDSRemoteStore::UDSRemoteStore(const Config & config) - : Store::Config{config} - , LocalFSStore::Config{config} - , RemoteStore::Config{config} - , UDSRemoteStore::Config{config} - , Store(static_cast(*this)) - , LocalFSStore(static_cast(*this)) - , RemoteStore(static_cast(*this)) +UDSRemoteStore::UDSRemoteStore(ref config) + : Store{*config} + , LocalFSStore{*config} + , RemoteStore{*config} + , config{config} { } std::string UDSRemoteStore::getUri() { - return path == settings.nixDaemonSocketFile + return config->path == settings.nixDaemonSocketFile ? // FIXME: Not clear why we return daemon here and not default // to settings.nixDaemonSocketFile // // unix:// with no path also works. Change what we return? "daemon" - : std::string(*uriSchemes().begin()) + "://" + path; + : std::string(*Config::uriSchemes().begin()) + "://" + config->path; } @@ -88,7 +75,7 @@ ref UDSRemoteStore::openConnection() /* Connect to a daemon that does the privileged work for us. */ conn->fd = createUnixDomainSocket(); - nix::connect(toSocket(conn->fd.get()), path); + nix::connect(toSocket(conn->fd.get()), config->path); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); @@ -109,7 +96,7 @@ void UDSRemoteStore::addIndirectRoot(const Path & path) ref UDSRemoteStore::Config::openStore() const { - return make_ref(*this); + return make_ref(ref{shared_from_this()}); } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index a584bb90385..75203cbd20c 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -53,7 +53,7 @@ struct UDSRemoteStore : ref config; - UDSRemoteStore(const Config &); + UDSRemoteStore(ref); std::string getUri() override; From 0c8330cae3cefa688abae2a8e65c11314dc075c4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 22:17:13 -0400 Subject: [PATCH 51/80] WIP --- src/libstore/uds-remote-store.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 5b3518c0d8f..e8ec4e8c31d 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -18,6 +18,16 @@ namespace nix { +config::SettingDescriptionMap UDSRemoteStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(LocalFSStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + return ret; +} + + UDSRemoteStoreConfig::UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, From f5df5e7c46f3c320017317c1bf5cac097c85253b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 22:26:40 -0400 Subject: [PATCH 52/80] WIP --- src/libstore/common-ssh-store-config.cc | 79 ++++++++++++++----------- src/libstore/common-ssh-store-config.hh | 19 ++---- 2 files changed, 51 insertions(+), 47 deletions(-) diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index ab328cdd3e2..72b44cc41ff 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -6,33 +6,51 @@ namespace nix { -CommonSSHStoreConfig::Descriptions::Descriptions() - : CommonSSHStoreConfigT{ - .sshKey{ - .name = "ssh-key", - .description = "Path to the SSH private key used to authenticate to the remote machine.", - }, - .sshPublicHostKey{ - .name = "base64-ssh-public-host-key", - .description = "The public host key of the remote machine.", - }, - .compress{ - .name = "compress", - .description = "Whether to enable SSH compression.", - }, - .remoteStore{ - .name = "remote-store", - .description = R"( - [Store URL](@docroot@/store/types/index.md#store-url-format) - to be used on the remote machine. The default is `auto` - (i.e. use the Nix daemon or `/nix/store` directly). - )", - }, - } +static const CommonSSHStoreConfigT commonSSHStoreConfigDescriptions = { + .sshKey{ + .name = "ssh-key", + .description = "Path to the SSH private key used to authenticate to the remote machine.", + }, + .sshPublicHostKey{ + .name = "base64-ssh-public-host-key", + .description = "The public host key of the remote machine.", + }, + .compress{ + .name = "compress", + .description = "Whether to enable SSH compression.", + }, + .remoteStore{ + .name = "remote-store", + .description = R"( + [Store URL](@docroot@/store/types/index.md#store-url-format) + to be used on the remote machine. The default is `auto` + (i.e. use the Nix daemon or `/nix/store` directly). + )", + }, +}; + +#define COMMON_SSH_STORE_CONFIG_FIELDS(X) X(sshKey), X(sshPublicHostKey), X(compress), X(remoteStore), + +MAKE_PARSE(CommonSSHStoreConfig, commonSSHStoreConfig, COMMON_SSH_STORE_CONFIG_FIELDS) + +static CommonSSHStoreConfigT commonSSHStoreConfigDefaults() { + return { + .sshKey = {""}, + .sshPublicHostKey = {""}, + .compress = {false}, + .remoteStore = {""}, + }; } -const CommonSSHStoreConfig::Descriptions CommonSSHStoreConfig::descriptions{}; +MAKE_APPLY_PARSE(CommonSSHStoreConfig, commonSSHStoreConfig, COMMON_SSH_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap CommonSSHStoreConfig::descriptions() +{ + constexpr auto & descriptions = commonSSHStoreConfigDescriptions; + auto defaults = commonSSHStoreConfigDefaults(); + return {COMMON_SSH_STORE_CONFIG_FIELDS(DESC_ROW)}; +} static std::string extractConnStr(std::string_view scheme, std::string_view _connStr) { @@ -52,16 +70,9 @@ static std::string extractConnStr(std::string_view scheme, std::string_view _con } CommonSSHStoreConfig::CommonSSHStoreConfig( - std::string_view scheme, - std::string_view host, - const StoreReference::Params & params) - : CommonSSHStoreConfigT{ - CONFIG_ROW(sshKey, ""), - CONFIG_ROW(sshPublicHostKey, ""), - CONFIG_ROW(compress, false), - CONFIG_ROW(remoteStore, ""), - } - , host(extractConnStr(scheme, host)) + std::string_view scheme, std::string_view host, const StoreReference::Params & params) + : CommonSSHStoreConfigT{commonSSHStoreConfigApplyParse(params)} + , host{extractConnStr(scheme, host)} { } diff --git a/src/libstore/common-ssh-store-config.hh b/src/libstore/common-ssh-store-config.hh index d9361e259a3..357c3ced5cb 100644 --- a/src/libstore/common-ssh-store-config.hh +++ b/src/libstore/common-ssh-store-config.hh @@ -10,22 +10,15 @@ class SSHMaster; template class F> struct CommonSSHStoreConfigT { - const F sshKey; - const F sshPublicHostKey; - const F compress; - const F remoteStore; + F sshKey; + F sshPublicHostKey; + F compress; + F remoteStore; }; -struct CommonSSHStoreConfig : - CommonSSHStoreConfigT +struct CommonSSHStoreConfig : CommonSSHStoreConfigT { - struct Descriptions : - CommonSSHStoreConfigT - { - Descriptions(); - }; - - static const Descriptions descriptions; + static config::SettingDescriptionMap descriptions(); /** * @param scheme Note this isn't stored by this mix-in class, but From 6ab4817f8802ff0c49f4c3c4f695b4e331f96ddc Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 22:48:06 -0400 Subject: [PATCH 53/80] WIP --- src/libstore/common-ssh-store-config.cc | 2 +- src/libstore/common-ssh-store-config.hh | 2 +- src/libstore/legacy-ssh-store.cc | 81 +++++++++++++------------ src/libstore/legacy-ssh-store.hh | 29 ++++----- 4 files changed, 56 insertions(+), 58 deletions(-) diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index 72b44cc41ff..96ec172998f 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -76,7 +76,7 @@ CommonSSHStoreConfig::CommonSSHStoreConfig( { } -SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) +SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) const { return { host, diff --git a/src/libstore/common-ssh-store-config.hh b/src/libstore/common-ssh-store-config.hh index 357c3ced5cb..d998e16a2b3 100644 --- a/src/libstore/common-ssh-store-config.hh +++ b/src/libstore/common-ssh-store-config.hh @@ -56,7 +56,7 @@ struct CommonSSHStoreConfig : CommonSSHStoreConfigT */ SSHMaster createSSHMaster( bool useMaster, - Descriptor logFD = INVALID_DESCRIPTOR); + Descriptor logFD = INVALID_DESCRIPTOR) const; }; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index ea49b6c8ac6..1b5aded4448 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -17,24 +17,32 @@ namespace nix { -LegacySSHStore::Config::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , CommonSSHStoreConfig::Descriptions{CommonSSHStoreConfig::descriptions} - , LegacySSHStoreConfigT{ - .remoteProgram{ - .name = "remote-program", - .description = "Path to the `nix-store` executable on the remote machine.", - }, - .maxConnections{ - .name = "max-connections", - .description = "Maximum number of concurrent SSH connections.", - }, - } -{} +static const LegacySSHStoreConfigT legacySSHStoreConfigDescriptions = { + .remoteProgram{ + .name = "remote-program", + .description = "Path to the `nix-store` executable on the remote machine.", + }, + .maxConnections{ + .name = "max-connections", + .description = "Maximum number of concurrent SSH connections.", + }, +}; +#define LEGACY_SSH_STORE_CONFIG_FIELDS(X) \ + X(remoteProgram), \ + X(maxConnections) -const LegacySSHStore::Config::Descriptions LegacySSHStore::Config::descriptions{}; +MAKE_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS) + +static LegacySSHStoreConfigT legacySSHStoreConfigDefaults() +{ + return { + .remoteProgram = {{"nix-store"}}, + .maxConnections = {1}, + }; +} +MAKE_APPLY_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS) LegacySSHStore::Config::LegacySSHStoreConfig( std::string_view scheme, @@ -42,10 +50,7 @@ LegacySSHStore::Config::LegacySSHStoreConfig( const StoreReference::Params & params) : Store::Config{params} , CommonSSHStoreConfig{scheme, authority, params} - , LegacySSHStoreConfigT{ - CONFIG_ROW(remoteProgram, Strings{"nix-store"}), - CONFIG_ROW(maxConnections, 1), - } + , LegacySSHStoreConfigT{legacySSHStoreConfigApplyParse(params)} { #ifndef _WIN32 if (auto * p = get(params, "log-fd")) { @@ -55,7 +60,7 @@ LegacySSHStore::Config::LegacySSHStoreConfig( } -std::string LegacySSHStoreConfig::doc() +std::string LegacySSHStoreConfig::doc() const { return #include "legacy-ssh-store.md" @@ -69,20 +74,18 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection bool good = true; }; -LegacySSHStore::LegacySSHStore(const Config & config) - : Store::Config{config} - , CommonSSHStoreConfig{config} - , LegacySSHStore::Config{config} - , Store{static_cast(*this)} +LegacySSHStore::LegacySSHStore(ref config) + : Store{*config} + , config{config} , connections(make_ref>( - std::max(1, (int) maxConnections), + std::max(1, (int) config->maxConnections), [this]() { return openConnection(); }, [](const ref & r) { return r->good; } )) - , master(createSSHMaster( + , master(config->createSSHMaster( // Use SSH master only if using more than 1 connection. connections->capacity() > 1, - logFD)) + config->logFD)) { } @@ -90,12 +93,12 @@ LegacySSHStore::LegacySSHStore(const Config & config) ref LegacySSHStore::openConnection() { auto conn = make_ref(); - Strings command = remoteProgram.get(); + Strings command = config->remoteProgram.get(); command.push_back("--serve"); command.push_back("--write"); - if (remoteStore.get() != "") { + if (config->remoteStore.get() != "") { command.push_back("--store"); - command.push_back(remoteStore.get()); + command.push_back(config->remoteStore.get()); } conn->sshConn = master.startCommand(std::move(command)); conn->to = FdSink(conn->sshConn->in.get()); @@ -105,7 +108,7 @@ ref LegacySSHStore::openConnection() TeeSource tee(conn->from, saved); try { conn->remoteVersion = ServeProto::BasicClientConnection::handshake( - conn->to, tee, SERVE_PROTOCOL_VERSION, host); + conn->to, tee, SERVE_PROTOCOL_VERSION, config->host); } catch (SerialisationError & e) { // in.close(): Don't let the remote block on us not writing. conn->sshConn->in.close(); @@ -114,9 +117,9 @@ ref LegacySSHStore::openConnection() tee.drainInto(nullSink); } throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", - host, chomp(saved.s)); + config->host, chomp(saved.s)); } catch (EndOfFile & e) { - throw Error("cannot connect to '%1%'", host); + throw Error("cannot connect to '%1%'", config->host); } return conn; @@ -125,7 +128,7 @@ ref LegacySSHStore::openConnection() std::string LegacySSHStore::getUri() { - return *uriSchemes().begin() + "://" + host; + return *Config::uriSchemes().begin() + "://" + config->host; } @@ -138,7 +141,7 @@ void LegacySSHStore::queryPathInfoUncached(const StorePath & path, /* No longer support missing NAR hash */ assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); - debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); + debug("querying remote host '%s' for info on '%s'", config->host, printStorePath(path)); auto infos = conn->queryPathInfos(*this, {path}); @@ -167,7 +170,7 @@ void LegacySSHStore::queryPathInfoUncached(const StorePath & path, void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { - debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); + debug("adding path '%s' to remote host '%s'", printStorePath(info.path), config->host); auto conn(connections->get()); @@ -194,7 +197,7 @@ void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, conn->to.flush(); if (readInt(conn->from) != 1) - throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); + throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), config->host); } else { @@ -346,7 +349,7 @@ std::optional LegacySSHStore::isTrustedClient() ref LegacySSHStore::Config::openStore() const { - return make_ref(*this); + return make_ref(ref{shared_from_this()}); } diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index 4da2d02dfa6..9dc744bfd49 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -12,24 +12,17 @@ namespace nix { template class F> struct LegacySSHStoreConfigT { - const F remoteProgram; - const F maxConnections; + F remoteProgram; + F maxConnections; }; struct LegacySSHStoreConfig : - virtual Store::Config, - virtual CommonSSHStoreConfig, + std::enable_shared_from_this, + Store::Config, + CommonSSHStoreConfig, LegacySSHStoreConfigT { - struct Descriptions : - virtual Store::Config::Descriptions, - virtual CommonSSHStoreConfig::Descriptions, - LegacySSHStoreConfigT - { - Descriptions(); - }; - - static const Descriptions descriptions; + static config::SettingDescriptionMap descriptions(); /** * Hack for getting remote build log output. Intentionally not a @@ -42,26 +35,28 @@ struct LegacySSHStoreConfig : std::string_view authority, const StoreReference::Params & params); - const std::string name() override { return "SSH Store"; } + const std::string name() const override { return "SSH Store"; } static std::set uriSchemes() { return {"ssh"}; } - std::string doc() override; + std::string doc() const override; ref openStore() const override; }; -struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store +struct LegacySSHStore : public virtual Store { using Config = LegacySSHStoreConfig; + ref config; + struct Connection; ref> connections; SSHMaster master; - LegacySSHStore(const Config &); + LegacySSHStore(ref); ref openConnection(); From cc1ce6a4c66ff2c2d875d7c6ccb1ae4fb8aec842 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 23:24:25 -0400 Subject: [PATCH 54/80] WIP --- src/libstore/legacy-ssh-store.cc | 23 ++++ src/libstore/mounted-ssh-store.md | 18 --- src/libstore/ssh-store.cc | 178 ++++++++++++++++-------------- src/libstore/ssh-store.hh | 52 ++------- src/libstore/ssh-store.md | 14 +++ src/libstore/uds-remote-store.cc | 6 +- 6 files changed, 145 insertions(+), 146 deletions(-) delete mode 100644 src/libstore/mounted-ssh-store.md diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 1b5aded4448..110a832e05d 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -28,12 +28,15 @@ static const LegacySSHStoreConfigT legacySSHStoreConfigDesc }, }; + #define LEGACY_SSH_STORE_CONFIG_FIELDS(X) \ X(remoteProgram), \ X(maxConnections) + MAKE_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS) + static LegacySSHStoreConfigT legacySSHStoreConfigDefaults() { return { @@ -42,8 +45,27 @@ static LegacySSHStoreConfigT legacySSHStoreConfigDefaults() }; } + MAKE_APPLY_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap LegacySSHStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(CommonSSHStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + { + constexpr auto & descriptions = legacySSHStoreConfigDescriptions; + auto defaults = legacySSHStoreConfigDefaults(); + ret.merge(decltype(ret){ + LEGACY_SSH_STORE_CONFIG_FIELDS(DESC_ROW) + }); + } + return ret; +} + + LegacySSHStore::Config::LegacySSHStoreConfig( std::string_view scheme, std::string_view authority, @@ -74,6 +96,7 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection bool good = true; }; + LegacySSHStore::LegacySSHStore(ref config) : Store{*config} , config{config} diff --git a/src/libstore/mounted-ssh-store.md b/src/libstore/mounted-ssh-store.md deleted file mode 100644 index 1ebfe3081dc..00000000000 --- a/src/libstore/mounted-ssh-store.md +++ /dev/null @@ -1,18 +0,0 @@ -R"( - -**Store URL format**: `mounted-ssh-ng://[username@]hostname` - -Experimental store type that allows full access to a Nix store on a remote machine, -and additionally requires that store be mounted in the local file system. - -The mounting of that store is not managed by Nix, and must by managed manually. -It could be accomplished with SSHFS or NFS, for example. - -The local file system is used to optimize certain operations. -For example, rather than serializing Nix archives and sending over the Nix channel, -we can directly access the file system data via the mount-point. - -The local file system is also used to make certain operations possible that wouldn't otherwise be. -For example, persistent GC roots can be created if they reside on the same file system as the remote store: -the remote side will create the symlinks necessary to avoid race conditions. -)" diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index a83a11b4fdd..25d9f7818ea 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -12,19 +12,61 @@ namespace nix { -SSHStoreConfig::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , CommonSSHStoreConfig::Descriptions{CommonSSHStoreConfig::descriptions} - , SSHStoreConfigT{ - .remoteProgram{ - .name = "remote-program", - .description = "Path to the `nix-daemon` executable on the remote machine.", - }, +static const SSHStoreConfigT sshStoreConfigDescriptions = { + .remoteProgram{ + .name = "remote-program", + .description = "Path to the `nix-daemon` executable on the remote machine.", + }, +}; + + +#define SSH_STORE_CONFIG_FIELDS(X) \ + X(remoteProgram) + + +MAKE_PARSE(SSHStoreConfig, sshStoreConfig, SSH_STORE_CONFIG_FIELDS) + + +static SSHStoreConfigT sshStoreConfigDefaults() +{ + return { + .remoteProgram = {{"nix-daemon"}}, + }; +} + + +MAKE_APPLY_PARSE(SSHStoreConfig, sshStoreConfig, SSH_STORE_CONFIG_FIELDS) + + +config::SettingDescriptionMap SSHStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(CommonSSHStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + { + constexpr auto & descriptions = sshStoreConfigDescriptions; + auto defaults = sshStoreConfigDefaults(); + ret.merge(decltype(ret){ + SSH_STORE_CONFIG_FIELDS(DESC_ROW) + }); } -{} + //ret.merge(LocalFSStoreConfig::descriptions()); + return ret; +} -const SSHStoreConfig::Descriptions SSHStoreConfig::descriptions{}; +static std::optional getMounted( + const Store::Config & storeConfig, + const StoreReference::Params & params) +{ + auto mountedParamsOpt = optionalValueAt(params, "mounted"); + if (!mountedParamsOpt) return {}; + auto * mountedParamsP = getNullable(*mountedParamsOpt); + if (!mountedParamsP) return {}; + auto & mountedParams = getObject(*mountedParamsP); + return {{storeConfig, mountedParams}}; +} SSHStoreConfig::SSHStoreConfig( @@ -34,34 +76,40 @@ SSHStoreConfig::SSHStoreConfig( : Store::Config{params} , RemoteStore::Config{params} , CommonSSHStoreConfig{scheme, authority, params} - , SSHStoreConfigT{ - CONFIG_ROW(remoteProgram, Strings{"nix-daemon"}), - } + , SSHStoreConfigT{sshStoreConfigApplyParse(params)} + , mounted{getMounted(*this, params)} { } -std::string SSHStoreConfig::doc() +const std::string SSHStoreConfig::name() const +{ + std::string ret = "Experimental SSH Store"; + if (mounted) + ret += " with filesystem mounted"; + return ret; +} + + +std::string SSHStoreConfig::doc() const { return #include "ssh-store.md" ; } -struct SSHStore : - public virtual SSHStoreConfig, - public virtual RemoteStore + +struct SSHStore : virtual RemoteStore { using Config = SSHStoreConfig; - SSHStore(const Config & config) - : Store::Config{config} - , RemoteStore::Config{config} - , CommonSSHStoreConfig{config} - , SSHStore::Config{config} - , Store{static_cast(*this)} - , RemoteStore{static_cast(*this)} - , master(createSSHMaster( + ref config; + + SSHStore(ref config) + : Store{*config} + , RemoteStore{*config} + , config{config} + , master(config->createSSHMaster( // Use SSH master only if using more than 1 connection. connections->capacity() > 1)) { @@ -69,7 +117,7 @@ struct SSHStore : std::string getUri() override { - return *uriSchemes().begin() + "://" + host; + return *Config::uriSchemes().begin() + "://" + host; } // FIXME extend daemon protocol, move implementation to RemoteStore @@ -108,41 +156,6 @@ struct SSHStore : }; -ref SSHStore::Config::openStore() const { - return make_ref(*this); -} - - -MountedSSHStoreConfig::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , CommonSSHStoreConfig::Descriptions{CommonSSHStoreConfig::descriptions} - , SSHStore::Config::Descriptions{SSHStoreConfig::descriptions} -{} - - -const MountedSSHStoreConfig::Descriptions MountedSSHStoreConfig::descriptions{}; - - -MountedSSHStoreConfig::MountedSSHStoreConfig( - std::string_view scheme, - std::string_view host, - const StoreReference::Params & params) - : Store::Config{params} - , RemoteStore::Config{params} - , CommonSSHStoreConfig{scheme, host, params} - , SSHStoreConfig{scheme, host, params} - , LocalFSStoreConfig{params} -{ -} - -std::string MountedSSHStoreConfig::doc() -{ - return - #include "mounted-ssh-store.md" - ; -} - - /** * The mounted ssh store assumes that filesystems on the remote host are * shared with the local host. This means that the remote nix store is @@ -157,32 +170,24 @@ std::string MountedSSHStoreConfig::doc() * The difference lies in how they manage GC roots. See addPermRoot * below for details. */ -struct MountedSSHStore : virtual MountedSSHStoreConfig, virtual SSHStore, virtual LocalFSStore +struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore { - using Config = MountedSSHStoreConfig; - - MountedSSHStore(const Config & config) - : Store::Config{config} - , RemoteStore::Config{config} - , CommonSSHStoreConfig{config} - , SSHStore::Config{config} - , LocalFSStore::Config{config} - , MountedSSHStore::Config{config} - , Store{static_cast(*this)} - , RemoteStore{static_cast(*this)} - , SSHStore{static_cast(*this)} - , LocalFSStore{static_cast(*this)} + using Config = SSHStore::Config; + + const LocalFSStore::Config & mountedConfig; + + MountedSSHStore(ref config, const LocalFSStore::Config & mountedConfig) + : Store{*config} + , RemoteStore{*config} + , SSHStore{config} + , LocalFSStore{mountedConfig} + , mountedConfig{mountedConfig} { extraRemoteProgramArgs = { "--process-ops", }; } - std::string getUri() override - { - return *uriSchemes().begin() + "://" + host; - } - void narFromPath(const StorePath & path, Sink & sink) override { return LocalFSStore::narFromPath(path, sink); @@ -226,18 +231,23 @@ struct MountedSSHStore : virtual MountedSSHStoreConfig, virtual SSHStore, virtua ref MountedSSHStore::Config::openStore() const { - return make_ref(*this); + ref config {shared_from_this()}; + + if (config->mounted) + return make_ref(config, *config->mounted); + else + return make_ref(config); } ref SSHStore::openConnection() { auto conn = make_ref(); - Strings command = remoteProgram.get(); + Strings command = config->remoteProgram.get(); command.push_back("--stdio"); - if (remoteStore.get() != "") { + if (config->remoteStore.get() != "") { command.push_back("--store"); - command.push_back(remoteStore.get()); + command.push_back(config->remoteStore.get()); } command.insert(command.end(), extraRemoteProgramArgs.begin(), extraRemoteProgramArgs.end()); diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh index e934876c393..4f5796728a2 100644 --- a/src/libstore/ssh-store.hh +++ b/src/libstore/ssh-store.hh @@ -11,63 +11,33 @@ namespace nix { template class F> struct SSHStoreConfigT { - const F remoteProgram; + F remoteProgram; }; -struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig, SSHStoreConfigT +struct SSHStoreConfig : std::enable_shared_from_this, + Store::Config, + RemoteStore::Config, + CommonSSHStoreConfig, + SSHStoreConfigT { - struct Descriptions : virtual RemoteStoreConfig::Descriptions, - virtual CommonSSHStoreConfig::Descriptions, - SSHStoreConfigT - { - Descriptions(); - }; + static config::SettingDescriptionMap descriptions(); - static const Descriptions descriptions; + std::optional mounted; SSHStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params); - const std::string name() override - { - return "Experimental SSH Store"; - } + const std::string name() const override; static std::set uriSchemes() { return {"ssh-ng"}; } - std::string doc() override; - - ref openStore() const override; -}; - -struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStore::Config -{ - struct Descriptions : virtual SSHStoreConfig::Descriptions, virtual LocalFSStore::Config::Descriptions - { - Descriptions(); - }; - - static const Descriptions descriptions; - - MountedSSHStoreConfig(std::string_view scheme, std::string_view host, const StoreReference::Params & params); - - const std::string name() override - { - return "Experimental SSH Store with filesystem mounted"; - } - - static std::set uriSchemes() - { - return {"mounted-ssh-ng"}; - } - - std::string doc() override; + std::string doc() const override; std::optional experimentalFeature() const override { - return ExperimentalFeature::MountedSSHStore; + return mounted ? std::optional{ExperimentalFeature::MountedSSHStore} : std::nullopt; } ref openStore() const override; diff --git a/src/libstore/ssh-store.md b/src/libstore/ssh-store.md index 881537e7114..bdd15ef99ed 100644 --- a/src/libstore/ssh-store.md +++ b/src/libstore/ssh-store.md @@ -5,4 +5,18 @@ R"( Experimental store type that allows full access to a Nix store on a remote machine. +### The `mounted` configuration subject + +This additionally requires that store be mounted in the local file system. + +The mounting of that store is not managed by Nix, and must by managed manually. +It could be accomplished with SSHFS or NFS, for example. + +The local file system is used to optimize certain operations. +For example, rather than serializing Nix archives and sending over the Nix channel, +we can directly access the file system data via the mount-point. + +The local file system is also used to make certain operations possible that wouldn't otherwise be. +For example, persistent GC roots can be created if they reside on the same file system as the remote store: +the remote side will create the symlinks necessary to avoid race conditions. )" diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index e8ec4e8c31d..eff0743a933 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -32,9 +32,9 @@ UDSRemoteStoreConfig::UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, const StoreReference::Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(*this, params) - , RemoteStoreConfig(params) + : Store::Config{params} + , LocalFSStore::Config{*this, params} + , RemoteStore::Config{params} , path{authority.empty() ? settings.nixDaemonSocketFile : authority} { if (uriSchemes().count(std::string{scheme}) == 0) { From 53f692ba602e3d1b3ba7a4cdbba221dc78740dff Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 20 Jul 2024 23:39:05 -0400 Subject: [PATCH 55/80] WIP --- src/libstore/store-registration.hh | 20 +++++++++++++++----- src/nix/main.cc | 13 +++++-------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/libstore/store-registration.hh b/src/libstore/store-registration.hh index 289bdf341ff..7c254d61b3f 100644 --- a/src/libstore/store-registration.hh +++ b/src/libstore/store-registration.hh @@ -13,7 +13,10 @@ namespace nix { struct StoreFactory { + std::string doc; std::set uriSchemes; + config::SettingDescriptionMap configDescriptions; + std::optional experimentalFeature; /** * The `authorityPath` parameter is `/`, or really * whatever comes after `://` and before `?`. @@ -21,26 +24,33 @@ struct StoreFactory std::function( std::string_view scheme, std::string_view authorityPath, const StoreReference::Params & params)> parseConfig; - config::SettingDescriptionMap configDescriptions; }; struct Implementations { - static std::vector * registered; +private: + + using V = std::vector>; + +public: + + static V * registered; template static void add() { if (!registered) - registered = new std::vector(); + registered = new V{}; StoreFactory factory{ + .doc = TConfig::doc(), .uriSchemes = TConfig::uriSchemes(), + .configDescriptions = TConfig::descriptions(), + .experimentalFeature = TConfig::experimentalFeature(), .parseConfig = ([](auto scheme, auto uri, auto & params) -> ref { return make_ref(scheme, uri, params); }), - .configDescriptions = TConfig::descriptions(), }; - registered->push_back(factory); + registered->push_back({TConfig::name(), std::move(factory)}); } }; diff --git a/src/nix/main.cc b/src/nix/main.cc index 524c4626350..158f758e589 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -222,15 +222,12 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs res["args"] = toJSON(); auto stores = nlohmann::json::object(); - for (auto & implem : *Implementations::registered) { -#if 0 - auto storeConfig = implem.getConfig(); - auto storeName = storeConfig->name(); + for (auto & [storeName, implem] : *Implementations::registered) { auto & j = stores[storeName]; - j["doc"] = storeConfig->doc(); - j["settings"] = storeConfig->toJSON(); - j["experimentalFeature"] = storeConfig->experimentalFeature(); -#endif + j["doc"] = implem.doc; + j["uri-schemes"] = implem.uriSchemes; + j["settings"] = implem.configDescriptions; + j["experimentalFeature"] = implem.experimentalFeature; } res["stores"] = std::move(stores); res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo(); From e6212087f04a869943f59a3368508031dfbf2f13 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 21 Jul 2024 10:05:50 -0400 Subject: [PATCH 56/80] WIP --- src/libstore/binary-cache-store.cc | 118 ++++++++++-------- src/libstore/binary-cache-store.hh | 25 ++-- src/libstore/config-parse.cc | 32 +++++ src/libstore/dummy-store.cc | 2 +- src/libstore/dummy-store.hh | 20 +++ src/libstore/http-binary-cache-store.cc | 8 ++ src/libstore/http-binary-cache-store.hh | 18 ++- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/legacy-ssh-store.hh | 4 +- src/libstore/local-binary-cache-store.hh | 9 +- src/libstore/local-overlay-store.cc | 2 +- src/libstore/local-overlay-store.hh | 6 +- src/libstore/local-store.cc | 2 +- src/libstore/local-store.hh | 4 +- src/libstore/s3-binary-cache-store.hh | 41 +++--- src/libstore/ssh-store.cc | 12 +- src/libstore/ssh-store.hh | 12 +- src/libstore/store-api.hh | 9 +- src/libstore/store-registration.cc | 4 +- src/libstore/store-registration.hh | 17 +++ src/libstore/uds-remote-store.cc | 2 +- src/libstore/uds-remote-store.hh | 4 +- .../unix/build/local-derivation-goal.cc | 2 - 23 files changed, 208 insertions(+), 147 deletions(-) create mode 100644 src/libstore/config-parse.cc create mode 100644 src/libstore/dummy-store.hh diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 1bd9363a272..23bd633296b 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -25,66 +25,78 @@ namespace nix { -BinaryCacheStore::Config::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , BinaryCacheStoreConfigT{ - .compression = { - .name = "compression", - .description = "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`).", - }, - .writeNARListing = { - .name = "write-nar-listing", - .description = "Whether to write a JSON file that lists the files in each NAR.", - }, - .writeDebugInfo = { - .name = "index-debug-info", - .description = R"( - Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to - fetch debug info on demand - )", - }, - .secretKeyFile{ - .name = "secret-key", - .description = "Path to the secret key used to sign the binary cache.", - }, - .localNarCache{ - .name = "local-nar-cache", - .description = "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`.", - }, - .parallelCompression{ - .name = "parallel-compression", - .description = "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`.", - }, - .compressionLevel{ - .name = "compression-level", - .description = R"( - The *preset level* to be used when compressing NARs. - The meaning and accepted values depend on the compression method selected. - `-1` specifies that the default compression level should be used. - )", - }, - } -{} +static const BinaryCacheStoreConfigT binaryCacheStoreConfigDescriptions = { + .compression = { + .name = "compression", + .description = "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`).", + }, + .writeNARListing = { + .name = "write-nar-listing", + .description = "Whether to write a JSON file that lists the files in each NAR.", + }, + .writeDebugInfo = { + .name = "index-debug-info", + .description = R"( + Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to + fetch debug info on demand + )", + }, + .secretKeyFile{ + .name = "secret-key", + .description = "Path to the secret key used to sign the binary cache.", + }, + .localNarCache{ + .name = "local-nar-cache", + .description = "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`.", + }, + .parallelCompression{ + .name = "parallel-compression", + .description = "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`.", + }, + .compressionLevel{ + .name = "compression-level", + .description = R"( + The *preset level* to be used when compressing NARs. + The meaning and accepted values depend on the compression method selected. + `-1` specifies that the default compression level should be used. + )", + }, +}; + +#define BINARY_CACHE_STORE_CONFIG_FIELDS(X) \ + X(compression), \ + X(writeNARListing), \ + X(writeDebugInfo), \ + X(secretKeyFile), \ + X(localNarCache), \ + X(parallelCompression), \ + X(compressionLevel), + +MAKE_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS) + +static BinaryCacheStoreConfigT binaryCacheStoreConfigDefaults() +{ + return { + .compression = {"xz"}, + .writeNARListing = {false}, + .writeDebugInfo = {false}, + .secretKeyFile = {""}, + .localNarCache = {""}, + .parallelCompression = {false}, + .compressionLevel = {-1}, + }; +} -const BinaryCacheStore::Config::Descriptions BinaryCacheStore::Config::descriptions{}; +MAKE_APPLY_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS) BinaryCacheStore::Config::BinaryCacheStoreConfig(const StoreReference::Params & params) - : StoreConfig{params} - , BinaryCacheStoreConfigT{ - CONFIG_ROW(compression, "xz"), - CONFIG_ROW(writeNARListing, false), - CONFIG_ROW(writeDebugInfo, false), - CONFIG_ROW(secretKeyFile, ""), - CONFIG_ROW(localNarCache, ""), - CONFIG_ROW(parallelCompression, false), - CONFIG_ROW(compressionLevel, -1), - } + : BinaryCacheStoreConfigT{binaryCacheStoreConfigApplyParse(params)} { } BinaryCacheStore::BinaryCacheStore(const Config & config) - : Config{config} - , Store{static_cast(*this)} + : Store{config} + , Config{config} { if (secretKeyFile != "") signer = std::make_unique( diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 877a5c3343c..c1c8a06b3f2 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -16,27 +16,20 @@ struct NarInfo; template class F> struct BinaryCacheStoreConfigT { - const F compression; - const F writeNARListing; - const F writeDebugInfo; - const F secretKeyFile; - const F localNarCache; - const F parallelCompression; - const F compressionLevel; + F compression; + F writeNARListing; + F writeDebugInfo; + F secretKeyFile; + F localNarCache; + F parallelCompression; + F compressionLevel; }; struct BinaryCacheStoreConfig : - virtual Store::Config, BinaryCacheStoreConfigT { - struct Descriptions : - virtual Store::Config::Descriptions, - BinaryCacheStoreConfigT - { - Descriptions(); - }; - - static const Descriptions descriptions; + virtual Store::Config, + static config::SettingDescriptionMap descriptions(); BinaryCacheStoreConfig(const StoreReference::Params &); }; diff --git a/src/libstore/config-parse.cc b/src/libstore/config-parse.cc new file mode 100644 index 00000000000..727bc391917 --- /dev/null +++ b/src/libstore/config-parse.cc @@ -0,0 +1,32 @@ +#include + +#include "config-parse.hh" + +namespace nix::config { + + +}; + +namespace nlohmann { + +using namespace nix::config; + +SettingDescription adl_serializer::from_json(const json & json) +{ + // TODO implement if we ever need (testing?) + assert(false); +} + +void adl_serializer::to_json(json & obj, SettingDescription s) +{ + obj.emplace("description", s.description); + //obj.emplace("aliases", s.aliases); + //obj.emplace("experimentalFeature", s.experimentalFeature); + + // Cannot use `null` because the default value might itself be + // `null`. + if (s.defaultValue) + obj.emplace("defaultValue", *s.defaultValue); +} + +} diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 503ec346d13..628b6bd3d07 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -12,7 +12,7 @@ DummyStoreConfig::DummyStoreConfig( throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); } -std::string DummyStoreConfig::doc() const +std::string DummyStoreConfig::doc() { return #include "dummy-store.md" diff --git a/src/libstore/dummy-store.hh b/src/libstore/dummy-store.hh new file mode 100644 index 00000000000..41683800b4a --- /dev/null +++ b/src/libstore/dummy-store.hh @@ -0,0 +1,20 @@ +#include "store-api.hh" + +namespace nix { + +struct DummyStoreConfig : std::enable_shared_from_this, StoreConfig +{ + DummyStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params); + + static const std::string name() { return "Dummy Store"; } + + static std::string doc(); + + static std::set uriSchemes() { + return {"dummy"}; + } + + ref openStore() const override; +}; + +} diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 3cdd7e8874d..6aee70a9c4a 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -9,6 +9,14 @@ namespace nix { MakeError(UploadToHTTP, Error); +std::set HttpBinaryCacheStoreConfig::uriSchemes() +{ + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; + auto ret = std::set({"http", "https"}); + if (forceHttp) + ret.insert("file"); + return ret; +} HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( std::string_view scheme, diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh index 397bd0592a6..c7cc1bccf03 100644 --- a/src/libstore/http-binary-cache-store.hh +++ b/src/libstore/http-binary-cache-store.hh @@ -2,28 +2,24 @@ namespace nix { -struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +struct HttpBinaryCacheStoreConfig : + std::enable_shared_from_this, + Store::Config, + BinaryCacheStoreConfig { HttpBinaryCacheStoreConfig( std::string_view scheme, std::string_view cacheUri, const StoreReference::Params & params); Path cacheUri; - const std::string name() override + static const std::string name() { return "HTTP Binary Cache Store"; } - static std::set uriSchemes() - { - static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; - auto ret = std::set({"http", "https"}); - if (forceHttp) - ret.insert("file"); - return ret; - } + static std::set uriSchemes(); - std::string doc() override; + static std::string doc(); ref openStore() const override; }; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 110a832e05d..85dba07c4a5 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -82,7 +82,7 @@ LegacySSHStore::Config::LegacySSHStoreConfig( } -std::string LegacySSHStoreConfig::doc() const +std::string LegacySSHStoreConfig::doc() { return #include "legacy-ssh-store.md" diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index 9dc744bfd49..d87b19aa118 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -35,11 +35,11 @@ struct LegacySSHStoreConfig : std::string_view authority, const StoreReference::Params & params); - const std::string name() const override { return "SSH Store"; } + static const std::string name() { return "SSH Store"; } static std::set uriSchemes() { return {"ssh"}; } - std::string doc() const override; + static std::string doc(); ref openStore() const override; }; diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh index 647cd83d099..b29dfc3c89f 100644 --- a/src/libstore/local-binary-cache-store.hh +++ b/src/libstore/local-binary-cache-store.hh @@ -2,7 +2,10 @@ namespace nix { -struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +struct LocalBinaryCacheStoreConfig : + std::enable_shared_from_this, + Store::Config, + BinaryCacheStoreConfig { /** * @param binaryCacheDir `file://` is a short-hand for `file:///` @@ -13,14 +16,14 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig Path binaryCacheDir; - const std::string name() override + static const std::string name() { return "Local Binary Cache Store"; } static std::set uriSchemes(); - std::string doc() override; + static std::string doc(); ref openStore() const override; }; diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 5f3d9281814..d00584263aa 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -86,7 +86,7 @@ LocalOverlayStore::Config::LocalOverlayStoreConfig( } -std::string LocalOverlayStoreConfig::doc() const +std::string LocalOverlayStoreConfig::doc() { return #include "local-overlay-store.md" diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 112abd6be6b..dd449c4fc9c 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -26,9 +26,9 @@ struct LocalOverlayStoreConfig : PathView path, const StoreReference::Params & params); - const std::string name() const override { return "Experimental Local Overlay Store"; } + static const std::string name() { return "Experimental Local Overlay Store"; } - std::optional experimentalFeature() const override + static std::optional experimentalFeature() { return ExperimentalFeature::LocalOverlayStore; } @@ -38,7 +38,7 @@ struct LocalOverlayStoreConfig : return { "local-overlay" }; } - std::string doc() const override; + static std::string doc(); protected: /** diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2572ed9e75a..b16d56ebb3d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -108,7 +108,7 @@ LocalStore::Config::LocalStoreConfig( { } -std::string LocalStoreConfig::doc() const +std::string LocalStoreConfig::doc() { return #include "local-store.md" diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index fd72249abe0..a37301ebfd3 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -58,12 +58,12 @@ struct LocalStoreConfig : std::string_view authority, const StoreReference::Params & params); - const std::string name() const override { return "Local Store"; } + static const std::string name() { return "Local Store"; } static std::set uriSchemes() { return {"local"}; } - std::string doc() const override; + static std::string doc(); ref openStore() const override; }; diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 2097bb5b427..e67f682d773 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -10,34 +10,31 @@ namespace nix { template class F> struct S3BinaryCacheStoreConfigT { - const F profile; - const F region; - const F scheme; - const F endpoint; - const F narinfoCompression; - const F lsCompression; - const F logCompression; - const F multipartUpload; - const F bufferSize; + F profile; + F region; + F scheme; + F endpoint; + F narinfoCompression; + F lsCompression; + F logCompression; + F multipartUpload; + F bufferSize; }; -struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig, S3BinaryCacheStoreConfigT +struct S3BinaryCacheStoreConfig : + std::enable_shared_from_this, + Store::Config, + BinaryCacheStoreConfig, + S3BinaryCacheStoreConfigT { - struct Descriptions : virtual Store::Config::Descriptions, - virtual BinaryCacheStore::Config::Descriptions, - S3BinaryCacheStoreConfigT - { - Descriptions(); - }; - - static const Descriptions descriptions; + static config::SettingDescriptionMap configDescriptions(); S3BinaryCacheStoreConfig( std::string_view uriScheme, std::string_view bucketName, const StoreReference::Params & params); std::string bucketName; - const std::string name() override + static std::string name() { return "S3 Binary Cache Store"; } @@ -47,7 +44,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig, S3BinaryCacheS return {"s3"}; } - std::string doc() override; + static std::string doc(); ref openStore() const override; }; @@ -56,9 +53,11 @@ struct S3BinaryCacheStore : virtual BinaryCacheStore { using Config = S3BinaryCacheStoreConfig; + ref config; + protected: - S3BinaryCacheStore(const Config &); + S3BinaryCacheStore(ref); public: diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 25d9f7818ea..53140955a46 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -82,18 +82,8 @@ SSHStoreConfig::SSHStoreConfig( } -const std::string SSHStoreConfig::name() const +std::string SSHStoreConfig::doc() { - std::string ret = "Experimental SSH Store"; - if (mounted) - ret += " with filesystem mounted"; - return ret; -} - - -std::string SSHStoreConfig::doc() const -{ - return #include "ssh-store.md" ; } diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh index 4f5796728a2..7f71c275a0b 100644 --- a/src/libstore/ssh-store.hh +++ b/src/libstore/ssh-store.hh @@ -26,19 +26,17 @@ struct SSHStoreConfig : std::enable_shared_from_this, SSHStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params); - const std::string name() const override; + static const std::string name() + { + return "Experimental SSH Store"; + } static std::set uriSchemes() { return {"ssh-ng"}; } - std::string doc() const override; - - std::optional experimentalFeature() const override - { - return mounted ? std::optional{ExperimentalFeature::MountedSSHStore} : std::nullopt; - } + static std::string doc(); ref openStore() const override; }; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 87b8335430a..099f2aee724 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -130,15 +130,10 @@ struct StoreConfig : static StringSet getDefaultSystemFeatures(); - /** - * The name of this type of store. - */ - virtual const std::string name() const = 0; - /** * Documentation for this type of store. */ - virtual std::string doc() const + static std::string doc() { return ""; } @@ -147,7 +142,7 @@ struct StoreConfig : * An experimental feature this type store is gated, if it is to be * experimental. */ - virtual std::optional experimentalFeature() const + static std::optional experimentalFeature() { return std::nullopt; } diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc index 00bd522d600..83f6255331b 100644 --- a/src/libstore/store-registration.cc +++ b/src/libstore/store-registration.cc @@ -59,7 +59,7 @@ ref resolveStoreConfig(StoreReference && storeURI) return make_ref(params); }, [&](const StoreReference::Specified & g) { - for (auto implem : *Implementations::registered) + for (auto & [name, implem] : *Implementations::registered) if (implem.uriSchemes.count(g.scheme)) return implem.parseConfig(g.scheme, g.authority, params); @@ -73,7 +73,7 @@ ref resolveStoreConfig(StoreReference && storeURI) return storeConfig; } -std::vector * Implementations::registered = 0; +Implementations::V * Implementations::registered = 0; std::list> getDefaultSubstituters() { diff --git a/src/libstore/store-registration.hh b/src/libstore/store-registration.hh index 7c254d61b3f..e479966250e 100644 --- a/src/libstore/store-registration.hh +++ b/src/libstore/store-registration.hh @@ -13,10 +13,24 @@ namespace nix { struct StoreFactory { + /** + * Documentation for this type of store. + */ std::string doc; + + /** + * URIs with these schemes should be handled by this factory + */ std::set uriSchemes; + config::SettingDescriptionMap configDescriptions; + + /** + * An experimental feature this type store is gated, if it is to be + * experimental. + */ std::optional experimentalFeature; + /** * The `authorityPath` parameter is `/`, or really * whatever comes after `://` and before `?`. @@ -30,6 +44,9 @@ struct Implementations { private: + /** + * The name of this type of store, and a factory for it. + */ using V = std::vector>; public: diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index eff0743a933..b6a108d2d9e 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -43,7 +43,7 @@ UDSRemoteStoreConfig::UDSRemoteStoreConfig( } -std::string UDSRemoteStoreConfig::doc() const +std::string UDSRemoteStoreConfig::doc() { return #include "uds-remote-store.md" diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 75203cbd20c..ce9fcbfeec8 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -27,9 +27,9 @@ struct UDSRemoteStoreConfig : std::string_view authority, const StoreReference::Params & params); - const std::string name() const override { return "Local Daemon Store"; } + static const std::string name() { return "Local Daemon Store"; } - std::string doc() const override; + static std::string doc(); /** * The path to the unix domain socket. diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 1d242f5927a..de30718481b 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -1256,8 +1256,6 @@ bool LocalDerivationGoal::isAllowed(const DerivedPath & req) struct RestrictedStoreConfig : virtual LocalFSStoreConfig { - const std::string name() override { return "Restricted Store"; } - ref openStore() const override; ref next; From 05861c4008951450ca5ba944c9f83e9d217edd0e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 21 Jul 2024 11:16:09 -0400 Subject: [PATCH 57/80] WIP --- src/build-remote/build-remote.cc | 4 +- src/libstore/binary-cache-store.cc | 47 +++++----- src/libstore/binary-cache-store.hh | 8 +- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/build/substitution-goal.cc | 4 +- src/libstore/config-parse.cc | 5 +- src/libstore/dummy-store.hh | 8 +- src/libstore/gc.cc | 24 +++--- src/libstore/http-binary-cache-store.cc | 39 +++++---- src/libstore/http-binary-cache-store.hh | 9 +- src/libstore/local-binary-cache-store.cc | 39 +++++---- src/libstore/local-binary-cache-store.hh | 9 +- src/libstore/local-store.hh | 5 ++ src/libstore/meson.build | 1 + src/libstore/optimise-store.cc | 8 +- src/libstore/parsed-derivations.cc | 2 +- src/libstore/remote-store.cc | 61 +++++++------ src/libstore/remote-store.hh | 4 +- src/libstore/s3-binary-cache-store.hh | 11 ++- src/libstore/ssh-store.cc | 2 +- src/libstore/uds-remote-store.cc | 2 +- .../unix/build/local-derivation-goal.cc | 85 ++++++++----------- tests/unit/libstore/ssh-store.cc | 14 ++- 23 files changed, 207 insertions(+), 186 deletions(-) diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 1824e20024e..929394dfd72 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -44,7 +44,7 @@ static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { for (auto & feature : requiredFeatures) - if (!store.systemFeatures.get().count(feature)) return false; + if (!store.config.systemFeatures.get().count(feature)) return false; return true; } @@ -85,7 +85,7 @@ static int main_build_remote(int argc, char * * argv) that gets cleared on reboot, but it wouldn't work on macOS. */ auto currentLoadName = "/current-load"; if (auto localStore = store.dynamic_pointer_cast()) - currentLoad = std::string { localStore->stateDir } + currentLoadName; + currentLoad = std::string { localStore->config.stateDir } + currentLoadName; else currentLoad = settings.nixStateDir + currentLoadName; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 23bd633296b..1246845130b 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -89,18 +89,21 @@ static BinaryCacheStoreConfigT binaryCacheStoreConfigDefaults MAKE_APPLY_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS) -BinaryCacheStore::Config::BinaryCacheStoreConfig(const StoreReference::Params & params) +BinaryCacheStore::Config::BinaryCacheStoreConfig( + const Store::Config & storeConfig, + const StoreReference::Params & params) : BinaryCacheStoreConfigT{binaryCacheStoreConfigApplyParse(params)} + , storeConfig{storeConfig} { } BinaryCacheStore::BinaryCacheStore(const Config & config) - : Store{config} - , Config{config} + : Store{config.storeConfig} + , config{config} { - if (secretKeyFile != "") + if (config.secretKeyFile != "") signer = std::make_unique( - SecretKey { readFile(secretKeyFile) }); + SecretKey { readFile(config.secretKeyFile) }); StringSink sink; sink << narVersionMagic1; @@ -127,11 +130,11 @@ void BinaryCacheStore::init() throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'", getUri(), value, storeDir); } else if (name == "WantMassQuery") { - if (defaultWantMassQuery) - wantMassQuery.value = value == "1"; + resolvedSubstConfig.wantMassQuery.value = + config.storeConfig.wantMassQuery.optValue.value_or(value == "1"); } else if (name == "Priority") { - if (defaultPriority) - priority.value = std::stoi(value); + resolvedSubstConfig.priority.value = + config.storeConfig.priority.optValue.value_or(std::stoi(value)); } } } @@ -218,7 +221,11 @@ ref BinaryCacheStore::addToStoreCommon( { FdSink fileSink(fdTemp.get()); TeeSink teeSinkCompressed { fileSink, fileHashSink }; - auto compressionSink = makeCompressionSink(compression, teeSinkCompressed, parallelCompression, compressionLevel); + auto compressionSink = makeCompressionSink( + config.compression, + teeSinkCompressed, + config.parallelCompression, + config.compressionLevel); TeeSink teeSinkUncompressed { *compressionSink, narHashSink }; TeeSource teeSource { narSource, teeSinkUncompressed }; narAccessor = makeNarAccessor(teeSource); @@ -230,17 +237,17 @@ ref BinaryCacheStore::addToStoreCommon( auto info = mkInfo(narHashSink.finish()); auto narInfo = make_ref(info); - narInfo->compression = compression; + narInfo->compression = config.compression; auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar" - + (compression == "xz" ? ".xz" : - compression == "bzip2" ? ".bz2" : - compression == "zstd" ? ".zst" : - compression == "lzip" ? ".lzip" : - compression == "lz4" ? ".lz4" : - compression == "br" ? ".br" : + + (config.compression == "xz" ? ".xz" : + config.compression == "bzip2" ? ".bz2" : + config.compression == "zstd" ? ".zst" : + config.compression == "lzip" ? ".lzip" : + config.compression == "lz4" ? ".lz4" : + config.compression == "br" ? ".br" : ""); auto duration = std::chrono::duration_cast(now2 - now1).count(); @@ -262,7 +269,7 @@ ref BinaryCacheStore::addToStoreCommon( /* Optionally write a JSON file containing a listing of the contents of the NAR. */ - if (writeNARListing) { + if (config.writeNARListing) { nlohmann::json j = { {"version", 1}, {"root", listNar(ref(narAccessor), CanonPath::root, true)}, @@ -274,7 +281,7 @@ ref BinaryCacheStore::addToStoreCommon( /* Optionally maintain an index of DWARF debug info files consisting of JSON files named 'debuginfo/' that specify the NAR file and member containing the debug info. */ - if (writeDebugInfo) { + if (config.writeDebugInfo) { CanonPath buildIdDir("lib/debug/.build-id"); @@ -586,7 +593,7 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) { ref BinaryCacheStore::getFSAccessor(bool requireValidPath) { - return make_ref(ref(shared_from_this()), requireValidPath, localNarCache); + return make_ref(ref(shared_from_this()), requireValidPath, config.localNarCache); } void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index c1c8a06b3f2..af649aaab86 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -28,10 +28,11 @@ struct BinaryCacheStoreConfigT struct BinaryCacheStoreConfig : BinaryCacheStoreConfigT { - virtual Store::Config, static config::SettingDescriptionMap descriptions(); - BinaryCacheStoreConfig(const StoreReference::Params &); + const Store::Config & storeConfig; + + BinaryCacheStoreConfig(const Store::Config &, const StoreReference::Params &); }; /** @@ -39,12 +40,13 @@ struct BinaryCacheStoreConfig : * virtual getFile() methods. */ struct BinaryCacheStore : - virtual BinaryCacheStoreConfig, virtual Store, virtual LogStore { using Config = BinaryCacheStoreConfig; + const Config & config; + private: std::unique_ptr signer; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index b809e3ffe3f..606151cf557 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1261,7 +1261,7 @@ Path DerivationGoal::openLogFile() /* Create a log file. */ Path logDir; if (auto localStore = dynamic_cast(&worker.store)) - logDir = localStore->logDir; + logDir = localStore->config->logDir; else logDir = settings.nixLogDir; Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2)); diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 32189ca7ae5..6d8cfe2d2cf 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -122,7 +122,7 @@ Goal::Co PathSubstitutionGoal::init() /* Bail out early if this substituter lacks a valid signature. LocalStore::addToStore() also checks for this, but only after we've downloaded the path. */ - if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) + if (!sub->config.isTrusted && worker.store.pathInfoIsUntrusted(*info)) { warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", worker.store.printStorePath(storePath), sub->getUri()); @@ -209,7 +209,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, PushActivity pact(act.id); copyStorePath(*sub, worker.store, - subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + subPath, repair, sub->config.isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { diff --git a/src/libstore/config-parse.cc b/src/libstore/config-parse.cc index 727bc391917..5d3a29574f6 100644 --- a/src/libstore/config-parse.cc +++ b/src/libstore/config-parse.cc @@ -4,7 +4,6 @@ namespace nix::config { - }; namespace nlohmann { @@ -20,8 +19,8 @@ SettingDescription adl_serializer::from_json(const json & js void adl_serializer::to_json(json & obj, SettingDescription s) { obj.emplace("description", s.description); - //obj.emplace("aliases", s.aliases); - //obj.emplace("experimentalFeature", s.experimentalFeature); + // obj.emplace("aliases", s.aliases); + // obj.emplace("experimentalFeature", s.experimentalFeature); // Cannot use `null` because the default value might itself be // `null`. diff --git a/src/libstore/dummy-store.hh b/src/libstore/dummy-store.hh index 41683800b4a..80f19a67a2a 100644 --- a/src/libstore/dummy-store.hh +++ b/src/libstore/dummy-store.hh @@ -6,11 +6,15 @@ struct DummyStoreConfig : std::enable_shared_from_this, StoreC { DummyStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params); - static const std::string name() { return "Dummy Store"; } + static const std::string name() + { + return "Dummy Store"; + } static std::string doc(); - static std::set uriSchemes() { + static std::set uriSchemes() + { return {"dummy"}; } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 54e09cb1639..ab8130ee221 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -40,7 +40,7 @@ static std::string gcRootsDir = "gcroots"; void LocalStore::addIndirectRoot(const Path & path) { std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Nix32, false); - Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); + Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", config->stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } @@ -79,7 +79,7 @@ void LocalStore::createTempRootsFile() void LocalStore::addTempRoot(const StorePath & path) { - if (readOnly) { + if (config->readOnly) { debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways."); return; } @@ -106,7 +106,7 @@ void LocalStore::addTempRoot(const StorePath & path) auto fdRootsSocket(_fdRootsSocket.lock()); if (!*fdRootsSocket) { - auto socketPath = stateDir.get() + gcSocketPath; + auto socketPath = config->stateDir.get() + gcSocketPath; debug("connecting to '%s'", socketPath); *fdRootsSocket = createUnixDomainSocket(); try { @@ -244,7 +244,7 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R else { target = absPath(target, dirOf(path)); if (!pathExists(target)) { - if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) { + if (isInDir(path, config->stateDir + "/" + gcRootsDir + "/auto")) { printInfo("removing stale link from '%1%' to '%2%'", path, target); unlink(path.c_str()); } @@ -285,8 +285,8 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R void LocalStore::findRootsNoTemp(Roots & roots, bool censor) { /* Process direct roots in {gcroots,profiles}. */ - findRoots(stateDir + "/" + gcRootsDir, std::filesystem::file_type::unknown, roots); - findRoots(stateDir + "/profiles", std::filesystem::file_type::unknown, roots); + findRoots(config->stateDir + "/" + gcRootsDir, std::filesystem::file_type::unknown, roots); + findRoots(config->stateDir + "/profiles", std::filesystem::file_type::unknown, roots); /* Add additional roots returned by different platforms-specific heuristics. This is typically used to add running programs to @@ -495,7 +495,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) readFile(*p); /* Start the server for receiving new roots. */ - auto socketPath = stateDir.get() + gcSocketPath; + auto socketPath = config->stateDir.get() + gcSocketPath; createDirs(dirOf(socketPath)); auto fdServer = createUnixDomainSocket(socketPath, 0666); @@ -632,7 +632,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) auto deleteFromStore = [&](std::string_view baseName) { Path path = storeDir + "/" + std::string(baseName); - Path realPath = realStoreDir + "/" + std::string(baseName); + Path realPath = config->realStoreDir + "/" + std::string(baseName); /* There may be temp directories in the store that are still in use by another process. We need to be sure that we can acquire an @@ -796,8 +796,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) printInfo("determining live/dead paths..."); try { - AutoCloseDir dir(opendir(realStoreDir.get().c_str())); - if (!dir) throw SysError("opening directory '%1%'", realStoreDir); + AutoCloseDir dir(opendir(config->realStoreDir.get().c_str())); + if (!dir) throw SysError("opening directory '%1%'", config->realStoreDir); /* Read the store and delete all paths that are invalid or unreachable. We don't use readDirectory() here so that @@ -899,8 +899,8 @@ void LocalStore::autoGC(bool sync) return std::stoll(readFile(*fakeFreeSpaceFile)); struct statvfs st; - if (statvfs(realStoreDir.get().c_str(), &st)) - throw SysError("getting filesystem info about '%s'", realStoreDir); + if (statvfs(config->realStoreDir.get().c_str(), &st)) + throw SysError("getting filesystem info about '%s'", config->realStoreDir); return (uint64_t) st.f_bavail * st.f_frsize; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 6aee70a9c4a..7a54218f579 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -22,8 +22,8 @@ HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( std::string_view scheme, std::string_view _cacheUri, const StoreReference::Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) + : Store::Config{params} + , BinaryCacheStoreConfig{*this, params} , cacheUri( std::string { scheme } + "://" @@ -45,7 +45,6 @@ std::string HttpBinaryCacheStoreConfig::doc() class HttpBinaryCacheStore : - public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore { struct State @@ -60,36 +59,36 @@ class HttpBinaryCacheStore : using Config = HttpBinaryCacheStoreConfig; - HttpBinaryCacheStore(const Config & config) - : Store::Config{config} - , BinaryCacheStore::Config{config} - , HttpBinaryCacheStore::Config{config} - , Store{static_cast(*this)} - , BinaryCacheStore{static_cast(*this)} + ref config; + + HttpBinaryCacheStore(ref config) + : Store{*config} + , BinaryCacheStore{*config} + , config{config} { diskCache = getNarInfoDiskCache(); } std::string getUri() override { - return cacheUri; + return config->cacheUri; } void init() override { // FIXME: do this lazily? - if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) { - if (defaultWantMassQuery) - wantMassQuery.value = cacheInfo->wantMassQuery; - if (defaultPriority) - priority.value = cacheInfo->priority; + if (auto cacheInfo = diskCache->upToDateCacheExists(config->cacheUri)) { + resolvedSubstConfig.wantMassQuery.value = + config->storeConfig.wantMassQuery.optValue.value_or(cacheInfo->wantMassQuery); + resolvedSubstConfig.priority.value = + config->storeConfig.priority.optValue.value_or(cacheInfo->priority); } else { try { BinaryCacheStore::init(); } catch (UploadToHTTP &) { - throw Error("'%s' does not appear to be a binary cache", cacheUri); + throw Error("'%s' does not appear to be a binary cache", config->cacheUri); } - diskCache->createCache(cacheUri, storeDir, wantMassQuery, priority); + diskCache->createCache(config->cacheUri, storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority); } } @@ -147,7 +146,7 @@ class HttpBinaryCacheStore : try { getFileTransfer()->upload(req); } catch (FileTransferError & e) { - throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); + throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", config->cacheUri, e.msg()); } } @@ -156,7 +155,7 @@ class HttpBinaryCacheStore : return FileTransferRequest( hasPrefix(path, "https://") || hasPrefix(path, "http://") || hasPrefix(path, "file://") ? path - : cacheUri + "/" + path); + : config->cacheUri + "/" + path); } @@ -219,7 +218,7 @@ class HttpBinaryCacheStore : ref HttpBinaryCacheStore::Config::openStore() const { - return make_ref(*this); + return make_ref(ref{shared_from_this()}); } static RegisterStoreImplementation regHttpBinaryCacheStore; diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh index c7cc1bccf03..0767e7dc8da 100644 --- a/src/libstore/http-binary-cache-store.hh +++ b/src/libstore/http-binary-cache-store.hh @@ -2,11 +2,12 @@ namespace nix { -struct HttpBinaryCacheStoreConfig : - std::enable_shared_from_this, - Store::Config, - BinaryCacheStoreConfig +struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this, + Store::Config, + BinaryCacheStoreConfig { + static config::SettingDescriptionMap descriptions(); + HttpBinaryCacheStoreConfig( std::string_view scheme, std::string_view cacheUri, const StoreReference::Params & params); diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 0997cfd77c1..143ff3c5445 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -12,8 +12,8 @@ LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig( std::string_view scheme, PathView binaryCacheDir, const StoreReference::Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) + : Store::Config{params} + , BinaryCacheStoreConfig{*this, params} , binaryCacheDir(binaryCacheDir) { } @@ -28,17 +28,16 @@ std::string LocalBinaryCacheStoreConfig::doc() struct LocalBinaryCacheStore : - virtual LocalBinaryCacheStoreConfig, virtual BinaryCacheStore { using Config = LocalBinaryCacheStoreConfig; - LocalBinaryCacheStore(const Config & config) - : Store::Config{config} - , BinaryCacheStore::Config{config} - , LocalBinaryCacheStore::Config{config} - , Store{static_cast(*this)} - , BinaryCacheStore{static_cast(*this)} + ref config; + + LocalBinaryCacheStore(ref config) + : Store{*config} + , BinaryCacheStore{*config} + , config{config} { } @@ -46,7 +45,7 @@ struct LocalBinaryCacheStore : std::string getUri() override { - return "file://" + binaryCacheDir; + return "file://" + config->binaryCacheDir; } protected: @@ -57,7 +56,7 @@ struct LocalBinaryCacheStore : std::shared_ptr> istream, const std::string & mimeType) override { - auto path2 = binaryCacheDir + "/" + path; + auto path2 = config->binaryCacheDir + "/" + path; static std::atomic counter{0}; Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter); AutoDelete del(tmp, false); @@ -70,7 +69,7 @@ struct LocalBinaryCacheStore : void getFile(const std::string & path, Sink & sink) override { try { - readFile(binaryCacheDir + "/" + path, sink); + readFile(config->binaryCacheDir + "/" + path, sink); } catch (SysError & e) { if (e.errNo == ENOENT) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); @@ -82,7 +81,7 @@ struct LocalBinaryCacheStore : { StorePathSet paths; - for (auto & entry : std::filesystem::directory_iterator{binaryCacheDir}) { + for (auto & entry : std::filesystem::directory_iterator{config->binaryCacheDir}) { checkInterrupt(); auto name = entry.path().filename().string(); if (name.size() != 40 || @@ -104,17 +103,17 @@ struct LocalBinaryCacheStore : void LocalBinaryCacheStore::init() { - createDirs(binaryCacheDir + "/nar"); - createDirs(binaryCacheDir + "/" + realisationsPrefix); - if (writeDebugInfo) - createDirs(binaryCacheDir + "/debuginfo"); - createDirs(binaryCacheDir + "/log"); + createDirs(config->binaryCacheDir + "/nar"); + createDirs(config->binaryCacheDir + "/" + realisationsPrefix); + if (config->writeDebugInfo) + createDirs(config->binaryCacheDir + "/debuginfo"); + createDirs(config->binaryCacheDir + "/log"); BinaryCacheStore::init(); } bool LocalBinaryCacheStore::fileExists(const std::string & path) { - return pathExists(binaryCacheDir + "/" + path); + return pathExists(config->binaryCacheDir + "/" + path); } std::set LocalBinaryCacheStoreConfig::uriSchemes() @@ -126,7 +125,7 @@ std::set LocalBinaryCacheStoreConfig::uriSchemes() } ref LocalBinaryCacheStoreConfig::openStore() const { - return make_ref(*this); + return make_ref(ref{shared_from_this()}); } static RegisterStoreImplementation regLocalBinaryCacheStore; diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh index b29dfc3c89f..9c12b4221c6 100644 --- a/src/libstore/local-binary-cache-store.hh +++ b/src/libstore/local-binary-cache-store.hh @@ -2,11 +2,12 @@ namespace nix { -struct LocalBinaryCacheStoreConfig : - std::enable_shared_from_this, - Store::Config, - BinaryCacheStoreConfig +struct LocalBinaryCacheStoreConfig : std::enable_shared_from_this, + Store::Config, + BinaryCacheStoreConfig { + static config::SettingDescriptionMap descriptions(); + /** * @param binaryCacheDir `file://` is a short-hand for `file:///` * for now. diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index a37301ebfd3..2bd4a5ea76e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -58,6 +58,11 @@ struct LocalStoreConfig : std::string_view authority, const StoreReference::Params & params); + /** + * For `RestrictedStore` + */ + LocalStoreConfig(const LocalStoreConfig &); + static const std::string name() { return "Local Store"; } static std::set uriSchemes() diff --git a/src/libstore/meson.build b/src/libstore/meson.build index ef01704de9a..d389655635f 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -55,6 +55,7 @@ configdata.set('CAN_LINK_SYMLINK', can_link_symlink.to_int()) check_funcs = [ # Optionally used for canonicalising files from the build 'lchown', + 'statvfs', ] foreach funcspec : check_funcs define_name = 'HAVE_' + funcspec.underscorify().to_upper() diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 9d903f21869..6ebd5b947d7 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -100,7 +100,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* HFS/macOS has some undocumented security feature disabling hardlinking for special files within .app dirs. Known affected paths include *.app/Contents/{PkgInfo,Resources/\*.lproj,_CodeSignature} and .DS_Store. - See https://github.com/NixOS/nix/issues/1443 and + See https://github.com/NixOS/nix/issues/1443 and https://github.com/NixOS/nix/pull/2230 for more discussion. */ if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) @@ -215,14 +215,14 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, the store itself (we don't want or need to mess with its permissions). */ const Path dirOfPath(dirOf(path)); - bool mustToggle = dirOfPath != realStoreDir.get(); + bool mustToggle = dirOfPath != config->realStoreDir.get(); if (mustToggle) makeWritable(dirOfPath); /* When we're done, make the directory read-only again and reset its timestamp back to 0. */ MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : ""); - std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), rand()); + std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", config->realStoreDir, getpid(), rand()); try { std::filesystem::create_hard_link(linkPath, tempLink); @@ -284,7 +284,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) if (!isValidPath(i)) continue; /* path was GC'ed, probably */ { Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i))); - optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash, NoRepair); + optimisePath_(&act, stats, config->realStoreDir + "/" + std::string(i.to_string()), inodeHash, NoRepair); } done++; act.progress(done, paths.size()); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index d8459d4d71c..c95d348d05c 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -110,7 +110,7 @@ bool ParsedDerivation::canBuildLocally(Store & localStore) const return false; for (auto & feature : getRequiredSystemFeatures()) - if (!localStore.systemFeatures.get().count(feature)) return false; + if (!localStore.config.systemFeatures.get().count(feature)) return false; return true; } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index cee331dde88..ba7e1a5d87b 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -24,40 +24,51 @@ namespace nix { -RemoteStore::Config::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , RemoteStoreConfigT{ - .maxConnections{ - .name = "max-connections", - .description = "Maximum number of concurrent connections to the Nix daemon.", - }, - .maxConnectionAge{ - .name = "max-connection-age", - .description = "Maximum age of a connection before it is closed.", - }, - } -{} +static const RemoteStoreConfigT remoteStoreConfigDescriptions = { + .maxConnections{ + .name = "max-connections", + .description = "Maximum number of concurrent connections to the Nix daemon.", + }, + .maxConnectionAge{ + .name = "max-connection-age", + .description = "Maximum age of a connection before it is closed.", + }, +}; -const RemoteStore::Config::Descriptions RemoteStore::Config::descriptions{}; +#define REMOTE_STORE_CONFIG_FIELDS(X) \ + X(maxConnections), \ + X(maxConnectionAge), -RemoteStore::Config::RemoteStoreConfig(const StoreReference::Params & params) - : Store::Config(params) - , RemoteStoreConfigT{ - CONFIG_ROW(maxConnections, 1), - CONFIG_ROW(maxConnectionAge, std::numeric_limits::max()), - } +MAKE_PARSE(RemoteStoreConfig, remoteStoreConfig, REMOTE_STORE_CONFIG_FIELDS) + + +static RemoteStoreConfigT remoteStoreConfigDefaults() +{ + return { + .maxConnections = {1}, + .maxConnectionAge = {std::numeric_limits::max()}, + }; +} + + +MAKE_APPLY_PARSE(RemoteStoreConfig, remoteStoreConfig, REMOTE_STORE_CONFIG_FIELDS) + + +RemoteStore::Config::RemoteStoreConfig(const Store::Config & storeConfig, const StoreReference::Params & params) + : RemoteStoreConfigT{remoteStoreConfigApplyParse(params)} + , storeConfig{storeConfig} { } /* TODO: Separate these store types into different files, give them better names */ RemoteStore::RemoteStore(const Config & config) - : RemoteStore::Config(config) - , Store(static_cast(*this)) + : Store{config.storeConfig} + , config{config} , connections(make_ref>( - std::max(1, (int) maxConnections), + std::max(1, (int) config.maxConnections), [this]() { auto conn = openConnectionWrapper(); try { @@ -68,12 +79,12 @@ RemoteStore::RemoteStore(const Config & config) } return conn; }, - [this](const ref & r) { + [config](const ref & r) { return r->to.good() && r->from.good() && std::chrono::duration_cast( - std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge; + std::chrono::steady_clock::now() - r->startTime).count() < config.maxConnectionAge; } )) { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index b79f68b3c53..41dd673e31c 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -29,7 +29,9 @@ struct RemoteStoreConfig : RemoteStoreConfigT { static config::SettingDescriptionMap descriptions(); - RemoteStoreConfig(const StoreReference::Params &); + const Store::Config & storeConfig; + + RemoteStoreConfig(const Store::Config &, const StoreReference::Params &); }; /** diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index e67f682d773..18ea4cd1592 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -21,13 +21,12 @@ struct S3BinaryCacheStoreConfigT F bufferSize; }; -struct S3BinaryCacheStoreConfig : - std::enable_shared_from_this, - Store::Config, - BinaryCacheStoreConfig, - S3BinaryCacheStoreConfigT +struct S3BinaryCacheStoreConfig : std::enable_shared_from_this, + Store::Config, + BinaryCacheStoreConfig, + S3BinaryCacheStoreConfigT { - static config::SettingDescriptionMap configDescriptions(); + static config::SettingDescriptionMap descriptions(); S3BinaryCacheStoreConfig( std::string_view uriScheme, std::string_view bucketName, const StoreReference::Params & params); diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 53140955a46..070da11666e 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -74,7 +74,7 @@ SSHStoreConfig::SSHStoreConfig( std::string_view authority, const StoreReference::Params & params) : Store::Config{params} - , RemoteStore::Config{params} + , RemoteStore::Config{*this, params} , CommonSSHStoreConfig{scheme, authority, params} , SSHStoreConfigT{sshStoreConfigApplyParse(params)} , mounted{getMounted(*this, params)} diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index b6a108d2d9e..5e8b40fe61d 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -34,7 +34,7 @@ UDSRemoteStoreConfig::UDSRemoteStoreConfig( const StoreReference::Params & params) : Store::Config{params} , LocalFSStore::Config{*this, params} - , RemoteStore::Config{params} + , RemoteStore::Config{*this, params} , path{authority.empty() ? settings.nixDaemonSocketFile : authority} { if (uriSchemes().count(std::string{scheme}) == 0) { diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index de30718481b..7c3c73cea1d 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -32,7 +32,7 @@ #include #if HAVE_STATVFS -#include +# include #endif /* Includes required for chroot support. */ @@ -56,8 +56,8 @@ #endif #if __APPLE__ -#include -#include +# include +# include #endif #include @@ -214,7 +214,7 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() } auto & localStore = getLocalStore(); - if (localStore.storeDir != localStore.realStoreDir.get()) { + if (localStore.storeDir != localStore.config->realStoreDir.get()) { #if __linux__ useChroot = true; #else @@ -354,7 +354,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() auto & localStore = getLocalStore(); uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; - if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 && + if (statvfs(localStore.config->realStoreDir.get().c_str(), &st) == 0 && (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; if (statvfs(tmpDir.c_str(), &st) == 0 && @@ -500,7 +500,7 @@ void LocalDerivationGoal::startBuilder() concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), worker.store.printStorePath(drvPath), settings.thisSystem, - concatStringsSep(", ", worker.store.systemFeatures)); + concatStringsSep(", ", worker.store.config.systemFeatures)); /* Create a temporary directory where the build will take place. */ @@ -1254,48 +1254,46 @@ bool LocalDerivationGoal::isAllowed(const DerivedPath & req) return this->isAllowed(pathPartOfReq(req)); } -struct RestrictedStoreConfig : virtual LocalFSStoreConfig +/** + * A wrapper around LocalStore that only allows building/querying of + * paths that are in the input closures of the build or were added via + * recursive Nix calls. + */ +struct RestrictedStore : + virtual IndirectRootStore, + virtual GcStore { - ref openStore() const override; + ref config; ref next; LocalDerivationGoal & goal; - RestrictedStoreConfig( - const StoreReference::Params & params, + RestrictedStore( + ref config, ref next, LocalDerivationGoal & goal) - : Store::Config{params} - , LocalFSStore::Config{params} + : Store{*config} + , LocalFSStore{*config} + , config{config} , next{next} , goal{goal} { } -}; -/** - * A wrapper around LocalStore that only allows building/querying of - * paths that are in the input closures of the build or were added via - * recursive Nix calls. - */ -struct RestrictedStore : - virtual RestrictedStoreConfig, - virtual IndirectRootStore, - virtual GcStore -{ - using Config = RestrictedStoreConfig; - - RestrictedStore(const RestrictedStoreConfig & config) - : Store::Config(config) - , LocalFSStore::Config(config) - , RestrictedStore::Config(config) - , Store(static_cast(*this)) - , LocalFSStore(static_cast(*this)) - { } + static ref make( + ref next, + LocalDerivationGoal & goal) + { + ref config = make_ref(*next->config); + config->pathInfoCacheSize.value = 0; + config->stateDir.value = "/no-such-path"; + config->logDir.value = "/no-such-path"; + return make_ref(std::move(config), std::move(next), goal); + } Path getRealStoreDir() override - { return next->realStoreDir; } + { return next->config->realStoreDir; } std::string getUri() override { return next->getUri(); } @@ -1495,28 +1493,13 @@ struct RestrictedStore : }; -ref RestrictedStore::Config::openStore() const -{ - return make_ref(*this); -} - - void LocalDerivationGoal::startDaemon() { experimentalFeatureSettings.require(Xp::RecursiveNix); - StoreReference::Params params; - params["path-info-cache-size"] = "0"; - params["store"] = worker.store.storeDir; - if (auto & optRoot = getLocalStore().rootDir.get()) - params["root"] = *optRoot; - params["state"] = "/no-such-path"; - params["log"] = "/no-such-path"; - auto store = RestrictedStore::Config{ - params, + auto store = RestrictedStore::make( ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), - *this, - }.openStore(); + *this); addedPaths.clear(); @@ -1844,7 +1827,7 @@ void LocalDerivationGoal::runChild() createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); - if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) + if (worker.store.config.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) ss.push_back("/dev/kvm"); ss.push_back("/dev/null"); ss.push_back("/dev/random"); diff --git a/tests/unit/libstore/ssh-store.cc b/tests/unit/libstore/ssh-store.cc index 0e2651f2e9b..8852273dcf9 100644 --- a/tests/unit/libstore/ssh-store.cc +++ b/tests/unit/libstore/ssh-store.cc @@ -7,7 +7,7 @@ namespace nix { TEST(SSHStore, constructConfig) { SSHStoreConfig config{ - "ssh", + "ssh-ng", "localhost", StoreReference::Params{ { @@ -30,8 +30,8 @@ TEST(SSHStore, constructConfig) TEST(MountedSSHStore, constructConfig) { - MountedSSHStoreConfig config{ - "mounted-ssh", + SSHStoreConfig config{ + "ssh-ng", "localhost", StoreReference::Params{ { @@ -41,6 +41,10 @@ TEST(MountedSSHStore, constructConfig) "bar", }, }, + { + "mounted", + nlohmann::json::object_t{}, + }, }, }; @@ -50,6 +54,10 @@ TEST(MountedSSHStore, constructConfig) "foo", "bar", })); + + ASSERT_TRUE(config.mounted); + + EXPECT_EQ(config.mounted->realStoreDir, "/nix/store"); } } From b0c6f187811d52e37d32014c087b72d04e2cc299 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 21 Jul 2024 13:50:43 -0400 Subject: [PATCH 58/80] Build `S3BinaryCacheStore` with Meson --- src/libstore/meson.build | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/libstore/meson.build b/src/libstore/meson.build index d389655635f..1eb413c1913 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -100,6 +100,20 @@ deps_public += nlohmann_json sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') deps_private += sqlite +# AWS C++ SDK has bad pkg-config +aws_s3 = dependency('aws-cpp-sdk-s3', required : false) +configdata.set('ENABLE_S3', aws_s3.found().to_int()) +if aws_s3.found() + aws_s3 = declare_dependency( + include_directories: include_directories(aws_s3.get_variable('includedir')), + link_args: [ + '-L' + aws_s3.get_variable('libdir'), + '-laws-cpp-sdk-s3', + ], + ) +endif +deps_other += aws_s3 + subdir('build-utils-meson/generate-header') generated_headers = [] From 40ebe0f5a76b52b5c1ba46bc9b3fccc1f34c8134 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 21 Jul 2024 17:34:48 -0400 Subject: [PATCH 59/80] S3 binary cache store --- configure.ac | 7 - src/libstore/http-binary-cache-store.cc | 3 +- src/libstore/s3-binary-cache-store.cc | 247 +++++++++++++----------- src/libstore/s3-binary-cache-store.hh | 4 - 4 files changed, 134 insertions(+), 127 deletions(-) diff --git a/configure.ac b/configure.ac index 4f66a3efcf6..5c22ed17636 100644 --- a/configure.ac +++ b/configure.ac @@ -340,13 +340,6 @@ AC_CHECK_HEADERS([aws/s3/S3Client.h], AC_SUBST(ENABLE_S3, [$enable_s3]) AC_LANG_POP(C++) -if test -n "$enable_s3"; then - declare -a aws_version_tokens=($(printf '#include \nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' ')) - AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.]) - AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.]) - AC_DEFINE_UNQUOTED([AWS_VERSION_PATCH], ${aws_version_tokens@<:@2@:>@}, [Patch version of aws-sdk-cpp.]) -fi - # Whether to use the Boehm garbage collector. AC_ARG_ENABLE(gc, AS_HELP_STRING([--enable-gc],[enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]), diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 7a54218f579..201cd8bba51 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -88,7 +88,8 @@ class HttpBinaryCacheStore : } catch (UploadToHTTP &) { throw Error("'%s' does not appear to be a binary cache", config->cacheUri); } - diskCache->createCache(config->cacheUri, storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority); + diskCache->createCache( + config->cacheUri, storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority); } } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 19f2a69c419..efd3a9bbd4d 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -60,7 +60,7 @@ class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem debug("AWS: %s", chomp(statement)); } -#if !(AWS_VERSION_MAJOR <= 1 && AWS_VERSION_MINOR <= 7 && AWS_VERSION_PATCH <= 115) +#if !(AWS_SDK_VERSION_MAJOR <= 1 && AWS_SDK_VERSION_MINOR <= 7 && AWS_SDK_VERSION_PATCH <= 115) void Flush() override {} #endif }; @@ -103,7 +103,7 @@ S3Helper::S3Helper( std::make_shared(profile.c_str())), *config, // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759 -#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3 +#if AWS_SDK_VERSION_MAJOR == 1 && AWS_SDK_VERSION_MINOR < 3 false, #else Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, @@ -188,96 +188,110 @@ S3Helper::FileTransferResult S3Helper::getObject( } -S3BinaryCacheStore::Config::Descriptions::Descriptions() - : Store::Config::Descriptions{Store::Config::descriptions} - , BinaryCacheStore::Config::Descriptions{BinaryCacheStore::Config::descriptions} - , S3BinaryCacheStoreConfigT{ - .profile{ - .name = "profile", - .description = R"( - The name of the AWS configuration profile to use. By default - Nix will use the `default` profile. - )", - }, - .region{ - .name = "region", - .description = R"( - The region of the S3 bucket. If your bucket is not in - `us–east-1`, you should always explicitly specify the region - parameter. - )", - }, - .scheme{ - .name = "scheme", - .description = R"( - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - )", - }, - .endpoint{ - .name = "endpoint", - .description = R"( - The URL of the endpoint of an S3-compatible service such as MinIO. - Do not specify this setting if you're using Amazon S3. - - > **Note** - > - > This endpoint must support HTTPS and will use path-based - > addressing instead of virtual host based addressing. - )", - }, - .narinfoCompression{ - .name = "narinfo-compression", - .description = "Compression method for `.narinfo` files.", - }, - .lsCompression{ - .name = "ls-compression", - .description = "Compression method for `.ls` files.", - }, - .logCompression{ - .name = "log-compression", - .description = R"( - Compression method for `log/*` files. It is recommended to - use a compression method supported by most web browsers - (e.g. `brotli`). - )", - }, - .multipartUpload{ - .name = "multipart-upload", - .description = "Whether to use multi-part uploads.", - }, - .bufferSize{ - .name = "buffer-size", - .description = "Size (in bytes) of each part in multi-part uploads.", - }, - } -{} +static const S3BinaryCacheStoreConfigT s3BinaryCacheStoreConfigDescriptions = { + .profile{ + .name = "profile", + .description = R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )", + }, + .region{ + .name = "region", + .description = R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )", + }, + .scheme{ + .name = "scheme", + .description = R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )", + }, + .endpoint{ + .name = "endpoint", + .description = R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )", + }, + .narinfoCompression{ + .name = "narinfo-compression", + .description = "Compression method for `.narinfo` files.", + }, + .lsCompression{ + .name = "ls-compression", + .description = "Compression method for `.ls` files.", + }, + .logCompression{ + .name = "log-compression", + .description = R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )", + }, + .multipartUpload{ + .name = "multipart-upload", + .description = "Whether to use multi-part uploads.", + }, + .bufferSize{ + .name = "buffer-size", + .description = "Size (in bytes) of each part in multi-part uploads.", + }, +}; + +#define S3_BINARY_CACHE_STORE_CONFIG_FIELDS(X) \ + X(profile), \ + X(region), \ + X(scheme), \ + X(endpoint), \ + X(narinfoCompression), \ + X(lsCompression), \ + X(logCompression), \ + X(multipartUpload), \ + X(bufferSize), + +MAKE_PARSE(S3BinaryCacheStoreConfig, s3BinaryCacheStoreConfig, S3_BINARY_CACHE_STORE_CONFIG_FIELDS) + +static S3BinaryCacheStoreConfigT s3BinaryCacheStoreConfigDefaults() +{ + return { + .profile = {""}, + .region = {Aws::Region::US_EAST_1}, + .scheme = {""}, + .endpoint = {""}, + .narinfoCompression = {""}, + .lsCompression = {""}, + .logCompression = {""}, + .multipartUpload = {false}, + .bufferSize = {5 * 1024 * 1024}, + }; +} -const S3BinaryCacheStore::Config::Descriptions S3BinaryCacheStore::Config::descriptions{}; +MAKE_APPLY_PARSE(S3BinaryCacheStoreConfig, s3BinaryCacheStoreConfig, S3_BINARY_CACHE_STORE_CONFIG_FIELDS) S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig( std::string_view scheme, std::string_view authority, const StoreReference::Params & params) - : Store::Config(params) - , BinaryCacheStore::Config(params) - , S3BinaryCacheStoreConfigT{ - CONFIG_ROW(profile, ""), - CONFIG_ROW(region, Aws::Region::US_EAST_1), - CONFIG_ROW(scheme, ""), - CONFIG_ROW(endpoint, ""), - CONFIG_ROW(narinfoCompression, ""), - CONFIG_ROW(lsCompression, ""), - CONFIG_ROW(logCompression, ""), - CONFIG_ROW(multipartUpload, false), - CONFIG_ROW(bufferSize, 5 * 1024 * 1024), - } + : Store::Config{params} + , BinaryCacheStore::Config{*this, params} + , S3BinaryCacheStoreConfigT{s3BinaryCacheStoreConfigApplyParse(params)} , bucketName{authority} { if (bucketName.empty()) @@ -285,8 +299,9 @@ S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig( } -S3BinaryCacheStore::S3BinaryCacheStore(const Config & config) - : BinaryCacheStore(config) +S3BinaryCacheStore::S3BinaryCacheStore(ref config) + : BinaryCacheStore(*config) + , config{config} { } std::string S3BinaryCacheStoreConfig::doc() @@ -297,39 +312,37 @@ std::string S3BinaryCacheStoreConfig::doc() } -struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore +struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore { Stats stats; S3Helper s3Helper; - S3BinaryCacheStoreImpl(const Config & config) - : Store::Config{config} - , BinaryCacheStore::Config{config} - , S3BinaryCacheStore::Config{config} - , Store{static_cast(*this)} - , BinaryCacheStore{static_cast(*this)} - , S3BinaryCacheStore{static_cast(*this)} - , s3Helper(profile, region, scheme, endpoint) + S3BinaryCacheStoreImpl(ref config) + : Store{*config} + , BinaryCacheStore{*config} + , S3BinaryCacheStore{config} + , s3Helper(config->profile, config->region, config->scheme, config->endpoint) { diskCache = getNarInfoDiskCache(); } std::string getUri() override { - return "s3://" + bucketName; + return "s3://" + config->bucketName; } void init() override { if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) { - if (defaultWantMassQuery) - wantMassQuery.value = cacheInfo->wantMassQuery; - if (defaultPriority) - priority.value = cacheInfo->priority; + resolvedSubstConfig.wantMassQuery.value = + config->storeConfig.wantMassQuery.optValue.value_or(cacheInfo->wantMassQuery); + resolvedSubstConfig.priority.value = + config->storeConfig.priority.optValue.value_or(cacheInfo->priority); } else { BinaryCacheStore::init(); - diskCache->createCache(getUri(), storeDir, wantMassQuery, priority); + diskCache->createCache( + getUri(), config->storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority); } } @@ -358,7 +371,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual auto res = s3Helper.client->HeadObject( Aws::S3::Model::HeadObjectRequest() - .WithBucket(bucketName) + .WithBucket(config->bucketName) .WithKey(path)); if (!res.IsSuccess()) { @@ -393,11 +406,11 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual std::call_once(transferManagerCreated, [&]() { - if (multipartUpload) { + if (config->multipartUpload) { TransferManagerConfiguration transferConfig(executor.get()); transferConfig.s3Client = s3Helper.client; - transferConfig.bufferSize = bufferSize; + transferConfig.bufferSize = config->bufferSize; transferConfig.uploadProgressCallback = [](const TransferManager *transferManager, @@ -418,6 +431,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual auto now1 = std::chrono::steady_clock::now(); + auto & bucketName = config->bucketName; + if (transferManager) { if (contentEncoding != "") @@ -481,12 +496,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual return std::make_shared(std::move(compressed)); }; - if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) - uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression); - else if (lsCompression != "" && hasSuffix(path, ".ls")) - uploadFile(path, compress(lsCompression), mimeType, lsCompression); - else if (logCompression != "" && hasPrefix(path, "log/")) - uploadFile(path, compress(logCompression), mimeType, logCompression); + if (config->narinfoCompression != "" && hasSuffix(path, ".narinfo")) + uploadFile(path, compress(config->narinfoCompression), mimeType, config->narinfoCompression); + else if (config->lsCompression != "" && hasSuffix(path, ".ls")) + uploadFile(path, compress(config->lsCompression), mimeType, config->lsCompression); + else if (config->logCompression != "" && hasPrefix(path, "log/")) + uploadFile(path, compress(config->logCompression), mimeType, config->logCompression); else uploadFile(path, istream, mimeType, ""); } @@ -496,14 +511,14 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual stats.get++; // FIXME: stream output to sink. - auto res = s3Helper.getObject(bucketName, path); + auto res = s3Helper.getObject(config->bucketName, path); stats.getBytes += res.data ? res.data->size() : 0; stats.getTimeMs += res.durationMs; if (res.data) { printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", - bucketName, path, res.data->size(), res.durationMs); + config->bucketName, path, res.data->size(), res.durationMs); sink(*res.data); } else @@ -515,6 +530,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual StorePathSet paths; std::string marker; + auto & bucketName = config->bucketName; + do { debug("listing bucket 's3://%s' from key '%s'...", bucketName, marker); @@ -555,7 +572,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual ref S3BinaryCacheStoreImpl::Config::openStore() const { - return make_ref(*this); + return make_ref(ref{shared_from_this()}); } static RegisterStoreImplementation regS3BinaryCacheStore; diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 18ea4cd1592..aa3b92bc99c 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -54,12 +54,8 @@ struct S3BinaryCacheStore : virtual BinaryCacheStore ref config; -protected: - S3BinaryCacheStore(ref); -public: - struct Stats { std::atomic put{0}; From 32292117df4fddd10bd8e497f2461b160f91688c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 1 Feb 2025 23:32:26 -0500 Subject: [PATCH 60/80] It links now --- src/libcmd/command.hh | 2 +- src/libstore/binary-cache-store.cc | 9 +++++++++ src/libstore/http-binary-cache-store.cc | 10 ++++++++++ src/libstore/local-binary-cache-store.cc | 9 +++++++++ src/libstore/local-fs-store.cc | 18 +++++++++--------- src/libstore/local-overlay-store.cc | 16 ++++++++++++++++ src/libstore/local-store.cc | 17 +++++++++++++++++ src/libstore/remote-store.cc | 10 ++++++++++ src/libstore/s3-binary-cache-store.cc | 15 +++++++++++++++ src/libstore/store-reference.cc | 2 ++ 10 files changed, 98 insertions(+), 10 deletions(-) diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 9570ce3e7ac..216d6a8bdba 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -18,7 +18,7 @@ extern char ** savedArgv; class EvalState; struct Pos; class Store; -class LocalFSStore; +struct LocalFSStore; static constexpr Command::Category catHelp = -1; static constexpr Command::Category catSecondary = 100; diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index cc54c60f500..a7984b690e9 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -97,6 +97,15 @@ BinaryCacheStore::Config::BinaryCacheStoreConfig( { } +config::SettingDescriptionMap BinaryCacheStoreConfig::descriptions() +{ + constexpr auto & descriptions = binaryCacheStoreConfigDescriptions; + auto defaults = binaryCacheStoreConfigDefaults(); + return { + BINARY_CACHE_STORE_CONFIG_FIELDS(DESC_ROW) + }; +} + BinaryCacheStore::BinaryCacheStore(const Config & config) : Store{config.storeConfig} , config{config} diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 36b6b85d9b3..c4a8daf19f6 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -7,8 +7,18 @@ namespace nix { +config::SettingDescriptionMap HttpBinaryCacheStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(BinaryCacheStoreConfig::descriptions()); + return ret; +} + + MakeError(UploadToHTTP, Error); + std::set HttpBinaryCacheStoreConfig::uriSchemes() { static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 143ff3c5445..a5730f699f3 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -8,6 +8,15 @@ namespace nix { +config::SettingDescriptionMap LocalBinaryCacheStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(BinaryCacheStoreConfig::descriptions()); + return ret; +} + + LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig( std::string_view scheme, PathView binaryCacheDir, diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 57a865cda8c..157705a5e85 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -62,6 +62,15 @@ static LocalFSStoreConfigT localFSStoreConfigApplyParse( return {LOCAL_FS_STORE_CONFIG_FIELDS(APPLY_ROW)}; } +config::SettingDescriptionMap LocalFSStoreConfig::descriptions() +{ + constexpr auto & descriptions = localFSStoreConfigDescriptions; + auto defaults = localFSStoreConfigDefaults(settings.nixStore, std::nullopt); + return { + LOCAL_FS_STORE_CONFIG_FIELDS(DESC_ROW) + }; +} + LocalFSStore::Config::LocalFSStoreConfig( const Store::Config & storeConfig, const StoreReference::Params & params) @@ -96,15 +105,6 @@ LocalFSStore::Config::LocalFSStoreConfig( { } -config::SettingDescriptionMap LocalFSStoreConfig::descriptions() -{ - constexpr auto & descriptions = localFSStoreConfigDescriptions; - auto defaults = localFSStoreConfigDefaults(settings.nixStore, std::nullopt); - return { - LOCAL_FS_STORE_CONFIG_FIELDS(DESC_ROW) - }; -} - LocalFSStore::LocalFSStore(const Config & config) : Store{static_cast(*this)} , config{config} diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 663f6f24a02..c4fde6375e2 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -76,6 +76,22 @@ static LocalOverlayStoreConfigT localOverlayStoreConfigDefaul MAKE_APPLY_PARSE(LocalOverlayStoreConfig, localOverlayStoreConfig, LOCAL_OVERLAY_STORE_CONFIG_FIELDS) +config::SettingDescriptionMap LocalOverlayStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(LocalFSStoreConfig::descriptions()); + ret.merge(LocalStoreConfig::descriptions()); + { + constexpr auto & descriptions = localOverlayStoreConfigDescriptions; + auto defaults = localOverlayStoreConfigDefaults(); + ret.merge(decltype(ret){ + LOCAL_OVERLAY_STORE_CONFIG_FIELDS(DESC_ROW) + }); + } + return ret; +} + LocalOverlayStore::Config::LocalOverlayStoreConfig( std::string_view scheme, std::string_view authority, diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ed2b3cea0bc..238fc7e6cf8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -99,6 +99,21 @@ static LocalStoreConfigT localStoreConfigDefaults() MAKE_APPLY_PARSE(LocalStoreConfig, localStoreConfig, LOCAL_STORE_CONFIG_FIELDS) +config::SettingDescriptionMap LocalStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(LocalFSStoreConfig::descriptions()); + { + constexpr auto & descriptions = localStoreConfigDescriptions; + auto defaults = localStoreConfigDefaults(); + ret.merge(decltype(ret){ + LOCAL_STORE_CONFIG_FIELDS(DESC_ROW) + }); + } + return ret; +} + LocalStore::Config::LocalStoreConfig( std::string_view scheme, std::string_view authority, @@ -109,6 +124,8 @@ LocalStore::Config::LocalStoreConfig( { } +LocalStoreConfig::LocalStoreConfig(const LocalStoreConfig &) = default; + std::string LocalStoreConfig::doc() { return diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index eb36a5da9ac..42eb4671623 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -56,6 +56,16 @@ static RemoteStoreConfigT remoteStoreConfigDefaults() MAKE_APPLY_PARSE(RemoteStoreConfig, remoteStoreConfig, REMOTE_STORE_CONFIG_FIELDS) +config::SettingDescriptionMap RemoteStoreConfig::descriptions() +{ + constexpr auto & descriptions = remoteStoreConfigDescriptions; + auto defaults = remoteStoreConfigDefaults(); + return { + REMOTE_STORE_CONFIG_FIELDS(DESC_ROW) + }; +} + + RemoteStore::Config::RemoteStoreConfig(const Store::Config & storeConfig, const StoreReference::Params & params) : RemoteStoreConfigT{remoteStoreConfigApplyParse(params)} , storeConfig{storeConfig} diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 3ee1ce16e60..4a5777fc734 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -292,6 +292,21 @@ static S3BinaryCacheStoreConfigT s3BinaryCacheStoreConfigDefa MAKE_APPLY_PARSE(S3BinaryCacheStoreConfig, s3BinaryCacheStoreConfig, S3_BINARY_CACHE_STORE_CONFIG_FIELDS) +config::SettingDescriptionMap S3BinaryCacheStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(BinaryCacheStoreConfig::descriptions()); + { + constexpr auto & descriptions = s3BinaryCacheStoreConfigDescriptions; + auto defaults = s3BinaryCacheStoreConfigDefaults(); + ret.merge(decltype(ret){ + S3_BINARY_CACHE_STORE_CONFIG_FIELDS(DESC_ROW) + }); + } + return ret; +} + S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig( std::string_view scheme, std::string_view authority, diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc index 7be513f4fcc..96586236859 100644 --- a/src/libstore/store-reference.cc +++ b/src/libstore/store-reference.cc @@ -10,6 +10,8 @@ namespace nix { +bool StoreReference::operator==(const StoreReference & rhs) const = default; + static bool isNonUriPath(const std::string & spec) { return From c5b11c5a36000668c7064b2cb205d7aade3e05e8 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 1 Feb 2025 23:57:04 -0500 Subject: [PATCH 61/80] Fix an issue with `BinaryCacheStore::init` --- src/libstore/binary-cache-store.cc | 6 +++++- src/libstore/http-binary-cache-store.cc | 2 ++ src/libstore/local-binary-cache-store.cc | 1 + src/libstore/s3-binary-cache-store.cc | 2 ++ 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index a7984b690e9..00be8a221fe 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -118,7 +118,11 @@ BinaryCacheStore::BinaryCacheStore(const Config & config) sink << narVersionMagic1; narMagic = sink.s; - init(); + // Want to call this but cannot, because virtual function lookup is + // disabled in a constructor. It is thus left to instances to call + // it instead. + + //init(); } void BinaryCacheStore::init() diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index c4a8daf19f6..e9ff90c7491 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -77,6 +77,8 @@ class HttpBinaryCacheStore : , config{config} { diskCache = getNarInfoDiskCache(); + + init(); } std::string getUri() override diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index a5730f699f3..cfeb05fdbae 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -48,6 +48,7 @@ struct LocalBinaryCacheStore : , BinaryCacheStore{*config} , config{config} { + init(); } void init() override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 4a5777fc734..601a13d533b 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -347,6 +347,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore , s3Helper(config->profile, config->region, config->scheme, config->endpoint) { diskCache = getNarInfoDiskCache(); + + init(); } std::string getUri() override From 1146db3672c79af872822fcf98fae040daa81b04 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sat, 1 Feb 2025 23:58:11 -0500 Subject: [PATCH 62/80] Fix store reference test parsed Now those are real numbers and booleans! --- src/libstore-tests/store-reference.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstore-tests/store-reference.cc b/src/libstore-tests/store-reference.cc index d4c42f0fda1..6150c0170e9 100644 --- a/src/libstore-tests/store-reference.cc +++ b/src/libstore-tests/store-reference.cc @@ -81,7 +81,7 @@ static StoreReference localExample_2{ }, .params = { - {"trusted", "true"}, + {"trusted", true}, }, }; @@ -100,8 +100,8 @@ static StoreReference unixExample{ }, .params = { - {"max-connections", "7"}, - {"trusted", "true"}, + {"max-connections", 7}, + {"trusted", true}, }, }; From cb60d2994bf42d582641896d6f0a1f80b2a6c6cf Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 00:06:14 -0500 Subject: [PATCH 63/80] Fix dummy store URL in test --- src/libstore-tests/nix_api_store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index a8b7b8e5fc8..14de1289efd 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -95,7 +95,7 @@ TEST_F(nix_api_util_context, nix_store_open_dummy) nix_libstore_init(ctx); Store * store = nix_store_open(ctx, "dummy://", nullptr); ASSERT_EQ(NIX_OK, ctx->last_err_code); - ASSERT_STREQ("dummy", store->ptr->getUri().c_str()); + ASSERT_STREQ("dummy://", store->ptr->getUri().c_str()); std::string str; nix_store_get_version(ctx, store, OBSERVE_STRING(str)); From 643620d34f7f96e0d87aa351fbe21dfa69be7160 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 00:35:03 -0500 Subject: [PATCH 64/80] `openUncachedStore`: Use number not string for JSON now --- src/nix/unix/daemon.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index ab546ea4b1b..c28c3563b66 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -246,7 +246,7 @@ static ref openUncachedStore() { StoreReference::Params params; // FIXME: get params from somewhere // Disable caching since the client already does that. - params["path-info-cache-size"] = "0"; + params["path-info-cache-size"] = 0; return openStore(settings.storeUri, params); } From 23123563421568533b63861da3cc5539f429ca9f Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 11:24:46 -0500 Subject: [PATCH 65/80] Fix some manual mangling of store reference params --- src/libstore/machines.cc | 16 ++++++---------- src/libstore/store-reference.cc | 2 +- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 35d81a7005f..8b14c0b35bf 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -71,8 +71,8 @@ StoreReference Machine::completeStoreReference() const auto * generic = std::get_if(&storeUri.variant); if (generic && generic->scheme == "ssh") { - storeUri.params["max-connections"] = "1"; - storeUri.params["log-fd"] = "4"; + storeUri.params["max-connections"] = 1; + storeUri.params["log-fd"] = 4; } if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) { @@ -84,14 +84,10 @@ StoreReference Machine::completeStoreReference() const { auto & fs = storeUri.params["system-features"]; - auto append = [&](auto feats) { - for (auto & f : feats) { - if (fs.size() > 0) fs += ' '; - fs += f; - } - }; - append(supportedFeatures); - append(mandatoryFeatures); + if (!fs.is_array()) fs = nlohmann::json::array(); + auto features = supportedFeatures; + features.insert(supportedFeatures.begin(), supportedFeatures.end()); + fs += features; } return storeUri; diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc index 96586236859..fcd2c4b5aee 100644 --- a/src/libstore/store-reference.cc +++ b/src/libstore/store-reference.cc @@ -64,7 +64,7 @@ static StoreReference::Params decodeParamsJson(StringMap paramsRaw) j = nlohmann::json::parse(v); } catch (nlohmann::json::exception &) { // if its not valid JSON... - if (k == "remoteProgram" || k == "systemFeatures") { + if (k == "remote-program" || k == "system-features") { // Back compat hack! Split and take that array j = tokenizeString>(v); } else { From 1b370d9e22aabf53bcb427e93733087e42eb999a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 11:43:12 -0500 Subject: [PATCH 66/80] Fix appending features Otherwise we get nested arrays! --- src/libstore/machines.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 8b14c0b35bf..036e8d87b8e 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -87,7 +87,7 @@ StoreReference Machine::completeStoreReference() const if (!fs.is_array()) fs = nlohmann::json::array(); auto features = supportedFeatures; features.insert(supportedFeatures.begin(), supportedFeatures.end()); - fs += features; + for (auto & feat : features) fs += feat; } return storeUri; From b8cdb215b09527ed0e83c8d541070f64c9f63cef Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 11:52:17 -0500 Subject: [PATCH 67/80] Fix mounted ssh store test --- tests/functional/build-remote-with-mounted-ssh-ng.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/functional/build-remote-with-mounted-ssh-ng.sh b/tests/functional/build-remote-with-mounted-ssh-ng.sh index e2627af394c..0f8172c95b8 100755 --- a/tests/functional/build-remote-with-mounted-ssh-ng.sh +++ b/tests/functional/build-remote-with-mounted-ssh-ng.sh @@ -7,15 +7,18 @@ requireSandboxSupport enableFeatures mounted-ssh-store +# N.B. encoded query param is `mounted={}`. In the future, we can just +# do `--store` with JSON, and then the nested structure will actually +# bring benefits. nix build -Lvf simple.nix \ --arg busybox "$busybox" \ --out-link "$TEST_ROOT/result-from-remote" \ - --store mounted-ssh-ng://localhost + --store 'ssh-ng://localhost?mounted=%7B%7D' nix build -Lvf simple.nix \ --arg busybox "$busybox" \ --out-link "$TEST_ROOT/result-from-remote-new-cli" \ - --store 'mounted-ssh-ng://localhost?remote-program=nix daemon' + --store 'ssh-ng://localhost?mounted=%7B%7D&remote-program=nix daemon' # This verifies that the out link was actually created and valid. The ability # to create out links (permanent gc roots) is the distinguishing feature of From cf5d50b26dffa9559f36a00ba9af2d6f04cadc39 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 12:17:56 -0500 Subject: [PATCH 68/80] Fix opening the local overlay store --- src/libstore/local-overlay-store.cc | 7 +++++++ src/libstore/local-overlay-store.hh | 2 ++ 2 files changed, 9 insertions(+) diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index c4fde6375e2..9ddb8d85892 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -109,6 +109,13 @@ std::string LocalOverlayStoreConfig::doc() ; } +ref LocalOverlayStoreConfig::openStore() const +{ + return make_ref(ref{ + std::dynamic_pointer_cast(shared_from_this()) + }); +} + Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) const { diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index dd449c4fc9c..eed3c69daac 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -40,6 +40,8 @@ struct LocalOverlayStoreConfig : static std::string doc(); + ref openStore() const override; + protected: /** * @return The host OS path corresponding to the store path for the From b19aa3739c76488fb5a96c3a2b1aff9b3be814d5 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 15:19:03 -0500 Subject: [PATCH 69/80] Get the store docs to render again --- doc/manual/generate-settings.nix | 22 ++++++++++++++++++---- doc/manual/generate-store-info.nix | 1 + src/libstore/config-parse.cc | 9 +++++++-- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 35ae73e5d1f..9bab34d8193 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -27,11 +27,22 @@ let prefix: setting: { description, + + # Whether we document the default, because it is machine agostic, + # or don't because because it is machine-specific. documentDefault, - defaultValue, - aliases, - value, - experimentalFeature, + + # The default value is JSON for new-style config, rather than then + # a string or boolean, for old-style config. + json ? false, + + defaultValue ? null, + + aliases ? [ ], + + # The current value for this setting. Purposefully unused. + value ? null, + experimentalFeature ? null, }: let result = squash '' @@ -75,6 +86,9 @@ let showDefault = documentDefault: defaultValue: if documentDefault then + if json then + "`${builtins.toJSON defaultValue}`" + else # a StringMap value type is specified as a string, but # this shows the value type. The empty stringmap is `null` in # JSON, but that converts to `{ }` here. diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index e8b7377dafd..e66611affe0 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -33,6 +33,7 @@ let { settings, doc, + uri-schemes, experimentalFeature, }: let diff --git a/src/libstore/config-parse.cc b/src/libstore/config-parse.cc index 5d3a29574f6..6e3c28169fb 100644 --- a/src/libstore/config-parse.cc +++ b/src/libstore/config-parse.cc @@ -22,9 +22,14 @@ void adl_serializer::to_json(json & obj, SettingDescription // obj.emplace("aliases", s.aliases); // obj.emplace("experimentalFeature", s.experimentalFeature); - // Cannot use `null` because the default value might itself be + // Indicate this is a JSON default, not a + obj.emplace("json", true); + + // Cannot just use `null` because the default value might itself be // `null`. - if (s.defaultValue) + obj.emplace("documentDefault", s.defaultValue.has_value()); + + if (s.defaultValue.has_value()) obj.emplace("defaultValue", *s.defaultValue); } From 4dbb0db9ec4ed4684244aa718e0ff99411c7ef90 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 15:39:12 -0500 Subject: [PATCH 70/80] Make all config description `constexpr` This avoids initialization order issues. --- src/libstore/binary-cache-store.cc | 2 +- src/libstore/common-ssh-store-config.cc | 2 +- src/libstore/config-parse-impl.hh | 4 ++-- src/libstore/config-parse.hh | 9 ++++++--- src/libstore/legacy-ssh-store.cc | 2 +- src/libstore/local-fs-store.cc | 2 +- src/libstore/local-store.cc | 2 +- src/libstore/remote-store.cc | 2 +- src/libstore/s3-binary-cache-store.cc | 2 +- src/libstore/ssh-store.cc | 2 +- src/libstore/store-api.cc | 4 ++-- src/libstore/store-dir-config.cc | 2 +- src/libutil/util.hh | 12 ++++++------ 13 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 00be8a221fe..c7cd64d821f 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -25,7 +25,7 @@ namespace nix { -static const BinaryCacheStoreConfigT binaryCacheStoreConfigDescriptions = { +constexpr static const BinaryCacheStoreConfigT binaryCacheStoreConfigDescriptions = { .compression = { .name = "compression", .description = "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`).", diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index 96ec172998f..86a227d81a8 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -6,7 +6,7 @@ namespace nix { -static const CommonSSHStoreConfigT commonSSHStoreConfigDescriptions = { +constexpr static const CommonSSHStoreConfigT commonSSHStoreConfigDescriptions = { .sshKey{ .name = "ssh-key", .description = "Path to the SSH private key used to authenticate to the remote machine.", diff --git a/src/libstore/config-parse-impl.hh b/src/libstore/config-parse-impl.hh index b73801793e8..b45e3a6ea91 100644 --- a/src/libstore/config-parse-impl.hh +++ b/src/libstore/config-parse-impl.hh @@ -19,9 +19,9 @@ template std::pair SettingInfo::describe(const JustValue & def) const { return { - name, + std::string { name }, SettingDescription{ - .description = description, + .description = stripIndentation(description), .defaultValue = documentDefault ? (std::optional{}) : (std::optional{nlohmann::json{def.value}}), }, diff --git a/src/libstore/config-parse.hh b/src/libstore/config-parse.hh index 300a649d7ba..4335d0edf2e 100644 --- a/src/libstore/config-parse.hh +++ b/src/libstore/config-parse.hh @@ -13,6 +13,9 @@ struct SettingDescription; /** * Typed version used as source of truth, and for operations like * defaulting configurations. + * + * It is important that this type support `constexpr` values to avoid + * running into issues with static initialization order. */ template struct SettingInfo @@ -20,19 +23,19 @@ struct SettingInfo /** * Name of the setting, used when parsing configuration maps */ - std::string name; + std::string_view name; /** * Description of the setting. It is used just for documentation. */ - std::string description; + std::string_view description; #if 0 /** * Other names of the setting also used when parsing configuration * maps. This is useful for back-compat, etc. */ - std::set aliases; + std::set aliases; /** * `ExperimentalFeature` that must be enabled if the setting is diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 85dba07c4a5..c9d880536df 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -17,7 +17,7 @@ namespace nix { -static const LegacySSHStoreConfigT legacySSHStoreConfigDescriptions = { +constexpr static const LegacySSHStoreConfigT legacySSHStoreConfigDescriptions = { .remoteProgram{ .name = "remote-program", .description = "Path to the `nix-store` executable on the remote machine.", diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 157705a5e85..2e68304c5ea 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -10,7 +10,7 @@ namespace nix { -static const LocalFSStoreConfigT localFSStoreConfigDescriptions = { +constexpr static const LocalFSStoreConfigT localFSStoreConfigDescriptions = { .rootDir = { .name = "root", .description = "Directory prefixed to all other paths.", diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 238fc7e6cf8..af87f6887fe 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -60,7 +60,7 @@ namespace nix { -static const LocalStoreConfigT localStoreConfigDescriptions = { +constexpr static const LocalStoreConfigT localStoreConfigDescriptions = { .requireSigs = { .name = "require-sigs", .description = "Whether store paths copied into this store should have a trusted signature.", diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 42eb4671623..c92dbe4b527 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -24,7 +24,7 @@ namespace nix { -static const RemoteStoreConfigT remoteStoreConfigDescriptions = { +constexpr static const RemoteStoreConfigT remoteStoreConfigDescriptions = { .maxConnections{ .name = "max-connections", .description = "Maximum number of concurrent connections to the Nix daemon.", diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 601a13d533b..06aeec81ae7 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -195,7 +195,7 @@ S3Helper::FileTransferResult S3Helper::getObject( } -static const S3BinaryCacheStoreConfigT s3BinaryCacheStoreConfigDescriptions = { +constexpr static const S3BinaryCacheStoreConfigT s3BinaryCacheStoreConfigDescriptions = { .profile{ .name = "profile", .description = R"( diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index ed001a8edc5..5603f469a04 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -13,7 +13,7 @@ namespace nix { -static const SSHStoreConfigT sshStoreConfigDescriptions = { +constexpr static const SSHStoreConfigT sshStoreConfigDescriptions = { .remoteProgram{ .name = "remote-program", .description = "Path to the `nix-daemon` executable on the remote machine.", diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 68e3cd3908d..5693c44521b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -189,7 +189,7 @@ std::pair MixStoreDirMethods::computeStorePath( } -static const StoreConfigT storeConfigDescriptions = { +constexpr static const StoreConfigT storeConfigDescriptions = { .pathInfoCacheSize{ .name = "path-info-cache-size", .description = "Size of the in-memory store path metadata cache.", @@ -216,7 +216,7 @@ static const StoreConfigT storeConfigDescriptions = { }, }; -static const SubstituterConfigT substituterConfigDescriptions = { +constexpr static const SubstituterConfigT substituterConfigDescriptions = { .priority{ .name = "priority", .description = R"( diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc index ae22ba159f1..281a7437f2b 100644 --- a/src/libstore/store-dir-config.cc +++ b/src/libstore/store-dir-config.cc @@ -5,7 +5,7 @@ namespace nix { -static const StoreDirConfigT storeDirConfigDescriptions = { +constexpr static const StoreDirConfigT storeDirConfigDescriptions = { ._storeDir{ .name = "store", .description = R"( diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 0d55cf93bed..e92d5a08a9c 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -218,16 +218,16 @@ std::pair getLine(std::string_view s); /** * Get a value for the specified key from an associate container. */ -template -const typename T::mapped_type * get(const T & map, const typename T::key_type & key) +template +const typename T::mapped_type * get(const T & map, K key) { auto i = map.find(key); if (i == map.end()) return nullptr; return &i->second; } -template -typename T::mapped_type * get(T & map, const typename T::key_type & key) +template +typename T::mapped_type * get(T & map, K key) { auto i = map.find(key); if (i == map.end()) return nullptr; @@ -237,9 +237,9 @@ typename T::mapped_type * get(T & map, const typename T::key_type & key) /** * Get a value for the specified key from an associate container, or a default value if the key isn't present. */ -template +template const typename T::mapped_type & getOr(T & map, - const typename T::key_type & key, + K key, const typename T::mapped_type & defaultValue) { auto i = map.find(key); From 287d0ef681a5b54a84cab73c0483dfb51971d2c4 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 15:54:31 -0500 Subject: [PATCH 71/80] Fix `documentDefault`, it was inverted --- src/libstore/config-parse-impl.hh | 2 +- src/libstore/local-overlay-store.cc | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/config-parse-impl.hh b/src/libstore/config-parse-impl.hh index b45e3a6ea91..8bfa0f372d0 100644 --- a/src/libstore/config-parse-impl.hh +++ b/src/libstore/config-parse-impl.hh @@ -23,7 +23,7 @@ std::pair SettingInfo::describe(const JustVa SettingDescription{ .description = stripIndentation(description), .defaultValue = - documentDefault ? (std::optional{}) : (std::optional{nlohmann::json{def.value}}), + documentDefault ? (std::optional{nlohmann::json{def.value}}) : (std::optional{}), }, }; } diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 9ddb8d85892..30374b4a8ee 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -21,6 +21,9 @@ static LocalOverlayStoreConfigT localOverlayStoreConfigDesc Must be a store with a store dir on the file system. Must be used as OverlayFS lower layer for this store's store dir. )", + // It's not actually machine-specific, but we don't yet have a + // `to_json` for `StoreConfig`. + .documentDefault = false, }, .upperLayer{ .name = "upper-layer", From 880c1e2dc4a4a5ec11b5c34274b1a24f3dbf105c Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 16:22:15 -0500 Subject: [PATCH 72/80] Make `JSON_IMPL(StoreReference)`, use for `JSON_IMPL(ref)` --- src/libstore/store-reference.cc | 50 ++++++++++++++++++++++++++++++ src/libstore/store-reference.hh | 3 ++ src/libstore/store-registration.cc | 36 +-------------------- src/libutil/json-impls.hh | 2 +- 4 files changed, 55 insertions(+), 36 deletions(-) diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc index fcd2c4b5aee..94279b60d27 100644 --- a/src/libstore/store-reference.cc +++ b/src/libstore/store-reference.cc @@ -7,6 +7,7 @@ #include "store-reference.hh" #include "file-system.hh" #include "util.hh" +#include "json-utils.hh" namespace nix { @@ -153,3 +154,52 @@ std::pair splitUriAndParams(const std::stri } } + +namespace nlohmann { + +StoreReference adl_serializer::from_json(const json & json) +{ + StoreReference ref; + switch (json.type()) { + + case json::value_t::string: { + ref = StoreReference::parse(json.get_ref()); + break; + } + + case json::value_t::object: { + auto & obj = json.get_ref(); + ref = StoreReference{ + .variant = + StoreReference::Specified{ + .scheme = getString(valueAt(obj, "scheme")), + .authority = getString(valueAt(obj, "authority")), + }, + .params = obj, + }; + break; + } + + case json::value_t::null: + case json::value_t::number_unsigned: + case json::value_t::number_integer: + case json::value_t::number_float: + case json::value_t::boolean: + case json::value_t::array: + case json::value_t::binary: + case json::value_t::discarded: + default: + throw UsageError( + "Invalid JSON for Store configuration: is type '%s' but must be string or object", json.type_name()); + }; + + return ref; +} + +void adl_serializer::to_json(json & obj, StoreReference s) +{ + // TODO, for tests maybe + assert(false); +} + +} diff --git a/src/libstore/store-reference.hh b/src/libstore/store-reference.hh index b14b6436e1d..38a5f84ac52 100644 --- a/src/libstore/store-reference.hh +++ b/src/libstore/store-reference.hh @@ -5,6 +5,7 @@ #include #include "types.hh" +#include "json-impls.hh" namespace nix { @@ -100,3 +101,5 @@ struct StoreReference std::pair splitUriAndParams(const std::string & uri); } + +JSON_IMPL(StoreReference) diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc index 8fa8dcc0674..f9a68fd659e 100644 --- a/src/libstore/store-registration.cc +++ b/src/libstore/store-registration.cc @@ -113,41 +113,7 @@ using namespace nix::config; ref adl_serializer>::from_json(const json & json) { - StoreReference ref; - switch (json.type()) { - - case json::value_t::string: { - ref = StoreReference::parse(json.get_ref()); - break; - } - - case json::value_t::object: { - auto & obj = json.get_ref(); - ref = StoreReference{ - .variant = - StoreReference::Specified{ - .scheme = getString(valueAt(obj, "scheme")), - .authority = getString(valueAt(obj, "authority")), - }, - .params = obj, - }; - break; - } - - case json::value_t::null: - case json::value_t::number_unsigned: - case json::value_t::number_integer: - case json::value_t::number_float: - case json::value_t::boolean: - case json::value_t::array: - case json::value_t::binary: - case json::value_t::discarded: - default: - throw UsageError( - "Invalid JSON for Store configuration: is type '%s' but must be string or object", json.type_name()); - }; - - return resolveStoreConfig(std::move(ref)); + return resolveStoreConfig(adl_serializer::from_json(json)); } void adl_serializer>::to_json(json & obj, ref s) diff --git a/src/libutil/json-impls.hh b/src/libutil/json-impls.hh index b26163a04ae..9dd344c508d 100644 --- a/src/libutil/json-impls.hh +++ b/src/libutil/json-impls.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "nlohmann/json_fwd.hpp" +#include // Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types #define JSON_IMPL(TYPE) \ From aae2c8e5e1f258777779b095576b4743e912bc8a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 17:58:18 -0500 Subject: [PATCH 73/80] Test store reference to/from JSON roundtrip --- .../data/store-reference/auto.json | 3 ++ .../data/store-reference/auto_param.json | 4 ++ .../data/store-reference/local_1.json | 5 ++ .../data/store-reference/local_2.json | 5 ++ .../data/store-reference/ssh.json | 4 ++ .../data/store-reference/unix.json | 6 +++ src/libstore-tests/store-reference.cc | 47 +++++++++++++++---- src/libstore/config-parse-impl.hh | 2 +- src/libstore/store-reference.cc | 31 ++++++++---- 9 files changed, 88 insertions(+), 19 deletions(-) create mode 100644 src/libstore-tests/data/store-reference/auto.json create mode 100644 src/libstore-tests/data/store-reference/auto_param.json create mode 100644 src/libstore-tests/data/store-reference/local_1.json create mode 100644 src/libstore-tests/data/store-reference/local_2.json create mode 100644 src/libstore-tests/data/store-reference/ssh.json create mode 100644 src/libstore-tests/data/store-reference/unix.json diff --git a/src/libstore-tests/data/store-reference/auto.json b/src/libstore-tests/data/store-reference/auto.json new file mode 100644 index 00000000000..da510f64949 --- /dev/null +++ b/src/libstore-tests/data/store-reference/auto.json @@ -0,0 +1,3 @@ +{ + "scheme": "auto" +} diff --git a/src/libstore-tests/data/store-reference/auto_param.json b/src/libstore-tests/data/store-reference/auto_param.json new file mode 100644 index 00000000000..41826c83b56 --- /dev/null +++ b/src/libstore-tests/data/store-reference/auto_param.json @@ -0,0 +1,4 @@ +{ + "root": "/foo/bar/baz", + "scheme": "auto" +} diff --git a/src/libstore-tests/data/store-reference/local_1.json b/src/libstore-tests/data/store-reference/local_1.json new file mode 100644 index 00000000000..3b44e382b7f --- /dev/null +++ b/src/libstore-tests/data/store-reference/local_1.json @@ -0,0 +1,5 @@ +{ + "authority": "", + "root": "/foo/bar/baz", + "scheme": "local" +} diff --git a/src/libstore-tests/data/store-reference/local_2.json b/src/libstore-tests/data/store-reference/local_2.json new file mode 100644 index 00000000000..346b0cf225f --- /dev/null +++ b/src/libstore-tests/data/store-reference/local_2.json @@ -0,0 +1,5 @@ +{ + "authority": "/foo/bar/baz", + "scheme": "local", + "trusted": true +} diff --git a/src/libstore-tests/data/store-reference/ssh.json b/src/libstore-tests/data/store-reference/ssh.json new file mode 100644 index 00000000000..e5b0b367ffd --- /dev/null +++ b/src/libstore-tests/data/store-reference/ssh.json @@ -0,0 +1,4 @@ +{ + "authority": "localhost", + "scheme": "ssh" +} diff --git a/src/libstore-tests/data/store-reference/unix.json b/src/libstore-tests/data/store-reference/unix.json new file mode 100644 index 00000000000..8943e959dd6 --- /dev/null +++ b/src/libstore-tests/data/store-reference/unix.json @@ -0,0 +1,6 @@ +{ + "authority": "", + "max-connections": 7, + "scheme": "unix", + "trusted": true +} diff --git a/src/libstore-tests/store-reference.cc b/src/libstore-tests/store-reference.cc index 6150c0170e9..08c4d60c1f8 100644 --- a/src/libstore-tests/store-reference.cc +++ b/src/libstore-tests/store-reference.cc @@ -17,14 +17,14 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest std::filesystem::path goldenMaster(PathView testStem) const override { - return unitTestData / (testStem + ".txt"); + return unitTestData / testStem; } }; #define URI_TEST_READ(STEM, OBJ) \ TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_uri) \ { \ - readTest(#STEM, ([&](const auto & encoded) { \ + readTest(#STEM ".txt", ([&](const auto & encoded) { \ StoreReference expected = OBJ; \ auto got = StoreReference::parse(encoded); \ ASSERT_EQ(got, expected); \ @@ -35,7 +35,7 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_uri) \ { \ writeTest( \ - #STEM, \ + #STEM ".txt", \ [&]() -> StoreReference { return OBJ; }, \ [](const auto & file) { return StoreReference::parse(readFile(file)); }, \ [](const auto & file, const auto & got) { return writeFile(file, got.render()); }); \ @@ -45,14 +45,43 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest URI_TEST_READ(STEM, OBJ) \ URI_TEST_WRITE(STEM, OBJ) -URI_TEST( +#define JSON_TEST_READ(STEM, OBJ) \ + TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_json) \ + { \ + readTest(#STEM ".json", ([&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + StoreReference expected = OBJ; \ + StoreReference got = encoded; \ + ASSERT_EQ(got, expected); \ + })); \ + } + +#define JSON_TEST_WRITE(STEM, OBJ) \ + TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_json) \ + { \ + writeTest( \ + #STEM ".json", \ + [&]() -> StoreReference { return OBJ; }, \ + [](const auto & file) -> StoreReference { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, json(got).dump(2) + "\n"); }); \ + } + +#define JSON_TEST(STEM, OBJ) \ + JSON_TEST_READ(STEM, OBJ) \ + JSON_TEST_WRITE(STEM, OBJ) + +#define BOTH_FORMATS_TEST(STEM, OBJ) \ + URI_TEST(STEM, OBJ) \ + JSON_TEST(STEM, OBJ) + +BOTH_FORMATS_TEST( auto, (StoreReference{ .variant = StoreReference::Auto{}, .params = {}, })) -URI_TEST( +BOTH_FORMATS_TEST( auto_param, (StoreReference{ .variant = StoreReference::Auto{}, @@ -85,9 +114,9 @@ static StoreReference localExample_2{ }, }; -URI_TEST(local_1, localExample_1) +BOTH_FORMATS_TEST(local_1, localExample_1) -URI_TEST(local_2, localExample_2) +BOTH_FORMATS_TEST(local_2, localExample_2) URI_TEST_READ(local_shorthand_1, localExample_1) @@ -105,11 +134,11 @@ static StoreReference unixExample{ }, }; -URI_TEST(unix, unixExample) +BOTH_FORMATS_TEST(unix, unixExample) URI_TEST_READ(unix_shorthand, unixExample) -URI_TEST( +BOTH_FORMATS_TEST( ssh, (StoreReference{ .variant = diff --git a/src/libstore/config-parse-impl.hh b/src/libstore/config-parse-impl.hh index 8bfa0f372d0..40989220a72 100644 --- a/src/libstore/config-parse-impl.hh +++ b/src/libstore/config-parse-impl.hh @@ -19,7 +19,7 @@ template std::pair SettingInfo::describe(const JustValue & def) const { return { - std::string { name }, + std::string{name}, SettingDescription{ .description = stripIndentation(description), .defaultValue = diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc index 94279b60d27..17257900c5c 100644 --- a/src/libstore/store-reference.cc +++ b/src/libstore/store-reference.cc @@ -168,14 +168,19 @@ StoreReference adl_serializer::from_json(const json & json) } case json::value_t::object: { - auto & obj = json.get_ref(); + auto & obj = getObject(json); + auto scheme = getString(valueAt(obj, "scheme")); + auto variant = scheme == "auto" ? (StoreReference::Variant{StoreReference::Auto{}}) + : (StoreReference::Variant{StoreReference::Specified{ + .scheme = scheme, + .authority = getString(valueAt(obj, "authority")), + }}); + auto params = obj; + params.erase("scheme"); + params.erase("authority"); ref = StoreReference{ - .variant = - StoreReference::Specified{ - .scheme = getString(valueAt(obj, "scheme")), - .authority = getString(valueAt(obj, "authority")), - }, - .params = obj, + .variant = std::move(variant), + .params = std::move(params), }; break; } @@ -198,8 +203,16 @@ StoreReference adl_serializer::from_json(const json & json) void adl_serializer::to_json(json & obj, StoreReference s) { - // TODO, for tests maybe - assert(false); + obj = s.params; + std::visit( + overloaded{ + [&](const StoreReference::Auto &) { obj.emplace("scheme", "auto"); }, + [&](const StoreReference::Specified & g) { + obj.emplace("scheme", g.scheme); + obj.emplace("authority", g.authority); + }, + }, + s.variant); } } From 34af9ca498a5effcfa0b2e69e5428de09d24966d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 18:12:04 -0500 Subject: [PATCH 74/80] Fix settings to JSON, write from JSON for completeness I guess we can use this in tests at some point. --- src/libstore/config-parse-impl.hh | 2 +- src/libstore/config-parse.cc | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/libstore/config-parse-impl.hh b/src/libstore/config-parse-impl.hh index 40989220a72..73fba8a070d 100644 --- a/src/libstore/config-parse-impl.hh +++ b/src/libstore/config-parse-impl.hh @@ -23,7 +23,7 @@ std::pair SettingInfo::describe(const JustVa SettingDescription{ .description = stripIndentation(description), .defaultValue = - documentDefault ? (std::optional{nlohmann::json{def.value}}) : (std::optional{}), + documentDefault ? (std::optional{nlohmann::json(def.value)}) : (std::optional{}), }, }; } diff --git a/src/libstore/config-parse.cc b/src/libstore/config-parse.cc index 6e3c28169fb..915bb35aa37 100644 --- a/src/libstore/config-parse.cc +++ b/src/libstore/config-parse.cc @@ -1,6 +1,7 @@ #include #include "config-parse.hh" +#include "json-utils.hh" namespace nix::config { @@ -12,8 +13,13 @@ using namespace nix::config; SettingDescription adl_serializer::from_json(const json & json) { - // TODO implement if we ever need (testing?) - assert(false); + auto & obj = getObject(json); + return { + .description = getString(valueAt(obj, "description")), + .defaultValue = getBoolean(valueAt(obj, "documentDefault")) + ? (std::optional{valueAt(obj, "defaultValue")}) + : (std::optional{}), + }; } void adl_serializer::to_json(json & obj, SettingDescription s) @@ -22,7 +28,11 @@ void adl_serializer::to_json(json & obj, SettingDescription // obj.emplace("aliases", s.aliases); // obj.emplace("experimentalFeature", s.experimentalFeature); - // Indicate this is a JSON default, not a + // Indicate the default value is JSON, rather than a legacy setting + // boolean or string. + // + // TODO remove if we no longer have the legacy setting system / the + // code handling doc rendering of the settings is decoupled. obj.emplace("json", true); // Cannot just use `null` because the default value might itself be From 5f79489420763c7b544c0905189c966fa1734d05 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 18:23:19 -0500 Subject: [PATCH 75/80] Fix docs, add test --- src/libstore-tests/ssh-store.cc | 34 +++++++++++++++++++++++++++++++++ src/libstore/ssh-store.md | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/libstore-tests/ssh-store.cc b/src/libstore-tests/ssh-store.cc index 8852273dcf9..c844736d4a6 100644 --- a/src/libstore-tests/ssh-store.cc +++ b/src/libstore-tests/ssh-store.cc @@ -60,4 +60,38 @@ TEST(MountedSSHStore, constructConfig) EXPECT_EQ(config.mounted->realStoreDir, "/nix/store"); } +TEST(MountedSSHStore, constructConfigWithFunnyRealStoreDir) +{ + SSHStoreConfig config{ + "ssh-ng", + "localhost", + StoreReference::Params{ + { + "remote-program", + { + "foo", + "bar", + }, + }, + { + "mounted", + nlohmann::json::object_t{ + {"real", "/foo/bar"}, + }, + }, + }, + }; + + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); + + ASSERT_TRUE(config.mounted); + + EXPECT_EQ(config.mounted->realStoreDir, "/foo/bar"); +} + } diff --git a/src/libstore/ssh-store.md b/src/libstore/ssh-store.md index bdd15ef99ed..e370103bb64 100644 --- a/src/libstore/ssh-store.md +++ b/src/libstore/ssh-store.md @@ -5,7 +5,7 @@ R"( Experimental store type that allows full access to a Nix store on a remote machine. -### The `mounted` configuration subject +### The `mounted` configuration setting This additionally requires that store be mounted in the local file system. From 8626dfe95e0989dd6bd93a271d8e4324601cab6b Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 18:39:03 -0500 Subject: [PATCH 76/80] Add test showing bug we are not catching the mounted SSH store being XP early enough. --- tests/functional/build-remote-with-mounted-ssh-ng.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/functional/build-remote-with-mounted-ssh-ng.sh b/tests/functional/build-remote-with-mounted-ssh-ng.sh index 0f8172c95b8..4c89b11d892 100755 --- a/tests/functional/build-remote-with-mounted-ssh-ng.sh +++ b/tests/functional/build-remote-with-mounted-ssh-ng.sh @@ -5,6 +5,11 @@ source common.sh requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" +# An example of a command that uses the store only for its settings, to +# make sure we catch needing the XP feature early. +touch "$TEST_ROOT/foo" +expect 1 nix --store 'ssh-ng://localhost?mounted=%7B%7D' store add "$TEST_ROOT/foo" --dry-run + enableFeatures mounted-ssh-store # N.B. encoded query param is `mounted={}`. In the future, we can just From 1d51e95dde69c12900aa0743a079695e773a057a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 21:32:35 -0500 Subject: [PATCH 77/80] Render nested setting descriptions --- doc/manual/generate-settings.nix | 28 ++++++++++--- src/libstore/config-parse-impl.hh | 8 +++- src/libstore/config-parse.cc | 67 +++++++++++++++++++++---------- src/libstore/config-parse.hh | 54 ++++++++++++++++++++----- src/libstore/ssh-store.cc | 24 ++++++++++- src/libstore/ssh-store.md | 15 ------- 6 files changed, 142 insertions(+), 54 deletions(-) diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 9bab34d8193..25a1600c849 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -19,7 +19,6 @@ in prefix, inlineHTML ? true, }: -settingsInfo: let @@ -28,9 +27,11 @@ let { description, + experimentalFeature, + # Whether we document the default, because it is machine agostic, # or don't because because it is machine-specific. - documentDefault, + documentDefault ? true, # The default value is JSON for new-style config, rather than then # a string or boolean, for old-style config. @@ -38,11 +39,12 @@ let defaultValue ? null, + subSettings ? null, + aliases ? [ ], # The current value for this setting. Purposefully unused. value ? null, - experimentalFeature ? null, }: let result = squash '' @@ -61,7 +63,7 @@ let ${description} - **Default:** ${showDefault documentDefault defaultValue} + ${showDefaultOrSubSettings} ${showAliases aliases} ''; @@ -83,8 +85,20 @@ let > ``` ''; + showDefaultOrSubSettings = + if !isAttrs subSettings then + # No subsettings, instead single setting. Show the default value. + '' + **Default:** ${showDefault} + '' + else + # Indent the nested sub-settings, and append the outer setting name onto the prefix + indent " " '' + **Nullable sub-settings**: ${if subSettings.nullable then "true" else "false"} + ${builtins.trace prefix (showSettings "${prefix}-${setting}" subSettings.map)} + ''; + showDefault = - documentDefault: defaultValue: if documentDefault then if json then "`${builtins.toJSON defaultValue}`" @@ -109,5 +123,7 @@ let in result; + showSettings = + prefix: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo)); in -concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo)) +showSettings prefix diff --git a/src/libstore/config-parse-impl.hh b/src/libstore/config-parse-impl.hh index 73fba8a070d..daadc1ff0d0 100644 --- a/src/libstore/config-parse-impl.hh +++ b/src/libstore/config-parse-impl.hh @@ -22,8 +22,12 @@ std::pair SettingInfo::describe(const JustVa std::string{name}, SettingDescription{ .description = stripIndentation(description), - .defaultValue = - documentDefault ? (std::optional{nlohmann::json(def.value)}) : (std::optional{}), + .experimentalFeature = experimentalFeature, + .info = + SettingDescription::Single{ + .defaultValue = documentDefault ? (std::optional{nlohmann::json(def.value)}) + : (std::optional{}), + }, }, }; } diff --git a/src/libstore/config-parse.cc b/src/libstore/config-parse.cc index 915bb35aa37..a32e32d061d 100644 --- a/src/libstore/config-parse.cc +++ b/src/libstore/config-parse.cc @@ -2,6 +2,7 @@ #include "config-parse.hh" #include "json-utils.hh" +#include "util.hh" namespace nix::config { @@ -16,31 +17,55 @@ SettingDescription adl_serializer::from_json(const json & js auto & obj = getObject(json); return { .description = getString(valueAt(obj, "description")), - .defaultValue = getBoolean(valueAt(obj, "documentDefault")) - ? (std::optional{valueAt(obj, "defaultValue")}) - : (std::optional{}), + .experimentalFeature = valueAt(obj, "experimentalFeature").get>(), + .info = [&]() -> decltype(SettingDescription::info) { + if (auto documentDefault = optionalValueAt(obj, "documentDefault")) { + return SettingDescription::Single{ + .defaultValue = *documentDefault ? (std::optional{valueAt(obj, "defaultValue")}) + : (std::optional{}), + }; + } else { + auto & subObj = getObject(valueAt(obj, "subSettings")); + return SettingDescription::Sub{ + .nullable = valueAt(subObj, "nullable"), + .map = valueAt(subObj, "map"), + }; + } + }(), }; } -void adl_serializer::to_json(json & obj, SettingDescription s) +void adl_serializer::to_json(json & obj, SettingDescription sd) { - obj.emplace("description", s.description); - // obj.emplace("aliases", s.aliases); - // obj.emplace("experimentalFeature", s.experimentalFeature); - - // Indicate the default value is JSON, rather than a legacy setting - // boolean or string. - // - // TODO remove if we no longer have the legacy setting system / the - // code handling doc rendering of the settings is decoupled. - obj.emplace("json", true); - - // Cannot just use `null` because the default value might itself be - // `null`. - obj.emplace("documentDefault", s.defaultValue.has_value()); - - if (s.defaultValue.has_value()) - obj.emplace("defaultValue", *s.defaultValue); + obj.emplace("description", sd.description); + // obj.emplace("aliases", sd.aliases); + obj.emplace("experimentalFeature", sd.experimentalFeature); + + std::visit( + overloaded{ + [&](const SettingDescription::Single & single) { + // Indicate the default value is JSON, rather than a legacy setting + // boolean or string. + // + // TODO remove if we no longer have the legacy setting system / the + // code handling doc rendering of the settings is decoupled. + obj.emplace("json", true); + + // Cannot just use `null` because the default value might itself be + // `null`. + obj.emplace("documentDefault", single.defaultValue.has_value()); + + if (single.defaultValue.has_value()) + obj.emplace("defaultValue", *single.defaultValue); + }, + [&](const SettingDescription::Sub & sub) { + json subJson; + subJson.emplace("nullable", sub.nullable); + subJson.emplace("map", sub.map); + obj.emplace("subSettings", std::move(subJson)); + }, + }, + sd.info); } } diff --git a/src/libstore/config-parse.hh b/src/libstore/config-parse.hh index 4335d0edf2e..e2de3db606a 100644 --- a/src/libstore/config-parse.hh +++ b/src/libstore/config-parse.hh @@ -5,6 +5,7 @@ #include "config-abstract.hh" #include "json-impls.hh" +#include "experimental-features.hh" namespace nix::config { @@ -36,13 +37,13 @@ struct SettingInfo * maps. This is useful for back-compat, etc. */ std::set aliases; +#endif /** * `ExperimentalFeature` that must be enabled if the setting is * allowed to be used */ std::optional experimentalFeature; -#endif /** * Whether to document the default value. (Some defaults are @@ -60,6 +61,13 @@ struct SettingInfo OptValue parseConfig(const nlohmann::json::object_t & map) const; }; +struct SettingDescription; + +/** + * Map of setting names to descriptions of those settings. + */ +using SettingDescriptionMap = std::map; + /** * Untyped version used for rendering docs. This is not the source of * truth, it is generated from the typed one. @@ -79,23 +87,51 @@ struct SettingDescription * @see SettingInfo::aliases */ std::set aliases; +#endif /** * @see SettingInfo::experimentalFeature */ std::optional experimentalFeature; -#endif /** - * Optional, for the `SettingInfo::documentDefault = false` case. + * A single leaf setting, to be optionally specified by arbitrary + * value (of some type) or left default. */ - std::optional defaultValue; -}; + struct Single + { + /** + * Optional, for the `SettingInfo::documentDefault = false` case. + */ + std::optional defaultValue; + }; -/** - * Map of setting names to descriptions of those settings. - */ -using SettingDescriptionMap = std::map; + /** + * A nested settings object + */ + struct Sub + { + /** + * If `false`, this is just pure namespaceing. If `true`, we + * have a distinction between `null` and `{}`, meaning + * enabling/disabling the entire settings group. + */ + bool nullable = true; + + SettingDescriptionMap map; + }; + + /** + * Variant for `info` below + */ + using Info = std::variant; + + /** + * More information about this setting, depending on whether its the + * single leaf setting or subsettings case + */ + Info info; +}; } diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 5603f469a04..0ec4c15a4a4 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -52,7 +52,29 @@ config::SettingDescriptionMap SSHStoreConfig::descriptions() SSH_STORE_CONFIG_FIELDS(DESC_ROW) }); } - //ret.merge(LocalFSStoreConfig::descriptions()); + ret.insert_or_assign( + "mounted", + config::SettingDescription{ + .description = stripIndentation(R"( + If this nested settings object is defined (`{..}` not `null`), additionally requires that store be mounted in the local file system. + + The mounting of that store is not managed by Nix, and must by managed manually. + It could be accomplished with SSHFS or NFS, for example. + + The local file system is used to optimize certain operations. + For example, rather than serializing Nix archives and sending over the Nix channel, + we can directly access the file system data via the mount-point. + + The local file system is also used to make certain operations possible that wouldn't otherwise be. + For example, persistent GC roots can be created if they reside on the same file system as the remote store: + the remote side will create the symlinks necessary to avoid race conditions. + )"), + .experimentalFeature = Xp::MountedSSHStore, + .info = config::SettingDescription::Sub{ + .nullable = true, + .map = LocalFSStoreConfig::descriptions() + }, + }); return ret; } diff --git a/src/libstore/ssh-store.md b/src/libstore/ssh-store.md index e370103bb64..ea34ef2b710 100644 --- a/src/libstore/ssh-store.md +++ b/src/libstore/ssh-store.md @@ -4,19 +4,4 @@ R"( Experimental store type that allows full access to a Nix store on a remote machine. - -### The `mounted` configuration setting - -This additionally requires that store be mounted in the local file system. - -The mounting of that store is not managed by Nix, and must by managed manually. -It could be accomplished with SSHFS or NFS, for example. - -The local file system is used to optimize certain operations. -For example, rather than serializing Nix archives and sending over the Nix channel, -we can directly access the file system data via the mount-point. - -The local file system is also used to make certain operations possible that wouldn't otherwise be. -For example, persistent GC roots can be created if they reside on the same file system as the remote store: -the remote side will create the symlinks necessary to avoid race conditions. )" From c4315422ad9457411c72a3222e547ed00863b46d Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 21:58:40 -0500 Subject: [PATCH 78/80] Require store settings experimental feature --- src/libstore/config-parse-impl.hh | 20 +++++++++++++------- src/libstore/config-parse.hh | 8 +++++++- src/libstore/ssh-store.cc | 1 + 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/libstore/config-parse-impl.hh b/src/libstore/config-parse-impl.hh index daadc1ff0d0..2bca7ee8855 100644 --- a/src/libstore/config-parse-impl.hh +++ b/src/libstore/config-parse-impl.hh @@ -5,13 +5,17 @@ #include "config-parse.hh" #include "util.hh" +#include "config.hh" namespace nix::config { template -OptValue SettingInfo::parseConfig(const nlohmann::json::object_t & map) const +OptValue +SettingInfo::parseConfig(const nlohmann::json::object_t & map, const ExperimentalFeatureSettings & xpSettings) const { const nlohmann::json * p = get(map, name); + if (p && experimentalFeature) + xpSettings.require(*experimentalFeature); return {.optValue = p ? (std::optional{p->get()}) : std::nullopt}; } @@ -36,7 +40,7 @@ std::pair SettingInfo::describe(const JustVa * Look up the setting's name in a map, falling back on the default if * it does not exist. */ -#define CONFIG_ROW(FIELD) .FIELD = descriptions.FIELD.parseConfig(params) +#define CONFIG_ROW(FIELD) .FIELD = descriptions.FIELD.parseConfig(params, xpSettings) #define APPLY_ROW(FIELD) .FIELD = {.value = parsed.FIELD.optValue.value_or(std::move(defaults.FIELD))} @@ -45,11 +49,13 @@ std::pair SettingInfo::describe(const JustVa descriptions.FIELD.describe(defaults.FIELD), \ } -#define MAKE_PARSE(CAPITAL, LOWER, FIELDS) \ - static CAPITAL##T LOWER##Parse(const StoreReference::Params & params) \ - { \ - constexpr auto & descriptions = LOWER##Descriptions; \ - return {FIELDS(CONFIG_ROW)}; \ +#define MAKE_PARSE(CAPITAL, LOWER, FIELDS) \ + static CAPITAL##T LOWER##Parse( \ + const StoreReference::Params & params, \ + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) \ + { \ + constexpr auto & descriptions = LOWER##Descriptions; \ + return {FIELDS(CONFIG_ROW)}; \ } #define MAKE_APPLY_PARSE(CAPITAL, LOWER, FIELDS) \ diff --git a/src/libstore/config-parse.hh b/src/libstore/config-parse.hh index e2de3db606a..e656bfba530 100644 --- a/src/libstore/config-parse.hh +++ b/src/libstore/config-parse.hh @@ -7,6 +7,12 @@ #include "json-impls.hh" #include "experimental-features.hh" +namespace nix { + +struct ExperimentalFeatureSettings; + +}; + namespace nix::config { struct SettingDescription; @@ -58,7 +64,7 @@ struct SettingInfo */ std::pair describe(const JustValue & def) const; - OptValue parseConfig(const nlohmann::json::object_t & map) const; + OptValue parseConfig(const nlohmann::json::object_t & map, const ExperimentalFeatureSettings & xpSettings) const; }; struct SettingDescription; diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 0ec4c15a4a4..d5d05c4490d 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -86,6 +86,7 @@ static std::optional getMounted( auto mountedParamsOpt = optionalValueAt(params, "mounted"); if (!mountedParamsOpt) return {}; auto * mountedParamsP = getNullable(*mountedParamsOpt); + experimentalFeatureSettings.require(Xp::MountedSSHStore); if (!mountedParamsP) return {}; auto & mountedParams = getObject(*mountedParamsP); return {{storeConfig, mountedParams}}; From 01d1acc8d73acb9600e9feedb8fb34d79f120525 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 23:01:46 -0500 Subject: [PATCH 79/80] Update tests given XP feature checks --- src/libstore-tests/ssh-store.cc | 8 ++++++++ src/libstore/ssh-store.cc | 9 +++++---- src/libstore/ssh-store.hh | 6 +++++- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/libstore-tests/ssh-store.cc b/src/libstore-tests/ssh-store.cc index c844736d4a6..0d18b351164 100644 --- a/src/libstore-tests/ssh-store.cc +++ b/src/libstore-tests/ssh-store.cc @@ -30,6 +30,9 @@ TEST(SSHStore, constructConfig) TEST(MountedSSHStore, constructConfig) { + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "mounted-ssh-store"); + SSHStoreConfig config{ "ssh-ng", "localhost", @@ -46,6 +49,7 @@ TEST(MountedSSHStore, constructConfig) nlohmann::json::object_t{}, }, }, + mockXpSettings, }; EXPECT_EQ( @@ -62,6 +66,9 @@ TEST(MountedSSHStore, constructConfig) TEST(MountedSSHStore, constructConfigWithFunnyRealStoreDir) { + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "mounted-ssh-store"); + SSHStoreConfig config{ "ssh-ng", "localhost", @@ -80,6 +87,7 @@ TEST(MountedSSHStore, constructConfigWithFunnyRealStoreDir) }, }, }, + mockXpSettings, }; EXPECT_EQ( diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index d5d05c4490d..12432a9b80d 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -81,12 +81,13 @@ config::SettingDescriptionMap SSHStoreConfig::descriptions() static std::optional getMounted( const Store::Config & storeConfig, - const StoreReference::Params & params) + const StoreReference::Params & params, + const ExperimentalFeatureSettings & xpSettings) { auto mountedParamsOpt = optionalValueAt(params, "mounted"); if (!mountedParamsOpt) return {}; auto * mountedParamsP = getNullable(*mountedParamsOpt); - experimentalFeatureSettings.require(Xp::MountedSSHStore); + xpSettings.require(Xp::MountedSSHStore); if (!mountedParamsP) return {}; auto & mountedParams = getObject(*mountedParamsP); return {{storeConfig, mountedParams}}; @@ -96,12 +97,12 @@ static std::optional getMounted( SSHStoreConfig::SSHStoreConfig( std::string_view scheme, std::string_view authority, - const StoreReference::Params & params) + const StoreReference::Params & params, const ExperimentalFeatureSettings & xpSettings) : Store::Config{params} , RemoteStore::Config{*this, params} , CommonSSHStoreConfig{scheme, authority, params} , SSHStoreConfigT{sshStoreConfigApplyParse(params)} - , mounted{getMounted(*this, params)} + , mounted{getMounted(*this, params, xpSettings)} { } diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh index 7f71c275a0b..e8b7f200fef 100644 --- a/src/libstore/ssh-store.hh +++ b/src/libstore/ssh-store.hh @@ -24,7 +24,11 @@ struct SSHStoreConfig : std::enable_shared_from_this, std::optional mounted; - SSHStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params); + SSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const StoreReference::Params & params, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); static const std::string name() { From ede1c2b1b3058ec2983f0e26788ef427bf4359f6 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Sun, 2 Feb 2025 23:39:09 -0500 Subject: [PATCH 80/80] Back compat for boolean fields --- src/libstore/store-reference.cc | 61 +++++++++++++++++++++++++++------ src/libutil/meson.build | 1 + 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc index 17257900c5c..12ed96a01cb 100644 --- a/src/libstore/store-reference.cc +++ b/src/libstore/store-reference.cc @@ -61,16 +61,57 @@ static StoreReference::Params decodeParamsJson(StringMap paramsRaw) StoreReference::Params params; for (auto && [k, v] : std::move(paramsRaw)) { nlohmann::json j; - try { - j = nlohmann::json::parse(v); - } catch (nlohmann::json::exception &) { - // if its not valid JSON... - if (k == "remote-program" || k == "system-features") { - // Back compat hack! Split and take that array - j = tokenizeString>(v); - } else { - // ...keep the literal string. - j = std::move(v); + /* We have to parse the URL un an "untyped" way before we do a + "typed" conversion to specific store-configuration types. As + such, the best we can do for back-compat is just white-list + specific query parameter names. + + These are all the boolean store parameters in use at the time + of the introduction of JSON store configuration, as evidenced + by `git grep 'F'`. So these will continue working with + "yes"/"no"/"1"/"0", whereas any new ones will require + "true"/"false". + */ + bool preJsonBool = + std::set{ + "check-mount", + "compress", + "trusted", + "multipart-upload", + "parallel-compression", + "read-only", + "require-sigs", + "want-mass-query", + "index-debug-info", + "write-nar-listing", + } + .contains(std::string_view{k}); + + auto warnPreJson = [&] { + warn( + "in query param '%s', using '%s' to mean a boolean is deprecated, please use valid JSON 'true' or 'false'", + k, + v); + }; + + if (preJsonBool && (v == "yes" || v == "1")) { + j = true; + warnPreJson(); + } else if (preJsonBool && (v == "no" || v == "0")) { + j = true; + warnPreJson(); + } else { + try { + j = nlohmann::json::parse(v); + } catch (nlohmann::json::exception &) { + // if its not valid JSON... + if (k == "remote-program" || k == "system-features") { + // Back compat hack! Split and take that array + j = tokenizeString>(v); + } else { + // ...keep the literal string. + j = std::move(v); + } } } params.insert_or_assign(std::move(k), std::move(j)); diff --git a/src/libutil/meson.build b/src/libutil/meson.build index ac701d8fd3b..974feb04b6a 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -187,6 +187,7 @@ headers = [config_h] + files( 'comparator.hh', 'compression.hh', 'compute-levels.hh', + 'config-abstract.hh', 'config-global.hh', 'config-impl.hh', 'config.hh',