diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 35ae73e5d1f..25a1600c849 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -19,7 +19,6 @@ in prefix, inlineHTML ? true, }: -settingsInfo: let @@ -27,11 +26,25 @@ let prefix: setting: { description, - documentDefault, - defaultValue, - aliases, - value, + experimentalFeature, + + # Whether we document the default, because it is machine agostic, + # or don't because because it is machine-specific. + documentDefault ? true, + + # The default value is JSON for new-style config, rather than then + # a string or boolean, for old-style config. + json ? false, + + defaultValue ? null, + + subSettings ? null, + + aliases ? [ ], + + # The current value for this setting. Purposefully unused. + value ? null, }: let result = squash '' @@ -50,7 +63,7 @@ let ${description} - **Default:** ${showDefault documentDefault defaultValue} + ${showDefaultOrSubSettings} ${showAliases aliases} ''; @@ -72,9 +85,24 @@ let > ``` ''; + showDefaultOrSubSettings = + if !isAttrs subSettings then + # No subsettings, instead single setting. Show the default value. + '' + **Default:** ${showDefault} + '' + else + # Indent the nested sub-settings, and append the outer setting name onto the prefix + indent " " '' + **Nullable sub-settings**: ${if subSettings.nullable then "true" else "false"} + ${builtins.trace prefix (showSettings "${prefix}-${setting}" subSettings.map)} + ''; + showDefault = - documentDefault: defaultValue: if documentDefault then + if json then + "`${builtins.toJSON defaultValue}`" + else # a StringMap value type is specified as a string, but # this shows the value type. The empty stringmap is `null` in # JSON, but that converts to `{ }` here. @@ -95,5 +123,7 @@ let in result; + showSettings = + prefix: settingsInfo: concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo)); in -concatStrings (attrValues (mapAttrs (showSetting prefix) settingsInfo)) +showSettings prefix diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index e8b7377dafd..e66611affe0 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -33,6 +33,7 @@ let { settings, doc, + uri-schemes, experimentalFeature, }: let diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 82ad7d86212..ff993fd9fcd 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -16,7 +16,7 @@ #include "globals.hh" #include "serialise.hh" #include "build-result.hh" -#include "store-api.hh" +#include "store-open.hh" #include "strings.hh" #include "derivations.hh" #include "local-store.hh" @@ -44,7 +44,7 @@ static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { for (auto & feature : requiredFeatures) - if (!store.systemFeatures.get().count(feature)) return false; + if (!store.config.systemFeatures.get().count(feature)) return false; return true; } @@ -85,7 +85,7 @@ static int main_build_remote(int argc, char * * argv) that gets cleared on reboot, but it wouldn't work on macOS. */ auto currentLoadName = "/current-load"; if (auto localStore = store.dynamic_pointer_cast()) - currentLoad = std::string { localStore->stateDir } + currentLoadName; + currentLoad = std::string { localStore->config.stateDir } + currentLoadName; else currentLoad = settings.nixStateDir + currentLoadName; diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 86d13fab796..75d12a6d4f3 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -3,7 +3,7 @@ #include "command.hh" #include "markdown.hh" -#include "store-api.hh" +#include "store-open.hh" #include "local-fs-store.hh" #include "derivations.hh" #include "nixexpr.hh" diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index 9570ce3e7ac..216d6a8bdba 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -18,7 +18,7 @@ extern char ** savedArgv; class EvalState; struct Pos; class Store; -class LocalFSStore; +struct LocalFSStore; static constexpr Command::Category catHelp = -1; static constexpr Command::Category catSecondary = 100; diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index de967e3fe49..ab183f6ae7d 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -9,7 +9,7 @@ #include "registry.hh" #include "flake/flakeref.hh" #include "flake/settings.hh" -#include "store-api.hh" +#include "store-open.hh" #include "command.hh" #include "tarball.hh" #include "fetch-to-store.hh" diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index f292f06bb81..c93d4c14571 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -12,7 +12,7 @@ #include "eval-settings.hh" #include "attr-path.hh" #include "signals.hh" -#include "store-api.hh" +#include "store-open.hh" #include "log-store.hh" #include "common-eval-args.hh" #include "get-drvs.hh" diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 04b8d059599..c7666abf470 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -1,5 +1,5 @@ #include "primops.hh" -#include "store-api.hh" +#include "store-open.hh" #include "realisation.hh" #include "make-content-addressed.hh" #include "url.hh" diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index bc306e0d0ad..cfa5adc9f76 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -5,6 +5,7 @@ #include "path.hh" #include "store-api.hh" +#include "store-open.hh" #include "build-result.hh" #include "globals.hh" @@ -42,7 +43,7 @@ Store * nix_store_open(nix_c_context * context, const char * uri, const char *** if (!params) return new Store{nix::openStore(uri_str)}; - nix::Store::Params params_map; + nix::StoreReference::Params params_map; for (size_t i = 0; params[i] != nullptr; i++) { params_map[params[i][0]] = params[i][1]; } diff --git a/src/libstore-test-support/tests/libstore.hh b/src/libstore-test-support/tests/libstore.hh index 699ba957ec8..800d07c343b 100644 --- a/src/libstore-test-support/tests/libstore.hh +++ b/src/libstore-test-support/tests/libstore.hh @@ -5,6 +5,7 @@ #include #include "store-api.hh" +#include "store-open.hh" namespace nix { diff --git a/src/libstore-tests/data/store-reference/auto.json b/src/libstore-tests/data/store-reference/auto.json new file mode 100644 index 00000000000..da510f64949 --- /dev/null +++ b/src/libstore-tests/data/store-reference/auto.json @@ -0,0 +1,3 @@ +{ + "scheme": "auto" +} diff --git a/src/libstore-tests/data/store-reference/auto_param.json b/src/libstore-tests/data/store-reference/auto_param.json new file mode 100644 index 00000000000..41826c83b56 --- /dev/null +++ b/src/libstore-tests/data/store-reference/auto_param.json @@ -0,0 +1,4 @@ +{ + "root": "/foo/bar/baz", + "scheme": "auto" +} diff --git a/src/libstore-tests/data/store-reference/local_1.json b/src/libstore-tests/data/store-reference/local_1.json new file mode 100644 index 00000000000..3b44e382b7f --- /dev/null +++ b/src/libstore-tests/data/store-reference/local_1.json @@ -0,0 +1,5 @@ +{ + "authority": "", + "root": "/foo/bar/baz", + "scheme": "local" +} diff --git a/src/libstore-tests/data/store-reference/local_2.json b/src/libstore-tests/data/store-reference/local_2.json new file mode 100644 index 00000000000..346b0cf225f --- /dev/null +++ b/src/libstore-tests/data/store-reference/local_2.json @@ -0,0 +1,5 @@ +{ + "authority": "/foo/bar/baz", + "scheme": "local", + "trusted": true +} diff --git a/src/libstore-tests/data/store-reference/ssh.json b/src/libstore-tests/data/store-reference/ssh.json new file mode 100644 index 00000000000..e5b0b367ffd --- /dev/null +++ b/src/libstore-tests/data/store-reference/ssh.json @@ -0,0 +1,4 @@ +{ + "authority": "localhost", + "scheme": "ssh" +} diff --git a/src/libstore-tests/data/store-reference/unix.json b/src/libstore-tests/data/store-reference/unix.json new file mode 100644 index 00000000000..8943e959dd6 --- /dev/null +++ b/src/libstore-tests/data/store-reference/unix.json @@ -0,0 +1,6 @@ +{ + "authority": "", + "max-connections": 7, + "scheme": "unix", + "trusted": true +} diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc new file mode 100644 index 00000000000..df005da61ca --- /dev/null +++ b/src/libstore-tests/dummy-store.cc @@ -0,0 +1,20 @@ +#include + +#include "dummy-store.hh" +#include "globals.hh" + +namespace nix { + +TEST(DummyStore, constructConfig) +{ + DummyStoreConfig config{"dummy", "", {}}; + + EXPECT_EQ(config.storeDir, settings.nixStore); +} + +TEST(DummyStore, constructConfigNoAuthority) +{ + EXPECT_THROW(DummyStoreConfig("dummy", "not-allowed", {}), UsageError); +} + +} // namespace nix diff --git a/src/libstore-tests/legacy-ssh-store.cc b/src/libstore-tests/legacy-ssh-store.cc index eb31a240804..ba877d10318 100644 --- a/src/libstore-tests/legacy-ssh-store.cc +++ b/src/libstore-tests/legacy-ssh-store.cc @@ -9,11 +9,13 @@ TEST(LegacySSHStore, constructConfig) LegacySSHStoreConfig config{ "ssh", "localhost", - StoreConfig::Params{ + StoreReference::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, }, }}; EXPECT_EQ( diff --git a/src/libstore-tests/local-overlay-store.cc b/src/libstore-tests/local-overlay-store.cc index b34ca92375e..4fe7e607f47 100644 --- a/src/libstore-tests/local-overlay-store.cc +++ b/src/libstore-tests/local-overlay-store.cc @@ -1,9 +1,6 @@ -// FIXME: Odd failures for templates that are causing the PR to break -// for now with discussion with @Ericson2314 to comment out. -#if 0 -# include +#include -# include "local-overlay-store.hh" +#include "local-overlay-store.hh" namespace nix { @@ -31,4 +28,3 @@ TEST(LocalOverlayStore, constructConfig_rootPath) } } // namespace nix -#endif diff --git a/src/libstore-tests/local-store.cc b/src/libstore-tests/local-store.cc index abc3ea7963f..e22f54bd45d 100644 --- a/src/libstore-tests/local-store.cc +++ b/src/libstore-tests/local-store.cc @@ -1,15 +1,6 @@ -// FIXME: Odd failures for templates that are causing the PR to break -// for now with discussion with @Ericson2314 to comment out. -#if 0 -# include +#include -# include "local-store.hh" - -// Needed for template specialisations. This is not good! When we -// overhaul how store configs work, this should be fixed. -# include "args.hh" -# include "config-impl.hh" -# include "abstract-setting-to-json.hh" +#include "local-store.hh" namespace nix { @@ -37,4 +28,3 @@ TEST(LocalStore, constructConfig_rootPath) } } // namespace nix -#endif diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index 3ba0795e9fa..81e122e63e8 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -58,6 +58,7 @@ sources = files( 'derivation-advanced-attrs.cc', 'derivation.cc', 'derived-path.cc', + 'dummy-store.cc', 'downstream-placeholder.cc', 'http-binary-cache-store.cc', 'legacy-ssh-store.cc', diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index a8b7b8e5fc8..14de1289efd 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -95,7 +95,7 @@ TEST_F(nix_api_util_context, nix_store_open_dummy) nix_libstore_init(ctx); Store * store = nix_store_open(ctx, "dummy://", nullptr); ASSERT_EQ(NIX_OK, ctx->last_err_code); - ASSERT_STREQ("dummy", store->ptr->getUri().c_str()); + ASSERT_STREQ("dummy://", store->ptr->getUri().c_str()); std::string str; nix_store_get_version(ctx, store, OBSERVE_STRING(str)); diff --git a/src/libstore-tests/ssh-store.cc b/src/libstore-tests/ssh-store.cc index b853a5f1fb9..0d18b351164 100644 --- a/src/libstore-tests/ssh-store.cc +++ b/src/libstore-tests/ssh-store.cc @@ -1,22 +1,21 @@ -// FIXME: Odd failures for templates that are causing the PR to break -// for now with discussion with @Ericson2314 to comment out. -#if 0 -# include +#include -# include "ssh-store.hh" +#include "ssh-store.hh" namespace nix { TEST(SSHStore, constructConfig) { SSHStoreConfig config{ - "ssh", + "ssh-ng", "localhost", - StoreConfig::Params{ + StoreReference::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, }, }, }; @@ -31,16 +30,64 @@ TEST(SSHStore, constructConfig) TEST(MountedSSHStore, constructConfig) { - MountedSSHStoreConfig config{ - "mounted-ssh", + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "mounted-ssh-store"); + + SSHStoreConfig config{ + "ssh-ng", + "localhost", + StoreReference::Params{ + { + "remote-program", + { + "foo", + "bar", + }, + }, + { + "mounted", + nlohmann::json::object_t{}, + }, + }, + mockXpSettings, + }; + + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); + + ASSERT_TRUE(config.mounted); + + EXPECT_EQ(config.mounted->realStoreDir, "/nix/store"); +} + +TEST(MountedSSHStore, constructConfigWithFunnyRealStoreDir) +{ + ExperimentalFeatureSettings mockXpSettings; + mockXpSettings.set("experimental-features", "mounted-ssh-store"); + + SSHStoreConfig config{ + "ssh-ng", "localhost", - StoreConfig::Params{ + StoreReference::Params{ { "remote-program", - // TODO #11106, no more split on space - "foo bar", + { + "foo", + "bar", + }, + }, + { + "mounted", + nlohmann::json::object_t{ + {"real", "/foo/bar"}, + }, }, }, + mockXpSettings, }; EXPECT_EQ( @@ -49,7 +96,10 @@ TEST(MountedSSHStore, constructConfig) "foo", "bar", })); + + ASSERT_TRUE(config.mounted); + + EXPECT_EQ(config.mounted->realStoreDir, "/foo/bar"); } } -#endif diff --git a/src/libstore-tests/store-reference.cc b/src/libstore-tests/store-reference.cc index d4c42f0fda1..08c4d60c1f8 100644 --- a/src/libstore-tests/store-reference.cc +++ b/src/libstore-tests/store-reference.cc @@ -17,14 +17,14 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest std::filesystem::path goldenMaster(PathView testStem) const override { - return unitTestData / (testStem + ".txt"); + return unitTestData / testStem; } }; #define URI_TEST_READ(STEM, OBJ) \ TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_uri) \ { \ - readTest(#STEM, ([&](const auto & encoded) { \ + readTest(#STEM ".txt", ([&](const auto & encoded) { \ StoreReference expected = OBJ; \ auto got = StoreReference::parse(encoded); \ ASSERT_EQ(got, expected); \ @@ -35,7 +35,7 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_uri) \ { \ writeTest( \ - #STEM, \ + #STEM ".txt", \ [&]() -> StoreReference { return OBJ; }, \ [](const auto & file) { return StoreReference::parse(readFile(file)); }, \ [](const auto & file, const auto & got) { return writeFile(file, got.render()); }); \ @@ -45,14 +45,43 @@ class StoreReferenceTest : public CharacterizationTest, public LibStoreTest URI_TEST_READ(STEM, OBJ) \ URI_TEST_WRITE(STEM, OBJ) -URI_TEST( +#define JSON_TEST_READ(STEM, OBJ) \ + TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_json) \ + { \ + readTest(#STEM ".json", ([&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + StoreReference expected = OBJ; \ + StoreReference got = encoded; \ + ASSERT_EQ(got, expected); \ + })); \ + } + +#define JSON_TEST_WRITE(STEM, OBJ) \ + TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_json) \ + { \ + writeTest( \ + #STEM ".json", \ + [&]() -> StoreReference { return OBJ; }, \ + [](const auto & file) -> StoreReference { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, json(got).dump(2) + "\n"); }); \ + } + +#define JSON_TEST(STEM, OBJ) \ + JSON_TEST_READ(STEM, OBJ) \ + JSON_TEST_WRITE(STEM, OBJ) + +#define BOTH_FORMATS_TEST(STEM, OBJ) \ + URI_TEST(STEM, OBJ) \ + JSON_TEST(STEM, OBJ) + +BOTH_FORMATS_TEST( auto, (StoreReference{ .variant = StoreReference::Auto{}, .params = {}, })) -URI_TEST( +BOTH_FORMATS_TEST( auto_param, (StoreReference{ .variant = StoreReference::Auto{}, @@ -81,13 +110,13 @@ static StoreReference localExample_2{ }, .params = { - {"trusted", "true"}, + {"trusted", true}, }, }; -URI_TEST(local_1, localExample_1) +BOTH_FORMATS_TEST(local_1, localExample_1) -URI_TEST(local_2, localExample_2) +BOTH_FORMATS_TEST(local_2, localExample_2) URI_TEST_READ(local_shorthand_1, localExample_1) @@ -100,16 +129,16 @@ static StoreReference unixExample{ }, .params = { - {"max-connections", "7"}, - {"trusted", "true"}, + {"max-connections", 7}, + {"trusted", true}, }, }; -URI_TEST(unix, unixExample) +BOTH_FORMATS_TEST(unix, unixExample) URI_TEST_READ(unix_shorthand, unixExample) -URI_TEST( +BOTH_FORMATS_TEST( ssh, (StoreReference{ .variant = diff --git a/src/libstore-tests/uds-remote-store.cc b/src/libstore-tests/uds-remote-store.cc index 5ccb208714f..4bd3bc18f5f 100644 --- a/src/libstore-tests/uds-remote-store.cc +++ b/src/libstore-tests/uds-remote-store.cc @@ -1,9 +1,6 @@ -// FIXME: Odd failures for templates that are causing the PR to break -// for now with discussion with @Ericson2314 to comment out. -#if 0 -# include +#include -# include "uds-remote-store.hh" +#include "uds-remote-store.hh" namespace nix { @@ -20,4 +17,3 @@ TEST(UDSRemoteStore, constructConfigWrongScheme) } } // namespace nix -#endif diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 896779f85fc..c7cd64d821f 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -13,6 +13,7 @@ #include "callback.hh" #include "signals.hh" #include "archive.hh" +#include "config-parse-impl.hh" #include #include @@ -24,17 +25,104 @@ namespace nix { -BinaryCacheStore::BinaryCacheStore(const Params & params) - : BinaryCacheStoreConfig(params) - , Store(params) +constexpr static const BinaryCacheStoreConfigT binaryCacheStoreConfigDescriptions = { + .compression = { + .name = "compression", + .description = "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`).", + }, + .writeNARListing = { + .name = "write-nar-listing", + .description = "Whether to write a JSON file that lists the files in each NAR.", + }, + .writeDebugInfo = { + .name = "index-debug-info", + .description = R"( + Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to + fetch debug info on demand + )", + }, + .secretKeyFile{ + .name = "secret-key", + .description = "Path to the secret key used to sign the binary cache.", + }, + .localNarCache{ + .name = "local-nar-cache", + .description = "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`.", + }, + .parallelCompression{ + .name = "parallel-compression", + .description = "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`.", + }, + .compressionLevel{ + .name = "compression-level", + .description = R"( + The *preset level* to be used when compressing NARs. + The meaning and accepted values depend on the compression method selected. + `-1` specifies that the default compression level should be used. + )", + }, +}; + +#define BINARY_CACHE_STORE_CONFIG_FIELDS(X) \ + X(compression), \ + X(writeNARListing), \ + X(writeDebugInfo), \ + X(secretKeyFile), \ + X(localNarCache), \ + X(parallelCompression), \ + X(compressionLevel), + +MAKE_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS) + +static BinaryCacheStoreConfigT binaryCacheStoreConfigDefaults() { - if (secretKeyFile != "") + return { + .compression = {"xz"}, + .writeNARListing = {false}, + .writeDebugInfo = {false}, + .secretKeyFile = {""}, + .localNarCache = {""}, + .parallelCompression = {false}, + .compressionLevel = {-1}, + }; +} + +MAKE_APPLY_PARSE(BinaryCacheStoreConfig, binaryCacheStoreConfig, BINARY_CACHE_STORE_CONFIG_FIELDS) + +BinaryCacheStore::Config::BinaryCacheStoreConfig( + const Store::Config & storeConfig, + const StoreReference::Params & params) + : BinaryCacheStoreConfigT{binaryCacheStoreConfigApplyParse(params)} + , storeConfig{storeConfig} +{ +} + +config::SettingDescriptionMap BinaryCacheStoreConfig::descriptions() +{ + constexpr auto & descriptions = binaryCacheStoreConfigDescriptions; + auto defaults = binaryCacheStoreConfigDefaults(); + return { + BINARY_CACHE_STORE_CONFIG_FIELDS(DESC_ROW) + }; +} + +BinaryCacheStore::BinaryCacheStore(const Config & config) + : Store{config.storeConfig} + , config{config} +{ + if (config.secretKeyFile != "") signer = std::make_unique( - SecretKey { readFile(secretKeyFile) }); + SecretKey { readFile(config.secretKeyFile) }); StringSink sink; sink << narVersionMagic1; narMagic = sink.s; + + // Want to call this but cannot, because virtual function lookup is + // disabled in a constructor. It is thus left to instances to call + // it instead. + + //init(); } void BinaryCacheStore::init() @@ -53,9 +141,11 @@ void BinaryCacheStore::init() throw Error("binary cache '%s' is for Nix stores with prefix '%s', not '%s'", getUri(), value, storeDir); } else if (name == "WantMassQuery") { - wantMassQuery.setDefault(value == "1"); + resolvedSubstConfig.wantMassQuery.value = + config.storeConfig.wantMassQuery.optValue.value_or(value == "1"); } else if (name == "Priority") { - priority.setDefault(std::stoi(value)); + resolvedSubstConfig.priority.value = + config.storeConfig.priority.optValue.value_or(std::stoi(value)); } } } @@ -147,7 +237,11 @@ ref BinaryCacheStore::addToStoreCommon( { FdSink fileSink(fdTemp.get()); TeeSink teeSinkCompressed { fileSink, fileHashSink }; - auto compressionSink = makeCompressionSink(compression, teeSinkCompressed, parallelCompression, compressionLevel); + auto compressionSink = makeCompressionSink( + config.compression, + teeSinkCompressed, + config.parallelCompression, + config.compressionLevel); TeeSink teeSinkUncompressed { *compressionSink, narHashSink }; TeeSource teeSource { narSource, teeSinkUncompressed }; narAccessor = makeNarAccessor(teeSource); @@ -159,17 +253,17 @@ ref BinaryCacheStore::addToStoreCommon( auto info = mkInfo(narHashSink.finish()); auto narInfo = make_ref(info); - narInfo->compression = compression; + narInfo->compression = config.compression; auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar" - + (compression == "xz" ? ".xz" : - compression == "bzip2" ? ".bz2" : - compression == "zstd" ? ".zst" : - compression == "lzip" ? ".lzip" : - compression == "lz4" ? ".lz4" : - compression == "br" ? ".br" : + + (config.compression == "xz" ? ".xz" : + config.compression == "bzip2" ? ".bz2" : + config.compression == "zstd" ? ".zst" : + config.compression == "lzip" ? ".lzip" : + config.compression == "lz4" ? ".lz4" : + config.compression == "br" ? ".br" : ""); auto duration = std::chrono::duration_cast(now2 - now1).count(); @@ -191,7 +285,7 @@ ref BinaryCacheStore::addToStoreCommon( /* Optionally write a JSON file containing a listing of the contents of the NAR. */ - if (writeNARListing) { + if (config.writeNARListing) { nlohmann::json j = { {"version", 1}, {"root", listNar(ref(narAccessor), CanonPath::root, true)}, @@ -203,7 +297,7 @@ ref BinaryCacheStore::addToStoreCommon( /* Optionally maintain an index of DWARF debug info files consisting of JSON files named 'debuginfo/' that specify the NAR file and member containing the debug info. */ - if (writeDebugInfo) { + if (config.writeDebugInfo) { CanonPath buildIdDir("lib/debug/.build-id"); @@ -515,7 +609,7 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) { ref BinaryCacheStore::getFSAccessor(bool requireValidPath) { - return make_ref(ref(shared_from_this()), requireValidPath, localNarCache); + return make_ref(ref(shared_from_this()), requireValidPath, config.localNarCache); } void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 6bd7fd14ac9..3cce6d9f949 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -13,48 +13,39 @@ namespace nix { struct NarInfo; -struct BinaryCacheStoreConfig : virtual StoreConfig +template class F> +struct BinaryCacheStoreConfigT { - using StoreConfig::StoreConfig; - - const Setting compression{this, "xz", "compression", - "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; - - const Setting writeNARListing{this, false, "write-nar-listing", - "Whether to write a JSON file that lists the files in each NAR."}; - - const Setting writeDebugInfo{this, false, "index-debug-info", - R"( - Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to - fetch debug info on demand - )"}; - - const Setting secretKeyFile{this, "", "secret-key", - "Path to the secret key used to sign the binary cache."}; + F compression; + F writeNARListing; + F writeDebugInfo; + F secretKeyFile; + F localNarCache; + F parallelCompression; + F compressionLevel; +}; - const Setting localNarCache{this, "", "local-nar-cache", - "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; +struct BinaryCacheStoreConfig : + BinaryCacheStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); - const Setting parallelCompression{this, false, "parallel-compression", - "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; + const Store::Config & storeConfig; - const Setting compressionLevel{this, -1, "compression-level", - R"( - The *preset level* to be used when compressing NARs. - The meaning and accepted values depend on the compression method selected. - `-1` specifies that the default compression level should be used. - )"}; + BinaryCacheStoreConfig(const Store::Config &, const StoreReference::Params &); }; - /** * @note subclasses must implement at least one of the two * virtual getFile() methods. */ -class BinaryCacheStore : public virtual BinaryCacheStoreConfig, - public virtual Store, - public virtual LogStore +struct BinaryCacheStore : + virtual Store, + virtual LogStore { + using Config = BinaryCacheStoreConfig; + + const Config & config; private: std::unique_ptr signer; @@ -66,7 +57,7 @@ protected: const std::string cacheInfoFile = "nix-cache-info"; - BinaryCacheStore(const Params & params); + BinaryCacheStore(const Config &); public: @@ -104,7 +95,11 @@ public: public: - virtual void init() override; + /** + * Perform any necessary effectful operation to make the store up and + * running + */ + virtual void init(); private: diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 4d97250d3af..fdedd504e09 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1265,7 +1265,7 @@ Path DerivationGoal::openLogFile() /* Create a log file. */ Path logDir; if (auto localStore = dynamic_cast(&worker.store)) - logDir = localStore->logDir; + logDir = localStore->config->logDir; else logDir = settings.nixLogDir; Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, baseName.substr(0, 2)); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index f069c0d9404..08e61bf47c5 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -3,6 +3,7 @@ #include "worker.hh" #include "substitution-goal.hh" #include "callback.hh" +#include "store-open.hh" namespace nix { diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index 983c86601d8..75deb4dfea7 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -1,4 +1,5 @@ #include "worker.hh" +#include "store-open.hh" #include "substitution-goal.hh" #include "nar-info.hh" #include "finally.hh" @@ -121,7 +122,7 @@ Goal::Co PathSubstitutionGoal::init() /* Bail out early if this substituter lacks a valid signature. LocalStore::addToStore() also checks for this, but only after we've downloaded the path. */ - if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) + if (!sub->config.isTrusted && worker.store.pathInfoIsUntrusted(*info)) { warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", worker.store.printStorePath(storePath), sub->getUri()); @@ -210,7 +211,7 @@ Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, PushActivity pact(act.id); copyStorePath(*sub, worker.store, - subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + subPath, repair, sub->config.isTrusted ? NoCheckSigs : CheckSigs); promise.set_value(); } catch (...) { diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc index 05332b9bb5c..86a227d81a8 100644 --- a/src/libstore/common-ssh-store-config.cc +++ b/src/libstore/common-ssh-store-config.cc @@ -2,9 +2,56 @@ #include "common-ssh-store-config.hh" #include "ssh.hh" +#include "config-parse-impl.hh" namespace nix { +constexpr static const CommonSSHStoreConfigT commonSSHStoreConfigDescriptions = { + .sshKey{ + .name = "ssh-key", + .description = "Path to the SSH private key used to authenticate to the remote machine.", + }, + .sshPublicHostKey{ + .name = "base64-ssh-public-host-key", + .description = "The public host key of the remote machine.", + }, + .compress{ + .name = "compress", + .description = "Whether to enable SSH compression.", + }, + .remoteStore{ + .name = "remote-store", + .description = R"( + [Store URL](@docroot@/store/types/index.md#store-url-format) + to be used on the remote machine. The default is `auto` + (i.e. use the Nix daemon or `/nix/store` directly). + )", + }, +}; + +#define COMMON_SSH_STORE_CONFIG_FIELDS(X) X(sshKey), X(sshPublicHostKey), X(compress), X(remoteStore), + +MAKE_PARSE(CommonSSHStoreConfig, commonSSHStoreConfig, COMMON_SSH_STORE_CONFIG_FIELDS) + +static CommonSSHStoreConfigT commonSSHStoreConfigDefaults() +{ + return { + .sshKey = {""}, + .sshPublicHostKey = {""}, + .compress = {false}, + .remoteStore = {""}, + }; +} + +MAKE_APPLY_PARSE(CommonSSHStoreConfig, commonSSHStoreConfig, COMMON_SSH_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap CommonSSHStoreConfig::descriptions() +{ + constexpr auto & descriptions = commonSSHStoreConfigDescriptions; + auto defaults = commonSSHStoreConfigDefaults(); + return {COMMON_SSH_STORE_CONFIG_FIELDS(DESC_ROW)}; +} + static std::string extractConnStr(std::string_view scheme, std::string_view _connStr) { if (_connStr.empty()) @@ -22,13 +69,14 @@ static std::string extractConnStr(std::string_view scheme, std::string_view _con return connStr; } -CommonSSHStoreConfig::CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params) - : StoreConfig(params) - , host(extractConnStr(scheme, host)) +CommonSSHStoreConfig::CommonSSHStoreConfig( + std::string_view scheme, std::string_view host, const StoreReference::Params & params) + : CommonSSHStoreConfigT{commonSSHStoreConfigApplyParse(params)} + , host{extractConnStr(scheme, host)} { } -SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) +SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) const { return { host, diff --git a/src/libstore/common-ssh-store-config.hh b/src/libstore/common-ssh-store-config.hh index 5deb6f4c9e9..d998e16a2b3 100644 --- a/src/libstore/common-ssh-store-config.hh +++ b/src/libstore/common-ssh-store-config.hh @@ -7,27 +7,27 @@ namespace nix { class SSHMaster; -struct CommonSSHStoreConfig : virtual StoreConfig +template class F> +struct CommonSSHStoreConfigT { - using StoreConfig::StoreConfig; - - CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params); - - const Setting sshKey{this, "", "ssh-key", - "Path to the SSH private key used to authenticate to the remote machine."}; - - const Setting sshPublicHostKey{this, "", "base64-ssh-public-host-key", - "The public host key of the remote machine."}; + F sshKey; + F sshPublicHostKey; + F compress; + F remoteStore; +}; - const Setting compress{this, false, "compress", - "Whether to enable SSH compression."}; +struct CommonSSHStoreConfig : CommonSSHStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); - const Setting remoteStore{this, "", "remote-store", - R"( - [Store URL](@docroot@/store/types/index.md#store-url-format) - to be used on the remote machine. The default is `auto` - (i.e. use the Nix daemon or `/nix/store` directly). - )"}; + /** + * @param scheme Note this isn't stored by this mix-in class, but + * just used for better error messages. + */ + CommonSSHStoreConfig( + std::string_view scheme, + std::string_view host, + const StoreReference::Params & params); /** * The `parseURL` function supports both IPv6 URIs as defined in @@ -56,7 +56,7 @@ struct CommonSSHStoreConfig : virtual StoreConfig */ SSHMaster createSSHMaster( bool useMaster, - Descriptor logFD = INVALID_DESCRIPTOR); + Descriptor logFD = INVALID_DESCRIPTOR) const; }; } diff --git a/src/libstore/config-parse-impl.hh b/src/libstore/config-parse-impl.hh new file mode 100644 index 00000000000..2bca7ee8855 --- /dev/null +++ b/src/libstore/config-parse-impl.hh @@ -0,0 +1,69 @@ +#pragma once +//@file + +#include + +#include "config-parse.hh" +#include "util.hh" +#include "config.hh" + +namespace nix::config { + +template +OptValue +SettingInfo::parseConfig(const nlohmann::json::object_t & map, const ExperimentalFeatureSettings & xpSettings) const +{ + const nlohmann::json * p = get(map, name); + if (p && experimentalFeature) + xpSettings.require(*experimentalFeature); + return {.optValue = p ? (std::optional{p->get()}) : std::nullopt}; +} + +template +std::pair SettingInfo::describe(const JustValue & def) const +{ + return { + std::string{name}, + SettingDescription{ + .description = stripIndentation(description), + .experimentalFeature = experimentalFeature, + .info = + SettingDescription::Single{ + .defaultValue = documentDefault ? (std::optional{nlohmann::json(def.value)}) + : (std::optional{}), + }, + }, + }; +} + +/** + * Look up the setting's name in a map, falling back on the default if + * it does not exist. + */ +#define CONFIG_ROW(FIELD) .FIELD = descriptions.FIELD.parseConfig(params, xpSettings) + +#define APPLY_ROW(FIELD) .FIELD = {.value = parsed.FIELD.optValue.value_or(std::move(defaults.FIELD))} + +#define DESC_ROW(FIELD) \ + { \ + descriptions.FIELD.describe(defaults.FIELD), \ + } + +#define MAKE_PARSE(CAPITAL, LOWER, FIELDS) \ + static CAPITAL##T LOWER##Parse( \ + const StoreReference::Params & params, \ + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) \ + { \ + constexpr auto & descriptions = LOWER##Descriptions; \ + return {FIELDS(CONFIG_ROW)}; \ + } + +#define MAKE_APPLY_PARSE(CAPITAL, LOWER, FIELDS) \ + static CAPITAL##T LOWER##ApplyParse(const StoreReference::Params & params) \ + { \ + auto defaults = LOWER##Defaults(); \ + auto parsed = LOWER##Parse(params); \ + return {FIELDS(APPLY_ROW)}; \ + } + +} diff --git a/src/libstore/config-parse.cc b/src/libstore/config-parse.cc new file mode 100644 index 00000000000..a32e32d061d --- /dev/null +++ b/src/libstore/config-parse.cc @@ -0,0 +1,71 @@ +#include + +#include "config-parse.hh" +#include "json-utils.hh" +#include "util.hh" + +namespace nix::config { + +}; + +namespace nlohmann { + +using namespace nix::config; + +SettingDescription adl_serializer::from_json(const json & json) +{ + auto & obj = getObject(json); + return { + .description = getString(valueAt(obj, "description")), + .experimentalFeature = valueAt(obj, "experimentalFeature").get>(), + .info = [&]() -> decltype(SettingDescription::info) { + if (auto documentDefault = optionalValueAt(obj, "documentDefault")) { + return SettingDescription::Single{ + .defaultValue = *documentDefault ? (std::optional{valueAt(obj, "defaultValue")}) + : (std::optional{}), + }; + } else { + auto & subObj = getObject(valueAt(obj, "subSettings")); + return SettingDescription::Sub{ + .nullable = valueAt(subObj, "nullable"), + .map = valueAt(subObj, "map"), + }; + } + }(), + }; +} + +void adl_serializer::to_json(json & obj, SettingDescription sd) +{ + obj.emplace("description", sd.description); + // obj.emplace("aliases", sd.aliases); + obj.emplace("experimentalFeature", sd.experimentalFeature); + + std::visit( + overloaded{ + [&](const SettingDescription::Single & single) { + // Indicate the default value is JSON, rather than a legacy setting + // boolean or string. + // + // TODO remove if we no longer have the legacy setting system / the + // code handling doc rendering of the settings is decoupled. + obj.emplace("json", true); + + // Cannot just use `null` because the default value might itself be + // `null`. + obj.emplace("documentDefault", single.defaultValue.has_value()); + + if (single.defaultValue.has_value()) + obj.emplace("defaultValue", *single.defaultValue); + }, + [&](const SettingDescription::Sub & sub) { + json subJson; + subJson.emplace("nullable", sub.nullable); + subJson.emplace("map", sub.map); + obj.emplace("subSettings", std::move(subJson)); + }, + }, + sd.info); +} + +} diff --git a/src/libstore/config-parse.hh b/src/libstore/config-parse.hh new file mode 100644 index 00000000000..e656bfba530 --- /dev/null +++ b/src/libstore/config-parse.hh @@ -0,0 +1,144 @@ +#pragma once +//@file + +#include + +#include "config-abstract.hh" +#include "json-impls.hh" +#include "experimental-features.hh" + +namespace nix { + +struct ExperimentalFeatureSettings; + +}; + +namespace nix::config { + +struct SettingDescription; + +/** + * Typed version used as source of truth, and for operations like + * defaulting configurations. + * + * It is important that this type support `constexpr` values to avoid + * running into issues with static initialization order. + */ +template +struct SettingInfo +{ + /** + * Name of the setting, used when parsing configuration maps + */ + std::string_view name; + + /** + * Description of the setting. It is used just for documentation. + */ + std::string_view description; + +#if 0 + /** + * Other names of the setting also used when parsing configuration + * maps. This is useful for back-compat, etc. + */ + std::set aliases; +#endif + + /** + * `ExperimentalFeature` that must be enabled if the setting is + * allowed to be used + */ + std::optional experimentalFeature; + + /** + * Whether to document the default value. (Some defaults are + * system-specific and should not be documented.) + */ + bool documentDefault = true; + + /** + * Describe the setting as a key-value pair (name -> other info). + * The default value will be rendered to JSON if it is to be + * documented. + */ + std::pair describe(const JustValue & def) const; + + OptValue parseConfig(const nlohmann::json::object_t & map, const ExperimentalFeatureSettings & xpSettings) const; +}; + +struct SettingDescription; + +/** + * Map of setting names to descriptions of those settings. + */ +using SettingDescriptionMap = std::map; + +/** + * Untyped version used for rendering docs. This is not the source of + * truth, it is generated from the typed one. + * + * @note No `name` field because this is intended to be used as the value type + * of a map + */ +struct SettingDescription +{ + /** + * @see SettingInfo::description + */ + std::string description; + +#if 0 + /** + * @see SettingInfo::aliases + */ + std::set aliases; +#endif + + /** + * @see SettingInfo::experimentalFeature + */ + std::optional experimentalFeature; + + /** + * A single leaf setting, to be optionally specified by arbitrary + * value (of some type) or left default. + */ + struct Single + { + /** + * Optional, for the `SettingInfo::documentDefault = false` case. + */ + std::optional defaultValue; + }; + + /** + * A nested settings object + */ + struct Sub + { + /** + * If `false`, this is just pure namespaceing. If `true`, we + * have a distinction between `null` and `{}`, meaning + * enabling/disabling the entire settings group. + */ + bool nullable = true; + + SettingDescriptionMap map; + }; + + /** + * Variant for `info` below + */ + using Info = std::variant; + + /** + * More information about this setting, depending on whether its the + * single leaf setting or subsettings case + */ + Info info; +}; + +} + +JSON_IMPL(config::SettingDescription) diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index c1e871e9384..628b6bd3d07 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -1,47 +1,39 @@ -#include "store-api.hh" +#include "dummy-store.hh" +#include "store-registration.hh" #include "callback.hh" namespace nix { -struct DummyStoreConfig : virtual StoreConfig { - using StoreConfig::StoreConfig; - - DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - { - if (!authority.empty()) - throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); - } - - const std::string name() override { return "Dummy Store"; } +DummyStoreConfig::DummyStoreConfig( + std::string_view scheme, std::string_view authority, const StoreReference::Params & params) + : StoreConfig{params} +{ + if (!authority.empty()) + throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); +} - std::string doc() override - { - return - #include "dummy-store.md" - ; - } +std::string DummyStoreConfig::doc() +{ + return + #include "dummy-store.md" + ; +} - static std::set uriSchemes() { - return {"dummy"}; - } -}; -struct DummyStore : public virtual DummyStoreConfig, public virtual Store +struct DummyStore : virtual Store { - DummyStore(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - , DummyStoreConfig(scheme, authority, params) - , Store(params) - { } + using Config = DummyStoreConfig; - DummyStore(const Params & params) - : DummyStore("dummy", "", params) + ref config; + + DummyStore(ref config) + : Store{*config} + , config(config) { } std::string getUri() override { - return *uriSchemes().begin(); + return *Config::uriSchemes().begin() + "://"; } void queryPathInfoUncached(const StorePath & path, @@ -86,6 +78,11 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store { unsupported("getFSAccessor"); } }; -static RegisterStoreImplementation regDummyStore; +ref DummyStore::Config::openStore() const +{ + return make_ref(ref{shared_from_this()}); +} + +static RegisterStoreImplementation regDummyStore; } diff --git a/src/libstore/dummy-store.hh b/src/libstore/dummy-store.hh new file mode 100644 index 00000000000..80f19a67a2a --- /dev/null +++ b/src/libstore/dummy-store.hh @@ -0,0 +1,24 @@ +#include "store-api.hh" + +namespace nix { + +struct DummyStoreConfig : std::enable_shared_from_this, StoreConfig +{ + DummyStoreConfig(std::string_view scheme, std::string_view authority, const StoreReference::Params & params); + + static const std::string name() + { + return "Dummy Store"; + } + + static std::string doc(); + + static std::set uriSchemes() + { + return {"dummy"}; + } + + ref openStore() const override; +}; + +} diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 8439cc39cc8..5622b49c8c9 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -727,7 +727,7 @@ struct curlFileTransfer : public FileTransfer } #if ENABLE_S3 - std::tuple parseS3Uri(std::string uri) + std::tuple parseS3Uri(std::string uri) { auto [path, params] = splitUriAndParams(uri); diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ac354f3faf7..bacd3ee2131 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -41,7 +41,7 @@ static std::string gcRootsDir = "gcroots"; void LocalStore::addIndirectRoot(const Path & path) { std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Nix32, false); - Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); + Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", config->stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } @@ -80,7 +80,7 @@ void LocalStore::createTempRootsFile() void LocalStore::addTempRoot(const StorePath & path) { - if (readOnly) { + if (config->readOnly) { debug("Read-only store doesn't support creating lock files for temp roots, but nothing can be deleted anyways."); return; } @@ -107,7 +107,7 @@ void LocalStore::addTempRoot(const StorePath & path) auto fdRootsSocket(_fdRootsSocket.lock()); if (!*fdRootsSocket) { - auto socketPath = stateDir.get() + gcSocketPath; + auto socketPath = config->stateDir.get() + gcSocketPath; debug("connecting to '%s'", socketPath); *fdRootsSocket = createUnixDomainSocket(); try { @@ -245,7 +245,7 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R else { target = absPath(target, dirOf(path)); if (!pathExists(target)) { - if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) { + if (isInDir(path, config->stateDir + "/" + gcRootsDir + "/auto")) { printInfo("removing stale link from '%1%' to '%2%'", path, target); unlink(path.c_str()); } @@ -286,8 +286,8 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R void LocalStore::findRootsNoTemp(Roots & roots, bool censor) { /* Process direct roots in {gcroots,profiles}. */ - findRoots(stateDir + "/" + gcRootsDir, std::filesystem::file_type::unknown, roots); - findRoots(stateDir + "/profiles", std::filesystem::file_type::unknown, roots); + findRoots(config->stateDir + "/" + gcRootsDir, std::filesystem::file_type::unknown, roots); + findRoots(config->stateDir + "/profiles", std::filesystem::file_type::unknown, roots); /* Add additional roots returned by different platforms-specific heuristics. This is typically used to add running programs to @@ -496,7 +496,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) readFile(*p); /* Start the server for receiving new roots. */ - auto socketPath = stateDir.get() + gcSocketPath; + auto socketPath = config->stateDir.get() + gcSocketPath; createDirs(dirOf(socketPath)); auto fdServer = createUnixDomainSocket(socketPath, 0666); @@ -633,7 +633,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) auto deleteFromStore = [&](std::string_view baseName) { Path path = storeDir + "/" + std::string(baseName); - Path realPath = realStoreDir + "/" + std::string(baseName); + Path realPath = config->realStoreDir + "/" + std::string(baseName); /* There may be temp directories in the store that are still in use by another process. We need to be sure that we can acquire an @@ -802,8 +802,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) printInfo("determining live/dead paths..."); try { - AutoCloseDir dir(opendir(realStoreDir.get().c_str())); - if (!dir) throw SysError("opening directory '%1%'", realStoreDir); + AutoCloseDir dir(opendir(config->realStoreDir.get().c_str())); + if (!dir) throw SysError("opening directory '%1%'", config->realStoreDir); /* Read the store and delete all paths that are invalid or unreachable. We don't use readDirectory() here so that @@ -905,8 +905,8 @@ void LocalStore::autoGC(bool sync) return std::stoll(readFile(*fakeFreeSpaceFile)); struct statvfs st; - if (statvfs(realStoreDir.get().c_str(), &st)) - throw SysError("getting filesystem info about '%s'", realStoreDir); + if (statvfs(config->realStoreDir.get().c_str(), &st)) + throw SysError("getting filesystem info about '%s'", config->realStoreDir); return (uint64_t) st.f_bavail * st.f_frsize; }; diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index f32616f94a6..e9ff90c7491 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -3,18 +3,37 @@ #include "globals.hh" #include "nar-info-disk-cache.hh" #include "callback.hh" +#include "store-registration.hh" namespace nix { +config::SettingDescriptionMap HttpBinaryCacheStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(BinaryCacheStoreConfig::descriptions()); + return ret; +} + + MakeError(UploadToHTTP, Error); +std::set HttpBinaryCacheStoreConfig::uriSchemes() +{ + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; + auto ret = std::set({"http", "https"}); + if (forceHttp) + ret.insert("file"); + return ret; +} + HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( std::string_view scheme, std::string_view _cacheUri, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) + const StoreReference::Params & params) + : Store::Config{params} + , BinaryCacheStoreConfig{*this, params} , cacheUri( std::string { scheme } + "://" @@ -35,10 +54,9 @@ std::string HttpBinaryCacheStoreConfig::doc() } -class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore +class HttpBinaryCacheStore : + public virtual BinaryCacheStore { -private: - struct State { bool enabled = true; @@ -49,37 +67,41 @@ class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public v public: - HttpBinaryCacheStore( - std::string_view scheme, - PathView cacheUri, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , HttpBinaryCacheStoreConfig(scheme, cacheUri, params) - , Store(params) - , BinaryCacheStore(params) + using Config = HttpBinaryCacheStoreConfig; + + ref config; + + HttpBinaryCacheStore(ref config) + : Store{*config} + , BinaryCacheStore{*config} + , config{config} { diskCache = getNarInfoDiskCache(); + + init(); } std::string getUri() override { - return cacheUri; + return config->cacheUri; } void init() override { // FIXME: do this lazily? - if (auto cacheInfo = diskCache->upToDateCacheExists(cacheUri)) { - wantMassQuery.setDefault(cacheInfo->wantMassQuery); - priority.setDefault(cacheInfo->priority); + if (auto cacheInfo = diskCache->upToDateCacheExists(config->cacheUri)) { + resolvedSubstConfig.wantMassQuery.value = + config->storeConfig.wantMassQuery.optValue.value_or(cacheInfo->wantMassQuery); + resolvedSubstConfig.priority.value = + config->storeConfig.priority.optValue.value_or(cacheInfo->priority); } else { try { BinaryCacheStore::init(); } catch (UploadToHTTP &) { - throw Error("'%s' does not appear to be a binary cache", cacheUri); + throw Error("'%s' does not appear to be a binary cache", config->cacheUri); } - diskCache->createCache(cacheUri, storeDir, wantMassQuery, priority); + diskCache->createCache( + config->cacheUri, storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority); } } @@ -137,7 +159,7 @@ class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public v try { getFileTransfer()->upload(req); } catch (FileTransferError & e) { - throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); + throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", config->cacheUri, e.msg()); } } @@ -146,7 +168,7 @@ class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public v return FileTransferRequest( hasPrefix(path, "https://") || hasPrefix(path, "http://") || hasPrefix(path, "file://") ? path - : cacheUri + "/" + path); + : config->cacheUri + "/" + path); } @@ -221,6 +243,11 @@ class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public v } }; -static RegisterStoreImplementation regHttpBinaryCacheStore; +ref HttpBinaryCacheStore::Config::openStore() const +{ + return make_ref(ref{shared_from_this()}); +} + +static RegisterStoreImplementation regHttpBinaryCacheStore; } diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh index d2fc43210a2..0767e7dc8da 100644 --- a/src/libstore/http-binary-cache-store.hh +++ b/src/libstore/http-binary-cache-store.hh @@ -2,29 +2,27 @@ namespace nix { -struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this, + Store::Config, + BinaryCacheStoreConfig { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + static config::SettingDescriptionMap descriptions(); - HttpBinaryCacheStoreConfig(std::string_view scheme, std::string_view _cacheUri, const Params & params); + HttpBinaryCacheStoreConfig( + std::string_view scheme, std::string_view cacheUri, const StoreReference::Params & params); Path cacheUri; - const std::string name() override + static const std::string name() { return "HTTP Binary Cache Store"; } - static std::set uriSchemes() - { - static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; - auto ret = std::set({"http", "https"}); - if (forceHttp) - ret.insert("file"); - return ret; - } + static std::set uriSchemes(); + + static std::string doc(); - std::string doc() override; + ref openStore() const override; }; } diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index eac360a4f7a..c9d880536df 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -12,18 +12,76 @@ #include "ssh.hh" #include "derivations.hh" #include "callback.hh" +#include "config-parse-impl.hh" +#include "store-registration.hh" namespace nix { -LegacySSHStoreConfig::LegacySSHStoreConfig( +constexpr static const LegacySSHStoreConfigT legacySSHStoreConfigDescriptions = { + .remoteProgram{ + .name = "remote-program", + .description = "Path to the `nix-store` executable on the remote machine.", + }, + .maxConnections{ + .name = "max-connections", + .description = "Maximum number of concurrent SSH connections.", + }, +}; + + +#define LEGACY_SSH_STORE_CONFIG_FIELDS(X) \ + X(remoteProgram), \ + X(maxConnections) + + +MAKE_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS) + + +static LegacySSHStoreConfigT legacySSHStoreConfigDefaults() +{ + return { + .remoteProgram = {{"nix-store"}}, + .maxConnections = {1}, + }; +} + + +MAKE_APPLY_PARSE(LegacySSHStoreConfig, legacySSHStoreConfig, LEGACY_SSH_STORE_CONFIG_FIELDS) + + +config::SettingDescriptionMap LegacySSHStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(CommonSSHStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + { + constexpr auto & descriptions = legacySSHStoreConfigDescriptions; + auto defaults = legacySSHStoreConfigDefaults(); + ret.merge(decltype(ret){ + LEGACY_SSH_STORE_CONFIG_FIELDS(DESC_ROW) + }); + } + return ret; +} + + +LegacySSHStore::Config::LegacySSHStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params) - : StoreConfig(params) - , CommonSSHStoreConfig(scheme, authority, params) + const StoreReference::Params & params) + : Store::Config{params} + , CommonSSHStoreConfig{scheme, authority, params} + , LegacySSHStoreConfigT{legacySSHStoreConfigApplyParse(params)} { +#ifndef _WIN32 + if (auto * p = get(params, "log-fd")) { + logFD = p->get(); + } +#endif } + std::string LegacySSHStoreConfig::doc() { return @@ -38,23 +96,19 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection bool good = true; }; -LegacySSHStore::LegacySSHStore( - std::string_view scheme, - std::string_view host, - const Params & params) - : StoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , LegacySSHStoreConfig(scheme, host, params) - , Store(params) + +LegacySSHStore::LegacySSHStore(ref config) + : Store{*config} + , config{config} , connections(make_ref>( - std::max(1, (int) maxConnections), + std::max(1, (int) config->maxConnections), [this]() { return openConnection(); }, [](const ref & r) { return r->good; } )) - , master(createSSHMaster( + , master(config->createSSHMaster( // Use SSH master only if using more than 1 connection. connections->capacity() > 1, - logFD)) + config->logFD)) { } @@ -62,12 +116,12 @@ LegacySSHStore::LegacySSHStore( ref LegacySSHStore::openConnection() { auto conn = make_ref(); - Strings command = remoteProgram.get(); + Strings command = config->remoteProgram.get(); command.push_back("--serve"); command.push_back("--write"); - if (remoteStore.get() != "") { + if (config->remoteStore.get() != "") { command.push_back("--store"); - command.push_back(remoteStore.get()); + command.push_back(config->remoteStore.get()); } conn->sshConn = master.startCommand(std::move(command)); conn->to = FdSink(conn->sshConn->in.get()); @@ -77,7 +131,7 @@ ref LegacySSHStore::openConnection() TeeSource tee(conn->from, saved); try { conn->remoteVersion = ServeProto::BasicClientConnection::handshake( - conn->to, tee, SERVE_PROTOCOL_VERSION, host); + conn->to, tee, SERVE_PROTOCOL_VERSION, config->host); } catch (SerialisationError & e) { // in.close(): Don't let the remote block on us not writing. conn->sshConn->in.close(); @@ -86,9 +140,9 @@ ref LegacySSHStore::openConnection() tee.drainInto(nullSink); } throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", - host, chomp(saved.s)); + config->host, chomp(saved.s)); } catch (EndOfFile & e) { - throw Error("cannot connect to '%1%'", host); + throw Error("cannot connect to '%1%'", config->host); } return conn; @@ -97,7 +151,7 @@ ref LegacySSHStore::openConnection() std::string LegacySSHStore::getUri() { - return *uriSchemes().begin() + "://" + host; + return *Config::uriSchemes().begin() + "://" + config->host; } @@ -110,7 +164,7 @@ void LegacySSHStore::queryPathInfoUncached(const StorePath & path, /* No longer support missing NAR hash */ assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); - debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); + debug("querying remote host '%s' for info on '%s'", config->host, printStorePath(path)); auto infos = conn->queryPathInfos(*this, {path}); @@ -139,7 +193,7 @@ void LegacySSHStore::queryPathInfoUncached(const StorePath & path, void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) { - debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); + debug("adding path '%s' to remote host '%s'", printStorePath(info.path), config->host); auto conn(connections->get()); @@ -166,7 +220,7 @@ void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, conn->to.flush(); if (readInt(conn->from) != 1) - throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); + throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), config->host); } else { @@ -311,12 +365,17 @@ unsigned int LegacySSHStore::getProtocol() * The legacy ssh protocol doesn't support checking for trusted-user. * Try using ssh-ng:// instead if you want to know. */ -std::optional isTrustedClient() +std::optional LegacySSHStore::isTrustedClient() { return std::nullopt; } -static RegisterStoreImplementation regLegacySSHStore; +ref LegacySSHStore::Config::openStore() const { + return make_ref(ref{shared_from_this()}); +} + + +static RegisterStoreImplementation regLegacySSHStore; } diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index b541455b4e5..d87b19aa118 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -9,38 +9,46 @@ namespace nix { -struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig +template class F> +struct LegacySSHStoreConfigT { - using CommonSSHStoreConfig::CommonSSHStoreConfig; + F remoteProgram; + F maxConnections; +}; + +struct LegacySSHStoreConfig : + std::enable_shared_from_this, + Store::Config, + CommonSSHStoreConfig, + LegacySSHStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); + + /** + * Hack for getting remote build log output. Intentionally not a + * documented user-visible setting. + */ + Descriptor logFD = INVALID_DESCRIPTOR; LegacySSHStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params); + const StoreReference::Params & params); - const Setting remoteProgram{this, {"nix-store"}, "remote-program", - "Path to the `nix-store` executable on the remote machine."}; - - const Setting maxConnections{this, 1, "max-connections", - "Maximum number of concurrent SSH connections."}; - - const std::string name() override { return "SSH Store"; } + static const std::string name() { return "SSH Store"; } static std::set uriSchemes() { return {"ssh"}; } - std::string doc() override; + static std::string doc(); + + ref openStore() const override; }; -struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store +struct LegacySSHStore : public virtual Store { -#ifndef _WIN32 - // Hack for getting remote build log output. - // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in - // the documentation - const Setting logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"}; -#else - Descriptor logFD = INVALID_DESCRIPTOR; -#endif + using Config = LegacySSHStoreConfig; + + ref config; struct Connection; @@ -48,10 +56,7 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor SSHMaster master; - LegacySSHStore( - std::string_view scheme, - std::string_view host, - const Params & params); + LegacySSHStore(ref); ref openConnection(); @@ -127,10 +132,7 @@ public: * The legacy ssh protocol doesn't support checking for trusted-user. * Try using ssh-ng:// instead if you want to know. */ - std::optional isTrustedClient() override - { - return std::nullopt; - } + std::optional isTrustedClient() override; void queryRealisationUncached(const DrvOutput &, Callback> callback) noexcept override diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index dcc6affe4a1..cfeb05fdbae 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -2,17 +2,27 @@ #include "globals.hh" #include "nar-info-disk-cache.hh" #include "signals.hh" +#include "store-registration.hh" #include namespace nix { +config::SettingDescriptionMap LocalBinaryCacheStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(BinaryCacheStoreConfig::descriptions()); + return ret; +} + + LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig( std::string_view scheme, PathView binaryCacheDir, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) + const StoreReference::Params & params) + : Store::Config{params} + , BinaryCacheStoreConfig{*this, params} , binaryCacheDir(binaryCacheDir) { } @@ -26,29 +36,26 @@ std::string LocalBinaryCacheStoreConfig::doc() } -struct LocalBinaryCacheStore : virtual LocalBinaryCacheStoreConfig, virtual BinaryCacheStore +struct LocalBinaryCacheStore : + virtual BinaryCacheStore { - /** - * @param binaryCacheDir `file://` is a short-hand for `file:///` - * for now. - */ - LocalBinaryCacheStore( - std::string_view scheme, - PathView binaryCacheDir, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , LocalBinaryCacheStoreConfig(scheme, binaryCacheDir, params) - , Store(params) - , BinaryCacheStore(params) + using Config = LocalBinaryCacheStoreConfig; + + ref config; + + LocalBinaryCacheStore(ref config) + : Store{*config} + , BinaryCacheStore{*config} + , config{config} { + init(); } void init() override; std::string getUri() override { - return "file://" + binaryCacheDir; + return "file://" + config->binaryCacheDir; } protected: @@ -59,7 +66,7 @@ struct LocalBinaryCacheStore : virtual LocalBinaryCacheStoreConfig, virtual Bina std::shared_ptr> istream, const std::string & mimeType) override { - auto path2 = binaryCacheDir + "/" + path; + auto path2 = config->binaryCacheDir + "/" + path; static std::atomic counter{0}; Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter); AutoDelete del(tmp, false); @@ -72,7 +79,7 @@ struct LocalBinaryCacheStore : virtual LocalBinaryCacheStoreConfig, virtual Bina void getFile(const std::string & path, Sink & sink) override { try { - readFile(binaryCacheDir + "/" + path, sink); + readFile(config->binaryCacheDir + "/" + path, sink); } catch (SysError & e) { if (e.errNo == ENOENT) throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); @@ -84,7 +91,7 @@ struct LocalBinaryCacheStore : virtual LocalBinaryCacheStoreConfig, virtual Bina { StorePathSet paths; - for (auto & entry : std::filesystem::directory_iterator{binaryCacheDir}) { + for (auto & entry : std::filesystem::directory_iterator{config->binaryCacheDir}) { checkInterrupt(); auto name = entry.path().filename().string(); if (name.size() != 40 || @@ -106,17 +113,17 @@ struct LocalBinaryCacheStore : virtual LocalBinaryCacheStoreConfig, virtual Bina void LocalBinaryCacheStore::init() { - createDirs(binaryCacheDir + "/nar"); - createDirs(binaryCacheDir + "/" + realisationsPrefix); - if (writeDebugInfo) - createDirs(binaryCacheDir + "/debuginfo"); - createDirs(binaryCacheDir + "/log"); + createDirs(config->binaryCacheDir + "/nar"); + createDirs(config->binaryCacheDir + "/" + realisationsPrefix); + if (config->writeDebugInfo) + createDirs(config->binaryCacheDir + "/debuginfo"); + createDirs(config->binaryCacheDir + "/log"); BinaryCacheStore::init(); } bool LocalBinaryCacheStore::fileExists(const std::string & path) { - return pathExists(binaryCacheDir + "/" + path); + return pathExists(config->binaryCacheDir + "/" + path); } std::set LocalBinaryCacheStoreConfig::uriSchemes() @@ -127,6 +134,10 @@ std::set LocalBinaryCacheStoreConfig::uriSchemes() return {"file"}; } -static RegisterStoreImplementation regLocalBinaryCacheStore; +ref LocalBinaryCacheStoreConfig::openStore() const { + return make_ref(ref{shared_from_this()}); +} + +static RegisterStoreImplementation regLocalBinaryCacheStore; } diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh index 997e8ecbb51..9c12b4221c6 100644 --- a/src/libstore/local-binary-cache-store.hh +++ b/src/libstore/local-binary-cache-store.hh @@ -2,22 +2,31 @@ namespace nix { -struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +struct LocalBinaryCacheStoreConfig : std::enable_shared_from_this, + Store::Config, + BinaryCacheStoreConfig { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + static config::SettingDescriptionMap descriptions(); - LocalBinaryCacheStoreConfig(std::string_view scheme, PathView binaryCacheDir, const Params & params); + /** + * @param binaryCacheDir `file://` is a short-hand for `file:///` + * for now. + */ + LocalBinaryCacheStoreConfig( + std::string_view scheme, PathView binaryCacheDir, const StoreReference::Params & params); Path binaryCacheDir; - const std::string name() override + static const std::string name() { return "Local Binary Cache Store"; } static std::set uriSchemes(); - std::string doc() override; + static std::string doc(); + + ref openStore() const override; }; } diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 5449b20eb3b..2e68304c5ea 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,3 +1,4 @@ +#include "json-utils.hh" #include "archive.hh" #include "posix-source-accessor.hh" #include "store-api.hh" @@ -5,25 +6,108 @@ #include "globals.hh" #include "compression.hh" #include "derivations.hh" +#include "config-parse-impl.hh" namespace nix { -LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params) - : StoreConfig(params) - // Default `?root` from `rootDir` if non set - // FIXME don't duplicate description once we don't have root setting - , rootDir{ - this, - !rootDir.empty() && params.count("root") == 0 - ? (std::optional{rootDir}) - : std::nullopt, - "root", - "Directory prefixed to all other paths."} +constexpr static const LocalFSStoreConfigT localFSStoreConfigDescriptions = { + .rootDir = { + .name = "root", + .description = "Directory prefixed to all other paths.", + }, + .stateDir = { + .name = "state", + .description = "Directory where Nix will store state.", + }, + .logDir = { + .name = "log", + .description = "directory where Nix will store log files.", + }, + .realStoreDir{ + .name = "real", + .description = "Physical path of the Nix store.", + }, +}; + +#define LOCAL_FS_STORE_CONFIG_FIELDS(X) \ + X(rootDir), \ + X(stateDir), \ + X(logDir), \ + X(realStoreDir), + +MAKE_PARSE(LocalFSStoreConfig, localFSStoreConfig, LOCAL_FS_STORE_CONFIG_FIELDS) + +/** + * @param rootDir Fallback if not in `params` + */ +static LocalFSStoreConfigT localFSStoreConfigDefaults( + const Path & storeDir, + const std::optional & rootDir) +{ + return { + .rootDir = {std::nullopt}, + .stateDir = {rootDir ? *rootDir + "/nix/var/nix" : settings.nixStateDir}, + .logDir = {rootDir ? *rootDir + "/nix/var/log/nix" : settings.nixLogDir}, + .realStoreDir = {rootDir ? *rootDir + "/nix/store" : storeDir}, + }; +} + +static LocalFSStoreConfigT localFSStoreConfigApplyParse( + const Path & storeDir, + LocalFSStoreConfigT parsed) +{ + auto defaults = localFSStoreConfigDefaults( + storeDir, + parsed.rootDir.optValue.value_or(std::nullopt)); + return {LOCAL_FS_STORE_CONFIG_FIELDS(APPLY_ROW)}; +} + +config::SettingDescriptionMap LocalFSStoreConfig::descriptions() +{ + constexpr auto & descriptions = localFSStoreConfigDescriptions; + auto defaults = localFSStoreConfigDefaults(settings.nixStore, std::nullopt); + return { + LOCAL_FS_STORE_CONFIG_FIELDS(DESC_ROW) + }; +} + +LocalFSStore::Config::LocalFSStoreConfig( + const Store::Config & storeConfig, + const StoreReference::Params & params) + : LocalFSStoreConfigT{ + localFSStoreConfigApplyParse( + storeConfig.storeDir, + localFSStoreConfigParse(params))} + , storeConfig{storeConfig} +{ +} + +static LocalFSStoreConfigT applyAuthority( + LocalFSStoreConfigT parsed, + PathView rootDir) +{ + if (!rootDir.empty()) + parsed.rootDir = {.optValue = {Path{rootDir}}}; + return parsed; +} + +LocalFSStore::Config::LocalFSStoreConfig( + const Store::Config & storeConfig, + PathView rootDir, + const StoreReference::Params & params) + : LocalFSStoreConfigT{ + localFSStoreConfigApplyParse( + storeConfig.storeDir, + applyAuthority( + localFSStoreConfigParse(params), + rootDir))} + , storeConfig{storeConfig} { } -LocalFSStore::LocalFSStore(const Params & params) - : Store(params) +LocalFSStore::LocalFSStore(const Config & config) + : Store{static_cast(*this)} + , config{config} { } @@ -97,8 +181,8 @@ std::optional LocalFSStore::getBuildLogExact(const StorePath & path Path logPath = j == 0 - ? fmt("%s/%s/%s/%s", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)) - : fmt("%s/%s/%s", logDir, drvsLogDir, baseName); + ? fmt("%s/%s/%s/%s", config.logDir.get(), drvsLogDir, baseName.substr(0, 2), baseName.substr(2)) + : fmt("%s/%s/%s", config.logDir.get(), drvsLogDir, baseName); Path logBz2Path = logPath + ".bz2"; if (pathExists(logPath)) diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 9bb569f0b25..8fbd98db99d 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -7,9 +7,24 @@ namespace nix { -struct LocalFSStoreConfig : virtual StoreConfig +template class F> +struct LocalFSStoreConfigT { - using StoreConfig::StoreConfig; + F> rootDir; + F stateDir; + F logDir; + F realStoreDir; +}; + +struct LocalFSStoreConfig : LocalFSStoreConfigT +{ + const Store::Config & storeConfig; + + static config::SettingDescriptionMap descriptions(); + + LocalFSStoreConfig( + const Store::Config & storeConfig, + const StoreReference::Params &); /** * Used to override the `root` settings. Can't be done via modifying @@ -18,38 +33,26 @@ struct LocalFSStoreConfig : virtual StoreConfig * * @todo Make this less error-prone with new store settings system. */ - LocalFSStoreConfig(PathView path, const Params & params); - - const OptionalPathSetting rootDir{this, std::nullopt, - "root", - "Directory prefixed to all other paths."}; - - const PathSetting stateDir{this, - rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir, - "state", - "Directory where Nix will store state."}; - - const PathSetting logDir{this, - rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir, - "log", - "directory where Nix will store log files."}; - - const PathSetting realStoreDir{this, - rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", - "Physical path of the Nix store."}; + LocalFSStoreConfig( + const Store::Config & storeConfig, + PathView path, + const StoreReference::Params & params); }; -class LocalFSStore : public virtual LocalFSStoreConfig, - public virtual Store, - public virtual GcStore, - public virtual LogStore +struct LocalFSStore : + virtual Store, + virtual GcStore, + virtual LogStore { -public: + using Config = LocalFSStoreConfig; + + const Config & config; + inline static std::string operationName = "Local Filesystem Store"; const static std::string drvsLogDir; - LocalFSStore(const Params & params); + LocalFSStore(const Config & params); void narFromPath(const StorePath & path, Sink & sink) override; ref getFSAccessor(bool requireValidPath = true) override; @@ -70,7 +73,7 @@ public: */ virtual Path addPermRoot(const StorePath & storePath, const Path & gcRoot) = 0; - virtual Path getRealStoreDir() { return realStoreDir; } + virtual Path getRealStoreDir() { return config.realStoreDir; } Path toRealPath(const Path & storePath) override { diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 56ff6bef3e5..30374b4a8ee 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -1,12 +1,110 @@ +#include + #include "local-overlay-store.hh" #include "callback.hh" #include "realisation.hh" #include "processes.hh" #include "url.hh" -#include +#include "store-api.hh" +#include "store-registration.hh" +#include "config-parse-impl.hh" namespace nix { +static LocalOverlayStoreConfigT localOverlayStoreConfigDescriptions = { + .lowerStoreConfig{ + .name = "lower-store", + .description = R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). + + Must be a store with a store dir on the file system. + Must be used as OverlayFS lower layer for this store's store dir. + )", + // It's not actually machine-specific, but we don't yet have a + // `to_json` for `StoreConfig`. + .documentDefault = false, + }, + .upperLayer{ + .name = "upper-layer", + .description = R"( + Directory containing the OverlayFS upper layer for this store's store dir. + )", + }, + .checkMount{ + .name = "check-mount", + .description = R"( + Check that the overlay filesystem is correctly mounted. + + Nix does not manage the overlayfs mount point itself, but the correct + functioning of the overlay store does depend on this mount point being set up + correctly. Rather than just assume this is the case, check that the lowerdir + and upperdir options are what we expect them to be. This check is on by + default, but can be disabled if needed. + )", + }, + .remountHook{ + .name = "remount-hook", + .description = R"( + Script or other executable to run when overlay filesystem needs remounting. + + This is occasionally necessary when deleting a store path that exists in both upper and lower layers. + In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly + is the only way to perform the deletion without creating a "whiteout". + However this causes the OverlayFS kernel data structures to get out-of-sync, + and can lead to 'stale file handle' errors; remounting solves the problem. + + The store directory is passed as an argument to the invoked executable. + )", + }, +}; + +#define LOCAL_OVERLAY_STORE_CONFIG_FIELDS(X) \ + X(lowerStoreConfig), \ + X(upperLayer), \ + X(checkMount), \ + X(remountHook), + +MAKE_PARSE(LocalOverlayStoreConfig, localOverlayStoreConfig, LOCAL_OVERLAY_STORE_CONFIG_FIELDS) + +static LocalOverlayStoreConfigT localOverlayStoreConfigDefaults() +{ + return { + .lowerStoreConfig = {make_ref(StoreReference::Params{})}, + .upperLayer = {""}, + .checkMount = {true}, + .remountHook = {""}, + }; +} + +MAKE_APPLY_PARSE(LocalOverlayStoreConfig, localOverlayStoreConfig, LOCAL_OVERLAY_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap LocalOverlayStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(LocalFSStoreConfig::descriptions()); + ret.merge(LocalStoreConfig::descriptions()); + { + constexpr auto & descriptions = localOverlayStoreConfigDescriptions; + auto defaults = localOverlayStoreConfigDefaults(); + ret.merge(decltype(ret){ + LOCAL_OVERLAY_STORE_CONFIG_FIELDS(DESC_ROW) + }); + } + return ret; +} + +LocalOverlayStore::Config::LocalOverlayStoreConfig( + std::string_view scheme, + std::string_view authority, + const StoreReference::Params & params) + : LocalStore::Config(scheme, authority, params) + , LocalOverlayStoreConfigT{localOverlayStoreConfigApplyParse(params)} +{ +} + + std::string LocalOverlayStoreConfig::doc() { return @@ -14,25 +112,32 @@ std::string LocalOverlayStoreConfig::doc() ; } -Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { +ref LocalOverlayStoreConfig::openStore() const +{ + return make_ref(ref{ + std::dynamic_pointer_cast(shared_from_this()) + }); +} + + +Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) const +{ return upperLayer + "/" + path.to_string(); } -LocalOverlayStore::LocalOverlayStore(std::string_view scheme, PathView path, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(path, params) - , LocalStoreConfig(params) - , LocalOverlayStoreConfig(scheme, path, params) - , Store(params) - , LocalFSStore(params) - , LocalStore(params) - , lowerStore(openStore(percentDecode(lowerStoreUri.get())).dynamic_pointer_cast()) + +LocalOverlayStore::LocalOverlayStore(ref config) + : Store{*config} + , LocalFSStore{*config} + , LocalStore{static_cast>(config)} + , config{config} + , lowerStore(config->lowerStoreConfig.value->openStore().dynamic_pointer_cast()) { - if (checkMount.get()) { + if (config->checkMount.get()) { std::smatch match; std::string mountInfo; auto mounts = readFile(std::filesystem::path{"/proc/self/mounts"}); - auto regex = std::regex(R"((^|\n)overlay )" + realStoreDir.get() + R"( .*(\n|$))"); + auto regex = std::regex(R"((^|\n)overlay )" + config->realStoreDir.get() + R"( .*(\n|$))"); // Mount points can be stacked, so there might be multiple matching entries. // Loop until the last match, which will be the current state of the mount point. @@ -45,13 +150,13 @@ LocalOverlayStore::LocalOverlayStore(std::string_view scheme, PathView path, con return std::regex_search(mountInfo, std::regex("\\b" + option + "=" + value + "( |,)")); }; - auto expectedLowerDir = lowerStore->realStoreDir.get(); - if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", upperLayer)) { + auto expectedLowerDir = lowerStore->config.realStoreDir.get(); + if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", config->upperLayer)) { debug("expected lowerdir: %s", expectedLowerDir); - debug("expected upperdir: %s", upperLayer); + debug("expected upperdir: %s", config->upperLayer); debug("actual mount: %s", mountInfo); throw Error("overlay filesystem '%s' mounted incorrectly", - realStoreDir.get()); + config->realStoreDir.get()); } } } @@ -201,14 +306,14 @@ void LocalOverlayStore::collectGarbage(const GCOptions & options, GCResults & re void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) { - auto mergedDir = realStoreDir.get() + "/"; + auto mergedDir = config->realStoreDir.get() + "/"; if (path.substr(0, mergedDir.length()) != mergedDir) { warn("local-overlay: unexpected gc path '%s' ", path); return; } StorePath storePath = {path.substr(mergedDir.length())}; - auto upperPath = toUpperPath(storePath); + auto upperPath = config->toUpperPath(storePath); if (pathExists(upperPath)) { debug("upper exists: %s", path); @@ -257,7 +362,7 @@ LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag StorePathSet done; auto existsInStoreDir = [&](const StorePath & storePath) { - return pathExists(realStoreDir.get() + "/" + storePath.to_string()); + return pathExists(config->realStoreDir.get() + "/" + storePath.to_string()); }; bool errors = false; @@ -277,16 +382,16 @@ void LocalOverlayStore::remountIfNecessary() { if (!_remountRequired) return; - if (remountHook.get().empty()) { - warn("'%s' needs remounting, set remount-hook to do this automatically", realStoreDir.get()); + if (config->remountHook.get().empty()) { + warn("'%s' needs remounting, set remount-hook to do this automatically", config->realStoreDir.get()); } else { - runProgram(remountHook, false, {realStoreDir}); + runProgram(config->remountHook, false, {config->realStoreDir}); } _remountRequired = false; } -static RegisterStoreImplementation regLocalOverlayStore; +static RegisterStoreImplementation regLocalOverlayStore; } diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh index 63628abed50..eed3c69daac 100644 --- a/src/libstore/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -2,63 +2,33 @@ namespace nix { + +template class F> +struct LocalOverlayStoreConfigT +{ + const F> lowerStoreConfig; + const F upperLayer; + const F checkMount; + const F remountHook; +}; + /** * Configuration for `LocalOverlayStore`. */ -struct LocalOverlayStoreConfig : virtual LocalStoreConfig +struct LocalOverlayStoreConfig : + LocalStoreConfig, + LocalOverlayStoreConfigT { - LocalOverlayStoreConfig(const StringMap & params) - : LocalOverlayStoreConfig("local-overlay", "", params) - { } - - LocalOverlayStoreConfig(std::string_view scheme, PathView path, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(path, params) - , LocalStoreConfig(scheme, path, params) - { - } - - const Setting lowerStoreUri{(StoreConfig*) this, "", "lower-store", - R"( - [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) - for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). - - Must be a store with a store dir on the file system. - Must be used as OverlayFS lower layer for this store's store dir. - )"}; - - const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer", - R"( - Directory containing the OverlayFS upper layer for this store's store dir. - )"}; + static config::SettingDescriptionMap descriptions(); - Setting checkMount{(StoreConfig*) this, true, "check-mount", - R"( - Check that the overlay filesystem is correctly mounted. + LocalOverlayStoreConfig( + std::string_view scheme, + PathView path, + const StoreReference::Params & params); - Nix does not manage the overlayfs mount point itself, but the correct - functioning of the overlay store does depend on this mount point being set up - correctly. Rather than just assume this is the case, check that the lowerdir - and upperdir options are what we expect them to be. This check is on by - default, but can be disabled if needed. - )"}; + static const std::string name() { return "Experimental Local Overlay Store"; } - const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook", - R"( - Script or other executable to run when overlay filesystem needs remounting. - - This is occasionally necessary when deleting a store path that exists in both upper and lower layers. - In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly - is the only way to perform the deletion without creating a "whiteout". - However this causes the OverlayFS kernel data structures to get out-of-sync, - and can lead to 'stale file handle' errors; remounting solves the problem. - - The store directory is passed as an argument to the invoked executable. - )"}; - - const std::string name() override { return "Experimental Local Overlay Store"; } - - std::optional experimentalFeature() const override + static std::optional experimentalFeature() { return ExperimentalFeature::LocalOverlayStore; } @@ -68,7 +38,9 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig return { "local-overlay" }; } - std::string doc() override; + static std::string doc(); + + ref openStore() const override; protected: /** @@ -79,7 +51,9 @@ protected: * at that file path. It might be stored in the lower layer instead, * or it might not be part of this store at all. */ - Path toUpperPath(const StorePath & path); + Path toUpperPath(const StorePath & path) const; + + friend struct LocalOverlayStore; }; /** @@ -88,24 +62,13 @@ protected: * Documentation on overridden methods states how they differ from their * `LocalStore` counterparts. */ -class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual LocalStore +struct LocalOverlayStore : virtual LocalStore { - /** - * The store beneath us. - * - * Our store dir should be an overlay fs where the lower layer - * is that store's store dir, and the upper layer is some - * scratch storage just for us. - */ - ref lowerStore; + using Config = LocalOverlayStoreConfig; -public: - LocalOverlayStore(const Params & params) - : LocalOverlayStore("local-overlay", "", params) - { - } + ref config; - LocalOverlayStore(std::string_view scheme, PathView path, const Params & params); + LocalOverlayStore(ref); std::string getUri() override { @@ -113,6 +76,15 @@ public: } private: + /** + * The store beneath us. + * + * Our store dir should be an overlay fs where the lower layer + * is that store's store dir, and the upper layer is some + * scratch storage just for us. + */ + ref lowerStore; + /** * First copy up any lower store realisation with the same key, so we * merge rather than mask it. diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9fa68303f2c..af87f6887fe 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -17,6 +17,9 @@ #include "posix-source-accessor.hh" #include "keys.hh" #include "users.hh" +#include "config-parse-impl.hh" +#include "store-open.hh" +#include "store-registration.hh" #include #include @@ -57,15 +60,72 @@ namespace nix { -LocalStoreConfig::LocalStoreConfig( +constexpr static const LocalStoreConfigT localStoreConfigDescriptions = { + .requireSigs = { + .name = "require-sigs", + .description = "Whether store paths copied into this store should have a trusted signature.", + }, + .readOnly = { + .name = "read-only", + .description = R"( + Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. + + Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. + + Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. + + > **Warning** + > Do not use this unless the filesystem is read-only. + > + > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. + > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. + )", + }, +}; + +#define LOCAL_STORE_CONFIG_FIELDS(X) \ + X(requireSigs), \ + X(readOnly), + +MAKE_PARSE(LocalStoreConfig, localStoreConfig, LOCAL_STORE_CONFIG_FIELDS) + +static LocalStoreConfigT localStoreConfigDefaults() +{ + return { + .requireSigs = {settings.requireSigs}, + .readOnly = {false}, + }; +} + +MAKE_APPLY_PARSE(LocalStoreConfig, localStoreConfig, LOCAL_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap LocalStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(LocalFSStoreConfig::descriptions()); + { + constexpr auto & descriptions = localStoreConfigDescriptions; + auto defaults = localStoreConfigDefaults(); + ret.merge(decltype(ret){ + LOCAL_STORE_CONFIG_FIELDS(DESC_ROW) + }); + } + return ret; +} + +LocalStore::Config::LocalStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(authority, params) + const StoreReference::Params & params) + : Store::Config(params) + , LocalFSStore::Config(*this, authority, params) + , LocalStoreConfigT{localStoreConfigApplyParse(params)} { } +LocalStoreConfig::LocalStoreConfig(const LocalStoreConfig &) = default; + std::string LocalStoreConfig::doc() { return @@ -73,6 +133,11 @@ std::string LocalStoreConfig::doc() ; } +ref LocalStore::Config::openStore() const +{ + return make_ref(ref{shared_from_this()}); +} + struct LocalStore::State::Stmts { /* Some precompiled SQLite statements. */ SQLiteStmt RegisterValidPath; @@ -95,20 +160,15 @@ struct LocalStore::State::Stmts { SQLiteStmt AddRealisationReference; }; -LocalStore::LocalStore( - std::string_view scheme, - PathView path, - const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(path, params) - , LocalStoreConfig(scheme, path, params) - , Store(params) - , LocalFSStore(params) - , dbDir(stateDir + "/db") - , linksDir(realStoreDir + "/.links") +LocalStore::LocalStore(ref config) + : Store{*config} + , LocalFSStore{*config} + , config{config} + , dbDir(config->stateDir + "/db") + , linksDir(config->realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") , schemaPath(dbDir + "/schema") - , tempRootsDir(stateDir + "/temproots") + , tempRootsDir(config->stateDir + "/temproots") , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) , locksHeld(tokenizeString(getEnv("NIX_HELD_LOCKS").value_or(""))) { @@ -116,18 +176,18 @@ LocalStore::LocalStore( state->stmts = std::make_unique(); /* Create missing state directories if they don't already exist. */ - createDirs(realStoreDir); - if (readOnly) { + createDirs(config->realStoreDir); + if (config->readOnly) { experimentalFeatureSettings.require(Xp::ReadOnlyLocalStore); } else { makeStoreWritable(); } createDirs(linksDir); - Path profilesDir = stateDir + "/profiles"; + Path profilesDir = config->stateDir + "/profiles"; createDirs(profilesDir); createDirs(tempRootsDir); createDirs(dbDir); - Path gcRootsDir = stateDir + "/gcroots"; + Path gcRootsDir = config->stateDir + "/gcroots"; if (!pathExists(gcRootsDir)) { createDirs(gcRootsDir); createSymlink(profilesDir, gcRootsDir + "/profiles"); @@ -135,7 +195,7 @@ LocalStore::LocalStore( for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { createDirs(perUserDir); - if (!readOnly) { + if (!config->readOnly) { if (chmod(perUserDir.c_str(), 0755) == -1) throw SysError("could not set permissions on '%s' to 755", perUserDir); } @@ -150,16 +210,16 @@ LocalStore::LocalStore( struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); - else if (!readOnly) { + else if (!config->readOnly) { struct stat st; - if (stat(realStoreDir.get().c_str(), &st)) - throw SysError("getting attributes of path '%1%'", realStoreDir); + if (stat(config->realStoreDir.get().c_str(), &st)) + throw SysError("getting attributes of path '%1%'", config->realStoreDir); if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { - if (chown(realStoreDir.get().c_str(), 0, gr->gr_gid) == -1) - throw SysError("changing ownership of path '%1%'", realStoreDir); - if (chmod(realStoreDir.get().c_str(), perm) == -1) - throw SysError("changing permissions on path '%1%'", realStoreDir); + if (chown(config->realStoreDir.get().c_str(), 0, gr->gr_gid) == -1) + throw SysError("changing ownership of path '%1%'", config->realStoreDir); + if (chmod(config->realStoreDir.get().c_str(), perm) == -1) + throw SysError("changing permissions on path '%1%'", config->realStoreDir); } } } @@ -167,7 +227,7 @@ LocalStore::LocalStore( /* Ensure that the store and its parents are not symlinks. */ if (!settings.allowSymlinkedStore) { - std::filesystem::path path = realStoreDir.get(); + std::filesystem::path path = config->realStoreDir.get(); std::filesystem::path root = path.root_path(); while (path != root) { if (std::filesystem::is_symlink(path)) @@ -214,12 +274,12 @@ LocalStore::LocalStore( /* Acquire the big fat lock in shared mode to make sure that no schema upgrade is in progress. */ - if (!readOnly) { + if (!config->readOnly) { Path globalLockPath = dbDir + "/big-lock"; globalLock = openLockFile(globalLockPath.c_str(), true); } - if (!readOnly && !lockFile(globalLock.get(), ltRead, false)) { + if (!config->readOnly && !lockFile(globalLock.get(), ltRead, false)) { printInfo("waiting for the big Nix store lock..."); lockFile(globalLock.get(), ltRead, true); } @@ -227,7 +287,7 @@ LocalStore::LocalStore( /* Check the current database schema and if necessary do an upgrade. */ int curSchema = getSchema(); - if (readOnly && curSchema < nixSchemaVersion) { + if (config->readOnly && curSchema < nixSchemaVersion) { debug("current schema version: %d", curSchema); debug("supported schema version: %d", nixSchemaVersion); throw Error(curSchema == 0 ? @@ -375,15 +435,9 @@ LocalStore::LocalStore( } -LocalStore::LocalStore(const Params & params) - : LocalStore("local", "", params) -{ -} - - AutoCloseFD LocalStore::openGCLock() { - Path fnGCLock = stateDir + "/gc.lock"; + Path fnGCLock = config->stateDir + "/gc.lock"; auto fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT #ifndef _WIN32 | O_CLOEXEC @@ -449,17 +503,17 @@ int LocalStore::getSchema() void LocalStore::openDB(State & state, bool create) { - if (create && readOnly) { + if (create && config->readOnly) { throw Error("cannot create database while in read-only mode"); } - if (access(dbDir.c_str(), R_OK | (readOnly ? 0 : W_OK))) + if (access(dbDir.c_str(), R_OK | (config->readOnly ? 0 : W_OK))) throw SysError("Nix database directory '%1%' is not writable", dbDir); /* Open the Nix database. */ std::string dbPath = dbDir + "/db.sqlite"; auto & db(state.db); - auto openMode = readOnly ? SQLiteOpenMode::Immutable + auto openMode = config->readOnly ? SQLiteOpenMode::Immutable : create ? SQLiteOpenMode::Normal : SQLiteOpenMode::NoCreate; state.db = SQLite(dbPath, openMode); @@ -572,12 +626,12 @@ void LocalStore::makeStoreWritable() if (!isRootUser()) return; /* Check if /nix/store is on a read-only mount. */ struct statvfs stat; - if (statvfs(realStoreDir.get().c_str(), &stat) != 0) + if (statvfs(config->realStoreDir.get().c_str(), &stat) != 0) throw SysError("getting info about the Nix store mount point"); if (stat.f_flag & ST_RDONLY) { - if (mount(0, realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) - throw SysError("remounting %1% writable", realStoreDir); + if (mount(0, config->realStoreDir.get().c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) + throw SysError("remounting %1% writable", config->realStoreDir); } #endif } @@ -917,7 +971,7 @@ StorePathSet LocalStore::querySubstitutablePaths(const StorePathSet & paths) for (auto & sub : getDefaultSubstituters()) { if (remaining.empty()) break; if (sub->storeDir != storeDir) continue; - if (!sub->wantMassQuery) continue; + if (!sub->resolvedSubstConfig.wantMassQuery) continue; auto valid = sub->queryValidPaths(remaining); @@ -1029,12 +1083,12 @@ const PublicKeys & LocalStore::getPublicKeys() bool LocalStore::pathInfoIsUntrusted(const ValidPathInfo & info) { - return requireSigs && !info.checkSignatures(*this, getPublicKeys()); + return config->requireSigs && !info.checkSignatures(*this, getPublicKeys()); } bool LocalStore::realisationIsUntrusted(const Realisation & realisation) { - return requireSigs && !realisation.checkSignatures(getPublicKeys()); + return config->requireSigs && !realisation.checkSignatures(getPublicKeys()); } void LocalStore::addToStore(const ValidPathInfo & info, Source & source, @@ -1331,7 +1385,7 @@ std::pair LocalStore::createTempDirInStore() /* There is a slight possibility that `tmpDir' gets deleted by the GC between createTempDir() and when we acquire a lock on it. We'll repeat until 'tmpDir' exists and we've locked it. */ - tmpDirFn = createTempDir(realStoreDir, "tmp"); + tmpDirFn = createTempDir(config->realStoreDir, "tmp"); tmpDirFd = openDirectory(tmpDirFn); if (!tmpDirFd) { continue; @@ -1472,7 +1526,7 @@ LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair database and the filesystem) in the loop below, in order to catch invalid states. */ - for (auto & i : std::filesystem::directory_iterator{realStoreDir.to_string()}) { + for (auto & i : std::filesystem::directory_iterator{config->realStoreDir.get()}) { checkInterrupt(); try { storePathsInStoreDir.insert({i.path().filename().string()}); @@ -1688,7 +1742,7 @@ void LocalStore::addBuildLog(const StorePath & drvPath, std::string_view log) auto baseName = drvPath.to_string(); - auto logPath = fmt("%s/%s/%s/%s.bz2", logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)); + auto logPath = fmt("%s/%s/%s/%s.bz2", config->logDir, drvsLogDir, baseName.substr(0, 2), baseName.substr(2)); if (pathExists(logPath)) return; @@ -1706,6 +1760,6 @@ std::optional LocalStore::getVersion() return nixVersion; } -static RegisterStoreImplementation regLocalStore; +static RegisterStoreImplementation regLocalStore; } // namespace nix diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 83154d65193..57dabff6658 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -34,49 +34,55 @@ struct OptimiseStats uint64_t bytesFreed = 0; }; -struct LocalStoreConfig : virtual LocalFSStoreConfig +template class F> +struct LocalStoreConfigT { - using LocalFSStoreConfig::LocalFSStoreConfig; + F requireSigs; + F readOnly; +}; + +struct LocalStoreConfig : + std::enable_shared_from_this, + Store::Config, + LocalFSStore::Config, + LocalStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); + + LocalStoreConfig(const StoreReference::Params & params) + : LocalStoreConfig{"local", "", params} + {} LocalStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params); - - Setting requireSigs{this, - settings.requireSigs, - "require-sigs", - "Whether store paths copied into this store should have a trusted signature."}; - - Setting readOnly{this, - false, - "read-only", - R"( - Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem. - - Normally Nix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem. - - Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set. + const StoreReference::Params & params); - > **Warning** - > Do not use this unless the filesystem is read-only. - > - > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process. - > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it. - )"}; + /** + * For `RestrictedStore` + */ + LocalStoreConfig(const LocalStoreConfig &); - const std::string name() override { return "Local Store"; } + static const std::string name() { return "Local Store"; } static std::set uriSchemes() { return {"local"}; } - std::string doc() override; + static std::string doc(); + + ref openStore() const override; }; -class LocalStore : public virtual LocalStoreConfig - , public virtual IndirectRootStore - , public virtual GcStore +class LocalStore : + public virtual IndirectRootStore, + public virtual GcStore { +public: + + using Config = LocalStoreConfig; + + ref config; + private: /** @@ -144,11 +150,7 @@ public: * Initialise the local store, upgrading the schema if * necessary. */ - LocalStore(const Params & params); - LocalStore( - std::string_view scheme, - PathView path, - const Params & params); + LocalStore(ref params); ~LocalStore(); diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index eb729b697f1..036e8d87b8e 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -1,6 +1,6 @@ #include "machines.hh" #include "globals.hh" -#include "store-api.hh" +#include "store-open.hh" #include @@ -71,8 +71,8 @@ StoreReference Machine::completeStoreReference() const auto * generic = std::get_if(&storeUri.variant); if (generic && generic->scheme == "ssh") { - storeUri.params["max-connections"] = "1"; - storeUri.params["log-fd"] = "4"; + storeUri.params["max-connections"] = 1; + storeUri.params["log-fd"] = 4; } if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) { @@ -84,14 +84,10 @@ StoreReference Machine::completeStoreReference() const { auto & fs = storeUri.params["system-features"]; - auto append = [&](auto feats) { - for (auto & f : feats) { - if (fs.size() > 0) fs += ' '; - fs += f; - } - }; - append(supportedFeatures); - append(mandatoryFeatures); + if (!fs.is_array()) fs = nlohmann::json::array(); + auto features = supportedFeatures; + features.insert(supportedFeatures.begin(), supportedFeatures.end()); + for (auto & feat : features) fs += feat; } return storeUri; diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index b70ab907806..16d6e0e43de 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + #include "ref.hh" #include "store-reference.hh" diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 79d91249722..56bb3e58816 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -193,6 +193,7 @@ sources = files( 'builtins/unpack-channel.cc', 'common-protocol.cc', 'common-ssh-store-config.cc', + 'config-parse.cc', 'content-address.cc', 'daemon.cc', 'derivations.cc', @@ -240,6 +241,8 @@ sources = files( 'ssh-store.cc', 'ssh.cc', 'store-api.cc', + 'store-dir-config.cc', + 'store-registration.cc', 'store-reference.cc', 'uds-remote-store.cc', 'worker-protocol-connection.cc', @@ -312,6 +315,8 @@ headers = [config_h] + files( 'sqlite.hh', 'ssh.hh', 'store-api.hh', + 'store-open.hh', + 'store-registration.hh', 'store-cast.hh', 'store-dir-config.hh', 'store-reference.hh', diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index bcc02206bc9..31762f34d2c 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -3,7 +3,7 @@ #include "derivations.hh" #include "parsed-derivations.hh" #include "globals.hh" -#include "store-api.hh" +#include "store-open.hh" #include "thread-pool.hh" #include "realisation.hh" #include "topo-sort.hh" diff --git a/src/libstore/mounted-ssh-store.md b/src/libstore/mounted-ssh-store.md deleted file mode 100644 index 1ebfe3081dc..00000000000 --- a/src/libstore/mounted-ssh-store.md +++ /dev/null @@ -1,18 +0,0 @@ -R"( - -**Store URL format**: `mounted-ssh-ng://[username@]hostname` - -Experimental store type that allows full access to a Nix store on a remote machine, -and additionally requires that store be mounted in the local file system. - -The mounting of that store is not managed by Nix, and must by managed manually. -It could be accomplished with SSHFS or NFS, for example. - -The local file system is used to optimize certain operations. -For example, rather than serializing Nix archives and sending over the Nix channel, -we can directly access the file system data via the mount-point. - -The local file system is also used to make certain operations possible that wouldn't otherwise be. -For example, persistent GC roots can be created if they reside on the same file system as the remote store: -the remote side will create the symlinks necessary to avoid race conditions. -)" diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index aeff24c642a..675c31d4985 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -100,7 +100,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, /* HFS/macOS has some undocumented security feature disabling hardlinking for special files within .app dirs. Known affected paths include *.app/Contents/{PkgInfo,Resources/\*.lproj,_CodeSignature} and .DS_Store. - See https://github.com/NixOS/nix/issues/1443 and + See https://github.com/NixOS/nix/issues/1443 and https://github.com/NixOS/nix/pull/2230 for more discussion. */ if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) @@ -215,14 +215,14 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, the store itself (we don't want or need to mess with its permissions). */ const Path dirOfPath(dirOf(path)); - bool mustToggle = dirOfPath != realStoreDir.get(); + bool mustToggle = dirOfPath != config->realStoreDir.get(); if (mustToggle) makeWritable(dirOfPath); /* When we're done, make the directory read-only again and reset its timestamp back to 0. */ MakeReadOnly makeReadOnly(mustToggle ? dirOfPath : ""); - std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", realStoreDir, getpid(), rand()); + std::filesystem::path tempLink = fmt("%1%/.tmp-link-%2%-%3%", config->realStoreDir, getpid(), rand()); try { std::filesystem::create_hard_link(linkPath, tempLink); @@ -284,7 +284,7 @@ void LocalStore::optimiseStore(OptimiseStats & stats) if (!isValidPath(i)) continue; /* path was GC'ed, probably */ { Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", printStorePath(i))); - optimisePath_(&act, stats, realStoreDir + "/" + std::string(i.to_string()), inodeHash, NoRepair); + optimisePath_(&act, stats, config->realStoreDir + "/" + std::string(i.to_string()), inodeHash, NoRepair); } done++; act.progress(done, paths.size()); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index d8459d4d71c..c95d348d05c 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -110,7 +110,7 @@ bool ParsedDerivation::canBuildLocally(Store & localStore) const return false; for (auto & feature : getRequiredSystemFeatures()) - if (!localStore.systemFeatures.get().count(feature)) return false; + if (!localStore.config.systemFeatures.get().count(feature)) return false; return true; } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 3e9d054778c..b0ec8db52c0 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -75,7 +75,7 @@ StorePath StorePath::random(std::string_view name) return StorePath(Hash::random(HashAlgorithm::SHA1), name); } -StorePath StoreDirConfig::parseStorePath(std::string_view path) const +StorePath MixStoreDirMethods::parseStorePath(std::string_view path) const { // On Windows, `/nix/store` is not a canonical path. More broadly it // is unclear whether this function should be using the native @@ -94,7 +94,7 @@ StorePath StoreDirConfig::parseStorePath(std::string_view path) const return StorePath(baseNameOf(p)); } -std::optional StoreDirConfig::maybeParseStorePath(std::string_view path) const +std::optional MixStoreDirMethods::maybeParseStorePath(std::string_view path) const { try { return parseStorePath(path); @@ -103,24 +103,24 @@ std::optional StoreDirConfig::maybeParseStorePath(std::string_view pa } } -bool StoreDirConfig::isStorePath(std::string_view path) const +bool MixStoreDirMethods::isStorePath(std::string_view path) const { return (bool) maybeParseStorePath(path); } -StorePathSet StoreDirConfig::parseStorePathSet(const PathSet & paths) const +StorePathSet MixStoreDirMethods::parseStorePathSet(const PathSet & paths) const { StorePathSet res; for (auto & i : paths) res.insert(parseStorePath(i)); return res; } -std::string StoreDirConfig::printStorePath(const StorePath & path) const +std::string MixStoreDirMethods::printStorePath(const StorePath & path) const { return (storeDir + "/").append(path.to_string()); } -PathSet StoreDirConfig::printStorePathSet(const StorePathSet & paths) const +PathSet MixStoreDirMethods::printStorePathSet(const StorePathSet & paths) const { PathSet res; for (auto & i : paths) res.insert(printStorePath(i)); diff --git a/src/libstore/profiles.hh b/src/libstore/profiles.hh index 33fcf04b3a8..c4d9698ccd6 100644 --- a/src/libstore/profiles.hh +++ b/src/libstore/profiles.hh @@ -86,7 +86,7 @@ typedef std::list Generations; */ std::pair> findGenerations(Path profile); -class LocalFSStore; +struct LocalFSStore; /** * Create a new generation of the given profile diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index d09762a53c4..6946074affa 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -19,7 +19,7 @@ class RemoteFSAccessor : public SourceAccessor std::pair, CanonPath> fetch(const CanonPath & path); - friend class BinaryCacheStore; + friend struct BinaryCacheStore; Path makeCacheFile(std::string_view hashPart, const std::string & ext); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b230079eb27..c92dbe4b527 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -18,17 +18,67 @@ #include "callback.hh" #include "filetransfer.hh" #include "signals.hh" +#include "config-parse-impl.hh" #include namespace nix { +constexpr static const RemoteStoreConfigT remoteStoreConfigDescriptions = { + .maxConnections{ + .name = "max-connections", + .description = "Maximum number of concurrent connections to the Nix daemon.", + }, + .maxConnectionAge{ + .name = "max-connection-age", + .description = "Maximum age of a connection before it is closed.", + }, +}; + + +#define REMOTE_STORE_CONFIG_FIELDS(X) \ + X(maxConnections), \ + X(maxConnectionAge), + + +MAKE_PARSE(RemoteStoreConfig, remoteStoreConfig, REMOTE_STORE_CONFIG_FIELDS) + + +static RemoteStoreConfigT remoteStoreConfigDefaults() +{ + return { + .maxConnections = {1}, + .maxConnectionAge = {std::numeric_limits::max()}, + }; +} + + +MAKE_APPLY_PARSE(RemoteStoreConfig, remoteStoreConfig, REMOTE_STORE_CONFIG_FIELDS) + + +config::SettingDescriptionMap RemoteStoreConfig::descriptions() +{ + constexpr auto & descriptions = remoteStoreConfigDescriptions; + auto defaults = remoteStoreConfigDefaults(); + return { + REMOTE_STORE_CONFIG_FIELDS(DESC_ROW) + }; +} + + +RemoteStore::Config::RemoteStoreConfig(const Store::Config & storeConfig, const StoreReference::Params & params) + : RemoteStoreConfigT{remoteStoreConfigApplyParse(params)} + , storeConfig{storeConfig} +{ +} + + /* TODO: Separate these store types into different files, give them better names */ -RemoteStore::RemoteStore(const Params & params) - : RemoteStoreConfig(params) - , Store(params) +RemoteStore::RemoteStore(const Config & config) + : Store{config.storeConfig} + , config{config} , connections(make_ref>( - std::max(1, (int) maxConnections), + std::max(1, (int) config.maxConnections), [this]() { auto conn = openConnectionWrapper(); try { @@ -39,12 +89,12 @@ RemoteStore::RemoteStore(const Params & params) } return conn; }, - [this](const ref & r) { + [config](const ref & r) { return r->to.good() && r->from.good() && std::chrono::duration_cast( - std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge; + std::chrono::steady_clock::now() - r->startTime).count() < config.maxConnectionAge; } )) { @@ -122,7 +172,7 @@ void RemoteStore::setOptions(Connection & conn) << settings.useSubstitutes; if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) { - std::map overrides; + std::map overrides; settings.getSettings(overrides, true); // libstore settings fileTransferSettings.getSettings(overrides, true); overrides.erase(settings.keepFailed.name); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index ea6cd471eb5..a78cc7a7f96 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -18,31 +18,36 @@ struct FdSink; struct FdSource; template class Pool; -struct RemoteStoreConfig : virtual StoreConfig +template class F> +struct RemoteStoreConfigT { - using StoreConfig::StoreConfig; + F maxConnections; + F maxConnectionAge; +}; + +struct RemoteStoreConfig : RemoteStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); - const Setting maxConnections{this, 1, "max-connections", - "Maximum number of concurrent connections to the Nix daemon."}; + const Store::Config & storeConfig; - const Setting maxConnectionAge{this, - std::numeric_limits::max(), - "max-connection-age", - "Maximum age of a connection before it is closed."}; + RemoteStoreConfig(const Store::Config &, const StoreReference::Params &); }; /** * \todo RemoteStore is a misnomer - should be something like * DaemonStore. */ -class RemoteStore : public virtual RemoteStoreConfig, +struct RemoteStore : public virtual Store, public virtual GcStore, public virtual LogStore { -public: + using Config = RemoteStoreConfig; + + const Config & config; - RemoteStore(const Params & params); + RemoteStore(const Config & config); /* Implementations of abstract store API methods. */ diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index cfa713b00c2..06aeec81ae7 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -1,7 +1,5 @@ #if ENABLE_S3 -#include - #include "s3.hh" #include "s3-binary-cache-store.hh" #include "nar-info.hh" @@ -10,6 +8,8 @@ #include "compression.hh" #include "filetransfer.hh" #include "signals.hh" +#include "config-parse-impl.hh" +#include "store-registration.hh" #include #include @@ -194,29 +194,138 @@ S3Helper::FileTransferResult S3Helper::getObject( return res; } -S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) - : BinaryCacheStoreConfig(params) - , BinaryCacheStore(params) -{ } +constexpr static const S3BinaryCacheStoreConfigT s3BinaryCacheStoreConfigDescriptions = { + .profile{ + .name = "profile", + .description = R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )", + }, + .region{ + .name = "region", + .description = R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )", + }, + .scheme{ + .name = "scheme", + .description = R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )", + }, + .endpoint{ + .name = "endpoint", + .description = R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )", + }, + .narinfoCompression{ + .name = "narinfo-compression", + .description = "Compression method for `.narinfo` files.", + }, + .lsCompression{ + .name = "ls-compression", + .description = "Compression method for `.ls` files.", + }, + .logCompression{ + .name = "log-compression", + .description = R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )", + }, + .multipartUpload{ + .name = "multipart-upload", + .description = "Whether to use multi-part uploads.", + }, + .bufferSize{ + .name = "buffer-size", + .description = "Size (in bytes) of each part in multi-part uploads.", + }, +}; + +#define S3_BINARY_CACHE_STORE_CONFIG_FIELDS(X) \ + X(profile), \ + X(region), \ + X(scheme), \ + X(endpoint), \ + X(narinfoCompression), \ + X(lsCompression), \ + X(logCompression), \ + X(multipartUpload), \ + X(bufferSize), + +MAKE_PARSE(S3BinaryCacheStoreConfig, s3BinaryCacheStoreConfig, S3_BINARY_CACHE_STORE_CONFIG_FIELDS) -S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( - std::string_view uriScheme, - std::string_view bucketName, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , bucketName(bucketName) +static S3BinaryCacheStoreConfigT s3BinaryCacheStoreConfigDefaults() { - // Don't want to use use AWS SDK in header, so we check the default - // here. TODO do this better after we overhaul the store settings - // system. - assert(std::string{defaultRegion} == std::string{Aws::Region::US_EAST_1}); + return { + .profile = {""}, + .region = {Aws::Region::US_EAST_1}, + .scheme = {""}, + .endpoint = {""}, + .narinfoCompression = {""}, + .lsCompression = {""}, + .logCompression = {""}, + .multipartUpload = {false}, + .bufferSize = {5 * 1024 * 1024}, + }; +} +MAKE_APPLY_PARSE(S3BinaryCacheStoreConfig, s3BinaryCacheStoreConfig, S3_BINARY_CACHE_STORE_CONFIG_FIELDS) + +config::SettingDescriptionMap S3BinaryCacheStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(BinaryCacheStoreConfig::descriptions()); + { + constexpr auto & descriptions = s3BinaryCacheStoreConfigDescriptions; + auto defaults = s3BinaryCacheStoreConfigDefaults(); + ret.merge(decltype(ret){ + S3_BINARY_CACHE_STORE_CONFIG_FIELDS(DESC_ROW) + }); + } + return ret; +} + +S3BinaryCacheStore::Config::S3BinaryCacheStoreConfig( + std::string_view scheme, + std::string_view authority, + const StoreReference::Params & params) + : Store::Config{params} + , BinaryCacheStore::Config{*this, params} + , S3BinaryCacheStoreConfigT{s3BinaryCacheStoreConfigApplyParse(params)} + , bucketName{authority} +{ if (bucketName.empty()) - throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); + throw UsageError("`%s` store requires a bucket name in its Store URI", scheme); } + +S3BinaryCacheStore::S3BinaryCacheStore(ref config) + : BinaryCacheStore(*config) + , config{config} +{ } + std::string S3BinaryCacheStoreConfig::doc() { return @@ -225,40 +334,39 @@ std::string S3BinaryCacheStoreConfig::doc() } -struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore +struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStore { Stats stats; S3Helper s3Helper; - S3BinaryCacheStoreImpl( - std::string_view uriScheme, - std::string_view bucketName, - const Params & params) - : StoreConfig(params) - , BinaryCacheStoreConfig(params) - , S3BinaryCacheStoreConfig(uriScheme, bucketName, params) - , Store(params) - , BinaryCacheStore(params) - , S3BinaryCacheStore(params) - , s3Helper(profile, region, scheme, endpoint) + S3BinaryCacheStoreImpl(ref config) + : Store{*config} + , BinaryCacheStore{*config} + , S3BinaryCacheStore{config} + , s3Helper(config->profile, config->region, config->scheme, config->endpoint) { diskCache = getNarInfoDiskCache(); + + init(); } std::string getUri() override { - return "s3://" + bucketName; + return "s3://" + config->bucketName; } void init() override { if (auto cacheInfo = diskCache->upToDateCacheExists(getUri())) { - wantMassQuery.setDefault(cacheInfo->wantMassQuery); - priority.setDefault(cacheInfo->priority); + resolvedSubstConfig.wantMassQuery.value = + config->storeConfig.wantMassQuery.optValue.value_or(cacheInfo->wantMassQuery); + resolvedSubstConfig.priority.value = + config->storeConfig.priority.optValue.value_or(cacheInfo->priority); } else { BinaryCacheStore::init(); - diskCache->createCache(getUri(), storeDir, wantMassQuery, priority); + diskCache->createCache( + getUri(), config->storeDir, resolvedSubstConfig.wantMassQuery, resolvedSubstConfig.priority); } } @@ -287,7 +395,7 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual auto res = s3Helper.client->HeadObject( Aws::S3::Model::HeadObjectRequest() - .WithBucket(bucketName) + .WithBucket(config->bucketName) .WithKey(path)); if (!res.IsSuccess()) { @@ -322,11 +430,11 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual std::call_once(transferManagerCreated, [&]() { - if (multipartUpload) { + if (config->multipartUpload) { TransferManagerConfiguration transferConfig(executor.get()); transferConfig.s3Client = s3Helper.client; - transferConfig.bufferSize = bufferSize; + transferConfig.bufferSize = config->bufferSize; transferConfig.uploadProgressCallback = [](const TransferManager *transferManager, @@ -347,6 +455,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual auto now1 = std::chrono::steady_clock::now(); + auto & bucketName = config->bucketName; + if (transferManager) { if (contentEncoding != "") @@ -410,12 +520,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual return std::make_shared(std::move(compressed)); }; - if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) - uploadFile(path, compress(narinfoCompression), mimeType, narinfoCompression); - else if (lsCompression != "" && hasSuffix(path, ".ls")) - uploadFile(path, compress(lsCompression), mimeType, lsCompression); - else if (logCompression != "" && hasPrefix(path, "log/")) - uploadFile(path, compress(logCompression), mimeType, logCompression); + if (config->narinfoCompression != "" && hasSuffix(path, ".narinfo")) + uploadFile(path, compress(config->narinfoCompression), mimeType, config->narinfoCompression); + else if (config->lsCompression != "" && hasSuffix(path, ".ls")) + uploadFile(path, compress(config->lsCompression), mimeType, config->lsCompression); + else if (config->logCompression != "" && hasPrefix(path, "log/")) + uploadFile(path, compress(config->logCompression), mimeType, config->logCompression); else uploadFile(path, istream, mimeType, ""); } @@ -425,14 +535,14 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual stats.get++; // FIXME: stream output to sink. - auto res = s3Helper.getObject(bucketName, path); + auto res = s3Helper.getObject(config->bucketName, path); stats.getBytes += res.data ? res.data->size() : 0; stats.getTimeMs += res.durationMs; if (res.data) { printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", - bucketName, path, res.data->size(), res.durationMs); + config->bucketName, path, res.data->size(), res.durationMs); sink(*res.data); } else @@ -444,6 +554,8 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual StorePathSet paths; std::string marker; + auto & bucketName = config->bucketName; + do { debug("listing bucket 's3://%s' from key '%s'...", bucketName, marker); @@ -482,7 +594,12 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual } }; -static RegisterStoreImplementation regS3BinaryCacheStore; +ref S3BinaryCacheStoreImpl::Config::openStore() const +{ + return make_ref(ref{shared_from_this()}); +} + +static RegisterStoreImplementation regS3BinaryCacheStore; } diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 7d303a115f4..aa3b92bc99c 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -7,89 +7,33 @@ namespace nix { -struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +template class F> +struct S3BinaryCacheStoreConfigT { + F profile; + F region; + F scheme; + F endpoint; + F narinfoCompression; + F lsCompression; + F logCompression; + F multipartUpload; + F bufferSize; +}; + +struct S3BinaryCacheStoreConfig : std::enable_shared_from_this, + Store::Config, + BinaryCacheStoreConfig, + S3BinaryCacheStoreConfigT +{ + static config::SettingDescriptionMap descriptions(); + + S3BinaryCacheStoreConfig( + std::string_view uriScheme, std::string_view bucketName, const StoreReference::Params & params); + std::string bucketName; - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - - S3BinaryCacheStoreConfig(std::string_view uriScheme, std::string_view bucketName, const Params & params); - - const Setting profile{ - this, - "", - "profile", - R"( - The name of the AWS configuration profile to use. By default - Nix will use the `default` profile. - )"}; - -protected: - - constexpr static const char * defaultRegion = "us-east-1"; - -public: - - const Setting region{ - this, - defaultRegion, - "region", - R"( - The region of the S3 bucket. If your bucket is not in - `us–east-1`, you should always explicitly specify the region - parameter. - )"}; - - const Setting scheme{ - this, - "", - "scheme", - R"( - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - )"}; - - const Setting endpoint{ - this, - "", - "endpoint", - R"( - The URL of the endpoint of an S3-compatible service such as MinIO. - Do not specify this setting if you're using Amazon S3. - - > **Note** - > - > This endpoint must support HTTPS and will use path-based - > addressing instead of virtual host based addressing. - )"}; - - const Setting narinfoCompression{ - this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; - - const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; - - const Setting logCompression{ - this, - "", - "log-compression", - R"( - Compression method for `log/*` files. It is recommended to - use a compression method supported by most web browsers - (e.g. `brotli`). - )"}; - - const Setting multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."}; - - const Setting bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; - - const std::string name() override + static std::string name() { return "S3 Binary Cache Store"; } @@ -99,16 +43,18 @@ public: return {"s3"}; } - std::string doc() override; + static std::string doc(); + + ref openStore() const override; }; -class S3BinaryCacheStore : public virtual BinaryCacheStore +struct S3BinaryCacheStore : virtual BinaryCacheStore { -protected: + using Config = S3BinaryCacheStoreConfig; - S3BinaryCacheStore(const Params & params); + ref config; -public: + S3BinaryCacheStore(ref); struct Stats { diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 954a9746774..12432a9b80d 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,3 +1,4 @@ +#include "json-utils.hh" #include "ssh-store.hh" #include "local-fs-store.hh" #include "remote-store-connection.hh" @@ -7,19 +8,105 @@ #include "worker-protocol-impl.hh" #include "pool.hh" #include "ssh.hh" +#include "config-parse-impl.hh" +#include "store-registration.hh" namespace nix { +constexpr static const SSHStoreConfigT sshStoreConfigDescriptions = { + .remoteProgram{ + .name = "remote-program", + .description = "Path to the `nix-daemon` executable on the remote machine.", + }, +}; + + +#define SSH_STORE_CONFIG_FIELDS(X) \ + X(remoteProgram) + + +MAKE_PARSE(SSHStoreConfig, sshStoreConfig, SSH_STORE_CONFIG_FIELDS) + + +static SSHStoreConfigT sshStoreConfigDefaults() +{ + return { + .remoteProgram = {{"nix-daemon"}}, + }; +} + + +MAKE_APPLY_PARSE(SSHStoreConfig, sshStoreConfig, SSH_STORE_CONFIG_FIELDS) + + +config::SettingDescriptionMap SSHStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(CommonSSHStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + { + constexpr auto & descriptions = sshStoreConfigDescriptions; + auto defaults = sshStoreConfigDefaults(); + ret.merge(decltype(ret){ + SSH_STORE_CONFIG_FIELDS(DESC_ROW) + }); + } + ret.insert_or_assign( + "mounted", + config::SettingDescription{ + .description = stripIndentation(R"( + If this nested settings object is defined (`{..}` not `null`), additionally requires that store be mounted in the local file system. + + The mounting of that store is not managed by Nix, and must by managed manually. + It could be accomplished with SSHFS or NFS, for example. + + The local file system is used to optimize certain operations. + For example, rather than serializing Nix archives and sending over the Nix channel, + we can directly access the file system data via the mount-point. + + The local file system is also used to make certain operations possible that wouldn't otherwise be. + For example, persistent GC roots can be created if they reside on the same file system as the remote store: + the remote side will create the symlinks necessary to avoid race conditions. + )"), + .experimentalFeature = Xp::MountedSSHStore, + .info = config::SettingDescription::Sub{ + .nullable = true, + .map = LocalFSStoreConfig::descriptions() + }, + }); + return ret; +} + + +static std::optional getMounted( + const Store::Config & storeConfig, + const StoreReference::Params & params, + const ExperimentalFeatureSettings & xpSettings) +{ + auto mountedParamsOpt = optionalValueAt(params, "mounted"); + if (!mountedParamsOpt) return {}; + auto * mountedParamsP = getNullable(*mountedParamsOpt); + xpSettings.require(Xp::MountedSSHStore); + if (!mountedParamsP) return {}; + auto & mountedParams = getObject(*mountedParamsP); + return {{storeConfig, mountedParams}}; +} + + SSHStoreConfig::SSHStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, authority, params) + const StoreReference::Params & params, const ExperimentalFeatureSettings & xpSettings) + : Store::Config{params} + , RemoteStore::Config{*this, params} + , CommonSSHStoreConfig{scheme, authority, params} + , SSHStoreConfigT{sshStoreConfigApplyParse(params)} + , mounted{getMounted(*this, params, xpSettings)} { } + std::string SSHStoreConfig::doc() { return @@ -27,21 +114,18 @@ std::string SSHStoreConfig::doc() ; } -class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore + +struct SSHStore : virtual RemoteStore { -public: - - SSHStore( - std::string_view scheme, - std::string_view host, - const Params & params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(scheme, host, params) - , Store(params) - , RemoteStore(params) - , master(createSSHMaster( + using Config = SSHStoreConfig; + + ref config; + + SSHStore(ref config) + : Store{*config} + , RemoteStore{*config} + , config{config} + , master(config->createSSHMaster( // Use SSH master only if using more than 1 connection. connections->capacity() > 1)) { @@ -49,7 +133,7 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore std::string getUri() override { - return *uriSchemes().begin() + "://" + host; + return *Config::uriSchemes().begin() + "://" + host; } // FIXME extend daemon protocol, move implementation to RemoteStore @@ -88,32 +172,6 @@ class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore }; -MountedSSHStoreConfig::MountedSSHStoreConfig(StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) -{ -} - -MountedSSHStoreConfig::MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) -{ -} - -std::string MountedSSHStoreConfig::doc() -{ - return - #include "mounted-ssh-store.md" - ; -} - - /** * The mounted ssh store assumes that filesystems on the remote host are * shared with the local host. This means that the remote nix store is @@ -128,35 +186,24 @@ std::string MountedSSHStoreConfig::doc() * The difference lies in how they manage GC roots. See addPermRoot * below for details. */ -class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSHStore, public virtual LocalFSStore +struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore { -public: - - MountedSSHStore( - std::string_view scheme, - std::string_view host, - const Params & params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(scheme, host, params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) - , MountedSSHStoreConfig(params) - , Store(params) - , RemoteStore(params) - , SSHStore(scheme, host, params) - , LocalFSStore(params) + using Config = SSHStore::Config; + + const LocalFSStore::Config & mountedConfig; + + MountedSSHStore(ref config, const LocalFSStore::Config & mountedConfig) + : Store{*config} + , RemoteStore{*config} + , SSHStore{config} + , LocalFSStore{mountedConfig} + , mountedConfig{mountedConfig} { extraRemoteProgramArgs = { "--process-ops", }; } - std::string getUri() override - { - return *uriSchemes().begin() + "://" + host; - } - void narFromPath(const StorePath & path, Sink & sink) override { return LocalFSStore::narFromPath(path, sink); @@ -198,14 +245,25 @@ class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSH } }; + +ref MountedSSHStore::Config::openStore() const { + ref config {shared_from_this()}; + + if (config->mounted) + return make_ref(config, *config->mounted); + else + return make_ref(config); +} + + ref SSHStore::openConnection() { auto conn = make_ref(); - Strings command = remoteProgram.get(); + Strings command = config->remoteProgram.get(); command.push_back("--stdio"); - if (remoteStore.get() != "") { + if (config->remoteStore.get() != "") { command.push_back("--store"); - command.push_back(remoteStore.get()); + command.push_back(config->remoteStore.get()); } command.insert(command.end(), extraRemoteProgramArgs.begin(), extraRemoteProgramArgs.end()); @@ -215,7 +273,7 @@ ref SSHStore::openConnection() return conn; } -static RegisterStoreImplementation regSSHStore; -static RegisterStoreImplementation regMountedSSHStore; +static RegisterStoreImplementation regSSHStore; +static RegisterStoreImplementation regMountedSSHStore; } diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh index 29a2a8b2c2d..e8b7f200fef 100644 --- a/src/libstore/ssh-store.hh +++ b/src/libstore/ssh-store.hh @@ -8,54 +8,41 @@ namespace nix { -struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig +template class F> +struct SSHStoreConfigT { - using CommonSSHStoreConfig::CommonSSHStoreConfig; - using RemoteStoreConfig::RemoteStoreConfig; - - SSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); - - const Setting remoteProgram{ - this, {"nix-daemon"}, "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; - - const std::string name() override - { - return "Experimental SSH Store"; - } - - static std::set uriSchemes() - { - return {"ssh-ng"}; - } - - std::string doc() override; + F remoteProgram; }; -struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig +struct SSHStoreConfig : std::enable_shared_from_this, + Store::Config, + RemoteStore::Config, + CommonSSHStoreConfig, + SSHStoreConfigT { - using LocalFSStoreConfig::LocalFSStoreConfig; - using SSHStoreConfig::SSHStoreConfig; + static config::SettingDescriptionMap descriptions(); - MountedSSHStoreConfig(StringMap params); + std::optional mounted; - MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params); + SSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const StoreReference::Params & params, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); - const std::string name() override + static const std::string name() { - return "Experimental SSH Store with filesystem mounted"; + return "Experimental SSH Store"; } static std::set uriSchemes() { - return {"mounted-ssh-ng"}; + return {"ssh-ng"}; } - std::string doc() override; + static std::string doc(); - std::optional experimentalFeature() const override - { - return ExperimentalFeature::MountedSSHStore; - } + ref openStore() const override; }; } diff --git a/src/libstore/ssh-store.md b/src/libstore/ssh-store.md index 881537e7114..ea34ef2b710 100644 --- a/src/libstore/ssh-store.md +++ b/src/libstore/ssh-store.md @@ -4,5 +4,4 @@ R"( Experimental store type that allows full access to a Nix store on a remote machine. - )" diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 236622eae37..5693c44521b 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -5,6 +5,7 @@ #include "realisation.hh" #include "derivations.hh" #include "store-api.hh" +#include "store-open.hh" #include "util.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" @@ -18,6 +19,7 @@ #include "worker-protocol.hh" #include "signals.hh" #include "users.hh" +#include "config-parse-impl.hh" #include #include @@ -28,14 +30,13 @@ using json = nlohmann::json; namespace nix { - -bool StoreDirConfig::isInStore(PathView path) const +bool MixStoreDirMethods::isInStore(PathView path) const { return isInDir(path, storeDir); } -std::pair StoreDirConfig::toStorePath(PathView path) const +std::pair MixStoreDirMethods::toStorePath(PathView path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); @@ -77,7 +78,7 @@ to match. */ -StorePath StoreDirConfig::makeStorePath(std::string_view type, +StorePath MixStoreDirMethods::makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ @@ -88,14 +89,14 @@ StorePath StoreDirConfig::makeStorePath(std::string_view type, } -StorePath StoreDirConfig::makeStorePath(std::string_view type, +StorePath MixStoreDirMethods::makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const { return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name); } -StorePath StoreDirConfig::makeOutputPath(std::string_view id, +StorePath MixStoreDirMethods::makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const { return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id)); @@ -106,7 +107,7 @@ StorePath StoreDirConfig::makeOutputPath(std::string_view id, hacky, but we can't put them in, say, (per the grammar above) since that would be ambiguous. */ static std::string makeType( - const StoreDirConfig & store, + const MixStoreDirMethods & store, std::string && type, const StoreReferences & references) { @@ -119,7 +120,7 @@ static std::string makeType( } -StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const +StorePath MixStoreDirMethods::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { if (info.method == FileIngestionMethod::Git && info.hash.algo != HashAlgorithm::SHA1) throw Error("Git file ingestion must use SHA-1 hash"); @@ -141,7 +142,7 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed } -StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const +StorePath MixStoreDirMethods::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { @@ -162,7 +163,7 @@ StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const } -std::pair StoreDirConfig::computeStorePath( +std::pair MixStoreDirMethods::computeStorePath( std::string_view name, const SourcePath & path, ContentAddressMethod method, @@ -188,6 +189,114 @@ std::pair StoreDirConfig::computeStorePath( } +constexpr static const StoreConfigT storeConfigDescriptions = { + .pathInfoCacheSize{ + .name = "path-info-cache-size", + .description = "Size of the in-memory store path metadata cache.", + }, + .isTrusted{ + .name = "trusted", + .description = R"( + Whether paths from this store can be used as substitutes + even if they are not signed by a key listed in the + [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) + setting. + )", + }, + .systemFeatures{ + .name = "system-features", + .description = R"( + Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. + + Example: `"kvm"` + )", + // The default value is CPU- and OS-specific, and thus + // unsuitable to be rendered in the documentation. + .documentDefault = false, + }, +}; + +constexpr static const SubstituterConfigT substituterConfigDescriptions = { + .priority{ + .name = "priority", + .description = R"( + Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + A lower value means a higher priority. + )", + }, + .wantMassQuery{ + .name = "want-mass-query", + .description = R"( + Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). + )", + }, +}; + + +#define STORE_CONFIG_FIELDS(X) \ + X(pathInfoCacheSize), \ + X(isTrusted), \ + X(systemFeatures), + +#define SUBSTITUTER_CONFIG_FIELDS(X) \ + X(priority), \ + X(wantMassQuery), + + +MAKE_PARSE(StoreConfig, storeConfig, STORE_CONFIG_FIELDS) +MAKE_PARSE(SubstituterConfig, substituterConfig, SUBSTITUTER_CONFIG_FIELDS) + + +static StoreConfigT storeConfigDefaults() +{ + return { + .pathInfoCacheSize = {65536}, + .isTrusted = {false}, + .systemFeatures = {StoreConfig::getDefaultSystemFeatures()}, + }; +}; + +static SubstituterConfigT substituterConfigDefaults() +{ + return { + .priority = {0}, + .wantMassQuery = {false}, + }; +}; + + +MAKE_APPLY_PARSE(StoreConfig, storeConfig, STORE_CONFIG_FIELDS) + + +Store::Config::StoreConfig(const StoreReference::Params & params) + : StoreDirConfig{params} + , StoreConfigT{storeConfigApplyParse(params)} + , SubstituterConfigT{substituterConfigParse(params)} +{ +} + + +config::SettingDescriptionMap StoreConfig::descriptions() +{ + auto ret = StoreDirConfig::descriptions(); + { + constexpr auto & descriptions = storeConfigDescriptions; + auto defaults = storeConfigDefaults(); + ret.merge(config::SettingDescriptionMap { + STORE_CONFIG_FIELDS(DESC_ROW) + }); + } + { + constexpr auto & descriptions = substituterConfigDescriptions; + auto defaults = substituterConfigDefaults(); + ret.merge(config::SettingDescriptionMap { + SUBSTITUTER_CONFIG_FIELDS(DESC_ROW) + }); + }; + return ret; +} + + StorePath Store::addToStore( std::string_view name, const SourcePath & path, @@ -419,7 +528,7 @@ ValidPathInfo Store::addToStoreSlow( return info; } -StringSet StoreConfig::getDefaultSystemFeatures() +StringSet Store::Config::getDefaultSystemFeatures() { auto res = settings.systemFeatures.get(); @@ -432,9 +541,10 @@ StringSet StoreConfig::getDefaultSystemFeatures() return res; } -Store::Store(const Params & params) - : StoreConfig(params) - , state({(size_t) pathInfoCacheSize}) +Store::Store(const Store::Config & config) + : MixStoreDirMethods{config} + , config{config} + , state({(size_t) config.pathInfoCacheSize}) { assertLibStoreInitialized(); } @@ -1204,7 +1314,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre } -std::string StoreDirConfig::showPaths(const StorePathSet & paths) +std::string MixStoreDirMethods::showPaths(const StorePathSet & paths) const { std::string s; for (auto & i : paths) { @@ -1274,102 +1384,3 @@ Derivation Store::readInvalidDerivation(const StorePath & drvPath) { return readDerivationCommon(*this, drvPath, false); } } - - -#include "local-store.hh" -#include "uds-remote-store.hh" - - -namespace nix { - -ref openStore(const std::string & uri, - const Store::Params & extraParams) -{ - return openStore(StoreReference::parse(uri, extraParams)); -} - -ref openStore(StoreReference && storeURI) -{ - auto & params = storeURI.params; - - auto store = std::visit(overloaded { - [&](const StoreReference::Auto &) -> std::shared_ptr { - auto stateDir = getOr(params, "state", settings.nixStateDir); - if (access(stateDir.c_str(), R_OK | W_OK) == 0) - return std::make_shared(params); - else if (pathExists(settings.nixDaemonSocketFile)) - return std::make_shared(params); - #if __linux__ - else if (!pathExists(stateDir) - && params.empty() - && !isRootUser() - && !getEnv("NIX_STORE_DIR").has_value() - && !getEnv("NIX_STATE_DIR").has_value()) - { - /* If /nix doesn't exist, there is no daemon socket, and - we're not root, then automatically set up a chroot - store in ~/.local/share/nix/root. */ - auto chrootStore = getDataDir() + "/root"; - if (!pathExists(chrootStore)) { - try { - createDirs(chrootStore); - } catch (SystemError & e) { - return std::make_shared(params); - } - warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); - } else - debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); - return std::make_shared("local", chrootStore, params); - } - #endif - else - return std::make_shared(params); - }, - [&](const StoreReference::Specified & g) { - for (const auto & implem : *Implementations::registered) - if (implem.uriSchemes.count(g.scheme)) - return implem.create(g.scheme, g.authority, params); - - throw Error("don't know how to open Nix store with scheme '%s'", g.scheme); - }, - }, storeURI.variant); - - experimentalFeatureSettings.require(store->experimentalFeature()); - store->warnUnknownSettings(); - store->init(); - - return ref { store }; -} - -std::list> getDefaultSubstituters() -{ - static auto stores([]() { - std::list> stores; - - StringSet done; - - auto addStore = [&](const std::string & uri) { - if (!done.insert(uri).second) return; - try { - stores.push_back(openStore(uri)); - } catch (Error & e) { - logWarning(e.info()); - } - }; - - for (const auto & uri : settings.substituters.get()) - addStore(uri); - - stores.sort([](ref & a, ref & b) { - return a->priority < b->priority; - }); - - return stores; - } ()); - - return stores; -} - -std::vector * Implementations::registered = 0; - -} diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 474dffcb5da..163a06778e8 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -9,7 +9,6 @@ #include "lru-cache.hh" #include "sync.hh" #include "globals.hh" -#include "config.hh" #include "path-info.hh" #include "repair-flag.hh" #include "store-dir-config.hh" @@ -41,7 +40,7 @@ namespace nix { * 2. A class `Foo : virtual Store, virtual FooConfig` that contains the * implementation of the store. * - * This class is expected to have a constructor `Foo(const Params & params)` + * This class is expected to have a constructor `Foo(const StoreReference::Params & params)` * that calls `StoreConfig(params)` (otherwise you're gonna encounter an * `assertion failure` when trying to instantiate it). * @@ -97,27 +96,44 @@ struct KeyedBuildResult; typedef std::map> StorePathCAMap; -struct StoreConfig : public StoreDirConfig +template class F> +struct StoreConfigT { - using Params = StoreReference::Params; + F pathInfoCacheSize; + F isTrusted; + F systemFeatures; +}; - using StoreDirConfig::StoreDirConfig; +template class F> +struct SubstituterConfigT +{ + F priority; + F wantMassQuery; +}; - StoreConfig() = delete; +/** + * @note `config::OptValue` rather than `config::JustValue` is applied to + * `SubstitutorConfigT` because these are overrides. Caches themselves (not our + * config) can update default settings, but aren't allowed to update settings + * specified by the client (i.e. us). + */ +struct StoreConfig : + StoreDirConfig, + StoreConfigT, + SubstituterConfigT +{ + static config::SettingDescriptionMap descriptions(); - static StringSet getDefaultSystemFeatures(); + StoreConfig(const StoreReference::Params &); virtual ~StoreConfig() { } - /** - * The name of this type of store. - */ - virtual const std::string name() = 0; + static StringSet getDefaultSystemFeatures(); /** * Documentation for this type of store. */ - virtual std::string doc() + static std::string doc() { return ""; } @@ -126,47 +142,50 @@ struct StoreConfig : public StoreDirConfig * An experimental feature this type store is gated, if it is to be * experimental. */ - virtual std::optional experimentalFeature() const + static std::optional experimentalFeature() { return std::nullopt; } - const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", - "Size of the in-memory store path metadata cache."}; - - const Setting isTrusted{this, false, "trusted", - R"( - Whether paths from this store can be used as substitutes - even if they are not signed by a key listed in the - [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys) - setting. - )"}; - - Setting priority{this, 0, "priority", - R"( - Priority of this store when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). - A lower value means a higher priority. - )"}; - - Setting wantMassQuery{this, false, "want-mass-query", - R"( - Whether this store can be queried efficiently for path validity when used as a [substituter](@docroot@/command-ref/conf-file.md#conf-substituters). - )"}; - - Setting systemFeatures{this, getDefaultSystemFeatures(), - "system-features", - R"( - Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. - - Example: `"kvm"` - )", - {}, - // Don't document the machine-specific default value - false}; + /** + * Open a store of the type corresponding to this configuration + * type. + */ + virtual ref openStore() const = 0; }; -class Store : public std::enable_shared_from_this, public virtual StoreConfig +/** + * A Store (client) + * + * This is an interface type allowing for create and read operations on + * a collection of store objects, and also building new store objects + * from `Derivation`s. See the manual for further details. + * + * "client" used is because this is just one view/actor onto an + * underlying resource, which could be an external process (daemon + * server), file system state, etc. + */ +class Store : public std::enable_shared_from_this, public MixStoreDirMethods { +public: + + using Config = StoreConfig; + + const Config & config; + + /** + * @note Avoid churn, since we used to inherit from `Config`. + */ + operator const Config &() const { return config; } + + /** + * Resolved substituter configuration. This is intentionally mutable + * as store clients may do IO to ask the underlying store for their + * default setting values if the client config did not statically + * override them. + */ + SubstituterConfigT resolvedSubstConfig; + protected: struct PathInfoCacheValue { @@ -205,14 +224,9 @@ protected: std::shared_ptr diskCache; - Store(const Params & params); + Store(const Store::Config & config); public: - /** - * Perform any necessary effectful operation to make the store up and - * running - */ - virtual void init() {}; virtual ~Store() { } @@ -857,74 +871,6 @@ StorePath resolveDerivedPath(Store &, const SingleDerivedPath &, Store * evalSto OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr); -/** - * @return a Store object to access the Nix store denoted by - * ‘uri’ (slight misnomer...). - */ -ref openStore(StoreReference && storeURI); - - -/** - * Opens the store at `uri`, where `uri` is in the format expected by `StoreReference::parse` - - */ -ref openStore(const std::string & uri = settings.storeUri.get(), - const Store::Params & extraParams = Store::Params()); - - -/** - * @return the default substituter stores, defined by the - * ‘substituters’ option and various legacy options. - */ -std::list> getDefaultSubstituters(); - -struct StoreFactory -{ - std::set uriSchemes; - /** - * The `authorityPath` parameter is `/`, or really - * whatever comes after `://` and before `?`. - */ - std::function ( - std::string_view scheme, - std::string_view authorityPath, - const Store::Params & params)> create; - std::function ()> getConfig; -}; - -struct Implementations -{ - static std::vector * registered; - - template - static void add() - { - if (!registered) registered = new std::vector(); - StoreFactory factory{ - .uriSchemes = TConfig::uriSchemes(), - .create = - ([](auto scheme, auto uri, auto & params) - -> std::shared_ptr - { return std::make_shared(scheme, uri, params); }), - .getConfig = - ([]() - -> std::shared_ptr - { return std::make_shared(StringMap({})); }) - }; - registered->push_back(factory); - } -}; - -template -struct RegisterStoreImplementation -{ - RegisterStoreImplementation() - { - Implementations::add(); - } -}; - - /** * Display a set of paths in human-readable form (i.e., between quotes * and separated by commas). @@ -946,3 +892,6 @@ std::map drvOutputReferences( Store * evalStore = nullptr); } + +// Parses a Store URL, uses global state not pure so think about this +JSON_IMPL(ref) diff --git a/src/libstore/store-dir-config.cc b/src/libstore/store-dir-config.cc new file mode 100644 index 00000000000..281a7437f2b --- /dev/null +++ b/src/libstore/store-dir-config.cc @@ -0,0 +1,45 @@ +#include "store-dir-config.hh" +#include "config-parse-impl.hh" +#include "util.hh" +#include "globals.hh" + +namespace nix { + +constexpr static const StoreDirConfigT storeDirConfigDescriptions = { + ._storeDir{ + .name = "store", + .description = R"( + Logical location of the Nix store, usually + `/nix/store`. Note that you can only copy store paths + between stores if they have the same `store` setting. + )", + }, +}; + +#define STORE_DIR_CONFIG_FIELDS(X) X(_storeDir), + +MAKE_PARSE(StoreDirConfig, storeDirConfig, STORE_DIR_CONFIG_FIELDS) + +static StoreDirConfigT storeDirConfigDefaults() +{ + return { + ._storeDir = {settings.nixStore}, + }; +} + +MAKE_APPLY_PARSE(StoreDirConfig, storeDirConfig, STORE_DIR_CONFIG_FIELDS) + +StoreDirConfig::StoreDirConfig(const StoreReference::Params & params) + : StoreDirConfigT{storeDirConfigApplyParse(params)} + , MixStoreDirMethods{_storeDir.value} +{ +} + +config::SettingDescriptionMap StoreDirConfig::descriptions() +{ + constexpr auto & descriptions = storeDirConfigDescriptions; + auto defaults = storeDirConfigDefaults(); + return {STORE_DIR_CONFIG_FIELDS(DESC_ROW)}; +} + +} diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index fd4332b918f..050ae579664 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -3,8 +3,8 @@ #include "path.hh" #include "hash.hh" #include "content-address.hh" -#include "globals.hh" -#include "config.hh" +#include "store-reference.hh" +#include "config-parse.hh" #include #include @@ -18,24 +18,32 @@ struct SourcePath; MakeError(BadStorePath, Error); MakeError(BadStorePathName, BadStorePath); -struct StoreDirConfig : public Config +/** + * Underlying store directory configuration type. + * + * Don't worry to much about the `F` parameter, it just some abstract + * nonsense for the "higher-kinded data" pattern. It is used in each + * settings record in order to ensure don't forgot to parse or document + * settings field. + */ +template class F> +struct StoreDirConfigT { - using Config::Config; - - StoreDirConfig() = delete; - - virtual ~StoreDirConfig() = default; - - const PathSetting storeDir_{this, settings.nixStore, - "store", - R"( - Logical location of the Nix store, usually - `/nix/store`. Note that you can only copy store paths - between stores if they have the same `store` setting. - )"}; - const Path storeDir = storeDir_; + F _storeDir; +}; - // pure methods +/** + * @todo This should just be part of `StoreDirConfig`. However, it would + * be a huge amount of churn if `Store` didn't have these methods + * anymore, forcing a bunch of code to go from `store.method(...)` to + * `store.config.method(...)`. + * + * So we instead pull out the methods into their own mix-in, so can put + * them directly on the Store too. + */ +struct MixStoreDirMethods +{ + const Path & storeDir; StorePath parseStorePath(std::string_view path) const; @@ -56,7 +64,7 @@ struct StoreDirConfig : public Config * Display a set of paths in human-readable form (i.e., between quotes * and separated by commas). */ - std::string showPaths(const StorePathSet & paths); + std::string showPaths(const StorePathSet & paths) const; /** * @return true if *path* is in the Nix store (but not the Nix @@ -104,4 +112,19 @@ struct StoreDirConfig : public Config PathFilter & filter = defaultPathFilter) const; }; +/** + * Store directory configuration type. + * + * Combines the underlying `*T` type (with plain values for the fields) + * and the methods. + */ +struct StoreDirConfig : StoreDirConfigT, MixStoreDirMethods +{ + static config::SettingDescriptionMap descriptions(); + + StoreDirConfig(const StoreReference::Params & params); + + virtual ~StoreDirConfig() = default; +}; + } diff --git a/src/libstore/store-open.hh b/src/libstore/store-open.hh new file mode 100644 index 00000000000..c074b846bcc --- /dev/null +++ b/src/libstore/store-open.hh @@ -0,0 +1,43 @@ +#pragma once +/** + * @file + * + * For opening a store described by an `StoreReference`, which is an "untyped" + * notion which needs to be decoded against a collection of specific + * implementations. + * + * For consumers of the store registration machinery defined in + * `store-registration.hh`. Not needed by store implementation definitions, or + * usages of a given `Store` which will be passed in. + */ + +#include "store-api.hh" + +namespace nix { + +/** + * @return The store config denoted `uri` (slight misnomer...). + */ +ref resolveStoreConfig(StoreReference && storeURI); + +/** + * @return a Store object to access the Nix store denoted by + * ‘uri’ (slight misnomer...). + */ +ref openStore(StoreReference && storeURI); + +/** + * Opens the store at `uri`, where `uri` is in the format expected by + * `StoreReference::parse` + */ +ref openStore( + const std::string & uri = settings.storeUri.get(), + const StoreReference::Params & extraParams = StoreReference::Params()); + +/** + * @return the default substituter stores, defined by the + * ‘substituters’ option and various legacy options. + */ +std::list> getDefaultSubstituters(); + +} diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc index b4968dfadbd..12ed96a01cb 100644 --- a/src/libstore/store-reference.cc +++ b/src/libstore/store-reference.cc @@ -1,13 +1,18 @@ #include +#include + #include "error.hh" #include "url.hh" #include "store-reference.hh" #include "file-system.hh" #include "util.hh" +#include "json-utils.hh" namespace nix { +bool StoreReference::operator==(const StoreReference & rhs) const = default; + static bool isNonUriPath(const std::string & spec) { return @@ -33,20 +38,96 @@ std::string StoreReference::render() const }, variant); + StringMap params2; + for (auto & [k, v] : params) { + auto * p = v.get_ptr(); + // if it is a JSON string, just use that + + // FIXME: Ensure the literal string isn't itself valid JSON. If + // it is, we still need to dump to escape it. + params2.insert_or_assign(k, p ? *p : v.dump()); + } + if (!params.empty()) { res += "?"; - res += encodeQuery(params); + res += encodeQuery(params2); } return res; } +static StoreReference::Params decodeParamsJson(StringMap paramsRaw) +{ + StoreReference::Params params; + for (auto && [k, v] : std::move(paramsRaw)) { + nlohmann::json j; + /* We have to parse the URL un an "untyped" way before we do a + "typed" conversion to specific store-configuration types. As + such, the best we can do for back-compat is just white-list + specific query parameter names. + + These are all the boolean store parameters in use at the time + of the introduction of JSON store configuration, as evidenced + by `git grep 'F'`. So these will continue working with + "yes"/"no"/"1"/"0", whereas any new ones will require + "true"/"false". + */ + bool preJsonBool = + std::set{ + "check-mount", + "compress", + "trusted", + "multipart-upload", + "parallel-compression", + "read-only", + "require-sigs", + "want-mass-query", + "index-debug-info", + "write-nar-listing", + } + .contains(std::string_view{k}); + + auto warnPreJson = [&] { + warn( + "in query param '%s', using '%s' to mean a boolean is deprecated, please use valid JSON 'true' or 'false'", + k, + v); + }; + + if (preJsonBool && (v == "yes" || v == "1")) { + j = true; + warnPreJson(); + } else if (preJsonBool && (v == "no" || v == "0")) { + j = true; + warnPreJson(); + } else { + try { + j = nlohmann::json::parse(v); + } catch (nlohmann::json::exception &) { + // if its not valid JSON... + if (k == "remote-program" || k == "system-features") { + // Back compat hack! Split and take that array + j = tokenizeString>(v); + } else { + // ...keep the literal string. + j = std::move(v); + } + } + } + params.insert_or_assign(std::move(k), std::move(j)); + } + return params; +} + StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams) { auto params = extraParams; try { auto parsedUri = parseURL(uri); - params.insert(parsedUri.query.begin(), parsedUri.query.end()); + { + auto params2 = decodeParamsJson(std::move(parsedUri.query)); + params.insert(params2.begin(), params2.end()); + } auto baseURI = parsedUri.authority.value_or("") + parsedUri.path; @@ -104,13 +185,75 @@ StoreReference StoreReference::parse(const std::string & uri, const StoreReferen std::pair splitUriAndParams(const std::string & uri_) { auto uri(uri_); - StoreReference::Params params; + StringMap params; auto q = uri.find('?'); if (q != std::string::npos) { params = decodeQuery(uri.substr(q + 1)); uri = uri_.substr(0, q); } - return {uri, params}; + return {uri, decodeParamsJson(std::move(params))}; +} + +} + +namespace nlohmann { + +StoreReference adl_serializer::from_json(const json & json) +{ + StoreReference ref; + switch (json.type()) { + + case json::value_t::string: { + ref = StoreReference::parse(json.get_ref()); + break; + } + + case json::value_t::object: { + auto & obj = getObject(json); + auto scheme = getString(valueAt(obj, "scheme")); + auto variant = scheme == "auto" ? (StoreReference::Variant{StoreReference::Auto{}}) + : (StoreReference::Variant{StoreReference::Specified{ + .scheme = scheme, + .authority = getString(valueAt(obj, "authority")), + }}); + auto params = obj; + params.erase("scheme"); + params.erase("authority"); + ref = StoreReference{ + .variant = std::move(variant), + .params = std::move(params), + }; + break; + } + + case json::value_t::null: + case json::value_t::number_unsigned: + case json::value_t::number_integer: + case json::value_t::number_float: + case json::value_t::boolean: + case json::value_t::array: + case json::value_t::binary: + case json::value_t::discarded: + default: + throw UsageError( + "Invalid JSON for Store configuration: is type '%s' but must be string or object", json.type_name()); + }; + + return ref; +} + +void adl_serializer::to_json(json & obj, StoreReference s) +{ + obj = s.params; + std::visit( + overloaded{ + [&](const StoreReference::Auto &) { obj.emplace("scheme", "auto"); }, + [&](const StoreReference::Specified & g) { + obj.emplace("scheme", g.scheme); + obj.emplace("authority", g.authority); + }, + }, + s.variant); } } diff --git a/src/libstore/store-reference.hh b/src/libstore/store-reference.hh index 7100a1db095..38a5f84ac52 100644 --- a/src/libstore/store-reference.hh +++ b/src/libstore/store-reference.hh @@ -2,8 +2,10 @@ ///@file #include +#include #include "types.hh" +#include "json-impls.hh" namespace nix { @@ -41,7 +43,17 @@ namespace nix { */ struct StoreReference { - using Params = std::map; + /** + * Would do + * + * ``` + * using Params = nlohmann::json::object_t; + * ``` + * + * but cannot because `` doesn't have that. + * + */ + using Params = std::map>; /** * Special store reference `""` or `"auto"` @@ -70,7 +82,7 @@ struct StoreReference Params params; - bool operator==(const StoreReference & rhs) const = default; + bool operator==(const StoreReference & rhs) const; /** * Render the whole store reference as a URI, including parameters. @@ -89,3 +101,5 @@ struct StoreReference std::pair splitUriAndParams(const std::string & uri); } + +JSON_IMPL(StoreReference) diff --git a/src/libstore/store-registration.cc b/src/libstore/store-registration.cc new file mode 100644 index 00000000000..f9a68fd659e --- /dev/null +++ b/src/libstore/store-registration.cc @@ -0,0 +1,125 @@ +#include "store-registration.hh" +#include "store-open.hh" +#include "local-store.hh" +#include "uds-remote-store.hh" +#include "json-utils.hh" + +namespace nix { + +ref openStore(const std::string & uri, const StoreReference::Params & extraParams) +{ + return openStore(StoreReference::parse(uri, extraParams)); +} + +ref openStore(StoreReference && storeURI) +{ + auto store = resolveStoreConfig(std::move(storeURI))->openStore(); + +#if 0 // FIXME + store->warnUnknownSettings(); + store->init(); +#endif + + return store; +} + +ref resolveStoreConfig(StoreReference && storeURI) +{ + auto & params = storeURI.params; + + auto storeConfig = std::visit( + overloaded{ + [&](const StoreReference::Auto &) -> ref { + auto stateDir = getString(getOr(params, "state", settings.nixStateDir)); + if (access(stateDir.c_str(), R_OK | W_OK) == 0) + return make_ref(params); + else if (pathExists(settings.nixDaemonSocketFile)) + return make_ref(params); +#if __linux__ + else if ( + !pathExists(stateDir) && params.empty() && !isRootUser() && !getEnv("NIX_STORE_DIR").has_value() + && !getEnv("NIX_STATE_DIR").has_value()) { + /* If /nix doesn't exist, there is no daemon socket, and + we're not root, then automatically set up a chroot + store in ~/.local/share/nix/root. */ + auto chrootStore = getDataDir() + "/root"; + if (!pathExists(chrootStore)) { + try { + createDirs(chrootStore); + } catch (SystemError & e) { + return make_ref(params); + } + warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); + } else + debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); + return make_ref("local", chrootStore, params); + } +#endif + else + return make_ref(params); + }, + [&](const StoreReference::Specified & g) { + for (auto & [name, implem] : *Implementations::registered) + if (implem.uriSchemes.count(g.scheme)) + return implem.parseConfig(g.scheme, g.authority, params); + + throw Error("don't know how to open Nix store with scheme '%s'", g.scheme); + }, + }, + storeURI.variant); + + experimentalFeatureSettings.require(storeConfig->experimentalFeature()); + + return storeConfig; +} + +Implementations::V * Implementations::registered = 0; + +std::list> getDefaultSubstituters() +{ + static auto stores([]() { + std::list> stores; + + StringSet done; + + auto addStore = [&](const std::string & uri) { + if (!done.insert(uri).second) + return; + try { + stores.push_back(openStore(uri)); + } catch (Error & e) { + logWarning(e.info()); + } + }; + + for (auto uri : settings.substituters.get()) + addStore(uri); + + stores.sort([](ref & a, ref & b) { + return a->resolvedSubstConfig.priority < b->resolvedSubstConfig.priority; + }); + + return stores; + }()); + + return stores; +} + +} + +namespace nlohmann { + +using namespace nix::config; + +ref adl_serializer>::from_json(const json & json) +{ + return resolveStoreConfig(adl_serializer::from_json(json)); +} + +void adl_serializer>::to_json(json & obj, ref s) +{ + // TODO, for tests maybe + assert(false); +} + +} diff --git a/src/libstore/store-registration.hh b/src/libstore/store-registration.hh new file mode 100644 index 00000000000..e479966250e --- /dev/null +++ b/src/libstore/store-registration.hh @@ -0,0 +1,83 @@ +#pragma once +/** + * @file + * + * Infrastructure for "registering" store implementations. Used by the + * store implementation definitions themselves but not by consumers of + * those implementations. + */ + +#include "store-api.hh" + +namespace nix { + +struct StoreFactory +{ + /** + * Documentation for this type of store. + */ + std::string doc; + + /** + * URIs with these schemes should be handled by this factory + */ + std::set uriSchemes; + + config::SettingDescriptionMap configDescriptions; + + /** + * An experimental feature this type store is gated, if it is to be + * experimental. + */ + std::optional experimentalFeature; + + /** + * The `authorityPath` parameter is `/`, or really + * whatever comes after `://` and before `?`. + */ + std::function( + std::string_view scheme, std::string_view authorityPath, const StoreReference::Params & params)> + parseConfig; +}; + +struct Implementations +{ +private: + + /** + * The name of this type of store, and a factory for it. + */ + using V = std::vector>; + +public: + + static V * registered; + + template + static void add() + { + if (!registered) + registered = new V{}; + StoreFactory factory{ + .doc = TConfig::doc(), + .uriSchemes = TConfig::uriSchemes(), + .configDescriptions = TConfig::descriptions(), + .experimentalFeature = TConfig::experimentalFeature(), + .parseConfig = ([](auto scheme, auto uri, auto & params) -> ref { + return make_ref(scheme, uri, params); + }), + }; + registered->push_back({TConfig::name(), std::move(factory)}); + } +}; + +template +struct RegisterStoreImplementation +{ + RegisterStoreImplementation() + { + Implementations::add(); + } +}; + +} diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 3c445eb1318..5e8b40fe61d 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,6 +1,7 @@ #include "uds-remote-store.hh" #include "unix-domain-socket.hh" #include "worker-protocol.hh" +#include "store-registration.hh" #include #include @@ -17,16 +18,26 @@ namespace nix { +config::SettingDescriptionMap UDSRemoteStoreConfig::descriptions() +{ + config::SettingDescriptionMap ret; + ret.merge(StoreConfig::descriptions()); + ret.merge(LocalFSStoreConfig::descriptions()); + ret.merge(RemoteStoreConfig::descriptions()); + return ret; +} + + UDSRemoteStoreConfig::UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RemoteStoreConfig(params) + const StoreReference::Params & params) + : Store::Config{params} + , LocalFSStore::Config{*this, params} + , RemoteStore::Config{*this, params} , path{authority.empty() ? settings.nixDaemonSocketFile : authority} { - if (scheme != UDSRemoteStoreConfig::scheme) { + if (uriSchemes().count(std::string{scheme}) == 0) { throw UsageError("Scheme must be 'unix'"); } } @@ -40,36 +51,24 @@ std::string UDSRemoteStoreConfig::doc() } -// A bit gross that we now pass empty string but this is knowing that -// empty string will later default to the same nixDaemonSocketFile. Why -// don't we just wire it all through? I believe there are cases where it -// will live reload so we want to continue to account for that. -UDSRemoteStore::UDSRemoteStore(const Params & params) - : UDSRemoteStore(scheme, "", params) -{} - - -UDSRemoteStore::UDSRemoteStore(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RemoteStoreConfig(params) - , UDSRemoteStoreConfig(scheme, authority, params) - , Store(params) - , LocalFSStore(params) - , RemoteStore(params) +UDSRemoteStore::UDSRemoteStore(ref config) + : Store{*config} + , LocalFSStore{*config} + , RemoteStore{*config} + , config{config} { } std::string UDSRemoteStore::getUri() { - return path == settings.nixDaemonSocketFile + return config->path == settings.nixDaemonSocketFile ? // FIXME: Not clear why we return daemon here and not default // to settings.nixDaemonSocketFile // // unix:// with no path also works. Change what we return? "daemon" - : std::string(scheme) + "://" + path; + : std::string(*Config::uriSchemes().begin()) + "://" + config->path; } @@ -86,7 +85,7 @@ ref UDSRemoteStore::openConnection() /* Connect to a daemon that does the privileged work for us. */ conn->fd = createUnixDomainSocket(); - nix::connect(toSocket(conn->fd.get()), path); + nix::connect(toSocket(conn->fd.get()), config->path); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); @@ -106,6 +105,11 @@ void UDSRemoteStore::addIndirectRoot(const Path & path) } -static RegisterStoreImplementation regUDSRemoteStore; +ref UDSRemoteStore::Config::openStore() const { + return make_ref(ref{shared_from_this()}); +} + + +static RegisterStoreImplementation regUDSRemoteStore; } diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index a8e57166416..ce9fcbfeec8 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -7,12 +7,17 @@ namespace nix { -struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig +struct UDSRemoteStoreConfig : + std::enable_shared_from_this, + Store::Config, + LocalFSStore::Config, + RemoteStore::Config { - // TODO(fzakaria): Delete this constructor once moved over to the factory pattern - // outlined in https://github.com/NixOS/nix/issues/10766 - using LocalFSStoreConfig::LocalFSStoreConfig; - using RemoteStoreConfig::RemoteStoreConfig; + static config::SettingDescriptionMap descriptions(); + + UDSRemoteStoreConfig(const StoreReference::Params & params) + : UDSRemoteStoreConfig{"unix", "", params} + {} /** * @param authority is the socket path. @@ -20,11 +25,11 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon UDSRemoteStoreConfig( std::string_view scheme, std::string_view authority, - const Params & params); + const StoreReference::Params & params); - const std::string name() override { return "Local Daemon Store"; } + static const std::string name() { return "Local Daemon Store"; } - std::string doc() override; + static std::string doc(); /** * The path to the unix domain socket. @@ -34,32 +39,21 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon */ Path path; -protected: - static constexpr char const * scheme = "unix"; - -public: static std::set uriSchemes() - { return {scheme}; } + { return {"unix"}; } + + ref openStore() const override; }; -class UDSRemoteStore : public virtual UDSRemoteStoreConfig - , public virtual IndirectRootStore - , public virtual RemoteStore +struct UDSRemoteStore : + virtual IndirectRootStore, + virtual RemoteStore { -public: + using Config = UDSRemoteStoreConfig; - /** - * @deprecated This is the old API to construct the store. - */ - UDSRemoteStore(const Params & params); + ref config; - /** - * @param authority is the socket path. - */ - UDSRemoteStore( - std::string_view scheme, - std::string_view authority, - const Params & params); + UDSRemoteStore(ref); std::string getUri() override; diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 9d26c0b0578..82904ec0496 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -32,7 +32,7 @@ #include #if HAVE_STATVFS -#include +# include #endif /* Includes required for chroot support. */ @@ -56,9 +56,9 @@ #endif #if __APPLE__ -#include -#include -#include +# include +# include +# include /* This definition is undocumented but depended upon by all major browsers. */ extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, const char *const parameters[], char **errorbuf); @@ -219,7 +219,7 @@ Goal::Co LocalDerivationGoal::tryLocalBuild() } auto & localStore = getLocalStore(); - if (localStore.storeDir != localStore.realStoreDir.get()) { + if (localStore.storeDir != localStore.config->realStoreDir.get()) { #if __linux__ useChroot = true; #else @@ -359,7 +359,7 @@ bool LocalDerivationGoal::cleanupDecideWhetherDiskFull() auto & localStore = getLocalStore(); uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable struct statvfs st; - if (statvfs(localStore.realStoreDir.get().c_str(), &st) == 0 && + if (statvfs(localStore.config->realStoreDir.get().c_str(), &st) == 0 && (uint64_t) st.f_bavail * st.f_bsize < required) diskFull = true; if (statvfs(tmpDir.c_str(), &st) == 0 && @@ -537,7 +537,7 @@ void LocalDerivationGoal::startBuilder() concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), worker.store.printStorePath(drvPath), settings.thisSystem, - concatStringsSep(", ", worker.store.systemFeatures)); + concatStringsSep(", ", worker.store.config.systemFeatures)); /* Create a temporary directory where the build will take place. */ @@ -1305,33 +1305,46 @@ bool LocalDerivationGoal::isAllowed(const DerivedPath & req) return this->isAllowed(pathPartOfReq(req)); } - -struct RestrictedStoreConfig : virtual LocalFSStoreConfig +/** + * A wrapper around LocalStore that only allows building/querying of + * paths that are in the input closures of the build or were added via + * recursive Nix calls. + */ +struct RestrictedStore : + virtual IndirectRootStore, + virtual GcStore { - using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() override { return "Restricted Store"; } -}; + ref config; -/* A wrapper around LocalStore that only allows building/querying of - paths that are in the input closures of the build or were added via - recursive Nix calls. */ -struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual IndirectRootStore, public virtual GcStore -{ ref next; LocalDerivationGoal & goal; - RestrictedStore(const Params & params, ref next, LocalDerivationGoal & goal) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RestrictedStoreConfig(params) - , Store(params) - , LocalFSStore(params) - , next(next), goal(goal) - { } + RestrictedStore( + ref config, + ref next, + LocalDerivationGoal & goal) + : Store{*config} + , LocalFSStore{*config} + , config{config} + , next{next} + , goal{goal} + { + } + + static ref make( + ref next, + LocalDerivationGoal & goal) + { + ref config = make_ref(*next->config); + config->pathInfoCacheSize.value = 0; + config->stateDir.value = "/no-such-path"; + config->logDir.value = "/no-such-path"; + return make_ref(std::move(config), std::move(next), goal); + } Path getRealStoreDir() override - { return next->realStoreDir; } + { return next->config->realStoreDir; } std::string getUri() override { return next->getUri(); } @@ -1535,14 +1548,7 @@ void LocalDerivationGoal::startDaemon() { experimentalFeatureSettings.require(Xp::RecursiveNix); - Store::Params params; - params["path-info-cache-size"] = "0"; - params["store"] = worker.store.storeDir; - if (auto & optRoot = getLocalStore().rootDir.get()) - params["root"] = *optRoot; - params["state"] = "/no-such-path"; - params["log"] = "/no-such-path"; - auto store = make_ref(params, + auto store = RestrictedStore::make( ref(std::dynamic_pointer_cast(worker.store.shared_from_this())), *this); @@ -1884,7 +1890,7 @@ void LocalDerivationGoal::runChild() createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); - if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) + if (worker.store.config.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) ss.push_back("/dev/kvm"); ss.push_back("/dev/null"); ss.push_back("/dev/random"); diff --git a/src/libutil/config-abstract.hh b/src/libutil/config-abstract.hh new file mode 100644 index 00000000000..2c34ff50bdd --- /dev/null +++ b/src/libutil/config-abstract.hh @@ -0,0 +1,51 @@ +#pragma once +///@type + +#include + +namespace nix::config { + +template +struct JustValue +{ + T value; + + operator const T &() const + { + return value; + } + + operator T &() + { + return value; + } + + const T & get() const + { + return value; + } + + bool operator==(auto && v2) const + { + return value == v2; + } + + bool operator!=(auto && v2) const + { + return value != v2; + } +}; + +template +auto && operator<<(auto && str, const JustValue & opt) +{ + return str << opt.get(); +} + +template +struct OptValue +{ + std::optional optValue; +}; + +} diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 502d2823e94..2001d3f594a 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -180,13 +180,12 @@ public: const std::string name; const std::string description; const std::set aliases; + std::optional experimentalFeature; int created = 123; bool overridden = false; - std::optional experimentalFeature; - protected: AbstractSetting( diff --git a/src/libutil/json-impls.hh b/src/libutil/json-impls.hh index b26163a04ae..9dd344c508d 100644 --- a/src/libutil/json-impls.hh +++ b/src/libutil/json-impls.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "nlohmann/json_fwd.hpp" +#include // Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types #define JSON_IMPL(TYPE) \ diff --git a/src/libutil/meson.build b/src/libutil/meson.build index ac701d8fd3b..974feb04b6a 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -187,6 +187,7 @@ headers = [config_h] + files( 'comparator.hh', 'compression.hh', 'compute-levels.hh', + 'config-abstract.hh', 'config-global.hh', 'config-impl.hh', 'config.hh', diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 0d55cf93bed..e92d5a08a9c 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -218,16 +218,16 @@ std::pair getLine(std::string_view s); /** * Get a value for the specified key from an associate container. */ -template -const typename T::mapped_type * get(const T & map, const typename T::key_type & key) +template +const typename T::mapped_type * get(const T & map, K key) { auto i = map.find(key); if (i == map.end()) return nullptr; return &i->second; } -template -typename T::mapped_type * get(T & map, const typename T::key_type & key) +template +typename T::mapped_type * get(T & map, K key) { auto i = map.find(key); if (i == map.end()) return nullptr; @@ -237,9 +237,9 @@ typename T::mapped_type * get(T & map, const typename T::key_type & key) /** * Get a value for the specified key from an associate container, or a default value if the key isn't present. */ -template +template const typename T::mapped_type & getOr(T & map, - const typename T::key_type & key, + K key, const typename T::mapped_type & defaultValue) { auto i = map.find(key); diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index de01e1afcde..789498fb5bf 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -11,7 +11,7 @@ #include "current-process.hh" #include "parsed-derivations.hh" -#include "store-api.hh" +#include "store-open.hh" #include "local-fs-store.hh" #include "globals.hh" #include "realisation.hh" diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 56d1d7abb77..1cb397cd006 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -2,7 +2,7 @@ #include "shared.hh" #include "globals.hh" #include "filetransfer.hh" -#include "store-api.hh" +#include "store-open.hh" #include "legacy.hh" #include "eval-settings.hh" // for defexpr #include "users.hh" diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 20d5161df09..ab26e575482 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -1,6 +1,6 @@ #include "file-system.hh" #include "signals.hh" -#include "store-api.hh" +#include "store-open.hh" #include "store-cast.hh" #include "gc-store.hh" #include "profiles.hh" diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc index b64af758fcb..d910adcf876 100644 --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -1,6 +1,6 @@ #include "shared.hh" #include "realisation.hh" -#include "store-api.hh" +#include "store-open.hh" #include "legacy.hh" using namespace nix; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index c99c1088ebb..e00092af556 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -9,7 +9,7 @@ #include "profiles.hh" #include "path-with-outputs.hh" #include "shared.hh" -#include "store-api.hh" +#include "store-open.hh" #include "local-fs-store.hh" #include "user-env.hh" #include "value-to-json.hh" diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 09d35483205..dabec26bde1 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -8,7 +8,7 @@ #include "signals.hh" #include "value-to-xml.hh" #include "value-to-json.hh" -#include "store-api.hh" +#include "store-open.hh" #include "local-fs-store.hh" #include "common-eval-args.hh" #include "legacy.hh" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 99bb2c72601..84a2ed06af1 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -2,6 +2,7 @@ #include "derivations.hh" #include "dotgraph.hh" #include "globals.hh" +#include "store-open.hh" #include "store-cast.hh" #include "local-fs-store.hh" #include "log-store.hh" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index adf391b97be..56e94a5c000 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -8,7 +8,7 @@ #include "flake/flake.hh" #include "get-drvs.hh" #include "signals.hh" -#include "store-api.hh" +#include "store-open.hh" #include "derivations.hh" #include "outputs-spec.hh" #include "attr-path.hh" diff --git a/src/nix/log.cc b/src/nix/log.cc index 1a6f48f5e29..3c0fa2a8c13 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "common-args.hh" #include "shared.hh" -#include "store-api.hh" +#include "store-open.hh" #include "log-store.hh" #include "progress-bar.hh" diff --git a/src/nix/main.cc b/src/nix/main.cc index 80ef53084a4..cc3e9b193bc 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -7,7 +7,8 @@ #include "globals.hh" #include "legacy.hh" #include "shared.hh" -#include "store-api.hh" +#include "store-open.hh" +#include "store-registration.hh" #include "filetransfer.hh" #include "finally.hh" #include "loggers.hh" @@ -222,13 +223,12 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs res["args"] = toJSON(); auto stores = nlohmann::json::object(); - for (auto & implem : *Implementations::registered) { - auto storeConfig = implem.getConfig(); - auto storeName = storeConfig->name(); + for (auto & [storeName, implem] : *Implementations::registered) { auto & j = stores[storeName]; - j["doc"] = storeConfig->doc(); - j["settings"] = storeConfig->toJSON(); - j["experimentalFeature"] = storeConfig->experimentalFeature(); + j["doc"] = implem.doc; + j["uri-schemes"] = implem.uriSchemes; + j["settings"] = implem.configDescriptions; + j["experimentalFeature"] = implem.experimentalFeature; } res["stores"] = std::move(stores); res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo(); diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc index d9c988a9f5d..0c3efd7f499 100644 --- a/src/nix/make-content-addressed.cc +++ b/src/nix/make-content-addressed.cc @@ -1,5 +1,5 @@ #include "command.hh" -#include "store-api.hh" +#include "store-open.hh" #include "make-content-addressed.hh" #include "common-args.hh" diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index db7d9e4efe6..fa087430f86 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "common-args.hh" #include "shared.hh" -#include "store-api.hh" +#include "store-open.hh" #include "filetransfer.hh" #include "finally.hh" #include "progress-bar.hh" diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 5a570749f4c..0433624797f 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -2,6 +2,7 @@ #include "eval-settings.hh" #include "config-global.hh" #include "globals.hh" +#include "store-open.hh" #include "command.hh" #include "installable-value.hh" #include "repl.hh" diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 134d4f34a17..23a533fc028 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -1,7 +1,7 @@ #include "signals.hh" #include "command.hh" #include "shared.hh" -#include "store-api.hh" +#include "store-open.hh" #include "thread-pool.hh" #include "progress-bar.hh" diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index 746963a0103..c28c3563b66 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -7,6 +7,7 @@ #include "local-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" +#include "store-open.hh" #include "serialise.hh" #include "archive.hh" #include "globals.hh" @@ -243,9 +244,9 @@ static PeerInfo getPeerInfo(int remote) */ static ref openUncachedStore() { - Store::Params params; // FIXME: get params from somewhere + StoreReference::Params params; // FIXME: get params from somewhere // Disable caching since the client already does that. - params["path-info-cache-size"] = "0"; + params["path-info-cache-size"] = 0; return openStore(settings.storeUri, params); } diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 52585fe08d5..5b78eddfb01 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -1,6 +1,6 @@ #include "command.hh" #include "shared.hh" -#include "store-api.hh" +#include "store-open.hh" #include "thread-pool.hh" #include "signals.hh" #include "keys.hh" diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index 172c3500de0..a6e33645125 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -12,7 +12,7 @@ #include "derivations.hh" #include "realisation.hh" #include "globals.hh" -#include "store-api.hh" +#include "store-open.hh" #include "posix-source-accessor.hh" #include diff --git a/tests/functional/build-remote-with-mounted-ssh-ng.sh b/tests/functional/build-remote-with-mounted-ssh-ng.sh index e2627af394c..4c89b11d892 100755 --- a/tests/functional/build-remote-with-mounted-ssh-ng.sh +++ b/tests/functional/build-remote-with-mounted-ssh-ng.sh @@ -5,17 +5,25 @@ source common.sh requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" +# An example of a command that uses the store only for its settings, to +# make sure we catch needing the XP feature early. +touch "$TEST_ROOT/foo" +expect 1 nix --store 'ssh-ng://localhost?mounted=%7B%7D' store add "$TEST_ROOT/foo" --dry-run + enableFeatures mounted-ssh-store +# N.B. encoded query param is `mounted={}`. In the future, we can just +# do `--store` with JSON, and then the nested structure will actually +# bring benefits. nix build -Lvf simple.nix \ --arg busybox "$busybox" \ --out-link "$TEST_ROOT/result-from-remote" \ - --store mounted-ssh-ng://localhost + --store 'ssh-ng://localhost?mounted=%7B%7D' nix build -Lvf simple.nix \ --arg busybox "$busybox" \ --out-link "$TEST_ROOT/result-from-remote-new-cli" \ - --store 'mounted-ssh-ng://localhost?remote-program=nix daemon' + --store 'ssh-ng://localhost?mounted=%7B%7D&remote-program=nix daemon' # This verifies that the out link was actually created and valid. The ability # to create out links (permanent gc roots) is the distinguishing feature of diff --git a/tests/functional/test-libstoreconsumer/main.cc b/tests/functional/test-libstoreconsumer/main.cc index c61489af69a..00763038e9f 100644 --- a/tests/functional/test-libstoreconsumer/main.cc +++ b/tests/functional/test-libstoreconsumer/main.cc @@ -1,5 +1,5 @@ #include "globals.hh" -#include "store-api.hh" +#include "store-open.hh" #include "build-result.hh" #include