Fine-grained access-tokens

src/libfetchers-tests/access-tokens.cc

@@ -0,0 +1,99 @@
+#include <gtest/gtest.h>
+#include "fetchers.hh"
+#include "fetch-settings.hh"
+#include "json-utils.hh"
+#include <nlohmann/json.hpp>
+#include "tests/characterization.hh"
+
+namespace nix::fetchers {
+
+using nlohmann::json;
+
+class AccessKeysTest : public ::testing::Test
+{
+protected:
+
+public:
+    void SetUp() override {
+      experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes);
+    }
+    void TearDown() override { }
+};
+
+TEST_F(AccessKeysTest, singleOrgGitHub)
+{
+    fetchers::Settings fetchSettings = fetchers::Settings{};
+    fetchSettings.accessTokens.get().insert({"github.com/a","token"});
+    auto i = Input::fromURL(fetchSettings, "github:a/b");
+
+    auto token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/b");
+    ASSERT_EQ(token,"token");
+}
+
+TEST_F(AccessKeysTest, nonMatches)
+{
+    fetchers::Settings fetchSettings = fetchers::Settings{};
+    fetchSettings.accessTokens.get().insert({"github.com","token"});
+    auto i = Input::fromURL(fetchSettings, "gitlab:github.com/evil");
+
+    auto token = i.scheme->getAccessToken(fetchSettings, "gitlab.com", "gitlab.com/github.com/evil");
+    ASSERT_EQ(token,std::nullopt);
+}
+
+TEST_F(AccessKeysTest, noPartialMatches)
+{
+    fetchers::Settings fetchSettings = fetchers::Settings{};
+    fetchSettings.accessTokens.get().insert({"github.com/partial","token"});
+    auto i = Input::fromURL(fetchSettings, "github:partial-match/repo");
+
+    auto token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/partial-match");
+    ASSERT_EQ(token,std::nullopt);
+}
+
+TEST_F(AccessKeysTest, repoGitHub)
+{
+    fetchers::Settings fetchSettings = fetchers::Settings{};
+    fetchSettings.accessTokens.get().insert({"github.com","token"});
+    fetchSettings.accessTokens.get().insert({"github.com/a/b","another_token"});
+    fetchSettings.accessTokens.get().insert({"github.com/a/c","yet_another_token"});
+    auto i = Input::fromURL(fetchSettings, "github:a/a");
+
+    auto token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/a");
+    ASSERT_EQ(token,"token");
+
+    token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/b");
+    ASSERT_EQ(token,"another_token");
+
+    token = i.scheme->getAccessToken(fetchSettings, "github.com", "github.com/a/c");
+    ASSERT_EQ(token,"yet_another_token");
+}
+
+TEST_F(AccessKeysTest, multipleGitLab)
+{
+    fetchers::Settings fetchSettings = fetchers::Settings{};
+    fetchSettings.accessTokens.get().insert({"gitlab.com","token"});
+    fetchSettings.accessTokens.get().insert({"gitlab.com/a/b","another_token"});
+    auto i = Input::fromURL(fetchSettings, "gitlab:a/b");
+
+    auto token = i.scheme->getAccessToken(fetchSettings, "gitlab.com", "gitlab.com/a/b");
+    ASSERT_EQ(token,"another_token");
+
+    token = i.scheme->getAccessToken(fetchSettings, "gitlab.com", "gitlab.com/a/c");
+    ASSERT_EQ(token,"token");
+}
+
+TEST_F(AccessKeysTest, multipleSourceHut)
+{
+    fetchers::Settings fetchSettings = fetchers::Settings{};
+    fetchSettings.accessTokens.get().insert({"git.sr.ht","token"});
+    fetchSettings.accessTokens.get().insert({"git.sr.ht/~a/b","another_token"});
+    auto i = Input::fromURL(fetchSettings, "sourcehut:a/b");
+
+    auto token = i.scheme->getAccessToken(fetchSettings, "git.sr.ht", "git.sr.ht/~a/b");
+    ASSERT_EQ(token,"another_token");
+
+    token = i.scheme->getAccessToken(fetchSettings, "git.sr.ht", "git.sr.ht/~a/c");
+    ASSERT_EQ(token,"token");
+}
+
+}

src/libfetchers-tests/meson.build

@@ -44,6 +44,7 @@ subdir('nix-meson-build-support/common')
 
 sources = files(
   'public-key.cc',
+  'access-tokens.cc',
 )
 
 include_dirs = [include_directories('.')]

src/libfetchers/fetch-settings.hh

@@ -23,9 +23,11 @@ struct Settings : public Config
           Access tokens are specified as a string made up of
           space-separated `host=token` values.  The specific token
           used is selected by matching the `host` portion against the
-          "host" specification of the input. The actual use of the
-          `token` value is determined by the type of resource being
-          accessed:
+          "host" specification of the input. The `host` portion may
+          contain a path element which will match against the prefix
+          URL for the input. (eg: `github.com/org=token`). The actual use
+          of the `token` value is determined by the type of resource
+          being accessed:
 
           * Github: the token value is the OAUTH-TOKEN string obtained
             as the Personal Access Token from the Github server (see

src/libfetchers/fetchers.hh

@@ -264,6 +264,9 @@ struct InputScheme
 
     virtual std::optional<std::string> isRelative(const Input & input) const
     { return std::nullopt; }
+
+    virtual std::optional<std::string> getAccessToken(const fetchers::Settings & settings, const std::string & host, const std::string & url) const
+    { return {};}
 };
 
 void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);

src/libfetchers/github.cc

@@ -172,20 +172,53 @@ struct GitArchiveInputScheme : InputScheme
         return input;
     }
 
-    std::optional<std::string> getAccessToken(const fetchers::Settings & settings, const std::string & host) const
+    // Search for the longest possible match starting from the begining and ending at either the end or a path segment.
+    std::optional<std::string> getAccessToken(const fetchers::Settings & settings, const std::string & host, const std::string & url) const override
     {
         auto tokens = settings.accessTokens.get();
+        std::string answer;
+        size_t answer_match_len = 0;
+        if(! url.empty()) {
+            for (auto & token : tokens) {
+                auto first = url.find(token.first);
+                if (
+                    first != std::string::npos
+                    && token.first.length() > answer_match_len
+                    && first == 0
+                    && url.substr(0,token.first.length()) == token.first
+                    && (url.length() == token.first.length() || url[token.first.length()] == '/')
+                    )
+                {
+                    answer = token.second;
+                    answer_match_len = token.first.length();
+                }
+            }
+            if (!answer.empty())
+                return answer;
+        }
         if (auto token = get(tokens, host))
             return *token;
         return {};
     }
 
     Headers makeHeadersWithAuthTokens(
         const fetchers::Settings & settings,
-        const std::string & host) const
+        const std::string & host,
+        const Input & input) const
+    {
+        auto owner = getStrAttr(input.attrs, "owner");
+        auto repo = getStrAttr(input.attrs, "repo");
+        auto urlGen = fmt( "%s/%s/%s", host, owner, repo);
+        return makeHeadersWithAuthTokens(settings, host, urlGen);
+    }
+
+    Headers makeHeadersWithAuthTokens(
+        const fetchers::Settings & settings,
+        const std::string & host,
+        const std::string & url) const
     {
         Headers headers;
-        auto accessToken = getAccessToken(settings, host);
+        auto accessToken = getAccessToken(settings, host, url);
         if (accessToken) {
             auto hdr = accessHeaderFromToken(*accessToken);
             if (hdr)
@@ -366,7 +399,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
             : "https://%s/api/v3/repos/%s/%s/commits/%s",
             host, getOwner(input), getRepo(input), *input.getRef());
 
-        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
 
         auto json = nlohmann::json::parse(
             readFile(
@@ -383,7 +416,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
     {
         auto host = getHost(input);
 
-        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
 
         // If we have no auth headers then we default to the public archive
         // urls so we do not run into rate limits.
@@ -440,7 +473,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
         auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
 
-        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
 
         auto json = nlohmann::json::parse(
             readFile(
@@ -470,7 +503,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(HashFormat::Base16, false));
 
-        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
         return DownloadUrl { url, headers };
     }
 
@@ -510,7 +543,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         auto base_url = fmt("https://%s/%s/%s",
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"));
 
-        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
 
         std::string refUri;
         if (ref == "HEAD") {
@@ -557,7 +590,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(HashFormat::Base16, false));
 
-        Headers headers = makeHeadersWithAuthTokens(*input.settings, host);
+        Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input);
         return DownloadUrl { url, headers };
     }