Lazy paths

src/libcmd/common-eval-args.cc

@@ -32,8 +32,7 @@ EvalSettings evalSettings {
                 // FIXME `parseFlakeRef` should take a `std::string_view`.
                 auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false);
                 debug("fetching flake search path element '%s''", rest);
-                auto storePath = flakeRef.resolve(store).fetchTree(store).first;
-                return store->toRealPath(storePath);
+                return flakeRef.resolve(store).lazyFetch(store).first;
             },
         },
     },
@@ -174,15 +173,15 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas
             state.store,
             state.fetchSettings,
             EvalSettings::resolvePseudoUrl(s));
-        auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy);
-        return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
+        state.registerAccessor(accessor);
+        return SourcePath(accessor);
     }
 
     else if (hasPrefix(s, "flake:")) {
         experimentalFeatureSettings.require(Xp::Flakes);
         auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false);
-        auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first;
-        return state.rootPath(CanonPath(state.store->toRealPath(storePath)));
+        auto accessor = flakeRef.resolve(state.store).lazyFetch(state.store).first;
+        return SourcePath(accessor);
     }
 
     else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {

src/libexpr/eval-settings.hh

@@ -7,6 +7,7 @@
 namespace nix {
 
 class Store;
+struct SourcePath;
 
 struct EvalSettings : Config
 {
@@ -22,7 +23,7 @@ struct EvalSettings : Config
      * @todo Return (`std::optional` of) `SourceAccssor` or something
      * more structured instead of mere `std::string`?
      */
-    using LookupPathHook = std::optional<std::string>(ref<Store> store, std::string_view);
+    using LookupPathHook = std::optional<SourcePath>(ref<Store> store, std::string_view);
 
     /**
      * Map from "scheme" to a `LookupPathHook`.

src/libexpr/eval.cc

@@ -315,6 +315,7 @@ EvalState::EvalState(
 {
     corepkgsFS->setPathDisplay("<nix", ">");
     internalFS->setPathDisplay("«nix-internal»", "");
+    rootFS->setPathDisplay("/", "");
 
     countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0";
 
@@ -859,6 +860,11 @@ void Value::mkPath(const SourcePath & path)
     mkPath(&*path.accessor, makeImmutableString(path.path.abs()));
 }
 
+void EvalState::registerAccessor(const ref<SourceAccessor> accessor)
+{
+    sourceAccessors.push_back(accessor);
+}
+
 
 inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
 {
@@ -1973,10 +1979,22 @@ void EvalState::concatLists(Value & v, size_t nrLists, Value * const * lists, co
     v.mkList(list);
 }
 
+// FIXME limit recursion
+Value * resolveOutPath(EvalState & state, Value * v, const PosIdx pos)
+{
+    state.forceValue(*v, pos);
+    if (v->type() != nAttrs)
+        return v;
+    auto found = v->attrs()->find(state.sOutPath);
+    if (found != v->attrs()->end())
+        return resolveOutPath(state, found->value, pos);
+    return v;
+}
 
 void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
 {
     NixStringContext context;
+    std::shared_ptr<SourceAccessor> accessor;
     std::vector<BackedStringView> s;
     size_t sSize = 0;
     NixInt n{0};
@@ -2010,15 +2028,19 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     Value * vTmpP = values.data();
 
     for (auto & [i_pos, i] : *es) {
-        Value & vTmp = *vTmpP++;
-        i->eval(state, env, vTmp);
+        Value & vTmp0 = *vTmpP++;
+        i->eval(state, env, vTmp0);
+        Value & vTmp = *resolveOutPath(state, &vTmp0, i_pos);
 
         /* If the first element is a path, then the result will also
            be a path, we don't copy anything (yet - that's done later,
            since paths are copied when they are used in a derivation),
            and none of the strings are allowed to have contexts. */
         if (first) {
             firstType = vTmp.type();
+            if (firstType == nPath) {
+                accessor = vTmp.path().accessor;
+            }
         }
 
         if (firstType == nInt) {
@@ -2065,7 +2087,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     else if (firstType == nPath) {
         if (!context.empty())
             state.error<EvalError>("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow();
-        v.mkPath(state.rootPath(CanonPath(canonPath(str()))));
+        v.mkPath({ref(accessor), CanonPath(str())});
     } else
         v.mkStringMove(c_str(), context);
 }
@@ -2315,6 +2337,8 @@ BackedStringView EvalState::coerceToString(
               v.payload.path.path
             : copyToStore
             ? store->printStorePath(copyPathToStore(context, v.path()))
+            : v.path().accessor->toStringReturnsStorePath()
+            ? store->printStorePath(copyPathToStore(context, SourcePath(v.path().accessor, CanonPath::root))) + v.path().path.absOrEmpty()
             : std::string(v.path().path.abs());
     }
 
@@ -2427,10 +2451,14 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext
     if (v.type() == nPath)
         return v.path();
 
-    /* Similarly, handle __toString where the result may be a path
+    /* Similarly, handle outPath and __toString where the result may be a path
        value. */
     if (v.type() == nAttrs) {
-        auto i = v.attrs()->find(sToString);
+        auto i = v.attrs()->find(sOutPath);
+        if (i != v.attrs()->end()) {
+            return coerceToPath(pos, *i->value, context, errorCtx);
+        }
+        i = v.attrs()->find(sToString);
         if (i != v.attrs()->end()) {
             Value v1;
             callFunction(*i->value, v, v1, pos);
@@ -3053,12 +3081,14 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_
         if (!suffixOpt) continue;
         auto suffix = *suffixOpt;
 
-        auto rOpt = resolveLookupPathPath(i.path);
+        auto rOpt = resolveLookupPathPath(
+                i.path,
+                true);
         if (!rOpt) continue;
         auto r = *rOpt;
 
-        Path res = suffix == "" ? r : concatStrings(r, "/", suffix);
-        if (pathExists(res)) return rootPath(CanonPath(canonPath(res)));
+        auto res = r / CanonPath(suffix);
+        if (res.pathExists()) return res;
     }
 
     if (hasPrefix(path, "nix/"))
@@ -3073,26 +3103,30 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_
 }
 
 
-std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl)
+std::optional<SourcePath> EvalState::resolveLookupPathPath(const LookupPath::Path & value0, bool initAccessControl)
 {
     auto & value = value0.s;
     auto i = lookupPathResolved.find(value);
     if (i != lookupPathResolved.end()) return i->second;
 
-    auto finish = [&](std::string res) {
+    auto finish = [&](SourcePath res) {
         debug("resolved search path element '%s' to '%s'", value, res);
         lookupPathResolved.emplace(value, res);
         return res;
     };
 
+    std::optional<SourcePath> res;
+
     if (EvalSettings::isPseudoUrl(value)) {
         try {
             auto accessor = fetchers::downloadTarball(
                 store,
                 fetchSettings,
                 EvalSettings::resolvePseudoUrl(value));
+            // Traditional search path lookups use the absolute path space for
+            // historical consistency.
             auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy);
-            return finish(store->toRealPath(storePath));
+            res.emplace(rootPath(CanonPath(store->toRealPath(storePath))));
         } catch (Error & e) {
             logWarning({
                 .msg = HintFmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value)
@@ -3111,22 +3145,22 @@ std::optional<std::string> EvalState::resolveLookupPathPath(const LookupPath::Pa
     }
 
     {
-        auto path = absPath(value);
+        auto path = rootPath(value);
 
         /* Allow access to paths in the search path. */
         if (initAccessControl) {
-            allowPath(path);
-            if (store->isInStore(path)) {
+            allowPath(path.path.abs());
+            if (store->isInStore(path.path.abs())) {
                 try {
                     StorePathSet closure;
-                    store->computeFSClosure(store->toStorePath(path).first, closure);
+                    store->computeFSClosure(store->toStorePath(path.path.abs()).first, closure);
                     for (auto & p : closure)
                         allowPath(p);
                 } catch (InvalidPath &) { }
             }
         }
 
-        if (pathExists(path))
+        if (path.pathExists())
             return finish(std::move(path));
         else {
             logWarning({

src/libexpr/eval.hh

@@ -249,6 +249,9 @@ public:
 
     const SourcePath callFlakeInternal;
 
+    /* A collection of InputAccessors, just to keep them alive. */
+    std::list<ref<SourceAccessor>> sourceAccessors;
+
     /**
      * Store used to materialise .drv files.
      */
@@ -339,7 +342,7 @@ private:
 
     LookupPath lookupPath;
 
-    std::map<std::string, std::optional<std::string>> lookupPathResolved;
+    std::map<std::string, std::optional<SourcePath>> lookupPathResolved;
 
     /**
      * Cache used by prim_match().
@@ -381,6 +384,8 @@ public:
      */
     SourcePath rootPath(PathView path);
 
+    void registerAccessor(ref<SourceAccessor> accessor);
+
     /**
      * Allow access to a path.
      */
@@ -446,7 +451,7 @@ public:
      *
      * If it is not found, return `std::nullopt`
      */
-    std::optional<std::string> resolveLookupPathPath(
+    std::optional<SourcePath> resolveLookupPathPath(
         const LookupPath::Path & elem,
         bool initAccessControl = false);
 

src/libexpr/nixexpr.cc

@@ -45,7 +45,7 @@ void ExprString::show(const SymbolTable & symbols, std::ostream & str) const
 
 void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const
 {
-    str << s;
+    str << path;
 }
 
 void ExprVar::show(const SymbolTable & symbols, std::ostream & str) const

src/libexpr/nixexpr.hh

@@ -131,12 +131,15 @@ struct ExprString : Expr
 
 struct ExprPath : Expr
 {
-    ref<SourceAccessor> accessor;
-    std::string s;
+    const SourcePath path;
     Value v;
-    ExprPath(ref<SourceAccessor> accessor, std::string s) : accessor(accessor), s(std::move(s))
+    ExprPath(SourcePath && path)
+        : path(path)
     {
-        v.mkPath(&*accessor, this->s.c_str());
+        v.mkPath(
+            &*path.accessor,
+            // TODO: GC_STRDUP
+            strdup(path.path.abs().c_str()));
     }
     Value * maybeThunk(EvalState & state, Env & env) override;
     COMMON_METHODS

src/libexpr/parser.y

@@ -136,6 +136,10 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
   std::vector<std::pair<nix::AttrName, nix::PosIdx>> * inheritAttrs;
   std::vector<std::pair<nix::PosIdx, nix::Expr *>> * string_parts;
   std::vector<std::pair<nix::PosIdx, std::variant<nix::Expr *, nix::StringToken>>> * ind_string_parts;
+  struct {
+      nix::Expr * e;
+      bool appendSlash;
+  } pathStart;
 }
 
 %type <e> start expr expr_function expr_if expr_op
@@ -149,7 +153,8 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) {
 %type <inheritAttrs> attrs
 %type <string_parts> string_parts_interpolated
 %type <ind_string_parts> ind_string_parts
-%type <e> path_start string_parts string_attr
+%type <pathStart> path_start
+%type <e> string_parts string_attr
 %type <id> attr
 %token <id> ID
 %token <str> STR IND_STR
@@ -295,9 +300,11 @@ expr_simple
       $$ = state->stripIndentation(CUR_POS, std::move(*$2));
       delete $2;
   }
-  | path_start PATH_END
+  | path_start PATH_END { $$ = $1.e; }
   | path_start string_parts_interpolated PATH_END {
-      $2->insert($2->begin(), {state->at(@1), $1});
+      if ($1.appendSlash)
+          $2->insert($2->begin(), {noPos, new ExprString("/")});
+      $2->insert($2->begin(), {state->at(@1), $1.e});
       $$ = new ExprConcatStrings(CUR_POS, false, $2);
   }
   | SPATH {
@@ -350,11 +357,17 @@ string_parts_interpolated
 
 path_start
   : PATH {
-    Path path(absPath({$1.p, $1.l}, state->basePath.path.abs()));
-    /* add back in the trailing '/' to the first segment */
-    if ($1.p[$1.l-1] == '/' && $1.l > 1)
-      path += "/";
-    $$ = new ExprPath(ref<SourceAccessor>(state->rootFS), std::move(path));
+      std::string_view path({$1.p, $1.l});
+      $$ = {
+          .e = new ExprPath(
+              /* Absolute paths are always interpreted relative to the
+                 root filesystem accessor, rather than the accessor of the
+                 current Nix expression. */
+              hasPrefix(path, "/")
+              ? SourcePath{state->rootFS, CanonPath(path)}
+              : SourcePath{state->basePath.accessor, CanonPath(path, state->basePath.path)}),
+          .appendSlash = hasSuffix(path, "/")
+      };
   }
   | HPATH {
     if (state->settings.pureEval) {
@@ -363,8 +376,8 @@ path_start
             std::string_view($1.p, $1.l)
         );
     }
-    Path path(getHome() + std::string($1.p + 1, $1.l - 1));
-    $$ = new ExprPath(ref<SourceAccessor>(state->rootFS), std::move(path));
+    CanonPath path(getHome() + std::string($1.p + 1, $1.l - 1));
+    $$ = {.e = new ExprPath(SourcePath{state->rootFS, std::move(path)}), .appendSlash = true};
   }
   ;
 

src/libexpr/primops.cc

@@ -178,7 +178,6 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
     auto path = realisePath(state, pos, vPath, std::nullopt);
     auto path2 = path.path.abs();
 
-    // FIXME
     auto isValidDerivationInStore = [&]() -> std::optional<StorePath> {
         if (!state.store->isStorePath(path2))
             return std::nullopt;
@@ -216,9 +215,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v
         state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh");
         v.mkApp(*state.vImportedDrvToDerivation, w);
         state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh");
-    }
-
-    else {
+    } else {
         if (!vScope)
             state.evalFile(path, v);
         else {
@@ -1700,7 +1697,10 @@ static std::string_view legacyBaseNameOf(std::string_view path)
 static void prim_baseNameOf(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     NixStringContext context;
-    v.mkString(legacyBaseNameOf(*state.coerceToString(pos, *args[0], context,
+    if (v.type() == nPath)
+        v.mkString(baseNameOf(v.path().path.abs()), context);
+    else
+        v.mkString(legacyBaseNameOf(*state.coerceToString(pos, *args[0], context,
             "while evaluating the first argument passed to builtins.baseNameOf",
             false, false)), context);
 }