libnixf/Parse: split parsers (#346)

libnixf/meson.build

@@ -7,7 +7,12 @@ libnixf = library(
     'src/Basic/Nodes.cpp',
     'src/Basic/Diagnostic.cpp',
     'src/Parse/Lexer.cpp',
-    'src/Parse/Parser.cpp',
+    'src/Parse/ParseAttrs.cpp',
+    'src/Parse/ParseExpr.cpp',
+    'src/Parse/ParseLambda.cpp',
+    'src/Parse/ParseSimple.cpp',
+    'src/Parse/ParseStrings.cpp',
+    'src/Parse/ParseSupport.cpp',
     'src/Sema/Lowering.cpp',
     include_directories: libnixf_inc,
     dependencies: libnixf_deps,
@@ -39,10 +44,10 @@ test('unit/libnixf/Basic',
 test('unit/libnixf/Parse',
     executable('unit-libnixf-parse',
         'test/Parse/Lexer.cpp',
-        'test/Parse/Parser.cpp',
-        'test/Parse/ParserAttrs.cpp',
-        'test/Parse/ParserExprDispatch.cpp',
-        'test/Parse/ParserLambda.cpp',
+        'test/Parse/ParseAttrs.cpp',
+        'test/Parse/ParseExpr.cpp',
+        'test/Parse/ParseLambda.cpp',
+        'test/Parse/ParseSimple.cpp',
         dependencies: [ nixf, gtest_main ],
         include_directories: [ 'src/Parse' ] # Private headers
     )

libnixf/src/Parse/ParseAttrs.cpp

@@ -0,0 +1,195 @@
+#include "Parser.h"
+
+namespace nixf {
+
+using namespace detail;
+
+std::unique_ptr<AttrName> Parser::parseAttrName() {
+  switch (Token Tok = peek(); Tok.kind()) {
+  case tok_kw_or:
+    Diags.emplace_back(Diagnostic::DK_OrIdentifier, Tok.range());
+    [[fallthrough]];
+  case tok_id: {
+    consume();
+    auto ID =
+        std::make_unique<Identifier>(Tok.range(), std::string(Tok.view()));
+    return std::make_unique<AttrName>(std::move(ID), Tok.range());
+  }
+  case tok_dquote: {
+    std::unique_ptr<ExprString> String = parseString(/*IsIndented=*/false);
+    return std::make_unique<AttrName>(std::move(String));
+  }
+  case tok_dollar_curly: {
+    std::unique_ptr<Interpolation> Expr = parseInterpolation();
+    return std::make_unique<AttrName>(std::move(Expr));
+  }
+  default:
+    return nullptr;
+  }
+}
+
+std::unique_ptr<AttrPath> Parser::parseAttrPath() {
+  auto First = parseAttrName();
+  if (!First)
+    return nullptr;
+  LexerCursor Begin = First->lCur();
+  assert(LastToken && "LastToken should be set after valid attrname");
+  std::vector<std::unique_ptr<AttrName>> AttrNames;
+  AttrNames.emplace_back(std::move(First));
+  while (true) {
+    if (Token Tok = peek(); Tok.kind() == tok_dot) {
+      consume();
+      auto Next = parseAttrName();
+      if (!Next) {
+        // extra ".", consider remove it.
+        Diagnostic &D =
+            Diags.emplace_back(Diagnostic::DK_AttrPathExtraDot, Tok.range());
+        D.fix("remove extra .").edit(TextEdit::mkRemoval(Tok.range()));
+        D.fix("insert dummy attrname")
+            .edit(TextEdit::mkInsertion(Tok.rCur(), R"("dummy")"));
+      }
+      AttrNames.emplace_back(std::move(Next));
+      continue;
+    }
+    break;
+  }
+  return std::make_unique<AttrPath>(LexerCursorRange{Begin, LastToken->rCur()},
+                                    std::move(AttrNames));
+}
+
+std::unique_ptr<Binding> Parser::parseBinding() {
+  auto Path = parseAttrPath();
+  if (!Path)
+    return nullptr;
+  assert(LastToken && "LastToken should be set after valid attrpath");
+  auto SyncEq = withSync(tok_eq);
+  auto SyncSemi = withSync(tok_semi_colon);
+  if (ExpectResult ER = expect(tok_eq); !ER.ok())
+    return std::make_unique<Binding>(
+        LexerCursorRange{Path->lCur(), LastToken->rCur()}, std::move(Path),
+        nullptr);
+  consume();
+  auto Expr = parseExpr();
+  if (!Expr)
+    diagNullExpr(Diags, LastToken->rCur(), "binding");
+  if (Token Tok = peek(); Tok.kind() == tok_semi_colon) {
+    consume();
+  } else {
+    // TODO: reset the cursor for error recovery.
+    // (https://github.com/nix-community/nixd/blob/2b0ca8cef0d13823132a52b6cd6f6d7372482664/libnixf/lib/Parse/Parser.cpp#L337)
+    // expected ";" for binding
+    Diagnostic &D = Diags.emplace_back(Diagnostic::DK_Expected,
+                                       LexerCursorRange(LastToken->rCur()));
+    D << std::string(tok::spelling(tok_semi_colon));
+    D.fix("insert ;").edit(TextEdit::mkInsertion(LastToken->rCur(), ";"));
+  }
+  return std::make_unique<Binding>(
+      LexerCursorRange{Path->lCur(), LastToken->rCur()}, std::move(Path),
+      std::move(Expr));
+}
+
+std::unique_ptr<Inherit> Parser::parseInherit() {
+  Token TokInherit = peek();
+  if (TokInherit.kind() != tok_kw_inherit)
+    return nullptr;
+  consume();
+  auto SyncSemiColon = withSync(tok_semi_colon);
+
+  // These tokens might be consumed as "inherited_attrs"
+  auto SyncID = withSync(tok_id);
+  auto SyncQuote = withSync(tok_dquote);
+  auto SyncDollarCurly = withSync(tok_dollar_curly);
+
+  assert(LastToken && "LastToken should be set after consume()");
+  std::vector<std::unique_ptr<AttrName>> AttrNames;
+  std::unique_ptr<Expr> Expr = nullptr;
+  if (Token Tok = peek(); Tok.kind() == tok_l_paren) {
+    consume();
+    Expr = parseExpr();
+    if (!Expr)
+      diagNullExpr(Diags, LastToken->rCur(), "inherit");
+    if (ExpectResult ER = expect(tok_r_paren); ER.ok())
+      consume();
+    else
+      ER.diag().note(Note::NK_ToMachThis, Tok.range())
+          << std::string(tok::spelling(tok_l_paren));
+  }
+  while (true) {
+    if (auto AttrName = parseAttrName()) {
+      AttrNames.emplace_back(std::move(AttrName));
+      continue;
+    }
+    break;
+  }
+  if (ExpectResult ER = expect(tok_semi_colon); ER.ok())
+    consume();
+  else
+    ER.diag().note(Note::NK_ToMachThis, TokInherit.range())
+        << std::string(tok::spelling(tok_kw_inherit));
+  return std::make_unique<Inherit>(
+      LexerCursorRange{TokInherit.lCur(), LastToken->rCur()},
+      std::move(AttrNames), std::move(Expr));
+}
+
+std::unique_ptr<Binds> Parser::parseBinds() {
+  // TODO: curently we don't support inherit
+  LexerCursor Begin = peek().lCur();
+  std::vector<std::unique_ptr<Node>> Bindings;
+  // attrpath
+  auto SyncID = withSync(tok_id);
+  auto SyncQuote = withSync(tok_dquote);
+  auto SyncDollarCurly = withSync(tok_dollar_curly);
+
+  // inherit
+  auto SyncInherit = withSync(tok_kw_inherit);
+  while (true) {
+    if (auto Binding = parseBinding()) {
+      Bindings.emplace_back(std::move(Binding));
+      continue;
+    }
+    if (auto Inherit = parseInherit()) {
+      Bindings.emplace_back(std::move(Inherit));
+      continue;
+    }
+    // If it is neither a binding, nor an inherit. Let's consume an "Unknown"
+    // For error recovery
+    if (removeUnexpected())
+      continue;
+    break;
+  }
+  if (Bindings.empty())
+    return nullptr;
+  assert(LastToken && "LastToken should be set after valid binding");
+  return std::make_unique<Binds>(LexerCursorRange{Begin, LastToken->rCur()},
+                                 std::move(Bindings));
+}
+
+std::unique_ptr<ExprAttrs> Parser::parseExprAttrs() {
+  std::unique_ptr<Misc> Rec;
+
+  auto Sync = withSync(tok_r_curly);
+
+  // "to match this ..."
+  // if "{" is missing, then use "rec", otherwise use "{"
+  Token Matcher = peek();
+  LexerCursor Begin = peek().lCur(); // rec or {
+  if (Token Tok = peek(); Tok.kind() == tok_kw_rec) {
+    consume();
+    Rec = std::make_unique<Misc>(Tok.range());
+  }
+  if (ExpectResult ER = expect(tok_l_curly); ER.ok()) {
+    consume();
+    Matcher = ER.tok();
+  }
+  assert(LastToken && "LastToken should be set after valid { or rec");
+  auto Binds = parseBinds();
+  if (ExpectResult ER = expect(tok_r_curly); ER.ok())
+    consume();
+  else
+    ER.diag().note(Note::NK_ToMachThis, Matcher.range())
+        << std::string(tok::spelling(Matcher.kind()));
+  return std::make_unique<ExprAttrs>(LexerCursorRange{Begin, LastToken->rCur()},
+                                     std::move(Binds), std::move(Rec));
+}
+
+} // namespace nixf

libnixf/src/Parse/ParseExpr.cpp

@@ -0,0 +1,115 @@
+#include "Parser.h"
+
+namespace nixf {
+
+using namespace detail;
+
+std::unique_ptr<Expr> Parser::parseExprSelect() {
+  std::unique_ptr<Expr> Expr = parseExprSimple();
+  if (!Expr)
+    return nullptr;
+  assert(LastToken && "LastToken should be set after valid expr");
+  LexerCursor Begin = Expr->lCur();
+
+  Token Tok = peek();
+  if (Tok.kind() != tok_dot) {
+    // expr_select : expr_simple
+    return Expr;
+  }
+
+  // expr_select : expr_simple '.' attrpath
+  //             | expr_simple '.' attrpath 'or' expr_select
+  consume(); // .
+  auto Path = parseAttrPath();
+  if (!Path) {
+    // extra ".", consider remove it.
+    Diagnostic &D =
+        Diags.emplace_back(Diagnostic::DK_SelectExtraDot, Tok.range());
+    D.fix("remove extra .").edit(TextEdit::mkRemoval(Tok.range()));
+    D.fix("insert dummy attrpath")
+        .edit(TextEdit::mkInsertion(Tok.rCur(), R"("dummy")"));
+  }
+
+  Token TokOr = peek();
+  if (TokOr.kind() != tok_kw_or) {
+    // expr_select : expr_simple '.' attrpath
+    return std::make_unique<ExprSelect>(
+        LexerCursorRange{Begin, LastToken->rCur()}, std::move(Expr),
+        std::move(Path), /*Default=*/nullptr);
+  }
+
+  // expr_select : expr_simple '.' attrpath 'or' expr_select
+  consume(); // `or`
+  auto Default = parseExprSelect();
+  if (!Default) {
+    Diagnostic &D = diagNullExpr(Diags, LastToken->rCur(), "default");
+    D.fix("remove `or` keyword").edit(TextEdit::mkRemoval(TokOr.range()));
+  }
+  return std::make_unique<ExprSelect>(
+      LexerCursorRange{Begin, LastToken->rCur()}, std::move(Expr),
+      std::move(Path), std::move(Default));
+}
+
+std::unique_ptr<Expr> Parser::parseExprApp(int Limit) {
+  std::unique_ptr<Expr> Fn = parseExprSelect();
+  // If fn cannot be evaluated to lambda, exit early.
+  if (!Fn || !Fn->maybeLambda())
+    return Fn;
+
+  std::vector<std::unique_ptr<Expr>> Args;
+  while (Limit--) {
+    std::unique_ptr<Expr> Arg = parseExprSelect();
+    if (!Arg)
+      break;
+    Args.emplace_back(std::move(Arg));
+  }
+
+  if (Args.empty())
+    return Fn;
+  return std::make_unique<ExprCall>(
+      LexerCursorRange{Fn->lCur(), Args.back()->rCur()}, std::move(Fn),
+      std::move(Args));
+}
+
+std::unique_ptr<Expr> Parser::parseExpr() {
+  // Look ahead 3 tokens.
+  switch (peek().kind()) {
+  case tok_id: {
+    switch (peek(1).kind()) {
+    case tok_at:    // ID @
+    case tok_colon: // ID :
+      return parseExprLambda();
+    default:
+      break;
+    }
+    break;
+  }
+  case tok_l_curly: {
+    switch (peek(1).kind()) {
+    case tok_id: {
+      switch (peek(2).kind()) {
+      case tok_colon:    // { a :
+      case tok_at:       // { a @
+      case tok_question: // { a ?
+      case tok_comma:    // { a ,
+      case tok_id:       // { a b
+      case tok_ellipsis: // { a ...
+        return parseExprLambda();
+      default:
+        break;
+      }
+      break;
+    }
+    default:
+      break;
+    }
+    break;
+  } // case tok_l_curly
+  default:
+    break;
+  }
+
+  return parseExprApp();
+}
+
+} // namespace nixf