fix: Added line continuation '\' and newline ignore in paranthesis

alinalihassan · Mar 14, 2022 · 73d0134 · 73d0134
1 parent 8107903
commit 73d0134
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 8 deletions.
diff --git a/src/Frontend/Lexer.cpp b/src/Frontend/Lexer.cpp
@@ -4,29 +4,34 @@ using namespace lesma;
 
 std::vector<Token> Lexer::ScanAll() {
     while (tokens.empty() || tokens.back()->type != TokenType::EOF_TOKEN)
-        tokens.push_back(ScanOne());
+        tokens.push_back(ScanOne(false));
     return tokens;
 }
 
-Token Lexer::ScanOne() {
+Token Lexer::ScanOne(bool continuation) {
     if (IsAtEnd())
         return Token{TokenType::EOF_TOKEN, "EOF", Span{begin_loc, loc}};
     ResetTokenBeg();
     char c = Advance();
-    bool continuation = false;
 
     switch (c) {
         case '(':
+            level_++;
             return AddToken(TokenType::LEFT_PAREN);
         case ')':
+            level_--;
             return AddToken(TokenType::RIGHT_PAREN);
         case '[':
+            level_++;
             return AddToken(TokenType::LEFT_SQUARE);
         case ']':
+            level_--;
             return AddToken(TokenType::RIGHT_SQUARE);
         case '{':
+            level_++;
             return AddToken(TokenType::LEFT_BRACE);
         case '}':
+            level_--;
             return AddToken(TokenType::RIGHT_BRACE);
         case ':':
             return AddToken(TokenType::COLON);
@@ -94,23 +99,44 @@ Token Lexer::ScanOne() {
         case '#': {
             // A comment goes until the end of the line.
             while (Peek() != '\n' && !IsAtEnd()) Advance();
-            return ScanOne();
+            return ScanOne(continuation);
         }
+        case '\\':
+            c = Advance();
+            continuation = true;
+
+            // TODO: Keep it DRY
+            while (true) {
+                if (c == ' ' || c == '\r' || c == '\t')
+                    c = Advance();
+                else if (c == '#') {
+                    while (Peek() != '\n' && !IsAtEnd()) c = Advance();
+                    c = Advance();
+                    break;
+                } else
+                    break;
+            }
+
+            if (c != '\n')
+                Error(fmt::format("Newline expected after line continuation, found {}", c));
+
+            // Newline should be parsed next
+            Fallback();
+            return ScanOne(continuation);
         case ' ':
         case '\r':
         case '\t':
             HandleWhitespace(c);
             if (loc.Col == 2)
                 HandleIndentation(false);
-            return ScanOne();
+            return ScanOne(continuation);
         case '\n':
             loc.Line++;
             loc.Col = 1;
             if (!continuation && level_ == 0)
                 tokens.push_back(AddToken({TokenType::NEWLINE, "NEWLINE", Span{begin_loc, loc}}));
             HandleIndentation(continuation);
-            continuation = false;
-            return ScanOne();
+            return ScanOne(false);
         case '"':
             return AddStringToken();
         default:

diff --git a/src/Frontend/Lexer.h b/src/Frontend/Lexer.h
@@ -20,7 +20,7 @@ namespace lesma {
             : srcs_(&srcs), src_file_name(std::move(src_file_name)) {}
 
         std::vector<Token> ScanAll();
-        Token ScanOne();
+        Token ScanOne(bool continuation = false);
         std::vector<Token> getTokens() { return tokens; };
 
         void Reset() { Reset(*srcs_); }