Add meta-information to lexer

XChy · Aug 7, 2023 · 0237141 · 0237141
1 parent f86a9ae
commit 0237141
Show file tree

Hide file tree

Showing 9 changed files with 706 additions and 705 deletions.
diff --git a/XSharp/Lexer.cpp b/XSharp/Lexer.cpp
@@ -30,6 +30,7 @@ Token Lexer::fetchFrom(XString::const_iterator& iter, Span& span)
     while (cur().isSpace() || cur() == '\n') {
         if (cur() == '\n') {
             span.row++;
+            span.col = 0;
         }
         next();
     }

diff --git a/XSharp/test/Lexer/arithmetic.xsharp_tokens.txt b/XSharp/test/Lexer/arithmetic.xsharp_tokens.txt
@@ -2,121 +2,121 @@
 1:6 Identifier:'main', 
 1:10 OpenParenthesis:'(', 
 1:11 CloseParenthesis:')', 
-2:13 OpenBrace:'{', 
-3:19 Identifier:'i32', 
-3:23 Identifier:'n', 
-3:24 SentenceEnd:';', 
-4:30 Identifier:'n', 
-4:32 Operator:'=', 
-4:34 Identifier:'inputI32', 
-4:42 OpenParenthesis:'(', 
-4:43 CloseParenthesis:')', 
-4:44 SentenceEnd:';', 
-6:51 Identifier:'i32', 
-6:55 Identifier:'i', 
-6:56 Operator:'=', 
-6:57 Integer:'0', 
-6:58 SentenceEnd:';', 
-7:64 Identifier:'i32', 
-7:68 Identifier:'a', 
-7:69 Operator:'=', 
-7:70 Integer:'1', 
-7:71 SentenceEnd:';', 
-8:77 Identifier:'i32', 
-8:81 Identifier:'b', 
-8:82 Operator:'=', 
-8:83 Integer:'1', 
-8:84 SentenceEnd:';', 
-9:90 Identifier:'i32', 
-9:94 Identifier:'c', 
-9:95 Operator:'=', 
-9:96 Integer:'1', 
-9:97 SentenceEnd:';', 
-10:103 Identifier:'u8', 
-10:106 Identifier:'d', 
-10:108 Operator:'=', 
-10:110 Integer:'0', 
-10:111 SentenceEnd:';', 
-11:117 Keyword:'while', 
-11:123 OpenParenthesis:'(', 
-11:124 Identifier:'i', 
-11:125 Operator:'<', 
-11:126 Identifier:'n', 
-11:127 CloseParenthesis:')', 
-11:128 OpenBrace:'{', 
-12:138 Identifier:'a', 
-12:140 Operator:'=', 
-12:142 Identifier:'c', 
-12:144 Operator:'=', 
-12:146 OpenParenthesis:'(', 
-12:147 Operator:'-', 
-12:148 Identifier:'a', 
-12:149 CloseParenthesis:')', 
-12:150 Operator:'+', 
-12:151 Identifier:'b', 
-12:152 Operator:'*', 
-12:153 Identifier:'c', 
-12:154 Operator:'/', 
-12:155 Integer:'66', 
-12:157 SentenceEnd:';', 
-13:167 Identifier:'a', 
-13:169 Operator:'=', 
-13:171 Identifier:'b', 
-13:173 Operator:'%', 
-13:175 Integer:'3', 
-13:177 Operator:'/', 
-13:179 Integer:'2', 
-13:181 Operator:'+', 
-13:183 Integer:'4', 
-13:185 Operator:'+', 
-13:187 OpenParenthesis:'(', 
-13:188 Identifier:'b', 
-13:190 Operator:'=', 
-13:192 Integer:'4', 
-13:193 CloseParenthesis:')', 
-13:194 SentenceEnd:';', 
-14:204 Identifier:'b', 
-14:206 Operator:'=', 
-14:208 Identifier:'c', 
-14:209 SentenceEnd:';', 
-15:219 Identifier:'i', 
-15:220 Operator:'=', 
-15:221 Identifier:'i', 
-15:222 Operator:'+', 
-15:223 Integer:'1', 
-15:224 SentenceEnd:';', 
-16:230 CloseBrace:'}', 
-17:236 Identifier:'print', 
-17:241 OpenParenthesis:'(', 
-17:242 Identifier:'a', 
-17:243 CloseParenthesis:')', 
-17:244 SentenceEnd:';', 
-18:250 Identifier:'print', 
-18:255 OpenParenthesis:'(', 
-18:256 Identifier:'b', 
-18:257 CloseParenthesis:')', 
-18:258 SentenceEnd:';', 
-19:264 Identifier:'print', 
-19:269 OpenParenthesis:'(', 
-19:270 Identifier:'c', 
-19:271 CloseParenthesis:')', 
-19:272 SentenceEnd:';', 
-20:278 Identifier:'print', 
-20:283 OpenParenthesis:'(', 
-20:284 Operator:'-', 
-20:285 Identifier:'c', 
-20:286 CloseParenthesis:')', 
-20:287 SentenceEnd:';', 
-21:293 Identifier:'print', 
-21:298 OpenParenthesis:'(', 
-21:299 Operator:'!', 
-21:300 OpenParenthesis:'(', 
-21:301 Identifier:'c', 
-21:303 Operator:'==', 
-21:306 Operator:'-', 
-21:307 Integer:'1', 
-21:308 CloseParenthesis:')', 
-21:309 CloseParenthesis:')', 
-21:310 SentenceEnd:';', 
-22:312 CloseBrace:'}', 
-23:314 Eof:'Eof'
+2:1 OpenBrace:'{', 
+3:5 Identifier:'i32', 
+3:9 Identifier:'n', 
+3:10 SentenceEnd:';', 
+4:5 Identifier:'n', 
+4:7 Operator:'=', 
+4:9 Identifier:'inputI32', 
+4:17 OpenParenthesis:'(', 
+4:18 CloseParenthesis:')', 
+4:19 SentenceEnd:';', 
+6:5 Identifier:'i32', 
+6:9 Identifier:'i', 
+6:10 Operator:'=', 
+6:11 Integer:'0', 
+6:12 SentenceEnd:';', 
+7:5 Identifier:'i32', 
+7:9 Identifier:'a', 
+7:10 Operator:'=', 
+7:11 Integer:'1', 
+7:12 SentenceEnd:';', 
+8:5 Identifier:'i32', 
+8:9 Identifier:'b', 
+8:10 Operator:'=', 
+8:11 Integer:'1', 
+8:12 SentenceEnd:';', 
+9:5 Identifier:'i32', 
+9:9 Identifier:'c', 
+9:10 Operator:'=', 
+9:11 Integer:'1', 
+9:12 SentenceEnd:';', 
+10:5 Identifier:'u8', 
+10:8 Identifier:'d', 
+10:10 Operator:'=', 
+10:12 Integer:'0', 
+10:13 SentenceEnd:';', 
+11:5 Keyword:'while', 
+11:11 OpenParenthesis:'(', 
+11:12 Identifier:'i', 
+11:13 Operator:'<', 
+11:14 Identifier:'n', 
+11:15 CloseParenthesis:')', 
+11:16 OpenBrace:'{', 
+12:9 Identifier:'a', 
+12:11 Operator:'=', 
+12:13 Identifier:'c', 
+12:15 Operator:'=', 
+12:17 OpenParenthesis:'(', 
+12:18 Operator:'-', 
+12:19 Identifier:'a', 
+12:20 CloseParenthesis:')', 
+12:21 Operator:'+', 
+12:22 Identifier:'b', 
+12:23 Operator:'*', 
+12:24 Identifier:'c', 
+12:25 Operator:'/', 
+12:26 Integer:'66', 
+12:28 SentenceEnd:';', 
+13:9 Identifier:'a', 
+13:11 Operator:'=', 
+13:13 Identifier:'b', 
+13:15 Operator:'%', 
+13:17 Integer:'3', 
+13:19 Operator:'/', 
+13:21 Integer:'2', 
+13:23 Operator:'+', 
+13:25 Integer:'4', 
+13:27 Operator:'+', 
+13:29 OpenParenthesis:'(', 
+13:30 Identifier:'b', 
+13:32 Operator:'=', 
+13:34 Integer:'4', 
+13:35 CloseParenthesis:')', 
+13:36 SentenceEnd:';', 
+14:9 Identifier:'b', 
+14:11 Operator:'=', 
+14:13 Identifier:'c', 
+14:14 SentenceEnd:';', 
+15:9 Identifier:'i', 
+15:10 Operator:'=', 
+15:11 Identifier:'i', 
+15:12 Operator:'+', 
+15:13 Integer:'1', 
+15:14 SentenceEnd:';', 
+16:5 CloseBrace:'}', 
+17:5 Identifier:'print', 
+17:10 OpenParenthesis:'(', 
+17:11 Identifier:'a', 
+17:12 CloseParenthesis:')', 
+17:13 SentenceEnd:';', 
+18:5 Identifier:'print', 
+18:10 OpenParenthesis:'(', 
+18:11 Identifier:'b', 
+18:12 CloseParenthesis:')', 
+18:13 SentenceEnd:';', 
+19:5 Identifier:'print', 
+19:10 OpenParenthesis:'(', 
+19:11 Identifier:'c', 
+19:12 CloseParenthesis:')', 
+19:13 SentenceEnd:';', 
+20:5 Identifier:'print', 
+20:10 OpenParenthesis:'(', 
+20:11 Operator:'-', 
+20:12 Identifier:'c', 
+20:13 CloseParenthesis:')', 
+20:14 SentenceEnd:';', 
+21:5 Identifier:'print', 
+21:10 OpenParenthesis:'(', 
+21:11 Operator:'!', 
+21:12 OpenParenthesis:'(', 
+21:13 Identifier:'c', 
+21:15 Operator:'==', 
+21:18 Operator:'-', 
+21:19 Integer:'1', 
+21:20 CloseParenthesis:')', 
+21:21 CloseParenthesis:')', 
+21:22 SentenceEnd:';', 
+22:1 CloseBrace:'}', 
+23:1 Eof:'Eof'