feat(scripting): multiline scripting feature (#259)

docs/scripting/language-syntax.md

@@ -80,6 +80,20 @@ Indentation needs to be respected and be consistent, because it's how the script
 For example if you've started indenting code with 2 spaces, you need to keep doing that for the rest of the script. If you start indenting with 4 spaces, you need to keep doing that for the rest of the script.
 :::
 
+## Multiline code
+
+If you want to split a line of code into multiple lines for readability, you can do so by adding a backslash `\` at the end of the line. This tells the engine that the code continues on the next line. It also works with strings. Example:
+
+```narrat
+main:
+  talk player idle "This is a very long string that \
+  I want to split into multiple lines for readability"
+  var text (concat \
+    "I am concatenating multiple lines of text " \
+    "together to make a longer string" \
+  )
+```
+
 ## Commands
 
 All lines of script in narrat are commands. A command is created by typing the name of the command, followed with arguments separated by spaces. Commands are also effectively expressions, they simply don't have parenthesis around them to be easier to write.

packages/narrat/src/examples/default/scripts/default.narrat

@@ -1,7 +1,16 @@
 dev_test:
-  "Hello"
+  run test_multiline
   menu_return
 
+test_multiline:
+  talk player idle "This is a very long string that \
+  I want to split into multiple lines for readability"
+  var text (concat \
+    "I am concatenating multiple lines of text " \
+    "together to make a longer string" \
+  )
+  talk player idle $text
+
 main:
   jump dev_test
   set_screen video

packages/narrat/src/vm/parser.test.narrat

@@ -0,0 +1,25 @@
+// Syntax test script for unit tests
+
+main:
+  "Hello world"
+  talk player idle "Multiline statement? \
+  Yes, they work!"
+  talk player idle "This is a single line statement."
+  choice:
+    "hello world 2":
+      jump test2
+    "hello world 3":
+      jump test3
+
+test2:
+  set data.counter (add (add "hello world (parenthesis in strings)" (set (add 1 2) (add 3 2) )) 2)
+
+test3:
+  if (&& (== (+ 1 2 3 4) 10) (== (+ 1 2 3) 6) (== (+ 1 2) 3 3 3 3)):
+    "hi"
+  else:
+    "bye"
+
+test4:
+  set data.test (add "multiline " \
+  "statement")

packages/narrat/src/vm/simple-script.test.narrat

@@ -0,0 +1,2 @@
+main:
+  talk player idle "Hello world"

packages/narrat/src/vm/vm-parser.test.ts

@@ -1,25 +1,84 @@
-import { test, expect } from 'vitest';
+import { test, expect, assert, vi } from 'vitest';
 
 import { Parser } from '@/types/parser';
-import { error } from '@/utils/error-handling';
 import {
+  getBranchesFromRawScript,
   parseCodeLine,
   parseCodeLineIntoTokens,
   ParserContext,
+  parseScript,
   splitIntoTokens,
+  splitScriptIntoLines,
   tokensToExpression,
 } from './vm-parser';
+import { readFile } from 'fs/promises';
+import { join } from 'path';
+import { createPinia, setActivePinia } from 'pinia';
+import { NarratScript } from '@/types/app-types';
+import { registerBaseCommands } from './commands';
+import { vm } from './vm';
+
+const onError = (line: number, text: string) => {
+  console.trace();
+  console.error(`Parser error: ${line + 1}: ${text}`);
+};
 
 const ctx: ParserContext = {
   fileName: 'test.js',
   currentLine: 1,
-  error: (line: number, text: string) =>
-    error(`Parser error: ${ctx.fileName}:${line + 1}: ${text}`),
-  indentSize: 0,
+  error: onError,
+  indentSize: 2,
   processCommandsFunction: (ctx: any, lines: any, parentLine: any) =>
-    ({} as any),
+    ({}) as any,
 };
 
+let simpleNarratFile: string = '';
+let complexNarratFile: string = '';
+const errorSpy = vi.spyOn(ctx, 'error');
+beforeEach(() => {
+  setActivePinia(createPinia());
+  errorSpy.mockClear();
+});
+beforeAll(async () => {
+  complexNarratFile = await readFile(
+    join(__dirname, 'parser.test.narrat'),
+    'utf-8',
+  );
+  simpleNarratFile = await readFile(
+    join(__dirname, 'simple-script.test.narrat'),
+    'utf-8',
+  );
+});
+afterEach(() => {
+  for (const call of errorSpy.mock.calls) {
+    assert.fail(call[1]);
+  }
+});
+
+test('splitScriptIntoLines', () => {
+  const lines1 = splitScriptIntoLines(ctx, simpleNarratFile);
+  const lines2 = splitScriptIntoLines(ctx, complexNarratFile);
+  expect(lines1.length).toBe(2);
+  expect(lines2[2].code).toBe(
+    `  talk player idle "Multiline statement?   Yes, they work!"`,
+  );
+  expect(lines2[3].line).toBe(6);
+});
+
+test('getBranchesFromRawScript', async () => {
+  const lines1 = getBranchesFromRawScript(ctx, simpleNarratFile);
+  const lines2 = getBranchesFromRawScript(ctx, complexNarratFile);
+  expect(lines1.length).toBe(1);
+  expect(lines2.find((line) => line.code === 'main:')).toBeTruthy();
+  expect(lines2.find((line) => line.code === 'test4:')).toBeTruthy();
+  expect(lines2[0].branch![1].code).toBe(
+    'talk player idle "Multiline statement?   Yes, they work!"',
+  );
+  expect(lines2[3].branch![0].code).toBe(
+    'set data.test (add "multiline "   "statement")',
+  );
+});
+
 test('parseCodeLine', () => {
   const line = 'set data.example.hello "Hello world"';
   const result = parseCodeLine(ctx, line);
@@ -109,3 +168,18 @@ test('complicated if line', () => {
   expect(r3[1]).toBe(1);
   expect(r2[2]).toBe(10);
 });
+
+test('parseScript', () => {
+  registerBaseCommands(vm);
+  const script: NarratScript = {
+    code: complexNarratFile,
+    fileName: 'main.narrat',
+    id: 'main',
+    type: 'script',
+  };
+  const parsed = parseScript(script);
+  expect(parsed.test4).toBeTruthy();
+  expect(parsed.test4.branch[0].code).toBe(
+    `set data.test (add "multiline "   "statement")`,
+  );
+});

packages/narrat/src/vm/vm-parser.ts

@@ -50,7 +50,7 @@ export function parseScriptFunction(
     indentSize: 0, // Will be overriden soon
   };
   ctx.indentSize = detectIndentation(ctx, code);
-  const lines = findLines(ctx, code);
+  const lines = getBranchesFromRawScript(ctx, code);
   ctx.currentLine = 0;
   logger.log(lines);
   const script: Parser.ParsedScript = {};
@@ -319,21 +319,64 @@ export function findExpressionStart(tokens: Parser.Primitive[]): number {
 export function findExpressionEnd(tokens: Parser.Primitive[]): number {
   return tokens.findIndex((token) => token === ')');
 }
-function findLines(ctx: ParserContext, data: string): Parser.Line[] {
-  const code = data.split(/\r?\n|$/).map((line) => {
-    const commentIndex = line.search(/ *\/\//g);
-    if (commentIndex !== -1) {
-      return line.substr(0, commentIndex);
-    }
-    return line;
-  });
-  const lines = findBranches(ctx, code, 0, 0);
-  return lines.lines;
+
+export interface LineData {
+  code: string;
+  line: number;
+  multiline: boolean;
+}
+
+/** Finds all the lines, handling merging multilines, removing comment and removing empty lines */
+export function splitScriptIntoLines(
+  ctx: ParserContext,
+  data: string,
+): LineData[] {
+  const result = data
+    .split(/\r?\n|$/)
+    .reduce<LineData[]>((lines, line, index) => {
+      const final = {
+        code: line,
+        line: index,
+        multiline: false,
+      };
+      const multilineIndex = line.search(/\\$/);
+      if (multilineIndex !== -1) {
+        final.multiline = true;
+        line = line.substring(0, multilineIndex);
+      }
+      const commentIndex = line.search(/ *\/\//g);
+      if (commentIndex !== -1) {
+        line = line.substring(0, commentIndex);
+      }
+      final.code = line;
+      if (final.code.search(/^\s*$/) === -1) {
+        // Skip empty lines
+        if (lines.length > 0 && lines[lines.length - 1].multiline) {
+          lines[lines.length - 1].code += line;
+          lines[lines.length - 1].multiline = final.multiline;
+        } else {
+          lines.push(final);
+        }
+      }
+      return lines;
+    }, [] as LineData[]);
+  return result;
+}
+
+export function getBranchesFromRawScript(
+  ctx: ParserContext,
+  data: string,
+): Parser.Line[] {
+  // First find all lines, combine multilines, remove comments
+  const lines = splitScriptIntoLines(ctx, data);
+  // Then parse them into branches
+  const parsedBranches = findBranches(ctx, lines, 0, 0);
+  return parsedBranches.lines;
 }
 
 function findBranches(
   ctx: ParserContext,
-  code: string[],
+  code: LineData[],
   startLine: number,
   indentLevel: number,
 ) {
@@ -344,35 +387,32 @@ function findBranches(
     if (currentLine >= code.length) {
       break;
     }
-    let lineText = code[currentLine];
-    if (lineText.search(/^\s*$/) !== -1) {
-      // Ignore empty lines
-      currentLine++;
-    } else {
-      const lineIndent = getIndentLevel(ctx, lineText);
-      lineText = lineText.substring(lineIndent * ctx.indentSize);
-      validateIndent(ctx, lineIndent, currentLine);
-      if (lineIndent < indentLevel) {
-        stillInBranch = false;
-      } else if (lineIndent > indentLevel) {
-        if (lines.length === 0 || lineIndent - indentLevel !== 1) {
-          ctx.error(currentLine, `Wrong double indentation`);
-        }
-        const branchLines = findBranches(ctx, code, currentLine, lineIndent);
-        lines[lines.length - 1].branch = branchLines.lines;
-        currentLine = branchLines.endLine;
-      } else {
-        const expression = parseCodeLine(ctx, lineText);
-        const line: Parser.Line = {
-          code: lineText,
-          indentation: lineIndent,
-          line: currentLine,
-          expression,
-        };
-        lines.push(line);
-        currentLine++;
-        ctx.currentLine = currentLine;
+    const codeLine = code[currentLine];
+    let lineText = codeLine.code;
+
+    const lineIndent = getIndentLevel(ctx, lineText);
+    lineText = lineText.substring(lineIndent * ctx.indentSize);
+    validateIndent(ctx, lineIndent, currentLine);
+    if (lineIndent < indentLevel) {
+      stillInBranch = false;
+    } else if (lineIndent > indentLevel) {
+      if (lines.length === 0 || lineIndent - indentLevel !== 1) {
+        ctx.error(codeLine.line, `Wrong double indentation`);
       }
+      const branchLines = findBranches(ctx, code, currentLine, lineIndent);
+      lines[lines.length - 1].branch = branchLines.lines;
+      currentLine = branchLines.endLine;
+    } else {
+      const expression = parseCodeLine(ctx, lineText);
+      const line: Parser.Line = {
+        code: lineText,
+        indentation: lineIndent,
+        line: codeLine.line,
+        expression,
+      };
+      lines.push(line);
+      currentLine++;
+      ctx.currentLine = currentLine;
     }
   }
   return {
@@ -399,8 +439,8 @@ function getIndentLevel(ctx: ParserContext, line: string) {
 }
 
 function detectIndentation(ctx: ParserContext, script: string): number {
-  const regex = /\n( *)/;
-  const result = script.match(regex);
+  const result = script.match(/: *[\n\r]+( *)/);
+
   if (!result || result.length < 2) {
     ctx.error(
       0,