grammar: changing interpolated strings to not need the backslash as p…

…art of the escape

Grammar.md

@@ -81,14 +81,18 @@ PositiveIntLit          := Base10Digit+ | (Esc EscNumDigits) | (HexEsc HexDigit+
 
 StrLit                  := PlainStr | SpacingStr | InterpStr | InterpSpacingStr;
 
-InterpEsc               := Esc "`";
+InterpEsc               := "`";
 InterpSpacingEsc        := Esc InterpEsc;
 
 PlainStr                := '"' (#'[^"]' | '""')* '"';
 SpacingStr              := Esc PlainStr;
 InterpStr               := InterpEsc InterpLit;
 InterpSpacingStr        := InterpSpacingEsc InterpLit;
-InterpLit               := '"' (#'[^"`]' | '""' | "`" WhSp* ExprAsOpt? WhSp* "`")* '"';
+InterpLit               := '"' (#'[^"`]' | '""' | "`" WhSp* InterpExprAsOpt? WhSp* "`")* '"';
+InterpExprAsOpt         := !('`"') ExprAsOpt;
+
+(* NOTE: the above `InterpExprAsOpt` production has a negative-lookahead *)
+(* to avoid a grammar ambiguity with nested interpolated-strings.        *)
 
 
 (*************** Data Structures ***************)
@@ -403,11 +407,11 @@ defn a~each() ^empty;
 \"A single line
     string with whitespace collapsing, defined across multiple
   lines";
-\\`"A single line (with
-   whitespace) collapsing, and a single `` backtick";
-\`"Special number: `-3.1415962`
+\`"A single line (with
+   whitespace collapsing), and a single `` backtick";
+`"Special number: `-3.1415962`
    Name: `name`
-   Greeting: `\\`"Hello world"`
+   Greeting: `\`"Hello world"`
    Reaction: `\"Yay!"`
    Reply: `"Ok."`
 !";

README.md

@@ -13,9 +13,9 @@ greetings("my friend");
 // Hello, my friend!
 
 defn greetings(who) {
-    def msg: \`"Hello, `who`!";
-    log(msg);
-};
+    def msg: `"Hello, `who`!";
+    log(msg)
+}
 ```
 
 The language is designed for general application programming purposes, but with a clear emphasis on FP (and de-emphasis on OOP). It's not trying to compete in performance or capability with systems languages like C or Rust. Eventually, **Foi** will compile to WASM so it should be usable for applications in a variety of environments, from the web to the server to mobile devices.
@@ -26,22 +26,22 @@ Hopefully, without too much heavy orientation, code like this will settle into f
 
 ```java
 defn getFavoriteMovies(userID) ^(IO ~<< {
-    def movieIDs:: fetch(\`"/movies/`userID`");
+    def movieIDs:: fetch(`"/movies/`userID`");
     def movieTitles:: all(
         movieIDs ~map (movieID) {
-            fetch(\`"/movie/`movieID`")
+            fetch(`"/movie/`movieID`")
         }
         ~map |. ,"title"|
     );
-    def itemsHTML: | ~fold
+    def itemsHTML: |~fold
         movieTitles,
         "",
-        (html,title) { html + \`"<li>`title`</li>" }
+        (html,title) { `"`html`<li>`title`</li>" }
     |;
-    document.body.innerHTML := \`"<ul>`itemsHTML`</ul>";
+    document.body.innerHTML := `"<ul>`itemsHTML`</ul>";
 });
 
-getFavoriteMovies(123).run();
+getFavoriteMovies(123).run()
 ```
 
 ### TL;DR
@@ -75,7 +75,7 @@ The syntax and design decisions attempt to diverge enough from familiar language
 ```java
 defn factorial(n)
     ![n ?> 1]: 1
-    ^(n * factorial(n-1));
+    ^(n*factorial(n-1));
 
 // or (tail-recursive):
 
@@ -156,7 +156,7 @@ To prepare for exploration of **Foi**, here are some aspects of the design philo
     defn myFn(x) :over(y) {
         y := x + z;
         // ..
-    };
+    }
     ```
 
     The compiler enforces this requirement: any lexical variable outside a function's scope (aka, "free variable") that appears as an assignment target must be added to the function's `:over` clause. **Note:** in the above snippet, `z` is also an outer variable, but since it's only read (not assigned to), it does not need to be declared in the `:over` clause.
@@ -186,11 +186,11 @@ To prepare for exploration of **Foi**, here are some aspects of the design philo
     **Foi** "preconditions" appear *in the function signature* to express these "early return preconditions", like this:
 
     ```java
-    defn myFn(x) ?[x ?< 0]: 0 { .. };
+    defn myFn(x) ?[x ?< 0]: 0 { .. }
 
     // or:
 
-    defn myFn(x) ![x ?>= 0]: 0 { .. };
+    defn myFn(x) ![x ?>= 0]: 0 { .. }
     ```
 
     In this function signature, the `x` parameter is checked, and if it's less than `0`, the fixed `0` value is substituted ("early returned") in place of the function being invoked.

foi-toy/src/tokenizer.js

@@ -261,6 +261,7 @@ async function *tokenize(charStream) {
 						"DOUBLE_QUOTE", "ESCAPE", "WHITESPACE", "GENERAL",
 						"STRING", "NUMBER", "FORWARD_SLASH", "COMMENT",
 						"PERIOD", "TILDE", "QMARK", "EXMARK", "COLON",
+						"BACKTICK"
 					].includes(curToken.type) ||
 
 					// hyphen that should not be a minus
@@ -269,14 +270,6 @@ async function *tokenize(charStream) {
 					(
 						curToken.type == "HYPHEN" &&
 						!minusOpAllowed
-					) ||
-
-					// backtick in an interpolated
-					// string literal?
-					(
-						curToken.type == "BACKTICK" &&
-						state.type == "escapedString" &&
-						[ "\\`", "\\\\`", ].includes(state.context.value)
 					)
 				)
 			) {
@@ -762,10 +755,10 @@ async function *tokenize(charStream) {
 			case "`": {
 				minusOpAllowed = false;
 
-				// interpolated string escape?
+				// interpolated-spacing string escape?
 				if (
 					escapeToken != null &&
-					[ "\\", "\\\\" ].includes(escapeToken.value)
+					escapeToken.value == "\\"
 				) {
 					escapeToken.value += char;
 					escapeToken.end++;
@@ -805,7 +798,7 @@ async function *tokenize(charStream) {
 				// starting an escaped string literal?
 				if (
 					escapeToken != null &&
-					[ "\\", "\\`", "\\\\`" ].includes(escapeToken.value)
+					[ "\\", "\\`" ].includes(escapeToken.value)
 				) {
 					let context = escapeToken;
 					escapeToken = null;
@@ -814,6 +807,20 @@ async function *tokenize(charStream) {
 						{ type: "escapedString", context, }
 					];
 				}
+				// starting an interpolated string literal?
+				else if (
+					pendingToken != null &&
+					pendingToken.type == "BACKTICK"
+				) {
+					pendingToken.type = "ESCAPE";
+					let context = pendingToken;
+					pendingToken2 = TOKEN("DOUBLE_QUOTE",char,position);
+					escapeToken = null;
+					return [
+						pendingToken2,
+						{ type: "escapedString", context, }
+					];
+				}
 				// actually, an escaped double-quote
 				// found inside a string?
 				else if (
@@ -871,9 +878,9 @@ async function *tokenize(charStream) {
 	function escapedString(char,position) {
 		var state = currentState[currentState.length-1];
 		var escapeType = (
-			state.context.value == "\\`" ? "interpolated" :
-			state.context.value == "\\\\`" ? "interpolatedSpacing" :
-			"regular"
+			state.context.value == "`" ? "interpolated" :
+			state.context.value == "\\`" ? "interpolatedSpacing" :
+			"basic"
 		);
 
 		minusOpAllowed = false;
@@ -883,9 +890,10 @@ async function *tokenize(charStream) {
 				// possibly starting an interpolated
 				// expression?
 				if ([ "interpolated", "interpolatedSpacing" ].includes(escapeType)) {
+					let context = TOKEN("BACKTICK",char,position);
 					return [
-						TOKEN("BACKTICK",char,position),
-						{ type: "interpolatedBase", context: null }
+						context,
+						{ type: "interpolatedBase", context, }
 					];
 				}
 				// otherwise, just general text in
@@ -899,7 +907,7 @@ async function *tokenize(charStream) {
 				// whitespace?
 				if (WHITESPACE.includes(char)) {
 					// in an escaped string that collapses certain whitespace?
-					if ([ "regular", "interpolatedSpacing" ].includes(escapeType)) {
+					if ([ "basic", "interpolatedSpacing" ].includes(escapeType)) {
 						return [ TOKEN("WHITESPACE",char,position), null ];
 					}
 					else {
@@ -932,11 +940,11 @@ async function *tokenize(charStream) {
 					// string-tokenizing state
 					return [ pendingToken, POP_STATE ];
 				}
-				// not part of an escape sequence, so
+				// cannot be part of an escape sequence, so
 				// must be ending the interpolated expression?
 				else if (
 					escapeToken == null ||
-					![ "\\", "\\\\" ].includes(escapeToken.value)
+					escapeToken.value != "\\"
 				) {
 					return [ TOKEN("BACKTICK",char,position), POP_STATE ];
 				}
@@ -956,7 +964,7 @@ async function *tokenize(charStream) {
 			(state.context.value == "\\h" || state.context.value == "\\u") ? "hex" :
 			state.context.value == "\\b" ? "binary" :
 			state.context.value == "\\o" ? "octal" :
-			"regular"
+			"basic"
 		);
 		escapeToken = null;
 
@@ -979,7 +987,7 @@ async function *tokenize(charStream) {
 
 				// tokening a octal-compatible number
 				// literal?
-				if ([ "regular", "hex", "octal", "monad" ].includes(escapeType)) {
+				if ([ "basic", "hex", "octal", "monad" ].includes(escapeType)) {
 					return [ TOKEN("NUMBER",char,position), null ];
 				}
 				// otherwise, no longer in a valid
@@ -996,7 +1004,7 @@ async function *tokenize(charStream) {
 
 				// tokening a octal-compatible number
 				// literal?
-				if ([ "regular", "hex", "monad" ].includes(escapeType)) {
+				if ([ "basic", "hex", "monad" ].includes(escapeType)) {
 					return [ TOKEN("NUMBER",char,position), null ];
 				}
 				// otherwise, no longer in a valid
@@ -1044,7 +1052,7 @@ async function *tokenize(charStream) {
 					pendingToken != null &&
 					pendingToken.type == "NUMBER" &&
 					!pendingToken.value.includes(".") &&
-					[ "regular", "monad" ].includes(escapeType)
+					[ "basic", "monad" ].includes(escapeType)
 				) {
 					let nextToken = TOKEN("PERIOD",char,position);
 					if (nextToken.type == "DOUBLE_PERIOD") {
@@ -1105,7 +1113,7 @@ async function *tokenize(charStream) {
 				// separator in valid position in
 				// escaped number literal?
 				if (
-					[ "regular", "monad" ].includes(escapeType) &&
+					[ "basic", "monad" ].includes(escapeType) &&
 					pendingToken != null &&
 					pendingToken.type == "NUMBER" &&
 					(

foi-toy/test/test-2.foi

@@ -16,9 +16,9 @@ def c: \"What do you
    of
   this?";
 
-def d: \`"My name is: `name`, and here is a `` back-tick character.";
+def d: `"My name is: `name`, and here is a `` back-tick character.";
 
-def e: \\`"You must go
+def e: \`"You must go
 
 	see the `showName` show, it's
 great!
@@ -27,4 +27,4 @@ great!
 def f: \`"Here is the `best(\`"label: `label`") #> uppercase`
    part!";
 
-def g: \`"A string with `some + /// inline comment in interpolated expr /// 42`!";
+def g: `"A string with `some + /// inline comment in interpolated expr /// 42`!";

foi-toy/test/test-3.foi

@@ -19,7 +19,7 @@ def ~e~: true;
 
 def f: "Hello world";
 
-def g: \`"Hello, `name`!";
+def g: `"Hello, `name`!";
 
 def _h_var: 32a + \h32a + \3_200;
 

foi-toy/test/test-5.foi

@@ -11,14 +11,14 @@ defn renderLargeOrders() ^(Promise ~<< {
 	def largeOrders: orders ~filter (order) { order.total ?>= 100; };
 	def customers:: all(
 		largeOrders ~map (order) {
-			fetch(\`"/customer/id/`order.custId`");
+			fetch(`"/customer/id/`order.custId`");
 		}
 	);
 	def records: largeOrders ~map (order,idx) {
 		< &order, customer: customers[idx] >;
 	};
-	\`"<ul>`
-		(| ~fold records, "", (html,record) { html + \`"
+	`"<ul>`
+		(| ~fold records, "", (html,record) { `"`html`
 			<li id=""order-`record.orderId`"">
 				<span>Order #`record.orderId`</span>
 				<span>Total: $`format(record.total)`</span>

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "foi-lang",
-	"version": "0.0.15",
+	"version": "0.0.16",
 	"description": "Foi: a different kind of functional programming language",
 	"repository": "getify/foi-lang",
 	"scripts": {