Add escape sequence parsing

Author: antonmedv

src/index.ts

@@ -17,6 +17,7 @@ export const mamlLanguage = LRLanguage.define({
         "PropertyName/Identifier": t.propertyName,
         "PropertyName/String": t.propertyName,
         String: t.string,
+        Escape: t.escape,
         RawString: t.special(t.string),
         Number: t.number,
         "True False": t.bool,

src/syntax.grammar

@@ -15,6 +15,8 @@ value {
   Null
 }
 
+String { '"' (stringContent | Escape)* '"' }
+
 Object { "{" (Property ","?)* "}" }
 
 Array { "[" (value ","?)* "]" }
@@ -27,17 +29,17 @@ True { @extend<Identifier, "true"> }
 False { @extend<Identifier, "false"> }
 Null { @extend<Identifier, "null"> }
 
+@external tokens stringTokenizer from "./tokens" { stringContent, Escape }
+
 @tokens {
   space { $[ \t\n\r]+ }
 
   LineComment { "#" ![\n\r]* }
 
-  @precedence { RawString, String, Number, Identifier }
+  @precedence { RawString, Number, Identifier }
 
   RawString { '"""' (![\"] | '"' ![\"] | '""' ![\"])* '"""' }
 
-  String { '"' (![\"\\\n\r] | "\\" _)* '"' }
-
   Number { "-"? ("0" | $[1-9] $[0-9]*) ("." $[0-9]+)? ($[eE] $[+\-]? $[0-9]+)? }
 
   Identifier { $[A-Za-z0-9_\-]+ }

src/syntax.grammar.terms.d.ts

@@ -0,0 +1,2 @@
+export declare const stringContent: number
+export declare const Escape: number

src/tokens.ts

@@ -0,0 +1,32 @@
+import {ExternalTokenizer} from "@lezer/lr"
+import {stringContent, Escape} from "./syntax.grammar.terms"
+
+const Backslash = 92 as number, Quote = 34 as number, LF = 10 as number, CR = 13 as number,
+  uChar = 117 as number, LBrace = 123 as number, RBrace = 125 as number
+
+function isHex(ch: number) {
+  return (ch >= 48 && ch <= 57) || (ch >= 65 && ch <= 70) || (ch >= 97 && ch <= 102)
+}
+
+export const stringTokenizer = new ExternalTokenizer((input) => {
+  if (input.next === Backslash) {
+    input.advance()
+    if (input.next === uChar) {
+      input.advance()
+      if (input.next === LBrace) {
+        input.advance()
+        while (isHex(input.next)) input.advance()
+        if (input.next === RBrace) input.advance()
+      }
+    } else if (input.next >= 0) {
+      input.advance()
+    }
+    input.acceptToken(Escape)
+  } else if (input.next !== Quote && input.next !== LF && input.next !== CR && input.next >= 0) {
+    while (input.next !== Quote && input.next !== Backslash &&
+    input.next !== LF && input.next !== CR && input.next >= 0) {
+      input.advance()
+    }
+    input.acceptToken(stringContent)
+  }
+}, {contextual: true})

test/test.spec.ts

@@ -141,6 +141,22 @@ describe("maml parser", () => {
   it("parses string with escape sequences", () => {
     const nodes = topNodes('"hello\\nworld"')
     expect(nodes).toContain("String")
+    expect(nodes).toContain("Escape")
+  })
+
+  it("parses unicode escape sequence", () => {
+    const nodes = topNodes('"\\u{263A}"')
+    expect(nodes).toContain("Escape")
+  })
+
+  it("parses all compact escape sequences", () => {
+    const input = '"\\n\\r\\t\\\\\\"\\/"'
+    const tree = parse(input)
+    let escapeCount = 0
+    tree.cursor().iterate((node) => {
+      if (node.name === "Escape") escapeCount++
+    })
+    expect(escapeCount).toBe(6)
   })
 
   it("parses raw string with quotes inside", () => {

vitest.config.ts

@@ -0,0 +1,6 @@
+import {defineConfig} from "vitest/config"
+import {lezer} from "@lezer/generator/rollup"
+
+export default defineConfig({
+  plugins: [lezer()],
+})