properly handle escape sequences in string literals

tree-sitter-grammars · Jun 12, 2021 · 52247de · 52247de
1 parent 67680e4
commit 52247de
Show file tree

Hide file tree

Showing 6 changed files with 1,945 additions and 1,596 deletions.
diff --git a/README.md b/README.md
@@ -14,4 +14,4 @@ To run tests simply run `nix-shell --run 'tree-sitter test'`.
 
 * use [Unicode® Standard Annex #31](https://www.unicode.org/reports/tr31/) (augmented with '-')for identifiers
 * add [operations](https://github.com/hashicorp/hcl/blob/main/hclsyntax/spec.md#operations)
-* add [template expressions](https://github.com/hashicorp/hcl/blob/main/hclsyntax/spec.md#template-expressions) and express string literals using them
+* add [template expressions](https://github.com/hashicorp/hcl/blob/main/hclsyntax/spec.md#template-expressions) 
diff --git a/grammar.js b/grammar.js
@@ -76,7 +76,24 @@ module.exports = grammar({
 
     numeric_lit: $ => /[0-9]+(\.[0-9]+([eE][-+]?[0-9]+)?)?/,
 
-    string_lit: $ => (seq('"', token.immediate(repeat(choice(/[^\\"\n]/, /\\(.|\n)/))), '"')),
+    string_lit: $ => seq(
+      '"', 
+      repeat(choice(token.immediate(prec(1, /[^\\"\n\r\t]+/)), $.escape_sequence)),
+      '"',
+    ),
+
+    escape_sequence: $ => token.immediate(seq(
+      '\\',
+      choice(
+        '\\',
+        '"',
+        'n',
+        'r',
+        't',
+        /u[0-9a-fA-F]{4}/,
+        /U[0-9a-fA-F]{8}/
+      )
+    )),
 
     bool_lit: $ => choice('true', 'false'),
 

diff --git a/src/grammar.json b/src/grammar.json
@@ -255,22 +255,26 @@
           "value": "\""
         },
         {
-          "type": "IMMEDIATE_TOKEN",
+          "type": "REPEAT",
           "content": {
-            "type": "REPEAT",
-            "content": {
-              "type": "CHOICE",
-              "members": [
-                {
-                  "type": "PATTERN",
-                  "value": "[^\\\\\"\\n]"
-                },
-                {
-                  "type": "PATTERN",
-                  "value": "\\\\(.|\\n)"
+            "type": "CHOICE",
+            "members": [
+              {
+                "type": "IMMEDIATE_TOKEN",
+                "content": {
+                  "type": "PREC",
+                  "value": 1,
+                  "content": {
+                    "type": "PATTERN",
+                    "value": "[^\\\\\"\\n\\r\\t]+"
+                  }
                 }
-              ]
-            }
+              },
+              {
+                "type": "SYMBOL",
+                "name": "escape_sequence"
+              }
+            ]
           }
         },
         {
@@ -279,6 +283,51 @@
         }
       ]
     },
+    "escape_sequence": {
+      "type": "IMMEDIATE_TOKEN",
+      "content": {
+        "type": "SEQ",
+        "members": [
+          {
+            "type": "STRING",
+            "value": "\\"
+          },
+          {
+            "type": "CHOICE",
+            "members": [
+              {
+                "type": "STRING",
+                "value": "\\"
+              },
+              {
+                "type": "STRING",
+                "value": "\""
+              },
+              {
+                "type": "STRING",
+                "value": "n"
+              },
+              {
+                "type": "STRING",
+                "value": "r"
+              },
+              {
+                "type": "STRING",
+                "value": "t"
+              },
+              {
+                "type": "PATTERN",
+                "value": "u[0-9a-fA-F]{4}"
+              },
+              {
+                "type": "PATTERN",
+                "value": "U[0-9a-fA-F]{8}"
+              }
+            ]
+          }
+        ]
+      }
+    },
     "bool_lit": {
       "type": "CHOICE",
       "members": [

diff --git a/src/node-types.json b/src/node-types.json
@@ -497,7 +497,17 @@
   {
     "type": "string_lit",
     "named": true,
-    "fields": {}
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": false,
+      "types": [
+        {
+          "type": "escape_sequence",
+          "named": true
+        }
+      ]
+    }
   },
   {
     "type": "tuple",
@@ -585,6 +595,10 @@
     "type": "ellipsis",
     "named": true
   },
+  {
+    "type": "escape_sequence",
+    "named": true
+  },
   {
     "type": "false",
     "named": false