Fix(tokenizer): treat quote as escape only if its followed by itself (#…

…2199) * Fix(tokenizer): treat quote as escape only if its followed by itself * Cleanup * Cleanup * Cleanup
tobymao · Sep 12, 2023 · a228656 · a228656
1 parent e2bb427
commit a228656
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 2 deletions.
diff --git a/sqlglot/dialects/mysql.py b/sqlglot/dialects/mysql.py
@@ -119,7 +119,7 @@ class Tokenizer(tokens.Tokenizer):
         QUOTES = ["'", '"']
         COMMENTS = ["--", "#", ("/*", "*/")]
         IDENTIFIERS = ["`"]
-        STRING_ESCAPES = ["'", "\\"]
+        STRING_ESCAPES = ["'", '"', "\\"]
         BIT_STRINGS = [("b'", "'"), ("B'", "'"), ("0b", "")]
         HEX_STRINGS = [("x'", "'"), ("X'", "'"), ("0x", "")]
 

diff --git a/sqlglot/tokens.py b/sqlglot/tokens.py
@@ -1168,7 +1168,11 @@ def _extract_string(self, delimiter: str, escapes=None) -> str:
         escapes = self._STRING_ESCAPES if escapes is None else escapes
 
         while True:
-            if self._char in escapes and (self._peek == delimiter or self._peek in escapes):
+            if (
+                self._char in escapes
+                and (self._peek == delimiter or self._peek in escapes)
+                and (self._char not in self._QUOTES or self._char == self._peek)
+            ):
                 if self._peek == delimiter:
                     text += self._peek
                 else:

diff --git a/tests/dialects/test_mysql.py b/tests/dialects/test_mysql.py
@@ -243,6 +243,15 @@ def test_canonical_functions(self):
         )
 
     def test_escape(self):
+        self.validate_identity("""'"abc"'""")
+        self.validate_identity(
+            r"'\'a'",
+            "'''a'",
+        )
+        self.validate_identity(
+            '''"'abc'"''',
+            "'''abc'''",
+        )
         self.validate_all(
             r"'a \' b '' '",
             write={