From b1c4dd96d769cb3ac3c8fc057e8dc60907b84c17 Mon Sep 17 00:00:00 2001
From: Tushar Sadhwani <tushar.sadhwani000@gmail.com>
Date: Fri, 2 Aug 2024 21:51:39 +0530
Subject: [PATCH] fix: respect braces better in f-string parsing (#4422)

---
 CHANGES.md                     |  2 ++
 src/blib2to3/pgen2/tokenize.py | 12 ++++++------
 tests/data/cases/pep_701.py    | 10 ++++++++++
 3 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 48fe337392d..00c210ba86f 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -39,6 +39,8 @@
 
 - Fix bug with Black incorrectly parsing empty lines with a backslash (#4343)
 
+- Fix bugs with Black's tokenizer not handling `\{` inside f-strings very well (#4422)
+
 - Fix incorrect line numbers in the tokenizer for certain tokens within f-strings
   (#4423)
 
diff --git a/src/blib2to3/pgen2/tokenize.py b/src/blib2to3/pgen2/tokenize.py
index ecd017b3148..d15dc03d945 100644
--- a/src/blib2to3/pgen2/tokenize.py
+++ b/src/blib2to3/pgen2/tokenize.py
@@ -136,12 +136,12 @@ def _combinations(*l: str) -> Set[str]:
 )
 
 # beginning of a single quoted f-string. must not end with `{{` or `\N{`
-SingleLbrace = r"(?:\\N{|\\.|{{|[^'\\{])*(?<!\\N){(?!{)"
-DoubleLbrace = r'(?:\\N{|\\.|{{|[^"\\{])*(?<!\\N){(?!{)'
+SingleLbrace = r"(?:\\N{|{{|\\'|[^\n'{])*(?<!\\N)({)(?!{)"
+DoubleLbrace = r'(?:\\N{|{{|\\"|[^\n"{])*(?<!\\N)({)(?!{)'
 
 # beginning of a triple quoted f-string. must not end with `{{` or `\N{`
-Single3Lbrace = r"(?:\\N{|\\[^{]|{{|'(?!'')|[^'{\\])*(?<!\\N){(?!{)"
-Double3Lbrace = r'(?:\\N{|\\[^{]|{{|"(?!"")|[^"{\\])*(?<!\\N){(?!{)'
+Single3Lbrace = r"(?:\\N{|{{|\\'|'(?!'')|[^'{])*(?<!\\N){(?!{)"
+Double3Lbrace = r'(?:\\N{|{{|\\"|"(?!"")|[^"{])*(?<!\\N){(?!{)'
 
 # ! format specifier inside an fstring brace, ensure it's not a `!=` token
 Bang = Whitespace + group("!") + r"(?!=)"
@@ -175,8 +175,8 @@ def _combinations(*l: str) -> Set[str]:
 _string_middle_double = r'(?:[^\n"\\]|\\.)*'
 
 # FSTRING_MIDDLE and LBRACE, must not end with a `{{` or `\N{`
-_fstring_middle_single = r"(?:\\N{|\\[^{]|{{|[^\n'{\\])*(?<!\\N)({)(?!{)"
-_fstring_middle_double = r'(?:\\N{|\\[^{]|{{|[^\n"{\\])*(?<!\\N)({)(?!{)'
+_fstring_middle_single = SingleLbrace
+_fstring_middle_double = DoubleLbrace
 
 # First (or only) line of ' or " string.
 ContStr = group(
diff --git a/tests/data/cases/pep_701.py b/tests/data/cases/pep_701.py
index 717927d206d..9acee951e71 100644
--- a/tests/data/cases/pep_701.py
+++ b/tests/data/cases/pep_701.py
@@ -131,6 +131,11 @@
 f"{'\''}"
 f"{f'\''}"
 
+f'{1}\{{'
+f'{2} foo \{{[\}}'
+f'\{3}'
+rf"\{"a"}"
+
 # output
 
 x = f"foo"
@@ -264,3 +269,8 @@
 
 f"{'\''}"
 f"{f'\''}"
+
+f"{1}\{{"
+f"{2} foo \{{[\}}"
+f"\{3}"
+rf"\{"a"}"