bpo-44010: IDLE: colorize pattern-matching soft keywords

Lib/idlelib/colorizer.py

@@ -1,6 +1,7 @@
 import builtins
 import keyword
 import re
+import textwrap
 import time
 
 from idlelib.config import idleConf
@@ -16,6 +17,20 @@ def any(name, alternates):
 
 def make_pat():
     kw = r"\b" + any("KEYWORD", keyword.kwlist) + r"\b"
+    match_softkw = (
+        r"^[ \t]*" +  # at beginning of line + possible indentation
+        r"(?P<PM_MATCH>match)\b" +
+        r"(?!([ \t]|\\\n)*[=,)\]}])"  # not followed by any of =,)]}
+    )
+    match_case_default = (
+        r"^[ \t]*" +  # at beginning of line + possible indentation
+        r"(?P<PM_DEFAULT_CASE>case)[ \t]+(?P<PM_DEFAULT_UNDERSCORE>_)[ \t]*:"
+    )
+    match_case_nondefault = (
+        r"^[ \t]*" +  # at beginning of line + possible indentation
+        r"(?P<PM_NONDEFAULT_CASE>case)\b" +
+        r"(?!([ \t]|\\\n)*[=,)\]}])"  # not followed by any of =,)]}
+    )
     builtinlist = [str(name) for name in dir(builtins)
                    if not name.startswith('_') and
                    name not in keyword.kwlist]
@@ -27,12 +42,20 @@ def make_pat():
     sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?"
     dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?'
     string = any("STRING", [sq3string, dq3string, sqstring, dqstring])
-    return (kw + "|" + builtin + "|" + comment + "|" + string +
-            "|" + any("SYNC", [r"\n"]))
+    return "|".join([
+        builtin, comment, string, any("SYNC", [r"\n"]),
+        kw, match_softkw, match_case_default, match_case_nondefault,
+    ])
 
 
-prog = re.compile(make_pat(), re.S)
-idprog = re.compile(r"\s+(\w+)", re.S)
+prog = re.compile(make_pat(), re.DOTALL | re.MULTILINE)
+idprog = re.compile(r"\s+(\w+)")
+prog_group_name_to_tag = {
+    "PM_MATCH": "KEYWORD",
+    "PM_DEFAULT_CASE": "KEYWORD",
+    "PM_DEFAULT_UNDERSCORE": "KEYWORD",
+    "PM_NONDEFAULT_CASE": "KEYWORD",
+}
 
 
 def color_config(text):
@@ -231,14 +254,10 @@ def recolorize(self):
     def recolorize_main(self):
         "Evaluate text and apply colorizing tags."
         next = "1.0"
-        while True:
-            item = self.tag_nextrange("TODO", next)
-            if not item:
-                break
-            head, tail = item
-            self.tag_remove("SYNC", head, tail)
-            item = self.tag_prevrange("SYNC", head)
-            head = item[1] if item else "1.0"
+        while todo_tag_range := self.tag_nextrange("TODO", next):
+            self.tag_remove("SYNC", todo_tag_range[0], todo_tag_range[1])
+            sync_tag_range = self.tag_prevrange("SYNC", todo_tag_range[0])
+            head = sync_tag_range[1] if sync_tag_range else "1.0"
 
             chars = ""
             next = head
@@ -257,22 +276,21 @@ def recolorize_main(self):
                 for tag in self.tagdefs:
                     self.tag_remove(tag, mark, next)
                 chars = chars + line
-                m = self.prog.search(chars)
-                while m:
+                for m in self.prog.finditer(chars):
                     for key, value in m.groupdict().items():
-                        if value:
-                            a, b = m.span(key)
-                            self.tag_add(key,
-                                         head + "+%dc" % a,
-                                         head + "+%dc" % b)
-                            if value in ("def", "class"):
-                                m1 = self.idprog.match(chars, b)
-                                if m1:
-                                    a, b = m1.span(1)
-                                    self.tag_add("DEFINITION",
-                                                 head + "+%dc" % a,
-                                                 head + "+%dc" % b)
-                    m = self.prog.search(chars, m.end())
+                        if not value:
+                            continue
+                        a, b = m.span(key)
+                        tag = prog_group_name_to_tag.get(key, key)
+                        self.tag_add(tag,
+                                     head + "+%dc" % a,
+                                     head + "+%dc" % b)
+                        if value in ("def", "class"):
+                            if m1 := self.idprog.match(chars, b):
+                                a, b = m1.span(1)
+                                self.tag_add("DEFINITION",
+                                             head + "+%dc" % a,
+                                             head + "+%dc" % b)
                 if "SYNC" in self.tag_names(next + "-1c"):
                     head = next
                     chars = ""
@@ -304,22 +322,39 @@ def _color_delegator(parent):  # htest #
     top = Toplevel(parent)
     top.title("Test ColorDelegator")
     x, y = map(int, parent.geometry().split('+')[1:])
-    top.geometry("700x250+%d+%d" % (x + 20, y + 175))
-    source = (
-        "if True: int ('1') # keyword, builtin, string, comment\n"
-        "elif False: print(0)\n"
-        "else: float(None)\n"
-        "if iF + If + IF: 'keyword matching must respect case'\n"
-        "if'': x or''  # valid keyword-string no-space combinations\n"
-        "async def f(): await g()\n"
-        "# All valid prefixes for unicode and byte strings should be colored.\n"
-        "'x', '''x''', \"x\", \"\"\"x\"\"\"\n"
-        "r'x', u'x', R'x', U'x', f'x', F'x'\n"
-        "fr'x', Fr'x', fR'x', FR'x', rf'x', rF'x', Rf'x', RF'x'\n"
-        "b'x',B'x', br'x',Br'x',bR'x',BR'x', rb'x', rB'x',Rb'x',RB'x'\n"
-        "# Invalid combinations of legal characters should be half colored.\n"
-        "ur'x', ru'x', uf'x', fu'x', UR'x', ufr'x', rfu'x', xf'x', fx'x'\n"
+    top.geometry("700x550+%d+%d" % (x + 20, y + 175))
+    source = textwrap.dedent("""\
+        if True: int ('1') # keyword, builtin, string, comment
+        elif False: print(0)
+        else: float(None)
+        if iF + If + IF: 'keyword matching must respect case'
+        if'': x or''  # valid keyword-string no-space combinations
+        async def f(): await g()
+        # All valid prefixes for unicode and byte strings should be colored.
+        'x', '''x''', "x", \"""x\"""
+        'abc\\
+        def'
+        '''abc\\
+        def'''
+        r'x', u'x', R'x', U'x', f'x', F'x'
+        fr'x', Fr'x', fR'x', FR'x', rf'x', rF'x', Rf'x', RF'x'
+        b'x',B'x', br'x',Br'x',bR'x',BR'x', rb'x', rB'x',Rb'x',RB'x'
+        # Invalid combinations of legal characters should be half colored.
+        ur'x', ru'x', uf'x', fu'x', UR'x', ufr'x', rfu'x', xf'x', fx'x'
+        match point:
+            case (x, 0):
+                print(f"X={x}")
+            case _:
+                raise ValueError("Not a point")
+        # The following statement should all be in the default color for code.
+        match = (
+            case,
+            _,
         )
+        '''
+        case _:'''
+        "match x:"
+        """)
     text = Text(top, background="white")
     text.pack(expand=1, fill="both")
     text.insert("insert", source)

Lib/idlelib/idle_test/test_colorizer.py

@@ -1,11 +1,11 @@
 "Test colorizer, coverage 93%."
-
 from idlelib import colorizer
 from test.support import requires
 import unittest
 from unittest import mock
 
 from functools import partial
+import textwrap
 from tkinter import Tk, Text
 from idlelib import config
 from idlelib.percolator import Percolator
@@ -19,15 +19,32 @@
     'extensions': config.IdleUserConfParser(''),
 }
 
-source = (
-    "if True: int ('1') # keyword, builtin, string, comment\n"
-    "elif False: print(0)  # 'string' in comment\n"
-    "else: float(None)  # if in comment\n"
-    "if iF + If + IF: 'keyword matching must respect case'\n"
-    "if'': x or''  # valid string-keyword no-space combinations\n"
-    "async def f(): await g()\n"
-    "'x', '''x''', \"x\", \"\"\"x\"\"\"\n"
+source = textwrap.dedent("""\
+    if True: int ('1') # keyword, builtin, string, comment
+    elif False: print(0)  # 'string' in comment
+    else: float(None)  # if in comment
+    if iF + If + IF: 'keyword matching must respect case'
+    if'': x or''  # valid string-keyword no-space combinations
+    async def f(): await g()
+    'x', '''x''', "x", \"""x\"""
+    'abc\\
+    def'
+    '''abc\\
+    def'''
+    match point:
+        case (x, 0):
+            print(f"X={x}")
+        case _:
+            raise ValueError("Not a point")
+    # The following statement should all be in the default color for code.
+    match = (
+        case,
+        _,
     )
+    '''
+    case _:'''
+    "match x:"
+""")
 
 
 def setUpModule():
@@ -366,6 +383,11 @@ def test_recolorize_main(self, mock_notify):
                     ('6.0', ('KEYWORD',)), ('6.10', ('DEFINITION',)), ('6.11', ()),
                     ('7.0', ('STRING',)), ('7.4', ()), ('7.5', ('STRING',)),
                     ('7.12', ()), ('7.14', ('STRING',)),
+                    ('12.0', ('KEYWORD',)),
+                    ('13.4', ('KEYWORD',)),
+                    ('15.4', ('KEYWORD',)), ('15.9', ('KEYWORD',)),
+                    ('18.0', ()), ('19.4', ()), ('20.4', ()),
+                    ('23.0', ('STRING',)), ('24.1', ('STRING',)),
                     # SYNC at the end of every line.
                     ('1.55', ('SYNC',)), ('2.50', ('SYNC',)), ('3.34', ('SYNC',)),
                    )
@@ -395,7 +417,8 @@ def test_recolorize_main(self, mock_notify):
         eq(text.tag_nextrange('STRING', '7.3'), ('7.5', '7.12'))
         eq(text.tag_nextrange('STRING', '7.12'), ('7.14', '7.17'))
         eq(text.tag_nextrange('STRING', '7.17'), ('7.19', '7.26'))
-        eq(text.tag_nextrange('SYNC', '7.0'), ('7.26', '9.0'))
+        eq(text.tag_nextrange('SYNC', '7.0'), ('7.26', '8.0'))
+        eq(text.tag_nextrange('SYNC', '24.0'), ('24.10', '26.0'))
 
     @mock.patch.object(colorizer.ColorDelegator, 'recolorize')
     @mock.patch.object(colorizer.ColorDelegator, 'notify_range')