From 1d6a5b1d2abf617b149e8bf8ff435f64dc507fd3 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 18 Nov 2022 10:48:27 +0000 Subject: [PATCH] Fix daemon crash on malformed NamedTuple (#14119) Fixes #14098 Having invalid statements in a NamedTuple is almost like a syntax error, we can remove them after giving an error (without further analysis). This PR does almost exactly the same as https://github.com/python/mypy/pull/13963 did for TypedDicts. Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- mypy/nodes.py | 4 ++ mypy/semanal_namedtuple.py | 19 ++++-- mypy/semanal_typeddict.py | 2 + mypy/server/aststrip.py | 2 + test-data/unit/check-class-namedtuple.test | 2 - test-data/unit/fine-grained.test | 72 ++++++++++++++++++++++ 6 files changed, 94 insertions(+), 7 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index cf711c45f587..ebf2f5cb271a 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -1068,6 +1068,7 @@ class ClassDef(Statement): "analyzed", "has_incompatible_baseclass", "deco_line", + "removed_statements", ) __match_args__ = ("name", "defs") @@ -1086,6 +1087,8 @@ class ClassDef(Statement): keywords: dict[str, Expression] analyzed: Expression | None has_incompatible_baseclass: bool + # Used by special forms like NamedTuple and TypedDict to store invalid statements + removed_statements: list[Statement] def __init__( self, @@ -1111,6 +1114,7 @@ def __init__( self.has_incompatible_baseclass = False # Used for error reporting (to keep backwad compatibility with pre-3.8) self.deco_line: int | None = None + self.removed_statements = [] def accept(self, visitor: StatementVisitor[T]) -> T: return visitor.visit_class_def(self) diff --git a/mypy/semanal_namedtuple.py b/mypy/semanal_namedtuple.py index 04308db99e63..ec5f13d0fce0 100644 --- a/mypy/semanal_namedtuple.py +++ b/mypy/semanal_namedtuple.py @@ -32,6 +32,7 @@ NameExpr, PassStmt, RefExpr, + Statement, StrExpr, SymbolTable, SymbolTableNode, @@ -111,7 +112,7 @@ def analyze_namedtuple_classdef( if result is None: # This is a valid named tuple, but some types are incomplete. return True, None - items, types, default_items = result + items, types, default_items, statements = result if is_func_scope and "@" not in defn.name: defn.name += "@" + str(defn.line) existing_info = None @@ -123,6 +124,7 @@ def analyze_namedtuple_classdef( defn.analyzed = NamedTupleExpr(info, is_typed=True) defn.analyzed.line = defn.line defn.analyzed.column = defn.column + defn.defs.body = statements # All done: this is a valid named tuple with all types known. return True, info # This can't be a valid named tuple. @@ -130,24 +132,27 @@ def analyze_namedtuple_classdef( def check_namedtuple_classdef( self, defn: ClassDef, is_stub_file: bool - ) -> tuple[list[str], list[Type], dict[str, Expression]] | None: + ) -> tuple[list[str], list[Type], dict[str, Expression], list[Statement]] | None: """Parse and validate fields in named tuple class definition. - Return a three tuple: + Return a four tuple: * field names * field types * field default values + * valid statements or None, if any of the types are not ready. """ if self.options.python_version < (3, 6) and not is_stub_file: self.fail("NamedTuple class syntax is only supported in Python 3.6", defn) - return [], [], {} + return [], [], {}, [] if len(defn.base_type_exprs) > 1: self.fail("NamedTuple should be a single base", defn) items: list[str] = [] types: list[Type] = [] default_items: dict[str, Expression] = {} + statements: list[Statement] = [] for stmt in defn.defs.body: + statements.append(stmt) if not isinstance(stmt, AssignmentStmt): # Still allow pass or ... (for empty namedtuples). if isinstance(stmt, PassStmt) or ( @@ -160,9 +165,13 @@ def check_namedtuple_classdef( # And docstrings. if isinstance(stmt, ExpressionStmt) and isinstance(stmt.expr, StrExpr): continue + statements.pop() + defn.removed_statements.append(stmt) self.fail(NAMEDTUP_CLASS_ERROR, stmt) elif len(stmt.lvalues) > 1 or not isinstance(stmt.lvalues[0], NameExpr): # An assignment, but an invalid one. + statements.pop() + defn.removed_statements.append(stmt) self.fail(NAMEDTUP_CLASS_ERROR, stmt) else: # Append name and type in this case... @@ -199,7 +208,7 @@ def check_namedtuple_classdef( ) else: default_items[name] = stmt.rvalue - return items, types, default_items + return items, types, default_items, statements def check_namedtuple( self, node: Expression, var_name: str | None, is_func_scope: bool diff --git a/mypy/semanal_typeddict.py b/mypy/semanal_typeddict.py index e8be82bd41be..fb45dcc0dfc4 100644 --- a/mypy/semanal_typeddict.py +++ b/mypy/semanal_typeddict.py @@ -283,9 +283,11 @@ def analyze_typeddict_classdef_fields( ): statements.append(stmt) else: + defn.removed_statements.append(stmt) self.fail(TPDICT_CLASS_ERROR, stmt) elif len(stmt.lvalues) > 1 or not isinstance(stmt.lvalues[0], NameExpr): # An assignment, but an invalid one. + defn.removed_statements.append(stmt) self.fail(TPDICT_CLASS_ERROR, stmt) else: name = stmt.lvalues[0].name diff --git a/mypy/server/aststrip.py b/mypy/server/aststrip.py index 1bfd820efb21..87ce63e9d543 100644 --- a/mypy/server/aststrip.py +++ b/mypy/server/aststrip.py @@ -140,6 +140,8 @@ def visit_class_def(self, node: ClassDef) -> None: ] with self.enter_class(node.info): super().visit_class_def(node) + node.defs.body.extend(node.removed_statements) + node.removed_statements = [] TypeState.reset_subtype_caches_for(node.info) # Kill the TypeInfo, since there is none before semantic analysis. node.info = CLASSDEF_NO_INFO diff --git a/test-data/unit/check-class-namedtuple.test b/test-data/unit/check-class-namedtuple.test index 8e0545953bd8..8ae7f6555f9d 100644 --- a/test-data/unit/check-class-namedtuple.test +++ b/test-data/unit/check-class-namedtuple.test @@ -393,8 +393,6 @@ class X(typing.NamedTuple): [out] main:6: error: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" main:7: error: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" -main:7: error: Type cannot be declared in assignment to non-self attribute -main:7: error: "int" has no attribute "x" main:9: error: Non-default NamedTuple fields cannot follow default fields [builtins fixtures/list.pyi] diff --git a/test-data/unit/fine-grained.test b/test-data/unit/fine-grained.test index a6d8f206fbba..c162f402486a 100644 --- a/test-data/unit/fine-grained.test +++ b/test-data/unit/fine-grained.test @@ -10205,3 +10205,75 @@ C [builtins fixtures/dict.pyi] [out] == + +[case testNamedTupleNestedCrash] +import m +[file m.py] +from typing import NamedTuple + +class NT(NamedTuple): + class C: ... + x: int + y: int + +[file m.py.2] +from typing import NamedTuple + +class NT(NamedTuple): + class C: ... + x: int + y: int +# change +[builtins fixtures/tuple.pyi] +[out] +m.py:4: error: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" +== +m.py:4: error: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" + +[case testNamedTupleNestedClassRecheck] +import n +[file n.py] +import m +x: m.NT +[file m.py] +from typing import NamedTuple +from f import A + +class NT(NamedTuple): + class C: ... + x: int + y: A + +[file f.py] +A = int +[file f.py.2] +A = str +[builtins fixtures/tuple.pyi] +[out] +m.py:5: error: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" +== +m.py:5: error: Invalid statement in NamedTuple definition; expected "field_name: field_type [= default]" + +[case testTypedDictNestedClassRecheck] +import n +[file n.py] +import m +x: m.TD +[file m.py] +from typing_extensions import TypedDict +from f import A + +class TD(TypedDict): + class C: ... + x: int + y: A + +[file f.py] +A = int +[file f.py.2] +A = str +[builtins fixtures/dict.pyi] +[out] +m.py:5: error: Invalid statement in TypedDict definition; expected "field_name: field_type" +== +m.py:5: error: Invalid statement in TypedDict definition; expected "field_name: field_type"