New semantic analyzer: fix corner cases in aliases to not ready classes (and other aliases) (#6390)

Fixes #6355

This does three non-trivial things:
* Creates placeholder with `becomes_typeinfo=True` if r.h.s. is such a placeholder (i.e. the definition is a potential type alias).
* Defers all `Invalid type` and use of corresponding unbound types until final iteration.
* Moves few tests in newly enabled test file to new semantic analyzer only (those where new behaviour is clearly better).

I added a bunch of new tests for things that was broken, plus some variations (although some of them are trivial, replacing `NameExpr` with `MemberExpr` or `IndexExpr`).

I also verified that this PR fixes the crash in mypy self-check.

Author: ilevkivskyi

mypy/newsemanal/semanal.py

@@ -213,6 +213,9 @@ class NewSemanticAnalyzer(NodeVisitor[None],
     # Stack of functions being analyzed
     function_stack = None  # type: List[FuncItem]
 
+    # Is this the final iteration of semantic analysis?
+    final_iteration = False
+
     loop_depth = 0         # Depth of breakable loops
     cur_mod_id = ''        # Current module id (or None) (phase 2)
     is_stub_file = False   # Are we analyzing a stub file?
@@ -1875,16 +1878,49 @@ def add_type_alias_deps(self, aliases_used: Iterable[str],
             target = self.scope.current_target()
         self.cur_mod_node.alias_deps[target].update(aliases_used)
 
+    def is_not_ready_type_ref(self, rv: Expression) -> bool:
+        """Does this expression refers to a not-ready class?
+
+        This includes 'Ref' and 'Ref[Arg1, Arg2, ...]', where 'Ref'
+        refers to a PlaceholderNode with becomes_typeinfo=True.
+        """
+        if isinstance(rv, IndexExpr) and isinstance(rv.base, RefExpr):
+            return self.is_not_ready_type_ref(rv.base)
+        if isinstance(rv, NameExpr):
+            n = self.lookup(rv.name, rv)
+            if n and isinstance(n.node, PlaceholderNode) and n.node.becomes_typeinfo:
+                return True
+        elif isinstance(rv, MemberExpr):
+            fname = get_member_expr_fullname(rv)
+            if fname:
+                n = self.lookup_qualified(fname, rv)
+                if n and isinstance(n.node, PlaceholderNode) and n.node.becomes_typeinfo:
+                    return True
+        return False
+
     def visit_assignment_stmt(self, s: AssignmentStmt) -> None:
         s.is_final_def = self.unwrap_final(s)
         tag = self.track_incomplete_refs()
         s.rvalue.accept(self)
-        if self.found_incomplete_ref(tag):
+        if isinstance(s.rvalue, IndexExpr) and isinstance(s.rvalue.base, RefExpr):
+            # Special case: analyze index expression _as a type_ to trigger
+            # incomplete refs for string forward references, for example
+            # Union['ClassA', 'ClassB'].
+            # We throw away the results of the analysis and we only care about
+            # the detection of incomplete references (this doesn't change the expression
+            # in place).
+            self.analyze_alias(s.rvalue, allow_placeholder=True)
+        top_level_not_ready = self.is_not_ready_type_ref(s.rvalue)
+        # NOTE: the first check is insufficient. We want to defer creation of a Var.
+        if self.found_incomplete_ref(tag) or top_level_not_ready:
             # Initializer couldn't be fully analyzed. Defer the current node and give up.
             # Make sure that if we skip the definition of some local names, they can't be
             # added later in this scope, since an earlier definition should take precedence.
             for expr in names_modified_by_assignment(s):
-                self.mark_incomplete(expr.name, expr)
+                # NOTE: Currently for aliases like 'X = List[Y]', where 'Y' is not ready
+                # we proceed forward and create a Var. The latter will be replaced with
+                # a type alias it r.h.s. is a valid alias.
+                self.mark_incomplete(expr.name, expr, becomes_typeinfo=top_level_not_ready)
             return
         if self.analyze_namedtuple_assign(s):
             return
@@ -2119,8 +2155,9 @@ def analyze_simple_literal_type(self, rvalue: Expression, is_final: bool) -> Opt
 
         return None
 
-    def analyze_alias(self, rvalue: Expression) -> Tuple[Optional[Type], List[str],
-                                                         Set[str], List[str]]:
+    def analyze_alias(self, rvalue: Expression,
+                      allow_placeholder: bool = False) -> Tuple[Optional[Type], List[str],
+                                                                Set[str], List[str]]:
         """Check if 'rvalue' is a valid type allowed for aliasing (e.g. not a type variable).
 
         If yes, return the corresponding type, a list of
@@ -2140,6 +2177,7 @@ def analyze_alias(self, rvalue: Expression) -> Tuple[Optional[Type], List[str],
                                  self.options,
                                  self.is_typeshed_stub_file,
                                  allow_unnormalized=self.is_stub_file,
+                                 allow_placeholder=allow_placeholder,
                                  in_dynamic_func=dynamic,
                                  global_scope=global_scope)
         typ = None  # type: Optional[Type]
@@ -2182,8 +2220,9 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None:
             # annotations (see the second rule).
             return
         rvalue = s.rvalue
+        tag = self.track_incomplete_refs()
         res, alias_tvars, depends_on, qualified_tvars = self.analyze_alias(rvalue)
-        if not res:
+        if not res or self.found_incomplete_ref(tag):
             return
         if (isinstance(res, Instance) and res.type.name() == lvalue.name and
                 res.type.module_name == self.cur_mod_id):
@@ -2220,7 +2259,7 @@ def check_and_set_up_type_alias(self, s: AssignmentStmt) -> None:
             s.rvalue.analyzed.column = res.column
         elif isinstance(s.rvalue, RefExpr):
             s.rvalue.is_alias_rvalue = True
-        node.node = TypeAlias(res, node.node.fullname(), s.line, s.column,
+        node.node = TypeAlias(res, self.qualified_name(lvalue.name), s.line, s.column,
                               alias_tvars=alias_tvars, no_args=no_args)
         if isinstance(rvalue, RefExpr) and isinstance(rvalue.node, TypeAlias):
             node.node.normalized = rvalue.node.normalized

mypy/newsemanal/semanal_main.py

@@ -138,8 +138,10 @@ def process_top_level_function(analyzer: 'NewSemanticAnalyzer',
             # OK, this is one last pass, now missing names will be reported.
             more_iterations = False
             analyzer.incomplete_namespaces.discard(module)
-        deferred, incomplete = semantic_analyze_target(target, state, node, active_type, False)
+        deferred, incomplete = semantic_analyze_target(target, state, node, active_type,
+                                                       not more_iterations)
 
+    analyzer.incomplete_namespaces.discard(module)
     # After semantic analysis is done, discard local namespaces
     # to avoid memory hoarding.
     analyzer.saved_locals.clear()

mypy/newsemanal/semanal_shared.py

@@ -68,6 +68,11 @@ def is_incomplete_namespace(self, fullname: str) -> bool:
         """Is a module or class namespace potentially missing some definitions?"""
         raise NotImplementedError
 
+    @abstractproperty
+    def final_iteration(self) -> bool:
+        """Is this the final iteration of semantic analysis?"""
+        raise NotImplementedError
+
 
 @trait
 class SemanticAnalyzerInterface(SemanticAnalyzerCoreInterface):

mypy/newsemanal/typeanal.py

@@ -65,6 +65,7 @@ def analyze_type_alias(node: Expression,
                        options: Options,
                        is_typeshed_stub: bool,
                        allow_unnormalized: bool = False,
+                       allow_placeholder: bool = False,
                        in_dynamic_func: bool = False,
                        global_scope: bool = True) -> Optional[Tuple[Type, Set[str]]]:
     """Analyze r.h.s. of a (potential) type alias definition.
@@ -121,7 +122,8 @@ def analyze_type_alias(node: Expression,
         api.fail('Invalid type alias', node)
         return None
     analyzer = TypeAnalyser(api, tvar_scope, plugin, options, is_typeshed_stub,
-                            allow_unnormalized=allow_unnormalized, defining_alias=True)
+                            allow_unnormalized=allow_unnormalized, defining_alias=True,
+                            allow_placeholder=allow_placeholder)
     analyzer.in_dynamic_func = in_dynamic_func
     analyzer.global_scope = global_scope
     res = type.accept(analyzer)
@@ -407,7 +409,13 @@ def analyze_unbound_type_without_type_info(self, t: UnboundType, sym: SymbolTabl
         if self.allow_unbound_tvars and unbound_tvar and not self.third_pass:
             return t
         # None of the above options worked, we give up.
-        self.fail('Invalid type "{}"'.format(name), t)
+        # NOTE: 'final_iteration' is iteration when we hit the maximum number of iterations limit.
+        if self.api.final_iteration:
+            # TODO: This is problematic, since we will have to wait until the maximum number
+            #       of iterations to report an invalid type.
+            self.fail('Invalid type "{}"'.format(name), t)
+        else:
+            self.api.defer()
         if self.third_pass and isinstance(sym.node, TypeVarExpr):
             self.note_func("Forward references to type variables are prohibited", t)
             return AnyType(TypeOfAny.from_error)
@@ -678,7 +686,7 @@ def analyze_literal_param(self, idx: int, arg: Type, ctx: Context) -> Optional[L
             # We report an error in only the first two cases. In the third case, we assume
             # some other region of the code has already reported a more relevant error.
             #
-            # TODO: Once we start adding support for enums, make sure we reprt a custom
+            # TODO: Once we start adding support for enums, make sure we report a custom
             # error for case 2 as well.
             if arg.type_of_any != TypeOfAny.from_error:
                 self.fail('Parameter {} of Literal[...] cannot be of type "Any"'.format(idx), ctx)

mypy/test/hacks.py

@@ -43,7 +43,6 @@
     'check-serialize.test',
     'check-statements.test',
     'check-tuples.test',
-    'check-type-aliases.test',
     'check-typeddict.test',
     'check-typevar-values.test',
     'check-unions.test',

test-data/unit/check-newsemanal.test

@@ -1055,6 +1055,174 @@ class Test:
     def __init__(self) -> None:
         some_module = self.a
 
+[case testNewAnalyzerAliasToNotReadyClass]
+import a
+[file a.py]
+from b import B
+
+x: A
+A = B
+[file b.py]
+from typing import List
+from a import x
+
+class B(List[B]): pass
+
+reveal_type(x[0][0])  # E: Revealed type is 'b.B*'
+[builtins fixtures/list.pyi]
+
+[case testNewAnalyzerAliasToNotReadyClass2]
+from typing import List
+
+x: A
+
+class A(List[B]): pass
+B = A
+
+reveal_type(x[0][0])  # E: Revealed type is '__main__.A*'
+[builtins fixtures/list.pyi]
+
+[case testNewAnalyzerAliasToNotReadyClass3]
+from typing import List
+
+x: B
+B = A
+A = C
+class C(List[B]): pass
+
+reveal_type(x[0][0])  # E: Revealed type is '__main__.C*'
+[builtins fixtures/list.pyi]
+
+[case testNewAnalyzerAliasToNotReadyNestedClass]
+import a
+[file a.py]
+from b import Out
+
+x: A
+A = Out.B
+[file b.py]
+from typing import List
+from a import x
+
+class Out:
+    class B(List[B]): pass
+
+reveal_type(x[0][0])  # E: Revealed type is 'b.Out.B*'
+[builtins fixtures/list.pyi]
+
+[case testNewAnalyzerAliasToNotReadyNestedClass2]
+from typing import List
+
+x: Out.A
+
+class Out:
+    class A(List[B]): pass
+B = Out.A
+
+reveal_type(x[0][0])  # E: Revealed type is '__main__.Out.A*'
+[builtins fixtures/list.pyi]
+
+[case testNewAnalyzerAliasToNotReadyClassGeneric]
+import a
+[file a.py]
+from typing import Tuple
+from b import B, T
+
+x: A[int]
+A = B[Tuple[T, T]]
+[file b.py]
+from typing import List, Generic, TypeVar
+from a import x
+
+class B(List[B], Generic[T]): pass
+T = TypeVar('T')
+reveal_type(x)  # E: Revealed type is 'b.B[Tuple[builtins.int, builtins.int]]'
+[builtins fixtures/list.pyi]
+
+[case testNewAnalyzerAliasToNotReadyClassInGeneric]
+import a
+[file a.py]
+from typing import Tuple
+from b import B
+
+x: A
+A = Tuple[B, B]
+[file b.py]
+from typing import List
+from a import x
+
+class B(List[B]): pass
+
+reveal_type(x)  # E: Revealed type is 'Tuple[b.B, b.B]'
+[builtins fixtures/list.pyi]
+
+[case testNewAnalyzerAliasToNotReadyClassDoubleGeneric]
+from typing import List, TypeVar, Union
+
+T = TypeVar('T')
+
+x: B[int]
+B = A[List[T]]
+A = Union[int, T]
+class C(List[B[int]]): pass
+
+reveal_type(x)  # E: Revealed type is 'Union[builtins.int, builtins.list[builtins.int]]'
+reveal_type(y[0])  # E: Revealed type is 'Union[builtins.int, builtins.list[builtins.int]]'
+y: C
+[builtins fixtures/list.pyi]
+
+[case testNewAnalyzerForwardAliasFromUnion]
+from typing import Union, List
+
+A = Union['B', 'C']
+
+class D:
+    x: List[A]
+
+    def test(self) -> None:
+        reveal_type(self.x[0].y)  # E: Revealed type is 'builtins.int'
+
+class B:
+    y: int
+class C:
+    y: int
+[builtins fixtures/list.pyi]
+
+[case testNewAnalyzerAliasToNotReadyTwoDeferrals-skip]
+from typing import List
+
+x: B
+B = List[C]
+A = C
+class C(List[A]): pass
+
+reveal_type(x)
+[builtins fixtures/list.pyi]
+
+[case testNewAnalyzerAliasToNotReadyDirectBase-skip]
+from typing import List
+
+x: B
+B = List[C]
+class C(B): pass
+
+reveal_type(x)  # E: Revealed type is 'builtins.list[__main__.C]'
+reveal_type(x[0][0])  # E: Revealed type is '__main__.C'
+[builtins fixtures/list.pyi]
+
+[case testNewAnalyzerAliasToNotReadyMixed]
+from typing import List, Union
+x: A
+
+A = Union[B, C]
+
+class B(List[A]): pass
+class C(List[A]): pass
+
+reveal_type(x)  # E: Revealed type is 'Union[__main__.B, __main__.C]'
+reveal_type(x[0])  # E: Revealed type is 'Union[__main__.B, __main__.C]'
+[builtins fixtures/list.pyi]
+
 [case testNewAnalyzerListComprehension]
 from typing import List
 a: List[A]