Turn on strict optional in CI (#3646)

Turn on strict optional for most files when self-checking (there are still 16 files 
that are skipped).

Refactor more code to be strict-optional clean.

Author: ilevkivskyi

mypy/binder.py

@@ -1,4 +1,4 @@
-from typing import (Dict, List, Set, Iterator, Union)
+from typing import Dict, List, Set, Iterator, Union, Optional, cast
 from contextlib import contextmanager
 
 from mypy.types import Type, AnyType, PartialType, UnionType, NoneTyp
@@ -31,6 +31,13 @@ def __init__(self) -> None:
         self.unreachable = False
 
 
+class DeclarationsFrame(Dict[Key, Optional[Type]]):
+    """Same as above, but allowed to have None values."""
+
+    def __init__(self) -> None:
+        self.unreachable = False
+
+
 class ConditionalTypeBinder:
     """Keep track of conditional types of variables.
 
@@ -68,9 +75,9 @@ def __init__(self) -> None:
         # has no corresponding element in this list.
         self.options_on_return = []  # type: List[List[Frame]]
 
-        # Maps expr.literal_hash] to get_declaration(expr)
+        # Maps expr.literal_hash to get_declaration(expr)
         # for every expr stored in the binder
-        self.declarations = Frame()
+        self.declarations = DeclarationsFrame()
         # Set of other keys to invalidate if a key is changed, e.g. x -> {x.a, x[0]}
         # Whenever a new key (e.g. x.a.b) is added, we update this
         self.dependencies = {}  # type: Dict[Key, Set[Key]]
@@ -101,7 +108,7 @@ def push_frame(self) -> Frame:
     def _put(self, key: Key, type: Type, index: int=-1) -> None:
         self.frames[index][key] = type
 
-    def _get(self, key: Key, index: int=-1) -> Type:
+    def _get(self, key: Key, index: int=-1) -> Optional[Type]:
         if index < 0:
             index += len(self.frames)
         for i in range(index, -1, -1):
@@ -124,7 +131,7 @@ def put(self, expr: Expression, typ: Type) -> None:
     def unreachable(self) -> None:
         self.frames[-1].unreachable = True
 
-    def get(self, expr: Expression) -> Type:
+    def get(self, expr: Expression) -> Optional[Type]:
         return self._get(expr.literal_hash)
 
     def is_unreachable(self) -> bool:
@@ -163,15 +170,17 @@ def update_from_options(self, frames: List[Frame]) -> bool:
                 # know anything about key in at least one possible frame.
                 continue
 
+            type = resulting_values[0]
+            assert type is not None
             if isinstance(self.declarations.get(key), AnyType):
-                type = resulting_values[0]
-                if not all(is_same_type(type, t) for t in resulting_values[1:]):
+                # At this point resulting values can't contain None, see continue above
+                if not all(is_same_type(type, cast(Type, t)) for t in resulting_values[1:]):
                     type = AnyType()
             else:
-                type = resulting_values[0]
                 for other in resulting_values[1:]:
+                    assert other is not None
                     type = join_simple(self.declarations[key], type, other)
-            if not is_same_type(type, current_value):
+            if current_value is None or not is_same_type(type, current_value):
                 self._put(key, type)
                 changed = True
 
@@ -252,7 +261,7 @@ def invalidate_dependencies(self, expr: BindableExpression) -> None:
         for dep in self.dependencies.get(expr.literal_hash, set()):
             self._cleanse_key(dep)
 
-    def most_recent_enclosing_type(self, expr: BindableExpression, type: Type) -> Type:
+    def most_recent_enclosing_type(self, expr: BindableExpression, type: Type) -> Optional[Type]:
         if isinstance(type, AnyType):
             return get_declaration(expr)
         key = expr.literal_hash
@@ -342,7 +351,7 @@ def top_frame_context(self) -> Iterator[Frame]:
         self.pop_frame(True, 0)
 
 
-def get_declaration(expr: BindableExpression) -> Type:
+def get_declaration(expr: BindableExpression) -> Optional[Type]:
     if isinstance(expr, RefExpr) and isinstance(expr.node, Var):
         type = expr.node.type
         if not isinstance(type, PartialType):

mypy/checkmember.py

@@ -212,7 +212,7 @@ def analyze_member_var_access(name: str, itype: Instance, info: TypeInfo,
                               not_ready_callback: Callable[[str, Context], None],
                               msg: MessageBuilder,
                               original_type: Type,
-                              chk: 'mypy.checker.TypeChecker' = None) -> Type:
+                              chk: 'mypy.checker.TypeChecker') -> Type:
     """Analyse attribute access that does not target a method.
 
     This is logically part of analyze_member_access and the arguments are similar.

mypy/errors.py

@@ -4,7 +4,7 @@
 from collections import OrderedDict, defaultdict
 from contextlib import contextmanager
 
-from typing import Tuple, List, TypeVar, Set, Dict, Iterator, Optional
+from typing import Tuple, List, TypeVar, Set, Dict, Iterator, Optional, cast
 
 from mypy.options import Options
 from mypy.version import __version__ as mypy_version
@@ -278,7 +278,7 @@ def report(self, line: int, column: int, message: str, blocker: bool = False,
         self.add_error_info(info)
 
     def add_error_info(self, info: ErrorInfo) -> None:
-        (file, line) = info.origin
+        (file, line) = cast(Tuple[str, int], info.origin)  # see issue 1855
         if not info.blocker:  # Blockers cannot be ignored
             if file in self.ignored_lines and line in self.ignored_lines[file]:
                 # Annotation requests us to ignore all errors on this line.

mypy/exprtotype.py

@@ -34,6 +34,7 @@ def expr_to_unanalyzed_type(expr: Expression, _parent: Optional[Expression] = No
     """
     # The `parent` paremeter is used in recursive calls to provide context for
     # understanding whether an CallableArgument is ok.
+    name = None  # type: Optional[str]
     if isinstance(expr, NameExpr):
         name = expr.name
         return UnboundType(name, line=expr.line, column=expr.column)

mypy/join.py

@@ -15,7 +15,7 @@
 from mypy import experiments
 
 
-def join_simple(declaration: Type, s: Type, t: Type) -> Type:
+def join_simple(declaration: Optional[Type], s: Type, t: Type) -> Type:
     """Return a simple least upper bound given the declared type."""
 
     if (s.can_be_true, s.can_be_false) != (t.can_be_true, t.can_be_false):

mypy/main.py

@@ -28,7 +28,7 @@ class InvalidPackageName(Exception):
     """Exception indicating that a package name was invalid."""
 
 
-def main(script_path: str, args: List[str] = None) -> None:
+def main(script_path: Optional[str], args: List[str] = None) -> None:
     """Main entry point to the type checker.
 
     Args:

mypy/messages.py

@@ -6,7 +6,7 @@
 import re
 import difflib
 
-from typing import cast, List, Dict, Any, Sequence, Iterable, Tuple
+from typing import cast, List, Dict, Any, Sequence, Iterable, Tuple, Optional
 
 from mypy.erasetype import erase_type
 from mypy.errors import Errors
@@ -591,7 +591,8 @@ def too_few_arguments(self, callee: CallableType, context: Context,
             else:
                 msg = 'Missing positional arguments'
             if callee.name and diff and all(d is not None for d in diff):
-                msg += ' "{}" in call to {}'.format('", "'.join(diff), callee.name)
+                msg += ' "{}" in call to {}'.format('", "'.join(cast(List[str], diff)),
+                                                    callee.name)
         else:
             msg = 'Too few arguments'
             if callee.name:
@@ -625,6 +626,7 @@ def unexpected_keyword_argument(self, callee: CallableType, name: str,
         self.fail(msg, context)
         module = find_defining_module(self.modules, callee)
         if module:
+            assert callee.definition is not None
             self.note('{} defined here'.format(callee.name), callee.definition,
                       file=module.path, origin=context)
 
@@ -636,9 +638,11 @@ def duplicate_argument_value(self, callee: CallableType, index: int,
 
     def does_not_return_value(self, callee_type: Type, context: Context) -> None:
         """Report an error about use of an unusable type."""
-        if isinstance(callee_type, FunctionLike) and callee_type.get_name() is not None:
-            self.fail('{} does not return a value'.format(
-                capitalize(callee_type.get_name())), context)
+        name = None  # type: Optional[str]
+        if isinstance(callee_type, FunctionLike):
+            name = callee_type.get_name()
+        if name is not None:
+            self.fail('{} does not return a value'.format(capitalize(name)), context)
         else:
             self.fail('Function does not return a value', context)
 
@@ -1011,7 +1015,7 @@ def callable_name(type: CallableType) -> str:
         return 'function'
 
 
-def find_defining_module(modules: Dict[str, MypyFile], typ: CallableType) -> MypyFile:
+def find_defining_module(modules: Dict[str, MypyFile], typ: CallableType) -> Optional[MypyFile]:
     if not typ.definition:
         return None
     fullname = typ.definition.fullname()

mypy/nodes.py

@@ -1954,7 +1954,7 @@ class is generic then it will be a type constructor of higher kind.
     mro = None  # type: List[TypeInfo]
 
     declared_metaclass = None  # type: Optional[mypy.types.Instance]
-    metaclass_type = None  # type: mypy.types.Instance
+    metaclass_type = None  # type: Optional[mypy.types.Instance]
 
     subtypes = None  # type: Set[TypeInfo] # Direct subclasses encountered so far
     names = None  # type: SymbolTable      # Names defined directly in this type
@@ -2506,9 +2506,9 @@ def check_arg_kinds(arg_kinds: List[int], nodes: List[T], fail: Callable[[str, T
             is_kw_arg = True
 
 
-def check_arg_names(names: List[str], nodes: List[T], fail: Callable[[str, T], None],
+def check_arg_names(names: List[Optional[str]], nodes: List[T], fail: Callable[[str, T], None],
                     description: str = 'function definition') -> None:
-    seen_names = set()  # type: Set[str]
+    seen_names = set()  # type: Set[Optional[str]]
     for name, node in zip(names, nodes):
         if name is not None and name in seen_names:
             fail("Duplicate argument '{}' in {}".format(name, description), node)

mypy/semanal.py

@@ -1713,6 +1713,7 @@ def analyze_lvalue(self, lval: Lvalue, nested: bool = False,
                 v.info = self.type
                 v.is_initialized_in_class = True
                 v.set_line(lval)
+                v._fullname = self.qualified_name(lval.name)
                 lval.node = v
                 lval.is_def = True
                 lval.kind = MDEF

mypy/stats.py

@@ -103,7 +103,10 @@ def visit_assignment_stmt(self, o: AssignmentStmt) -> None:
                     items = [lvalue]
                 for item in items:
                     if isinstance(item, RefExpr) and item.is_def:
-                        t = self.typemap.get(item)
+                        if self.typemap is not None:
+                            t = self.typemap.get(item)
+                        else:
+                            t = None
                         if t:
                             self.type(t)
                         else:
@@ -151,10 +154,11 @@ def visit_unary_expr(self, o: UnaryExpr) -> None:
 
     def process_node(self, node: Expression) -> None:
         if self.all_nodes:
-            typ = self.typemap.get(node)
-            if typ:
-                self.line = node.line
-                self.type(typ)
+            if self.typemap is not None:
+                typ = self.typemap.get(node)
+                if typ:
+                    self.line = node.line
+                    self.type(typ)
 
     def type(self, t: Type) -> None:
         if isinstance(t, AnyType):

mypy/test/data.py

@@ -164,7 +164,9 @@ def parse_test_cases(
                 for file_path, contents in files:
                     expand_errors(contents.split('\n'), tcout, file_path)
                 lastline = p[i].line if i < len(p) else p[i - 1].line + 9999
-                tc = DataDrivenTestCase(p[i0].arg, input, tcout, tcout2, path,
+                arg0 = p[i0].arg
+                assert arg0 is not None
+                tc = DataDrivenTestCase(arg0, input, tcout, tcout2, path,
                                         p[i0].line, lastline, perform,
                                         files, output_files, stale_modules,
                                         rechecked_modules, deleted_paths, native_sep)
@@ -200,7 +202,7 @@ def __init__(self,
                  file: str,
                  line: int,
                  lastline: int,
-                 perform: Callable[['DataDrivenTestCase'], None],
+                 perform: Optional[Callable[['DataDrivenTestCase'], None]],
                  files: List[Tuple[str, str]],
                  output_files: List[Tuple[str, str]],
                  expected_stale_modules: Dict[int, Set[str]],
@@ -270,6 +272,7 @@ def run(self) -> None:
         if self.name.endswith('-skip'):
             raise SkipTestCaseException()
         else:
+            assert self.perform is not None, 'Tests without `perform` should not be `run`'
             self.perform(self)
 
     def tear_down(self) -> None:

mypy/test/testdeps.py

@@ -1,7 +1,7 @@
 """Test cases for generating node-level dependencies (for fine-grained incremental checking)"""
 
 import os
-from typing import List, Tuple, Dict
+from typing import List, Tuple, Dict, Optional
 
 from mypy import build
 from mypy.build import BuildSource
@@ -35,6 +35,8 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
         src = '\n'.join(testcase.input)
         messages, files, type_map = self.build(src)
         a = messages
+        assert files is not None and type_map is not None, ('cases where CompileError'
+                                                            ' occurred should not be run')
         deps = get_dependencies('__main__', files['__main__'], type_map)
 
         for source, targets in sorted(deps.items()):
@@ -49,8 +51,8 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
                                                   testcase.line))
 
     def build(self, source: str) -> Tuple[List[str],
-                                          Dict[str, MypyFile],
-                                          Dict[Expression, Type]]:
+                                          Optional[Dict[str, MypyFile]],
+                                          Optional[Dict[Expression, Type]]]:
         options = Options()
         options.use_builtins_fixtures = True
         options.show_traceback = True

mypy/test/testdiff.py

@@ -1,7 +1,7 @@
 """Test cases for AST diff (used for fine-grained incremental checking)"""
 
 import os
-from typing import List, Tuple, Dict
+from typing import List, Tuple, Dict, Optional
 
 from mypy import build
 from mypy.build import BuildSource
@@ -46,6 +46,8 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
             a.append('== next ==')
             a.extend(messages2)
 
+        assert files1 is not None and files2 is not None, ('cases where CompileError'
+                                                           ' occurred should not be run')
         diff = compare_symbol_tables(
             '__main__',
             files1['__main__'].names,
@@ -58,7 +60,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
             'Invalid output ({}, line {})'.format(testcase.file,
                                                   testcase.line))
 
-    def build(self, source: str) -> Tuple[List[str], Dict[str, MypyFile]]:
+    def build(self, source: str) -> Tuple[List[str], Optional[Dict[str, MypyFile]]]:
         options = Options()
         options.use_builtins_fixtures = True
         options.show_traceback = True

mypy/test/testsemanal.py

@@ -201,6 +201,7 @@ def run_test(self, testcase: DataDrivenTestCase) -> None:
             for f in result.files.values():
                 for n in f.names.values():
                     if isinstance(n.node, TypeInfo):
+                        assert n.fullname is not None
                         typeinfos[n.fullname] = n.node
 
             # The output is the symbol table converted into a string.

mypy/test/testsolve.py

@@ -1,6 +1,6 @@
 """Test cases for the constraint solver used in type inference."""
 
-from typing import List, Union, Tuple
+from typing import List, Union, Tuple, Optional
 
 from mypy.myunit import Suite, assert_equal
 from mypy.constraints import SUPERTYPE_OF, SUBTYPE_OF, Constraint
@@ -114,9 +114,9 @@ def test_both_normal_and_any_types_in_results(self) -> None:
     def assert_solve(self,
                      vars: List[TypeVarId],
                      constraints: List[Constraint],
-                     results: List[Union[Type, Tuple[Type, Type]]],
+                     results: List[Union[None, Type, Tuple[Type, Type]]],
                      ) -> None:
-        res = []
+        res = []  # type: List[Optional[Type]]
         for r in results:
             if isinstance(r, tuple):
                 res.append(r[0])

mypy/traverser.py

@@ -122,8 +122,9 @@ def visit_raise_stmt(self, o: RaiseStmt) -> None:
     def visit_try_stmt(self, o: TryStmt) -> None:
         o.body.accept(self)
         for i in range(len(o.types)):
-            if o.types[i]:
-                o.types[i].accept(self)
+            tp = o.types[i]
+            if tp is not None:
+                tp.accept(self)
             o.handlers[i].accept(self)
         if o.else_body is not None:
             o.else_body.accept(self)
@@ -133,8 +134,9 @@ def visit_try_stmt(self, o: TryStmt) -> None:
     def visit_with_stmt(self, o: WithStmt) -> None:
         for i in range(len(o.expr)):
             o.expr[i].accept(self)
-            if o.target[i] is not None:
-                o.target[i].accept(self)
+            targ = o.target[i]
+            if targ is not None:
+                targ.accept(self)
         o.body.accept(self)
 
     def visit_member_expr(self, o: MemberExpr) -> None: