Turn on strict optional in CI (#3646)

Turn on strict optional for most files when self-checking (there are still 16 files 
that are skipped).

Refactor more code to be strict-optional clean.

Author: ilevkivskyi

mypy/binder.py

@@ -1,4 +1,4 @@
-from typing import (Dict, List, Set, Iterator, Union)
+from typing import Dict, List, Set, Iterator, Union, Optional, cast
 from contextlib import contextmanager
 
 from mypy.types import Type, AnyType, PartialType, UnionType, NoneTyp
@@ -31,6 +31,13 @@ def __init__(self) -> None:
         self.unreachable = False
 
 
+class DeclarationsFrame(Dict[Key, Optional[Type]]):
+    """Same as above, but allowed to have None values."""
+
+    def __init__(self) -> None:
+        self.unreachable = False
+
+
 class ConditionalTypeBinder:
     """Keep track of conditional types of variables.
 
@@ -68,9 +75,9 @@ def __init__(self) -> None:
         # has no corresponding element in this list.
         self.options_on_return = []  # type: List[List[Frame]]
 
-        # Maps expr.literal_hash] to get_declaration(expr)
+        # Maps expr.literal_hash to get_declaration(expr)
         # for every expr stored in the binder
-        self.declarations = Frame()
+        self.declarations = DeclarationsFrame()
         # Set of other keys to invalidate if a key is changed, e.g. x -> {x.a, x[0]}
         # Whenever a new key (e.g. x.a.b) is added, we update this
         self.dependencies = {}  # type: Dict[Key, Set[Key]]
@@ -101,7 +108,7 @@ def push_frame(self) -> Frame:
     def _put(self, key: Key, type: Type, index: int=-1) -> None:
         self.frames[index][key] = type
 
-    def _get(self, key: Key, index: int=-1) -> Type:
+    def _get(self, key: Key, index: int=-1) -> Optional[Type]:
         if index < 0:
             index += len(self.frames)
         for i in range(index, -1, -1):
@@ -124,7 +131,7 @@ def put(self, expr: Expression, typ: Type) -> None:
     def unreachable(self) -> None:
         self.frames[-1].unreachable = True
 
-    def get(self, expr: Expression) -> Type:
+    def get(self, expr: Expression) -> Optional[Type]:
         return self._get(expr.literal_hash)
 
     def is_unreachable(self) -> bool:
@@ -163,15 +170,17 @@ def update_from_options(self, frames: List[Frame]) -> bool:
                 # know anything about key in at least one possible frame.
                 continue
 
+            type = resulting_values[0]
+            assert type is not None
             if isinstance(self.declarations.get(key), AnyType):
-                type = resulting_values[0]
-                if not all(is_same_type(type, t) for t in resulting_values[1:]):
+                # At this point resulting values can't contain None, see continue above
+                if not all(is_same_type(type, cast(Type, t)) for t in resulting_values[1:]):
                     type = AnyType()
             else:
-                type = resulting_values[0]
                 for other in resulting_values[1:]:
+                    assert other is not None
                     type = join_simple(self.declarations[key], type, other)
-            if not is_same_type(type, current_value):
+            if current_value is None or not is_same_type(type, current_value):
                 self._put(key, type)
                 changed = True
 
@@ -252,7 +261,7 @@ def invalidate_dependencies(self, expr: BindableExpression) -> None:
         for dep in self.dependencies.get(expr.literal_hash, set()):
             self._cleanse_key(dep)
 
-    def most_recent_enclosing_type(self, expr: BindableExpression, type: Type) -> Type:
+    def most_recent_enclosing_type(self, expr: BindableExpression, type: Type) -> Optional[Type]:
         if isinstance(type, AnyType):
             return get_declaration(expr)
         key = expr.literal_hash
@@ -342,7 +351,7 @@ def top_frame_context(self) -> Iterator[Frame]:
         self.pop_frame(True, 0)
 
 
-def get_declaration(expr: BindableExpression) -> Type:
+def get_declaration(expr: BindableExpression) -> Optional[Type]:
     if isinstance(expr, RefExpr) and isinstance(expr.node, Var):
         type = expr.node.type
         if not isinstance(type, PartialType):

mypy/checkmember.py

@@ -212,7 +212,7 @@ def analyze_member_var_access(name: str, itype: Instance, info: TypeInfo,
                               not_ready_callback: Callable[[str, Context], None],
                               msg: MessageBuilder,
                               original_type: Type,
-                              chk: 'mypy.checker.TypeChecker' = None) -> Type:
+                              chk: 'mypy.checker.TypeChecker') -> Type:
     """Analyse attribute access that does not target a method.
 
     This is logically part of analyze_member_access and the arguments are similar.

mypy/errors.py

@@ -4,7 +4,7 @@
 from collections import OrderedDict, defaultdict
 from contextlib import contextmanager
 
-from typing import Tuple, List, TypeVar, Set, Dict, Iterator, Optional
+from typing import Tuple, List, TypeVar, Set, Dict, Iterator, Optional, cast
 
 from mypy.options import Options
 from mypy.version import __version__ as mypy_version
@@ -278,7 +278,7 @@ def report(self, line: int, column: int, message: str, blocker: bool = False,
         self.add_error_info(info)
 
     def add_error_info(self, info: ErrorInfo) -> None:
-        (file, line) = info.origin
+        (file, line) = cast(Tuple[str, int], info.origin)  # see issue 1855
         if not info.blocker:  # Blockers cannot be ignored
             if file in self.ignored_lines and line in self.ignored_lines[file]:
                 # Annotation requests us to ignore all errors on this line.

mypy/exprtotype.py

@@ -34,6 +34,7 @@ def expr_to_unanalyzed_type(expr: Expression, _parent: Optional[Expression] = No
     """
     # The `parent` paremeter is used in recursive calls to provide context for
     # understanding whether an CallableArgument is ok.
+    name = None  # type: Optional[str]
     if isinstance(expr, NameExpr):
         name = expr.name
         return UnboundType(name, line=expr.line, column=expr.column)

mypy/join.py

@@ -15,7 +15,7 @@
 from mypy import experiments
 
 
-def join_simple(declaration: Type, s: Type, t: Type) -> Type:
+def join_simple(declaration: Optional[Type], s: Type, t: Type) -> Type:
     """Return a simple least upper bound given the declared type."""
 
     if (s.can_be_true, s.can_be_false) != (t.can_be_true, t.can_be_false):

mypy/main.py

@@ -28,7 +28,7 @@ class InvalidPackageName(Exception):
     """Exception indicating that a package name was invalid."""
 
 
-def main(script_path: str, args: List[str] = None) -> None:
+def main(script_path: Optional[str], args: List[str] = None) -> None:
     """Main entry point to the type checker.
 
     Args:

mypy/messages.py

@@ -6,7 +6,7 @@
 import re
 import difflib
 
-from typing import cast, List, Dict, Any, Sequence, Iterable, Tuple
+from typing import cast, List, Dict, Any, Sequence, Iterable, Tuple, Optional
 
 from mypy.erasetype import erase_type
 from mypy.errors import Errors
@@ -591,7 +591,8 @@ def too_few_arguments(self, callee: CallableType, context: Context,
             else:
                 msg = 'Missing positional arguments'
             if callee.name and diff and all(d is not None for d in diff):
-                msg += ' "{}" in call to {}'.format('", "'.join(diff), callee.name)
+                msg += ' "{}" in call to {}'.format('", "'.join(cast(List[str], diff)),
+                                                    callee.name)
         else:
             msg = 'Too few arguments'
             if callee.name:
@@ -625,6 +626,7 @@ def unexpected_keyword_argument(self, callee: CallableType, name: str,
         self.fail(msg, context)
         module = find_defining_module(self.modules, callee)
         if module:
+            assert callee.definition is not None
             self.note('{} defined here'.format(callee.name), callee.definition,
                       file=module.path, origin=context)
 
@@ -636,9 +638,11 @@ def duplicate_argument_value(self, callee: CallableType, index: int,
 
     def does_not_return_value(self, callee_type: Type, context: Context) -> None:
         """Report an error about use of an unusable type."""
-        if isinstance(callee_type, FunctionLike) and callee_type.get_name() is not None:
-            self.fail('{} does not return a value'.format(
-                capitalize(callee_type.get_name())), context)
+        name = None  # type: Optional[str]
+        if isinstance(callee_type, FunctionLike):
+            name = callee_type.get_name()
+        if name is not None:
+            self.fail('{} does not return a value'.format(capitalize(name)), context)
         else:
             self.fail('Function does not return a value', context)
 
@@ -1011,7 +1015,7 @@ def callable_name(type: CallableType) -> str:
         return 'function'
 
 
-def find_defining_module(modules: Dict[str, MypyFile], typ: CallableType) -> MypyFile:
+def find_defining_module(modules: Dict[str, MypyFile], typ: CallableType) -> Optional[MypyFile]:
     if not typ.definition:
         return None
     fullname = typ.definition.fullname()

mypy/nodes.py

@@ -1954,7 +1954,7 @@ class is generic then it will be a type constructor of higher kind.
     mro = None  # type: List[TypeInfo]
 
     declared_metaclass = None  # type: Optional[mypy.types.Instance]
-    metaclass_type = None  # type: mypy.types.Instance
+    metaclass_type = None  # type: Optional[mypy.types.Instance]
 
     subtypes = None  # type: Set[TypeInfo] # Direct subclasses encountered so far
     names = None  # type: SymbolTable      # Names defined directly in this type
@@ -2506,9 +2506,9 @@ def check_arg_kinds(arg_kinds: List[int], nodes: List[T], fail: Callable[[str, T
             is_kw_arg = True
 
 
-def check_arg_names(names: List[str], nodes: List[T], fail: Callable[[str, T], None],
+def check_arg_names(names: List[Optional[str]], nodes: List[T], fail: Callable[[str, T], None],
                     description: str = 'function definition') -> None:
-    seen_names = set()  # type: Set[str]
+    seen_names = set()  # type: Set[Optional[str]]
     for name, node in zip(names, nodes):
         if name is not None and name in seen_names:
             fail("Duplicate argument '{}' in {}".format(name, description), node)

mypy/semanal.py

@@ -1713,6 +1713,7 @@ def analyze_lvalue(self, lval: Lvalue, nested: bool = False,
                 v.info = self.type
                 v.is_initialized_in_class = True
                 v.set_line(lval)
+                v._fullname = self.qualified_name(lval.name)
                 lval.node = v
                 lval.is_def = True
                 lval.kind = MDEF

mypy/stats.py

@@ -103,7 +103,10 @@ def visit_assignment_stmt(self, o: AssignmentStmt) -> None:
                     items = [lvalue]
                 for item in items:
                     if isinstance(item, RefExpr) and item.is_def:
-                        t = self.typemap.get(item)
+                        if self.typemap is not None:
+                            t = self.typemap.get(item)
+                        else:
+                            t = None
                         if t:
                             self.type(t)
                         else:
@@ -151,10 +154,11 @@ def visit_unary_expr(self, o: UnaryExpr) -> None:
 
     def process_node(self, node: Expression) -> None:
         if self.all_nodes:
-            typ = self.typemap.get(node)
-            if typ:
-                self.line = node.line
-                self.type(typ)
+            if self.typemap is not None:
+                typ = self.typemap.get(node)
+                if typ:
+                    self.line = node.line
+                    self.type(typ)
 
     def type(self, t: Type) -> None:
         if isinstance(t, AnyType):