diff --git a/mypy/binder.py b/mypy/binder.py index 2f83ffb095fc..adfa779f479b 100644 --- a/mypy/binder.py +++ b/mypy/binder.py @@ -10,7 +10,7 @@ from mypy.subtypes import is_subtype from mypy.join import join_simple from mypy.sametypes import is_same_type -from mypy.erasetype import remove_instance_last_known_values +from mypy.erasetype import remove_instance_transient_info from mypy.nodes import Expression, Var, RefExpr from mypy.literals import Key, literal, literal_hash, subkeys from mypy.nodes import IndexExpr, MemberExpr, AssignmentExpr, NameExpr @@ -259,7 +259,7 @@ def assign_type(self, expr: Expression, restrict_any: bool = False) -> None: # We should erase last known value in binder, because if we are using it, # it means that the target is not final, and therefore can't hold a literal. - type = remove_instance_last_known_values(type) + type = remove_instance_transient_info(type) type = get_proper_type(type) declared_type = get_proper_type(declared_type) diff --git a/mypy/checker.py b/mypy/checker.py index 95af9e5f01e9..fea9b47e8586 100644 --- a/mypy/checker.py +++ b/mypy/checker.py @@ -71,7 +71,7 @@ from mypy.typevars import fill_typevars, has_no_typevars, fill_typevars_with_any from mypy.semanal import set_callable_name, refers_to_fullname from mypy.mro import calculate_mro, MroError -from mypy.erasetype import erase_typevars, remove_instance_last_known_values, erase_type +from mypy.erasetype import erase_typevars, remove_instance_transient_info, erase_type from mypy.expandtype import expand_type, expand_type_by_instance from mypy.visitor import NodeVisitor from mypy.join import join_types @@ -2188,7 +2188,7 @@ def check_assignment(self, lvalue: Lvalue, rvalue: Expression, infer_lvalue_type if partial_types is not None: if not self.current_node_deferred: # Partial type can't be final, so strip any literal values. - rvalue_type = remove_instance_last_known_values(rvalue_type) + rvalue_type = remove_instance_transient_info(rvalue_type) inferred_type = make_simplified_union( [rvalue_type, NoneType()]) self.set_inferred_type(var, lvalue, inferred_type) @@ -2270,7 +2270,7 @@ def check_assignment(self, lvalue: Lvalue, rvalue: Expression, infer_lvalue_type if inferred: rvalue_type = self.expr_checker.accept(rvalue) if not inferred.is_final: - rvalue_type = remove_instance_last_known_values(rvalue_type) + rvalue_type = remove_instance_transient_info(rvalue_type) self.infer_variable_type(inferred, lvalue, rvalue_type, rvalue) self.check_assignment_to_slots(lvalue) @@ -4988,7 +4988,7 @@ def named_generic_type(self, name: str, args: List[Type]) -> Instance: the name refers to a compatible generic type. """ info = self.lookup_typeinfo(name) - args = [remove_instance_last_known_values(arg) for arg in args] + args = [remove_instance_transient_info(arg) for arg in args] # TODO: assert len(args) == len(info.defn.type_vars) return Instance(info, args) diff --git a/mypy/checkexpr.py b/mypy/checkexpr.py index dfac5be27d95..2963cacfaaf8 100644 --- a/mypy/checkexpr.py +++ b/mypy/checkexpr.py @@ -41,7 +41,7 @@ import mypy.checker from mypy import types from mypy.sametypes import is_same_type -from mypy.erasetype import replace_meta_vars, erase_type, remove_instance_last_known_values +from mypy.erasetype import replace_meta_vars, erase_type, remove_instance_transient_info from mypy.maptype import map_instance_to_supertype from mypy.messages import MessageBuilder from mypy import message_registry @@ -3334,7 +3334,7 @@ def check_lst_expr(self, items: List[Expression], fullname: str, [(nodes.ARG_STAR if isinstance(i, StarExpr) else nodes.ARG_POS) for i in items], context)[0] - return remove_instance_last_known_values(out) + return remove_instance_transient_info(out) def visit_tuple_expr(self, e: TupleExpr) -> Type: """Type check a tuple expression.""" diff --git a/mypy/erasetype.py b/mypy/erasetype.py index 8acebbd783d8..f4c99bd22277 100644 --- a/mypy/erasetype.py +++ b/mypy/erasetype.py @@ -139,20 +139,22 @@ def visit_type_alias_type(self, t: TypeAliasType) -> Type: return t.copy_modified(args=[a.accept(self) for a in t.args]) -def remove_instance_last_known_values(t: Type) -> Type: - return t.accept(LastKnownValueEraser()) - +def remove_instance_transient_info(t: Type) -> Type: + """Recursively removes any info from Instances that exist + on a per-instance basis. Currently, this means erasing the + last-known literal type and any plugin metadata. + """ + return t.accept(TransientInstanceInfoEraser()) -class LastKnownValueEraser(TypeTranslator): - """Removes the Literal[...] type that may be associated with any - Instance types.""" +class TransientInstanceInfoEraser(TypeTranslator): def visit_instance(self, t: Instance) -> Type: - if not t.last_known_value and not t.args: + if not t.last_known_value and not t.args and not t.metadata: return t new_t = t.copy_modified( args=[a.accept(self) for a in t.args], last_known_value=None, + metadata={}, ) new_t.can_be_true = t.can_be_true new_t.can_be_false = t.can_be_false diff --git a/mypy/plugins/default.py b/mypy/plugins/default.py index 67587090e0ba..e8d327cae1c2 100644 --- a/mypy/plugins/default.py +++ b/mypy/plugins/default.py @@ -23,7 +23,7 @@ class DefaultPlugin(Plugin): def get_function_hook(self, fullname: str ) -> Optional[Callable[[FunctionContext], Type]]: - from mypy.plugins import ctypes, singledispatch + from mypy.plugins import ctypes, regex, singledispatch if fullname in ('contextlib.contextmanager', 'contextlib.asynccontextmanager'): return contextmanager_callback @@ -31,6 +31,10 @@ def get_function_hook(self, fullname: str return open_callback elif fullname == 'ctypes.Array': return ctypes.array_constructor_callback + elif fullname == 're.compile': + return regex.re_compile_callback + elif fullname in regex.FUNCTIONS_PRODUCING_MATCH_OBJECT: + return regex.re_direct_match_callback elif fullname == 'functools.singledispatch': return singledispatch.create_singledispatch_function_callback return None @@ -55,7 +59,7 @@ def get_method_signature_hook(self, fullname: str def get_method_hook(self, fullname: str ) -> Optional[Callable[[MethodContext], Type]]: - from mypy.plugins import ctypes, singledispatch + from mypy.plugins import ctypes, regex, singledispatch if fullname == 'typing.Mapping.get': return typed_dict_get_callback @@ -77,6 +81,12 @@ def get_method_hook(self, fullname: str return ctypes.array_iter_callback elif fullname == 'pathlib.Path.open': return path_open_callback + elif fullname in regex.METHODS_PRODUCING_MATCH_OBJECT: + return regex.re_get_match_callback + elif fullname == 'typing.Match.groups': + return regex.re_match_groups_callback + elif fullname in regex.METHODS_PRODUCING_GROUP: + return regex.re_match_group_callback elif fullname == singledispatch.SINGLEDISPATCH_REGISTER_METHOD: return singledispatch.singledispatch_register_callback elif fullname == singledispatch.REGISTER_CALLABLE_CALL_METHOD: diff --git a/mypy/plugins/regex.py b/mypy/plugins/regex.py new file mode 100644 index 000000000000..8b1777a031ae --- /dev/null +++ b/mypy/plugins/regex.py @@ -0,0 +1,346 @@ +""" +A plugin for analyzing regexes to determine how many groups a +regex can contain and whether those groups are always matched or not. +For example: + + pattern: Final = re.compile("(foo)(bar)?") + match: Final = pattern.match(input_text) + if match: + reveal_type(match.groups()) + +Without the plugin, the best we can really do is determine revealed +type is either Sequence[str] or Tuple[str, ...]. But with this plugin, +we can obtain a more precise type of Tuple[str, Optionl[str]]. We were +able to deduce th first group is mandatory and the second optional. + +Broadly, this plugin works by using the underlying builtin regex +parsing engine to obtain the regex AST. We can then crawl this AST +to obtain the mandatory groups, total number of groups, and any +named groups. + +We then inject this obtained data into the Pattern or Match objects +into a "metadata" field on a per-instance basis. + +Note that while we parse the regex, we at no point will ever actually +try matching anything against it. +""" + +from typing import Union, Iterator, Tuple, List, Any, Optional, Dict +from typing_extensions import Final + +import sys + +from mypy.types import ( + Type, ProperType, Instance, NoneType, LiteralType, + TupleType, remove_optional, +) +from mypy.typeops import make_simplified_union, coerce_to_literal, get_proper_type +import mypy.plugin # To avoid circular imports. + +from sre_parse import parse, SubPattern +from sre_constants import ( + SUBPATTERN, MIN_REPEAT, MAX_REPEAT, GROUPREF_EXISTS, BRANCH, + error as SreError, _NamedIntConstant as NIC, +) + +STR_LIKE_TYPES: Final = { + 'builtins.unicode', + 'builtins.str', + 'builtins.bytes', +} + +FUNCTIONS_PRODUCING_MATCH_OBJECT = { + 're.search', + 're.match', + 're.fullmatch', +} + +METHODS_PRODUCING_MATCH_OBJECT: Final = { + 'typing.Pattern.search', + 'typing.Pattern.match', + 'typing.Pattern.fullmatch', +} + +METHODS_PRODUCING_GROUP = { + 'typing.Match.group', + 'typing.Match.__getitem__', +} + +OBJECTS_SUPPORTING_REGEX_METADATA = { + 'typing.Pattern', + 'typing.Match', +} + + +class RegexPluginException(Exception): + def __init__(self, msg: str) -> None: + super().__init__(msg) + self.msg = msg + + +def find_mandatory_groups(ast: Union[SubPattern, Tuple[NIC, Any]]) -> Iterator[int]: + """Yields the all group numbers that are guaranteed to match something + in the Match object corresponding to the given regex. + + For example, if the provided AST corresponds to the regex + "(a)(?:(b)|(c))(d)?(e)+(f)", this function would yield 1, 5, and 6. + + We do not yield 0 even though that group will always have a match. This + function only group numbers that can actually be found in the AST. + """ + if isinstance(ast, tuple): + data: List[Tuple[NIC, Any]] = [ast] + elif isinstance(ast, SubPattern): + data = ast.data + else: + raise RegexPluginException("Internal error: unexpected regex AST item '{}'".format(ast)) + + for op, av in data: + if op is SUBPATTERN: + # Use relative indexing for maximum compatibility: + # av contains just these two elements in Python 3.5 + # but four elements for newer Pythons. + group, children = av[0], av[-1] + + # This can be 'None' for "extension notation groups" + if group is not None: + yield group + for child in children: + yield from find_mandatory_groups(child) + elif op in (MIN_REPEAT, MAX_REPEAT): + min_repeats, _, children = av + if min_repeats == 0: + continue + for child in children: + yield from find_mandatory_groups(child) + elif op in (BRANCH, GROUPREF_EXISTS): + # Note: We deliberately ignore branches (e.g. "(a)|(b)") or + # conditional matches (e.g. "(?(named-group)yes-branch|no-branch)". + # The whole point of a branch is that it'll be matched only + # some of the time, therefore no subgroups in either branch can + # ever be mandatory. + continue + elif isinstance(av, list): + for child in av: + yield from find_mandatory_groups(child) + + +def extract_regex_group_info(pattern: str) -> Tuple[List[int], int, Dict[str, int]]: + """Analyzes the given regex pattern and returns a tuple of: + + 1. A list of all mandatory group indexes in sorted order (including 0). + 2. The total number of groups, including optional groups and the zero-th group. + 3. A mapping of named groups to group indices. + + If the given str is not a valid regex, raises RegexPluginException. + """ + try: + ast = parse(pattern) + except SreError as ex: + raise RegexPluginException("Invalid regex: {}".format(ex.msg)) + + mandatory_groups = [0] + list(sorted(find_mandatory_groups(ast))) + + if sys.version_info >= (3, 8): + state = ast.state + else: + state = ast.pattern + total_groups = state.groups + named_groups = state.groupdict + + return mandatory_groups, total_groups, named_groups + + +def analyze_regex_pattern_call(pattern_type: Type, + default_return_type: Type) -> Type: + """The re module contains several methods or functions + that accept some string containing a regex pattern and returns + either a typing.Pattern or typing.Match object. + + This function handles the core logic for extracting and + attaching this regex metadata to the return object in all + these cases. + """ + + pattern_type = get_proper_type(coerce_to_literal(pattern_type)) + if not isinstance(pattern_type, LiteralType): + return default_return_type + if pattern_type.fallback.type.fullname not in STR_LIKE_TYPES: + return default_return_type + + return_type = get_proper_type(default_return_type) + if not isinstance(return_type, Instance): + return default_return_type + if return_type.type.fullname not in OBJECTS_SUPPORTING_REGEX_METADATA: + return default_return_type + + pattern = pattern_type.value + assert isinstance(pattern, str) + mandatory_groups, total_groups, named_groups = extract_regex_group_info(pattern) + + metadata = { + "default_re_plugin": { + "mandatory_groups": mandatory_groups, + "total_groups": total_groups, + "named_groups": named_groups, + } + } + + return return_type.copy_modified( + metadata={**return_type.metadata, **metadata}, + ) + + +def extract_metadata(typ: ProperType) -> Optional[Tuple[Dict[str, Any], Instance]]: + """Returns the regex metadata from the given type, if it exists. + Otherwise returns None. + + This function is the dual of 'analyze_regex_pattern_call'. That function + tries finding and attaching the metadata to Pattern or Match objects; + this function tries extracting the attached metadata. + """ + if not isinstance(typ, Instance): + return None + + metadata = typ.metadata.get('default_re_plugin', None) + if metadata is None: + return None + + arg_type = get_proper_type(typ.args[0]) + if not isinstance(arg_type, Instance): + return None + + return metadata, arg_type + + +def re_direct_match_callback(ctx: mypy.plugin.FunctionContext) -> Type: + """Analyzes functions such as 're.match(PATTERN, INPUT)'""" + try: + return analyze_regex_pattern_call( + ctx.arg_types[0][0], + remove_optional(ctx.default_return_type), + ) + except RegexPluginException as ex: + ctx.api.fail(ex.msg, ctx.context) + return ctx.default_return_type + + +def re_compile_callback(ctx: mypy.plugin.FunctionContext) -> Type: + """Analyzes the 're.compile(PATTERN)' function.""" + try: + return analyze_regex_pattern_call( + ctx.arg_types[0][0], + ctx.default_return_type, + ) + except RegexPluginException as ex: + ctx.api.fail(ex.msg, ctx.context) + return ctx.default_return_type + + +def re_get_match_callback(ctx: mypy.plugin.MethodContext) -> Type: + """Analyzes the 'typing.Pattern.match(...)' method.""" + self_type = ctx.type + return_type = ctx.default_return_type + + if not isinstance(self_type, Instance) or 'default_re_plugin' not in self_type.metadata: + return return_type + + match_object = get_proper_type(remove_optional(return_type)) + assert isinstance(match_object, Instance) + + pattern_metadata = self_type.metadata['default_re_plugin'] + new_match_object = match_object.copy_modified(metadata={'default_re_plugin': pattern_metadata}) + return make_simplified_union([new_match_object, NoneType()]) + + +def re_match_groups_callback(ctx: mypy.plugin.MethodContext) -> Type: + """Analyzes the 'typing.Match.group(...)' method, which returns + a tuple of all matched groups.""" + info = extract_metadata(ctx.type) + if info is None: + return ctx.default_return_type + + metadata, mandatory_match_type = info + mandatory = set(metadata['mandatory_groups']) + total = metadata['total_groups'] + + if len(ctx.arg_types) > 0 and len(ctx.arg_types[0]) > 0: + default_type = ctx.arg_types[0][0] + else: + default_type = NoneType() + + optional_match_type = make_simplified_union([mandatory_match_type, default_type]) + + items: List[Type] = [] + for i in range(1, total): + if i in mandatory: + items.append(mandatory_match_type) + else: + items.append(optional_match_type) + + fallback = ctx.api.named_generic_type("builtins.tuple", [mandatory_match_type]) + return TupleType(items, fallback) + + +def re_match_group_callback(ctx: mypy.plugin.MethodContext) -> Type: + """Analyzes the 'typing.Match.group()' and '__getitem__(...)' methods.""" + info = extract_metadata(ctx.type) + if info is None: + return ctx.default_return_type + + metadata, mandatory_match_type = info + mandatory = set(metadata['mandatory_groups']) + total = metadata['total_groups'] + named_groups = metadata['named_groups'] + + if len(mandatory) != total: + optional_match_type = make_simplified_union([mandatory_match_type, NoneType()]) + else: + optional_match_type = mandatory_match_type + + possible_indices = [] + for arg_type in ctx.arg_types: + if len(arg_type) >= 1: + possible_indices.append(get_proper_type(coerce_to_literal(arg_type[0]))) + + outputs: List[Type] = [] + for possible_index in possible_indices: + if not isinstance(possible_index, LiteralType): + outputs.append(optional_match_type) + continue + + value = possible_index.value + fallback_name = possible_index.fallback.type.fullname + + if isinstance(value, str) and fallback_name in STR_LIKE_TYPES: + if value not in named_groups: + ctx.api.fail("Regex does not contain group named '{}'".format(value), ctx.context) + outputs.append(optional_match_type) + continue + + index = named_groups[value] + elif isinstance(value, int): + if value < 0: + ctx.api.fail("Regex group number should not be negative", ctx.context) + outputs.append(optional_match_type) + continue + elif value >= total: + msg = "Regex has {} total groups, given group number {} is too big" + ctx.api.fail(msg.format(total, value), ctx.context) + outputs.append(optional_match_type) + continue + index = value + else: + outputs.append(optional_match_type) + continue + + if index in mandatory: + outputs.append(mandatory_match_type) + else: + outputs.append(optional_match_type) + + if len(outputs) == 1: + return outputs[0] + else: + fallback = ctx.api.named_generic_type("builtins.tuple", [mandatory_match_type]) + return TupleType(outputs, fallback) diff --git a/mypy/test/testplugin.py b/mypy/test/testplugin.py new file mode 100644 index 000000000000..0f9cba204cf5 --- /dev/null +++ b/mypy/test/testplugin.py @@ -0,0 +1,95 @@ +from typing import List, Dict +import sys + +from mypy.test.helpers import Suite, assert_equal +from mypy.plugins.regex import extract_regex_group_info, RegexPluginException + + +class RegexPluginSuite(Suite): + def test_regex_group_analysis(self) -> None: + def check(pattern: str, + expected_mandatory: List[int], + expected_total: int, + expected_named: Dict[str, int], + ) -> None: + actual_mandatory, actual_total, actual_named = extract_regex_group_info(pattern) + assert_equal(actual_mandatory, expected_mandatory) + assert_equal(actual_total, expected_total) + assert_equal(actual_named, expected_named) + + # Some conventions, to make reading these more clear: + # + # m1, m2, m3... -- text meant to be a part of mandatory groups + # o1, o2, o3... -- text meant to be a part of optional groups + # x, y, z -- other dummy filter text + # n1, n2, n3... -- names for named groups + + # Basic sanity checks + check(r"x", [0], 1, {}) + check(r"", [0], 1, {}) + check(r"(m1(m2(m3)(m4)))", [0, 1, 2, 3, 4], 5, {}) + + # Named groups + check(r"(?Pm1)(?P=n1)(?Po2)*", [0, 1], 3, {'n1': 1, 'n2': 2}) + check(r"(?Pfoo){0,4} (?Pbar)", [0, 2], 3, {'n1': 1, 'n2': 2}) + + # Repetition checks + check(r"(m1)(o2)?(m3)(o4)*(r5)+(o6)??", [0, 1, 3, 5], 7, {}) + check(r"(m1(o2)?)+", [0, 1], 3, {}) + check(r"(o1){0,3} (m2){2} (m3){1,2}", [0, 2, 3], 4, {}) + check(r"(o1){0,3}? (m2){2}? (m3){1,2}?", [0, 2, 3], 4, {}) + + # Branching + check(r"(o1)|(o2)(o3|x)", [0], 4, {}) + check(r"(m1(o2)|(o3))(m4|x)", [0, 1, 4], 5, {}) + check(r"(?:(o1)|(o2))(m3|x)", [0, 3], 4, {}) + + # Non-capturing groups + check(r"(?:x)(m1)", [0, 1], 2, {}) + check(r"(?:x)", [0], 1, {}) + + # Flag groups, added in Python 3.6. + # Note: Doing re.compile("(?a)foo") is equivalent to doing + # re.compile("foo", flags=re.A). You can also use inline + # flag groups "(?FLAGS:PATTERN)" to apply flags just for + # the specified pattern. + if sys.version_info >= (3, 6): + check(r"(?s)(?i)x", [0], 1, {}) + check(r"(?si)x", [0], 1, {}) + check(r"(?s:(m1)(o2)*(?Pm3))", [0, 1, 3], 4, {'n3': 3}) + + # Lookahead assertions + check(r"(m1) (?=x) (m2)", [0, 1, 2], 3, {}) + check(r"(m1) (m2(?=x)) (m3)", [0, 1, 2, 3], 4, {}) + + # Negative lookahead assertions + check(r"(m1) (?!x) (m2)", [0, 1, 2], 3, {}) + check(r"(m1) (m2(?!x)) (m3)", [0, 1, 2, 3], 4, {}) + + # Positive lookbehind assertions + check(r"(m1)+ (?<=x)(m2)", [0, 1, 2], 3, {}) + check(r"(?<=x)x", [0], 1, {}) + + # Conditional matches + check(r"(?Pm1) (?(n1)x|y) (m2)", [0, 1, 2], 3, {"n1": 1}) + check(r"(?Po1)? (?(n1)x|y) (m2)", [0, 2], 3, {"n1": 1}) + check(r"(?Pm1) (?(n1)x) (m2)", [0, 1, 2], 3, {"n1": 1}) + check(r"(?Po1)? (?(n1)x) (m2)", [0, 2], 3, {"n1": 1}) + check(r"(m1) (?(1)x|y) (m2)", [0, 1, 2], 3, {}) + check(r"(o1)? (?(1)x|y) (m2)", [0, 2], 3, {}) + + # Comments + check(r"(m1)(?#comment)(r2)", [0, 1, 2], 3, {}) + + def test_regex_errors(self) -> None: + def check(pattern: str) -> None: + try: + extract_regex_group_info(pattern) + except RegexPluginException: + pass + else: + raise AssertionError("Did not throw expection for regex '{}'".format(pattern)) + + check(r"(unbalanced") + check(r"unbalanced)") + check(r"(?P=badgroupname)") diff --git a/mypy/types.py b/mypy/types.py index 15ae47e341d4..66a8911c6a72 100644 --- a/mypy/types.py +++ b/mypy/types.py @@ -930,10 +930,11 @@ def try_getting_instance_fallback(typ: ProperType) -> Optional[Instance]: """ - __slots__ = ('type', 'args', 'erased', 'invalid', 'type_ref', 'last_known_value') + __slots__ = ('type', 'args', 'erased', 'invalid', 'type_ref', 'metadata', 'last_known_value') def __init__(self, typ: mypy.nodes.TypeInfo, args: Sequence[Type], line: int = -1, column: int = -1, erased: bool = False, + metadata: Optional[Dict[str, JsonDict]] = None, last_known_value: Optional['LiteralType'] = None) -> None: super().__init__(line, column) self.type = typ @@ -946,11 +947,18 @@ def __init__(self, typ: mypy.nodes.TypeInfo, args: Sequence[Type], # True if recovered after incorrect number of type arguments error self.invalid = False + # This is a dictionary that will be serialized and un-serialized as is. + # It is useful for plugins to add their data to save in the cache. + if metadata: + self.metadata: Dict[str, JsonDict] = metadata + else: + self.metadata = {} + # This field keeps track of the underlying Literal[...] value associated with # this instance, if one is known. # # This field is set whenever possible within expressions, but is erased upon - # variable assignment (see erasetype.remove_instance_last_known_values) unless + # variable assignment (see erasetype.remove_instance_transient_info) unless # the variable is declared to be final. # # For example, consider the following program: @@ -1009,11 +1017,11 @@ def serialize(self) -> Union[JsonDict, str]: type_ref = self.type.fullname if not self.args and not self.last_known_value: return type_ref - data: JsonDict = { - ".class": "Instance", - } - data["type_ref"] = type_ref - data["args"] = [arg.serialize() for arg in self.args] + data: JsonDict = {".class": "Instance", + "type_ref": type_ref, + "args": [arg.serialize() for arg in self.args]} + if self.metadata: + data['metadata'] = self.metadata if self.last_known_value is not None: data['last_known_value'] = self.last_known_value.serialize() return data @@ -1032,12 +1040,15 @@ def deserialize(cls, data: Union[JsonDict, str]) -> 'Instance': args = [deserialize_type(arg) for arg in args_list] inst = Instance(NOT_READY, args) inst.type_ref = data['type_ref'] # Will be fixed up by fixup.py later. + if 'metadata' in data: + inst.metadata = data['metadata'] if 'last_known_value' in data: inst.last_known_value = LiteralType.deserialize(data['last_known_value']) return inst def copy_modified(self, *, args: Bogus[List[Type]] = _dummy, + metadata: Bogus[Dict[str, JsonDict]] = _dummy, erased: Bogus[bool] = _dummy, last_known_value: Bogus[Optional['LiteralType']] = _dummy) -> 'Instance': return Instance( @@ -1046,6 +1057,7 @@ def copy_modified(self, *, self.line, self.column, erased if erased is not _dummy else self.erased, + metadata if metadata is not _dummy else self.metadata, last_known_value if last_known_value is not _dummy else self.last_known_value, ) diff --git a/test-data/unit/pythoneval.test b/test-data/unit/pythoneval.test index 9bf511e1aba3..d8ae6c829ba5 100644 --- a/test-data/unit/pythoneval.test +++ b/test-data/unit/pythoneval.test @@ -1558,7 +1558,6 @@ _testNoPython3StubAvailable.py:3: error: Library stubs not installed for "maxmin _testNoPython3StubAvailable.py:3: note: Hint: "python3 -m pip install types-maxminddb" _testNoPython3StubAvailable.py:3: note: (or run "mypy --install-types" to install all missing stub packages) - [case testTypingOrderedDictAlias] # flags: --python-version 3.7 from typing import OrderedDict @@ -1573,3 +1572,133 @@ x: OrderedDict[str, str] = OrderedDict({}) reveal_type(x) # Revealed type is "collections.OrderedDict[builtins.str, builtins.int]" [out] _testTypingExtensionsOrderedDictAlias.py:3: note: Revealed type is "collections.OrderedDict[builtins.str, builtins.str]" + +[case testRegexPluginBasicCase] +# mypy: strict-optional +import re +from typing_extensions import Final + +pattern1: Final = re.compile("(foo)*(bar)") +match1: Final = pattern1.match("blah") +if match1: + reveal_type(match1.groups()) + reveal_type(match1.groups(default="test")) + reveal_type(match1.group(0)) + reveal_type(match1.group(1)) + reveal_type(match1.group(2)) + reveal_type(match1.group(0, 1, 2)) + +pattern2: Final = re.compile(b"(?Pfoo){0,4} (?Pbar)") +match2: Final = pattern2.search(b"blah") +if match2: + reveal_type(match2.groups()) + reveal_type(match2[0]) + reveal_type(match2[1]) + reveal_type(match2[2]) + reveal_type(match2["n1"]) + reveal_type(match2["n2"]) +[out] +_testRegexPluginBasicCase.py:8: note: Revealed type is "Tuple[Union[builtins.str*, None], builtins.str*]" +_testRegexPluginBasicCase.py:9: note: Revealed type is "Tuple[builtins.str*, builtins.str*]" +_testRegexPluginBasicCase.py:10: note: Revealed type is "builtins.str*" +_testRegexPluginBasicCase.py:11: note: Revealed type is "Union[builtins.str*, None]" +_testRegexPluginBasicCase.py:12: note: Revealed type is "builtins.str*" +_testRegexPluginBasicCase.py:13: note: Revealed type is "Tuple[builtins.str*, Union[builtins.str*, None], builtins.str*]" +_testRegexPluginBasicCase.py:18: note: Revealed type is "Tuple[Union[builtins.bytes*, None], builtins.bytes*]" +_testRegexPluginBasicCase.py:19: note: Revealed type is "builtins.bytes*" +_testRegexPluginBasicCase.py:20: note: Revealed type is "Union[builtins.bytes*, None]" +_testRegexPluginBasicCase.py:21: note: Revealed type is "builtins.bytes*" +_testRegexPluginBasicCase.py:22: note: Revealed type is "Union[builtins.bytes*, None]" +_testRegexPluginBasicCase.py:23: note: Revealed type is "builtins.bytes*" + +[case testRegexPluginNoFinal] +# mypy: strict-optional +import re + +pattern = re.compile("(foo)*(bar)") +match = pattern.match("blah") +if match: + # TODO: Consider typeshed so we default to using stricter types given ambiguity + reveal_type(match.groups()) + reveal_type(match[1]) +[out] +_testRegexPluginNoFinal.py:8: note: Revealed type is "builtins.tuple[Union[builtins.str*, Any], ...]" +_testRegexPluginNoFinal.py:9: note: Revealed type is "Union[builtins.str*, Any]" + +[case testRegexPluginErrors] +# mypy: strict-optional +import re +from typing_extensions import Final + +invalid1 = re.compile("(bad") +invalid2: Final = re.compile("(bad") + +pattern: Final = re.compile("(a)(b)*(?Pc)") +match: Final = pattern.fullmatch("blah") +if match: + match.group(5) + match.group("bad") +[out] +_testRegexPluginErrors.py:5: error: Invalid regex: missing ), unterminated subpattern +_testRegexPluginErrors.py:6: error: Invalid regex: missing ), unterminated subpattern +_testRegexPluginErrors.py:11: error: Regex has 4 total groups, given group number 5 is too big +_testRegexPluginErrors.py:12: error: Regex does not contain group named 'bad' + +[case testRegexPluginDirectMethods] +# mypy: strict-optional +import re +from typing_extensions import Final + +match: Final = re.search("(foo)*(bar)", "blah") +if match: + reveal_type(match.groups()) + reveal_type(match.groups(default="test")) + reveal_type(match[0]) + reveal_type(match[1]) + reveal_type(match[2]) + reveal_type(match.group(0, 1, 2)) +[out] +_testRegexPluginDirectMethods.py:7: note: Revealed type is "Tuple[Union[builtins.str*, None], builtins.str*]" +_testRegexPluginDirectMethods.py:8: note: Revealed type is "Tuple[builtins.str*, builtins.str*]" +_testRegexPluginDirectMethods.py:9: note: Revealed type is "builtins.str*" +_testRegexPluginDirectMethods.py:10: note: Revealed type is "Union[builtins.str*, None]" +_testRegexPluginDirectMethods.py:11: note: Revealed type is "builtins.str*" +_testRegexPluginDirectMethods.py:12: note: Revealed type is "Tuple[builtins.str*, Union[builtins.str*, None], builtins.str*]" + +[case testRegexPluginUnknownArg] +# mypy: strict-optional +import re +from typing_extensions import Final + +index: int +name: str + +pattern1: Final = re.compile("(foo)*(bar)(?Pbaz)?(?Pqux)") +match1: Final = pattern1.match("blah") +if match1: + reveal_type(match1.groups()) + reveal_type(match1[index]) + reveal_type(match1[name]) + reveal_type(match1.group(0, index, name)) + +pattern2: Final = re.compile("(foo)(?Pbar)") +match2: Final = pattern2.match("blah") +if match2: + # No optional groups, so we can always return str + reveal_type(match2.groups()) + reveal_type(match2[index]) + reveal_type(match2[name]) + reveal_type(match2.group(0, index, name)) + match2["bad"] + match2[5] +[out] +_testRegexPluginUnknownArg.py:11: note: Revealed type is "Tuple[Union[builtins.str*, None], builtins.str*, Union[builtins.str*, None], builtins.str*]" +_testRegexPluginUnknownArg.py:12: note: Revealed type is "Union[builtins.str*, None]" +_testRegexPluginUnknownArg.py:13: note: Revealed type is "Union[builtins.str*, None]" +_testRegexPluginUnknownArg.py:14: note: Revealed type is "Tuple[builtins.str*, Union[builtins.str*, None], Union[builtins.str*, None]]" +_testRegexPluginUnknownArg.py:20: note: Revealed type is "Tuple[builtins.str*, builtins.str*]" +_testRegexPluginUnknownArg.py:21: note: Revealed type is "builtins.str*" +_testRegexPluginUnknownArg.py:22: note: Revealed type is "builtins.str*" +_testRegexPluginUnknownArg.py:23: note: Revealed type is "Tuple[builtins.str*, builtins.str*, builtins.str*]" +_testRegexPluginUnknownArg.py:24: error: Regex does not contain group named 'bad' +_testRegexPluginUnknownArg.py:25: error: Regex has 3 total groups, given group number 5 is too big