From 5aca8d8aafbba16e4176883533a97ae38a15aaf3 Mon Sep 17 00:00:00 2001 From: David Ross Date: Thu, 20 Jul 2017 03:27:24 -0700 Subject: [PATCH 1/3] Support exported names starting with an underscore in __all__ Fixes #3745 --- mypy/nodes.py | 12 +++++++++++- mypy/semanal.py | 9 ++++++--- mypy/server/astdiff.py | 3 ++- test-data/unit/check-modules.test | 19 +++++++++++++++++++ 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 54c3a5522e99..721cd4fcb1d5 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2286,6 +2286,10 @@ class SymbolTableNode: # If False, this name won't be imported via 'from import *'. # This has no effect on names within classes. module_public = True + # If True, this name will be imported via 'from import *' + # even if it starts with an underscore. This has no effect on names + # within classes nor names which do not start with an underscore. + module_public_even_with_underscore = False # For deserialized MODULE_REF nodes, the referenced module name; # for other nodes, optionally the name of the referenced object. cross_ref = None # type: Optional[str] @@ -2302,12 +2306,14 @@ def __init__(self, module_public: bool = True, normalized: bool = False, alias_tvars: Optional[List[str]] = None, - implicit: bool = False) -> None: + implicit: bool = False, + module_public_even_with_underscore: bool = False) -> None: self.kind = kind self.node = node self.type_override = typ self.mod_id = mod_id self.module_public = module_public + self.module_public_even_with_underscore = module_public_even_with_underscore self.normalized = normalized self.alias_tvars = alias_tvars self.implicit = implicit @@ -2354,6 +2360,8 @@ def serialize(self, prefix: str, name: str) -> JsonDict: } # type: JsonDict if not self.module_public: data['module_public'] = False + if self.module_public_even_with_underscore: + data['module_public_even_with_underscore'] = True if self.normalized: data['normalized'] = True if self.implicit: @@ -2395,6 +2403,8 @@ def deserialize(cls, data: JsonDict) -> 'SymbolTableNode': stnode.alias_tvars = data['alias_tvars'] if 'module_public' in data: stnode.module_public = data['module_public'] + if 'module_public_even_with_underscore' in data: + stnode.module_public_even_with_underscore = data['module_public_even_with_underscore'] if 'normalized' in data: stnode.normalized = data['normalized'] if 'implicit' in data: diff --git a/mypy/semanal.py b/mypy/semanal.py index 4f0c9793f4d8..7ac7caa3bee3 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -301,9 +301,11 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options, if '__all__' in self.globals: for name, g in self.globals.items(): - if name not in self.all_exports: + if name in self.all_exports: + if name.startswith('_'): + g.module_public_even_with_underscore = True + else: g.module_public = False - del self.options del self.patches @@ -1483,7 +1485,8 @@ def visit_import_all(self, i: ImportAll) -> None: self.add_submodules_to_parent_modules(i_id, True) for name, node in m.names.items(): node = self.normalize_type_alias(node, i) - if not name.startswith('_') and node.module_public: + if ((node.module_public_even_with_underscore or not name.startswith('_')) + and node.module_public): existing_symbol = self.globals.get(name) if existing_symbol: # Import can redefine a variable. They get special treatment. diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index 422f46af127a..0aa413be2752 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -57,7 +57,8 @@ def is_similar_node_shallow(n: SymbolTableNode, m: SymbolTableNode) -> bool: # type_override if (n.kind != m.kind or n.mod_id != m.mod_id - or n.module_public != m.module_public): + or n.module_public != m.module_public + or n.module_public_even_with_underscore != m.module_public_even_with_underscore): return False if type(n.node) != type(m.node): # noqa return False diff --git a/test-data/unit/check-modules.test b/test-data/unit/check-modules.test index 8f212b27a7ad..546010eb7f2a 100644 --- a/test-data/unit/check-modules.test +++ b/test-data/unit/check-modules.test @@ -427,6 +427,25 @@ __all__ = [u'a', u'b', u'c'] [out] +[case testUnderscoreExportedValuesInImportAll] +import typing +from m import * +_ = a +_ = _b +_ = __c__ +_ = ___d +_ = e +_ = f # E: Name 'f' is not defined +_ = _g # E: Name '_g' is not defined +[file m.py] +__all__ = ['a'] +__all__ += ('_b',) +__all__.append('__c__') +__all__.extend(('___d', 'e')) + +a = _b = __c__ = ___d = e = f = _g = 1 +[builtins fixtures/module_all.pyi] + [case testEllipsisInitializerInStubFileWithType] import m m.x = '' # E: Incompatible types in assignment (expression has type "str", variable has type "int") From 86a085a862df6e978bc3d460a34ca4fbf9e8f6cf Mon Sep 17 00:00:00 2001 From: David Ross Date: Thu, 20 Jul 2017 03:53:23 -0700 Subject: [PATCH 2/3] Undo whitespace change. --- mypy/semanal.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mypy/semanal.py b/mypy/semanal.py index 7ac7caa3bee3..966bed9a2c21 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -306,6 +306,7 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options, g.module_public_even_with_underscore = True else: g.module_public = False + del self.options del self.patches From 74e42ff6be4e8cdf63718e8e498fa0b1b96dfdf2 Mon Sep 17 00:00:00 2001 From: David Ross Date: Thu, 20 Jul 2017 05:16:37 -0700 Subject: [PATCH 3/3] Simplify logic to only modify a single section of the code. Thanks to @ilevkivskyi for this! --- mypy/nodes.py | 12 +----------- mypy/semanal.py | 10 ++++------ mypy/server/astdiff.py | 3 +-- 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/mypy/nodes.py b/mypy/nodes.py index 721cd4fcb1d5..54c3a5522e99 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -2286,10 +2286,6 @@ class SymbolTableNode: # If False, this name won't be imported via 'from import *'. # This has no effect on names within classes. module_public = True - # If True, this name will be imported via 'from import *' - # even if it starts with an underscore. This has no effect on names - # within classes nor names which do not start with an underscore. - module_public_even_with_underscore = False # For deserialized MODULE_REF nodes, the referenced module name; # for other nodes, optionally the name of the referenced object. cross_ref = None # type: Optional[str] @@ -2306,14 +2302,12 @@ def __init__(self, module_public: bool = True, normalized: bool = False, alias_tvars: Optional[List[str]] = None, - implicit: bool = False, - module_public_even_with_underscore: bool = False) -> None: + implicit: bool = False) -> None: self.kind = kind self.node = node self.type_override = typ self.mod_id = mod_id self.module_public = module_public - self.module_public_even_with_underscore = module_public_even_with_underscore self.normalized = normalized self.alias_tvars = alias_tvars self.implicit = implicit @@ -2360,8 +2354,6 @@ def serialize(self, prefix: str, name: str) -> JsonDict: } # type: JsonDict if not self.module_public: data['module_public'] = False - if self.module_public_even_with_underscore: - data['module_public_even_with_underscore'] = True if self.normalized: data['normalized'] = True if self.implicit: @@ -2403,8 +2395,6 @@ def deserialize(cls, data: JsonDict) -> 'SymbolTableNode': stnode.alias_tvars = data['alias_tvars'] if 'module_public' in data: stnode.module_public = data['module_public'] - if 'module_public_even_with_underscore' in data: - stnode.module_public_even_with_underscore = data['module_public_even_with_underscore'] if 'normalized' in data: stnode.normalized = data['normalized'] if 'implicit' in data: diff --git a/mypy/semanal.py b/mypy/semanal.py index 966bed9a2c21..d9fe795326dd 100644 --- a/mypy/semanal.py +++ b/mypy/semanal.py @@ -301,10 +301,7 @@ def visit_file(self, file_node: MypyFile, fnam: str, options: Options, if '__all__' in self.globals: for name, g in self.globals.items(): - if name in self.all_exports: - if name.startswith('_'): - g.module_public_even_with_underscore = True - else: + if name not in self.all_exports: g.module_public = False del self.options @@ -1486,8 +1483,9 @@ def visit_import_all(self, i: ImportAll) -> None: self.add_submodules_to_parent_modules(i_id, True) for name, node in m.names.items(): node = self.normalize_type_alias(node, i) - if ((node.module_public_even_with_underscore or not name.startswith('_')) - and node.module_public): + # if '__all__' exists, all nodes not included have had module_public set to + # False, and we can skip checking '_' because it's been explicitly included. + if node.module_public and (not name.startswith('_') or '__all__' in m.names): existing_symbol = self.globals.get(name) if existing_symbol: # Import can redefine a variable. They get special treatment. diff --git a/mypy/server/astdiff.py b/mypy/server/astdiff.py index 0aa413be2752..422f46af127a 100644 --- a/mypy/server/astdiff.py +++ b/mypy/server/astdiff.py @@ -57,8 +57,7 @@ def is_similar_node_shallow(n: SymbolTableNode, m: SymbolTableNode) -> bool: # type_override if (n.kind != m.kind or n.mod_id != m.mod_id - or n.module_public != m.module_public - or n.module_public_even_with_underscore != m.module_public_even_with_underscore): + or n.module_public != m.module_public): return False if type(n.node) != type(m.node): # noqa return False