diff --git a/mypy/server/update.py b/mypy/server/update.py index c8803ef854fc..4b32acf6b285 100644 --- a/mypy/server/update.py +++ b/mypy/server/update.py @@ -93,6 +93,7 @@ def __init__(self, # Modules that had blocking errors in the previous run. # TODO: Handle blocking errors in the initial build self.blocking_errors = [] # type: List[str] + mark_all_meta_as_memory_only(graph, manager) manager.saved_cache = preserve_full_cache(graph, manager) def update(self, changed_modules: List[Tuple[str, str]]) -> List[str]: @@ -112,58 +113,94 @@ def update(self, changed_modules: List[Tuple[str, str]]) -> List[str]: Returns: A list of errors. """ - changed_ids = [id for id, _ in changed_modules] if DEBUG: + changed_ids = [id for id, _ in changed_modules] print('==== update %s ====' % changed_ids) + while changed_modules: + id, path = changed_modules.pop(0) + if DEBUG: + print('-- %s --' % id) + result, remaining = self.update_single(id, path) + changed_modules.extend(remaining) + return result + + def update_single(self, module: str, path: str) -> Tuple[List[str], + List[Tuple[str, str]]]: + """Update a single modified module. + + If the module contains imports of previously unseen modules, only process one of + the new modules and return the remaining work to be done. + + Returns: + Tuple with these items: + + - Error messages + - Remaining modules to process as (module id, path) tuples + """ + # TODO: If new module brings in other modules, we parse some files multiple times. if self.blocking_errors: # TODO: Relax this requirement - assert self.blocking_errors == changed_ids + assert self.blocking_errors == [module] manager = self.manager graph = self.graph - # Record symbol table snaphots of old versions of changed moduiles. + # Record symbol table snaphots of old versions of changed modules. old_snapshots = {} - for id, _ in changed_modules: - if id in manager.modules: - snapshot = snapshot_symbol_table(id, manager.modules[id].names) - old_snapshots[id] = snapshot - else: - old_snapshots[id] = {} + if module in manager.modules: + snapshot = snapshot_symbol_table(module, manager.modules[module].names) + old_snapshots[module] = snapshot + else: + old_snapshots[module] = {} manager.errors.reset() try: - new_modules, graph = build_incremental_step(manager, changed_modules, graph) + module, tree, graph, remaining = update_single_isolated(module, path, manager, graph) except CompileError as err: - self.blocking_errors = changed_ids - return err.messages + # TODO: Remaining modules + self.blocking_errors = [module] + return err.messages, [] self.blocking_errors = [] + if module not in old_snapshots: + old_snapshots[module] = {} + # TODO: What to do with stale dependencies? - triggered = calculate_active_triggers(manager, old_snapshots, new_modules) + triggered = calculate_active_triggers(manager, old_snapshots, {module: tree}) if DEBUG: print('triggered:', sorted(triggered)) - update_dependencies(new_modules, self.deps, graph, self.options) + update_dependencies({module: tree}, self.deps, graph, self.options) propagate_changes_using_dependencies(manager, graph, self.deps, triggered, - set(changed_ids), + {module}, self.previous_targets_with_errors, graph) # Preserve state needed for the next update. self.previous_targets_with_errors = manager.errors.targets() - for id, _ in changed_modules: - # If deleted, module won't be in the graph. - if id in graph: - # Generate metadata so that we can reuse the AST in the next run. - graph[id].write_cache() + # If deleted, module won't be in the graph. + if module in graph: + # Generate metadata so that we can reuse the AST in the next run. + graph[module].write_cache() for id, state in graph.items(): # Look up missing ASTs from saved cache. if state.tree is None and id in manager.saved_cache: meta, tree, type_map = manager.saved_cache[id] state.tree = tree + mark_all_meta_as_memory_only(graph, manager) manager.saved_cache = preserve_full_cache(graph, manager) self.graph = graph - return manager.errors.messages() + return manager.errors.messages(), remaining + + +def mark_all_meta_as_memory_only(graph: Dict[str, State], + manager: BuildManager) -> None: + for id, state in graph.items(): + if id in manager.saved_cache: + # Don't look at disk. + old = manager.saved_cache[id] + manager.saved_cache[id] = (old[0]._replace(memory_only=True), + old[1], + old[2]) def get_all_dependencies(manager: BuildManager, graph: Dict[str, State], @@ -174,56 +211,70 @@ def get_all_dependencies(manager: BuildManager, graph: Dict[str, State], return deps -def build_incremental_step(manager: BuildManager, - changed_modules: List[Tuple[str, str]], - graph: Dict[str, State]) -> Tuple[Dict[str, Optional[MypyFile]], - Graph]: - """Build new versions of changed modules only. - - Raise CompleError on encountering a blocking error. - - Return the new ASTs for the changed modules and the entire build graph. +def update_single_isolated(module: str, + path: str, + manager: BuildManager, + graph: Dict[str, State]) -> Tuple[str, + Optional[MypyFile], + Graph, + List[Tuple[str, str]]]: + """Build a new version of one changed module only. + + Don't propagate changes to elsewhere in the program. Raise CompleError on + encountering a blocking error. + + Args: + module: Changed module (modified, created or deleted) + path: Path of the changed module + manager: Build manager + graph: Build graph + + Returns: + A 4-tuple with these items: + + - Id of the changed module (can be different from the argument) + - New AST for the changed module (None if module was deleted) + - The entire build graph + - Remaining changed modules that are not processed yet as (module id, path) + tuples (non-empty if the original changed module imported other new + modules) """ - # TODO: Handle multiple changed modules per step - assert len(changed_modules) == 1 - id, path = changed_modules[0] - if id in manager.modules: - path1 = os.path.normpath(path) - path2 = os.path.normpath(manager.modules[id].path) - assert path1 == path2, '%s != %s' % (path1, path2) + if module in manager.modules: + assert_equivalent_paths(path, manager.modules[module].path) old_modules = dict(manager.modules) - - sources = get_sources(graph, changed_modules) - changed_set = {id for id, _ in changed_modules} - - invalidate_stale_cache_entries(manager.saved_cache, changed_modules) + sources = get_sources(graph, [(module, path)]) + invalidate_stale_cache_entries(manager.saved_cache, [(module, path)]) if not os.path.isfile(path): - graph = delete_module(id, graph, manager) - return {id: None}, graph + graph = delete_module(module, graph, manager) + return module, None, graph, [] old_graph = graph manager.missing_modules = set() graph = load_graph(sources, manager) # Find any other modules brought in by imports. - for st in graph.values(): - if st.id not in old_graph and st.id not in changed_set: - changed_set.add(st.id) - assert st.path - changed_modules.append((st.id, st.path)) - # TODO: Handle multiple changed modules per step - assert len(changed_modules) == 1, changed_modules + changed_modules = get_all_changed_modules(module, path, old_graph, graph) + # If there are multiple modules to process, only process one of them and return the + # remaining ones to the caller. + if len(changed_modules) > 1: + remaining_modules = changed_modules[:-1] + # The remaining modules haven't been processed yet so drop them. + for id, _ in remaining_modules: + del manager.modules[id] + del graph[id] + module, path = changed_modules[-1] + if DEBUG: + print('--> %s (newly imported)' % module) + else: + remaining_modules = [] - state = graph[id] + state = graph[module] - # Parse file and run first pass of semantic analysis. + # Process the changed file. state.parse_file() - # TODO: state.fix_suppressed_dependencies()? - - # Run remaining passes of semantic analysis. try: state.semantic_analysis() except CompileError as err: @@ -231,13 +282,14 @@ def build_incremental_step(manager: BuildManager, # There was a blocking error, so module AST is incomplete. Restore old modules. manager.modules.clear() manager.modules.update(old_modules) + # TODO: Propagate remaining modules raise err state.semantic_analysis_pass_three() state.semantic_analysis_apply_patches() # Merge old and new ASTs. assert state.tree is not None, "file must be at least parsed" - new_modules = {id: state.tree} # type: Dict[str, Optional[MypyFile]] + new_modules = {module: state.tree} # type: Dict[str, Optional[MypyFile]] replace_modules_with_new_variants(manager, graph, old_modules, new_modules) # Perform type checking. @@ -246,11 +298,16 @@ def build_incremental_step(manager: BuildManager, state.finish_passes() # TODO: state.write_cache()? # TODO: state.mark_as_rechecked()? - # TODO: Store new State in graph, as it has updated dependencies etc. - graph[id] = state + graph[module] = state + + return module, state.tree, graph, remaining_modules - return new_modules, graph + +def assert_equivalent_paths(path1: str, path2: str) -> None: + path1 = os.path.normpath(path1) + path2 = os.path.normpath(path2) + assert path1 == path2, '%s != %s' % (path1, path2) def delete_module(module_id: str, @@ -279,6 +336,20 @@ def get_sources(graph: Graph, changed_modules: List[Tuple[str, str]]) -> List[Bu return sources +def get_all_changed_modules(root_module: str, + root_path: str, + old_graph: Dict[str, State], + new_graph: Dict[str, State]) -> List[Tuple[str, str]]: + changed_set = {root_module} + changed_modules = [(root_module, root_path)] + for st in new_graph.values(): + if st.id not in old_graph and st.id not in changed_set: + changed_set.add(st.id) + assert st.path + changed_modules.append((st.id, st.path)) + return changed_modules + + def preserve_full_cache(graph: Graph, manager: BuildManager) -> SavedCache: """Preserve every module with an AST in the graph, including modules with errors.""" saved_cache = {} @@ -301,6 +372,8 @@ def preserve_full_cache(graph: Graph, manager: BuildManager) -> SavedCache: state.source_hash, state.ignore_all, manager) + else: + meta = meta._replace(memory_only=True) saved_cache[id] = (meta, state.tree, state.type_map()) return saved_cache diff --git a/test-data/unit/fine-grained-modules.test b/test-data/unit/fine-grained-modules.test index 2c765cf1e358..2b6b7e9342ad 100644 --- a/test-data/unit/fine-grained-modules.test +++ b/test-data/unit/fine-grained-modules.test @@ -259,3 +259,275 @@ a/b.py:2: error: Incompatible return value type (got "str", expected "int") == main:1: error: Cannot find module named 'a.b' main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help) + +[case testModifyTwoFilesNoError1] +import a +[file a.py] +import b +b.f() +[file b.py] +def f() -> None: pass +[file a.py.2] +import b +b.f(1) +[file b.py.2] +def f(x: int) -> None: pass +[out] +== + +[case testModifyTwoFilesNoError2] +import a +[file a.py] +from b import g +def f() -> None: pass +[file b.py] +import a +def g() -> None: pass +a.f() +[file a.py.2] +from b import g +def f(x: int) -> None: pass +[file b.py.2] +import a +def g() -> None: pass +a.f(1) +[out] +== + +[case testModifyTwoFilesErrorsElsewhere] +import a +import b +a.f() +b.g(1) +[file a.py] +def f() -> None: pass +[file b.py] +def g(x: int) -> None: pass +[file a.py.2] +def f(x: int) -> None: pass +[file b.py.2] +def g() -> None: pass +[out] +== +main:3: error: Too few arguments for "f" +main:4: error: Too many arguments for "g" + +[case testModifyTwoFilesErrorsInBoth] +import a +[file a.py] +import b +def f() -> None: pass +b.g(1) +[file b.py] +import a +def g(x: int) -> None: pass +a.f() +[file a.py.2] +import b +def f(x: int) -> None: pass +b.g(1) +[file b.py.2] +import a +def g() -> None: pass +a.f() +[out] +== +b.py:3: error: Too few arguments for "f" +a.py:3: error: Too many arguments for "g" + +[case testModifyTwoFilesFixErrorsInBoth] +import a +[file a.py] +import b +def f(x: int) -> None: pass +b.g(1) +[file b.py] +import a +def g() -> None: pass +a.f() +[file a.py.2] +import b +def f() -> None: pass +b.g(1) +[file b.py.2] +import a +def g(x: int) -> None: pass +a.f() +[out] +b.py:3: error: Too few arguments for "f" +a.py:3: error: Too many arguments for "g" +== + +[case testAddTwoFilesNoError] +import a +[file a.py] +import b +import c +b.f() +c.g() +[file b.py.2] +import c +def f() -> None: pass +c.g() +[file c.py.2] +import b +def g() -> None: pass +b.f() +[out] +a.py:1: error: Cannot find module named 'b' +a.py:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help) +a.py:2: error: Cannot find module named 'c' +== + +[case testAddTwoFilesErrorsInBoth] +import a +[file a.py] +import b +import c +b.f() +c.g() +[file b.py.2] +import c +def f() -> None: pass +c.g(1) +[file c.py.2] +import b +def g() -> None: pass +b.f(1) +[out] +a.py:1: error: Cannot find module named 'b' +a.py:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help) +a.py:2: error: Cannot find module named 'c' +== +c.py:3: error: Too many arguments for "f" +b.py:3: error: Too many arguments for "g" + +[case testAddTwoFilesErrorsElsewhere] +import a +import b +a.f(1) +b.g(1) +[file a.py.2] +def f() -> None: pass +[file b.py.2] +def g() -> None: pass +[out] +main:1: error: Cannot find module named 'a' +main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help) +main:2: error: Cannot find module named 'b' +== +main:3: error: Too many arguments for "f" +main:4: error: Too many arguments for "g" + +[case testDeleteTwoFilesErrorsElsewhere] +import a +import b +a.f() +b.g() +[file a.py] +def f() -> None: pass +[file b.py] +def g() -> None: pass +[delete a.py.2] +[delete b.py.2] +[out] +== +main:1: error: Cannot find module named 'a' +main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help) +main:2: error: Cannot find module named 'b' + +[case testDeleteTwoFilesNoErrors] +import a +[file a.py] +import b +import c +b.f() +c.g() +[file b.py] +def f() -> None: pass +[file c.py] +def g() -> None: pass +[file a.py.2] +[delete b.py.3] +[delete c.py.3] +[out] +== +== + +[case testDeleteTwoFilesFixErrors] +import a +import b +a.f() +b.g() +[file a.py] +import b +def f() -> None: pass +b.g(1) +[file b.py] +import a +def g() -> None: pass +a.f(1) +[delete a.py.2] +[delete b.py.2] +[out] +b.py:3: error: Too many arguments for "f" +a.py:3: error: Too many arguments for "g" +== +main:1: error: Cannot find module named 'a' +main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help) +main:2: error: Cannot find module named 'b' + +[case testAddFileWhichImportsLibModule] +import a +a.x = 0 +[file a.py.2] +import sys +x = sys.platform +[out] +main:1: error: Cannot find module named 'a' +main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help) +== +main:2: error: Incompatible types in assignment (expression has type "int", variable has type "str") + +[case testAddFileWhichImportsLibModuleWithErrors] +import a +a.x = 0 +[file a.py.2] +import broken +x = broken.x +z +[out] +main:1: error: Cannot find module named 'a' +main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help) +== +a.py:3: error: Name 'z' is not defined +/test-data/unit/lib-stub/broken.pyi:2: error: Name 'y' is not defined + +[case testRenameModule] +import a +[file a.py] +import b +b.f() +[file b.py] +def f() -> None: pass +[file a.py.2] +import c +c.f() +[file c.py.2] +def f() -> None: pass +[file a.py.3] +import c +c.f(1) +[out] +== +== +a.py:2: error: Too many arguments for "f" + +-- TODO: +-- - add one file which imports another new file, blocking error in new file +-- - arbitrary blocking errors +-- - packages +-- - add two files that form a package +-- - delete two files that form a package +-- - order of processing makes a difference +-- - mix of modify, add and delete in one iteration diff --git a/test-data/unit/lib-stub/broken.pyi b/test-data/unit/lib-stub/broken.pyi new file mode 100644 index 000000000000..22cfc72ed744 --- /dev/null +++ b/test-data/unit/lib-stub/broken.pyi @@ -0,0 +1,2 @@ +# Stub file that generates an error +x = y