From 471c635682ead95080d21f61a69c61f2439c0192 Mon Sep 17 00:00:00 2001 From: "Michael J. Sullivan" Date: Thu, 26 Apr 2018 23:32:19 -0700 Subject: [PATCH] Recompute lib_path on every fine-grained incremental run This fixes some crashes and false positives when creating or removing __init__.py files that do not appear in the build. This just factors out the existing implementation of lib_path computation and reuses it. It is somewhat inefficient and should be improved on if we are going to call it on every fine-grained update. --- mypy/build.py | 40 +++++++++++++++--------- mypy/dmypy_server.py | 7 ++++- mypy/test/testerrorstream.py | 1 - mypy/test/testfinegrained.py | 10 ++++-- test-data/unit/fine-grained-modules.test | 22 +++++++++++++ 5 files changed, 61 insertions(+), 19 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index c65db7900692..1aed00d2a9bb 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -195,19 +195,11 @@ def default_flush_errors(new_messages: List[str], is_serious: bool) -> None: raise -def _build(sources: List[BuildSource], - options: Options, - alt_lib_path: Optional[str], - bin_dir: Optional[str], - flush_errors: Callable[[List[str], bool], None], - fscache: Optional[FileSystemCache], - ) -> BuildResult: - # This seems the most reasonable place to tune garbage collection. - gc.set_threshold(50000) - - data_dir = default_data_dir(bin_dir) - fscache = fscache or FileSystemCache() - +def compute_lib_path(sources: List[BuildSource], + options: Options, + alt_lib_path: Optional[str], + data_dir: str, + fscache: FileSystemCache) -> List[str]: # Determine the default module search path. lib_path = default_lib_path(data_dir, options.python_version, @@ -219,7 +211,9 @@ def _build(sources: List[BuildSource], # as in the source tree. root_dir = dirname(dirname(__file__)) lib_path.insert(0, os.path.join(root_dir, 'test-data', 'unit', 'lib-stub')) - else: + # alt_lib_path is used by some tests to bypass the normal lib_path mechanics. + # If we don't have one, grab directories of source files. + if not alt_lib_path: for source in sources: if source.path: # Include directory of the program file in the module search path. @@ -246,6 +240,24 @@ def _build(sources: List[BuildSource], if alt_lib_path: lib_path.insert(0, alt_lib_path) + return lib_path + + +def _build(sources: List[BuildSource], + options: Options, + alt_lib_path: Optional[str], + bin_dir: Optional[str], + flush_errors: Callable[[List[str], bool], None], + fscache: Optional[FileSystemCache], + ) -> BuildResult: + # This seems the most reasonable place to tune garbage collection. + gc.set_threshold(50000) + + data_dir = default_data_dir(bin_dir) + fscache = fscache or FileSystemCache() + + lib_path = compute_lib_path(sources, options, alt_lib_path, data_dir, fscache) + reports = Reports(data_dir, options.report_dirs) source_set = BuildSourceSet(sources) errors = Errors(options.show_error_context, options.show_column_numbers) diff --git a/mypy/dmypy_server.py b/mypy/dmypy_server.py index 0878136998bb..7924cf874ae8 100644 --- a/mypy/dmypy_server.py +++ b/mypy/dmypy_server.py @@ -309,14 +309,19 @@ def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict def fine_grained_increment(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]: assert self.fine_grained_manager is not None + manager = self.fine_grained_manager.manager t0 = time.time() self.update_sources(sources) changed, removed = self.find_changed(sources) + # Update the lib_path, which can change when sources do. + # TODO: This is slow. + manager.lib_path = tuple(mypy.build.compute_lib_path( + sources, manager.options, self.alt_lib_path, manager.data_dir, self.fscache)) t1 = time.time() messages = self.fine_grained_manager.update(changed, removed) t2 = time.time() - self.fine_grained_manager.manager.log( + manager.log( "fine-grained increment: find_changed: {:.3f}s, update: {:.3f}s".format( t1 - t0, t2 - t1)) status = 1 if messages else 0 diff --git a/mypy/test/testerrorstream.py b/mypy/test/testerrorstream.py index b67451db787f..0671e98e56eb 100644 --- a/mypy/test/testerrorstream.py +++ b/mypy/test/testerrorstream.py @@ -40,7 +40,6 @@ def flush_errors(msgs: List[str], serious: bool) -> None: try: build.build(sources=sources, options=options, - alt_lib_path=test_temp_dir, flush_errors=flush_errors) except CompileError as e: assert e.messages == [] diff --git a/mypy/test/testfinegrained.py b/mypy/test/testfinegrained.py index 3aa21388dbf0..4b1b20c9faa0 100644 --- a/mypy/test/testfinegrained.py +++ b/mypy/test/testfinegrained.py @@ -5,6 +5,11 @@ See the comment at the top of test-data/unit/fine-grained.test for more information. + +N.B.: Unlike most of the other test suites, testfinegrained does not +rely on an alt_lib_path for finding source files. This means that they +can test interactions with the lib_path that is built implicitly based +on specified sources. """ import os @@ -84,7 +89,7 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: config = None if config: parse_config_file(options, config) - server = Server(options, alt_lib_path=test_temp_dir) + server = Server(options) step = 1 sources = self.parse_sources(main_src, step, options) @@ -186,8 +191,7 @@ def build(self, sources: List[BuildSource]) -> List[str]: try: result = build.build(sources=sources, - options=options, - alt_lib_path=test_temp_dir) + options=options) except CompileError as e: return e.messages return result.errors diff --git a/test-data/unit/fine-grained-modules.test b/test-data/unit/fine-grained-modules.test index 21777daf656e..601f32226c3b 100644 --- a/test-data/unit/fine-grained-modules.test +++ b/test-data/unit/fine-grained-modules.test @@ -484,6 +484,28 @@ def f(x: int) -> None: pass [out] == +[case testDeleteSubpackageInit1] +# cmd: mypy q/r/s.py +# flags: --follow-imports=skip --ignore-missing-imports +[file q/__init__.py] +[file q/r/__init__.py] +[file q/r/s.py] +[delete q/__init__.py.2] +[out] +== + +[case testAddSubpackageInit2] +# cmd: mypy q/r/s.py +# flags: --follow-imports=skip --ignore-missing-imports +[file q/r/__init__.py] +[file q/r/s.py] +1 +[file q/r/s.py.2] +2 +[file q/__init__.py.2] +[out] +== + [case testModifyTwoFilesNoError2] import a [file a.py]