From c555e90120016842074e9072965cea403264fbda Mon Sep 17 00:00:00 2001 From: Maarten ter Huurne Date: Mon, 17 Dec 2018 06:08:54 +0100 Subject: [PATCH 1/3] Support multiple steps in command line tests PythonCmdlineSuite uses DataDrivenTestCase, but inspects the test case properties rather than calling runtest(). Therefore it could parse tests with multiple steps, but during execution only one step was run. --- mypy/test/testcmdline.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/mypy/test/testcmdline.py b/mypy/test/testcmdline.py index 660bffd09321..677a00e511b0 100644 --- a/mypy/test/testcmdline.py +++ b/mypy/test/testcmdline.py @@ -32,10 +32,20 @@ class PythonCmdlineSuite(DataSuite): native_sep = True def run_case(self, testcase: DataDrivenTestCase) -> None: - test_python_cmdline(testcase) - - -def test_python_cmdline(testcase: DataDrivenTestCase) -> None: + if testcase.output2: + for step in [1] + sorted(testcase.output2): + try: + test_python_cmdline(testcase, step) + except AssertionError as ex: + if ex.args: + new_msg = '(step {}) {}'.format(step, ex.args[0]) + ex.args = (new_msg,) + ex.args[1:] + raise + else: + test_python_cmdline(testcase) + + +def test_python_cmdline(testcase: DataDrivenTestCase, step: int = 1) -> None: assert testcase.old_cwd is not None, "test was not properly set up" # Write the program to a file. program = '_program.py' @@ -101,7 +111,8 @@ def test_python_cmdline(testcase: DataDrivenTestCase) -> None: obvious_result = 1 if out else 0 if obvious_result != result: out.append('== Return code: {}'.format(result)) - assert_string_arrays_equal(testcase.output, out, + expected_out = testcase.output if step == 1 else testcase.output2[step] + assert_string_arrays_equal(expected_out, out, 'Invalid output ({}, line {})'.format( testcase.file, testcase.line)) From 2844779509b3568f425d66c4bf494e3335e91701 Mon Sep 17 00:00:00 2001 From: Maarten ter Huurne Date: Mon, 17 Dec 2018 06:48:58 +0100 Subject: [PATCH 2/3] Disable cache when producing reports If the cache is used, coverage information will only be reported for modules that were not in the cache. Fixes #5103 --- mypy/build.py | 6 ++++-- test-data/unit/reports.test | 12 ++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/mypy/build.py b/mypy/build.py index 5deb6451a25d..beadba811ec8 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -504,8 +504,10 @@ def __init__(self, data_dir: str, self.stale_modules = set() # type: Set[str] self.rechecked_modules = set() # type: Set[str] self.flush_errors = flush_errors - self.cache_enabled = options.incremental and ( - not options.fine_grained_incremental or options.use_fine_grained_cache) + self.cache_enabled = (options.incremental + and (not options.fine_grained_incremental + or options.use_fine_grained_cache) + and not reports.reporters) self.fscache = fscache self.find_module_cache = FindModuleCache(self.search_paths, self.fscache, self.options) if options.sqlite_cache: diff --git a/test-data/unit/reports.test b/test-data/unit/reports.test index 29ff53857edc..46f64ce0107e 100644 --- a/test-data/unit/reports.test +++ b/test-data/unit/reports.test @@ -357,3 +357,15 @@ class Foo: @attr.s class Z(object): pass + +[case testCoverageIgnoresCache] +-- Performs two runs to verify that cached information does not prevent +-- modules from being included in reports. +# cmd: mypy --linecount-report report a.py +[file a.py] +empty = False +[out] +[out2] +[outfile report/linecount.txt] + 1 1 0 0 total + 1 1 0 0 a From f01058d4ef18f4758f07dfca960cd18b88b7797b Mon Sep 17 00:00:00 2001 From: Maarten ter Huurne Date: Fri, 21 Dec 2018 01:02:55 +0100 Subject: [PATCH 3/3] Update command line test exception messages to include step number Instead of intercepting and altering the exception messages, the step number is now inserted into the message when it is initially constructed. --- mypy/test/testcmdline.py | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/mypy/test/testcmdline.py b/mypy/test/testcmdline.py index 677a00e511b0..b256966b9f33 100644 --- a/mypy/test/testcmdline.py +++ b/mypy/test/testcmdline.py @@ -32,20 +32,11 @@ class PythonCmdlineSuite(DataSuite): native_sep = True def run_case(self, testcase: DataDrivenTestCase) -> None: - if testcase.output2: - for step in [1] + sorted(testcase.output2): - try: - test_python_cmdline(testcase, step) - except AssertionError as ex: - if ex.args: - new_msg = '(step {}) {}'.format(step, ex.args[0]) - ex.args = (new_msg,) + ex.args[1:] - raise - else: - test_python_cmdline(testcase) - - -def test_python_cmdline(testcase: DataDrivenTestCase, step: int = 1) -> None: + for step in [1] + sorted(testcase.output2): + test_python_cmdline(testcase, step) + + +def test_python_cmdline(testcase: DataDrivenTestCase, step: int) -> None: assert testcase.old_cwd is not None, "test was not properly set up" # Write the program to a file. program = '_program.py' @@ -85,12 +76,14 @@ def test_python_cmdline(testcase: DataDrivenTestCase, step: int = 1) -> None: # Ignore stdout, but we insist on empty stderr and zero status. if err or result: raise AssertionError( - 'Expected zero status and empty stderr, got %d and\n%s' % - (result, '\n'.join(err + out))) + 'Expected zero status and empty stderr%s, got %d and\n%s' % + (' on step %d' % step if testcase.output2 else '', + result, '\n'.join(err + out))) for path, expected_content in testcase.output_files: if not os.path.exists(path): raise AssertionError( - 'Expected file {} was not produced by test case'.format(path)) + 'Expected file {} was not produced by test case{}'.format( + path, ' on step %d' % step if testcase.output2 else '')) with open(path, 'r', encoding='utf8') as output_file: actual_output_content = output_file.read().splitlines() normalized_output = normalize_file_output(actual_output_content, @@ -103,8 +96,8 @@ def test_python_cmdline(testcase: DataDrivenTestCase, step: int = 1) -> None: for line in normalized_output] normalized_output = normalize_error_messages(normalized_output) assert_string_arrays_equal(expected_content.splitlines(), normalized_output, - 'Output file {} did not match its expected output'.format( - path)) + 'Output file {} did not match its expected output{}'.format( + path, ' on step %d' % step if testcase.output2 else '')) else: if testcase.normalize_output: out = normalize_error_messages(err + out) @@ -113,8 +106,9 @@ def test_python_cmdline(testcase: DataDrivenTestCase, step: int = 1) -> None: out.append('== Return code: {}'.format(result)) expected_out = testcase.output if step == 1 else testcase.output2[step] assert_string_arrays_equal(expected_out, out, - 'Invalid output ({}, line {})'.format( - testcase.file, testcase.line)) + 'Invalid output ({}, line {}){}'.format( + testcase.file, testcase.line, + ' on step %d' % step if testcase.output2 else '')) def parse_args(line: str) -> List[str]: