Lazy tests: parse using regex first (#5459)

Parse tests on collection only enough to find the name, so small number of tests run faster

On my machine, `pytest -n0 -k testAttrsSimple` takes at least 2.24 seconds to finish on master, and at most 0.95 seconds to finish with this PR.

- 'skip-cache' is changed to 'only_when_nocache' and similarly 'skip-nocache'
- I have replace while loops with "if True" to make the diff simpler. Further cleanup and optimizations are also possible 

Fixes #1073

Author: elazarg

mypy/test/data.py

@@ -144,7 +144,7 @@ def run_case_once(self, testcase: DataDrivenTestCase,
         else:
             options.incremental = False
             # Don't waste time writing cache unless we are specifically looking for it
-            if 'writescache' not in testcase.name.lower():
+            if not testcase.writescache:
                 options.cache_dir = os.devnull
 
         sources = []

mypy/test/testcheck.py

@@ -78,7 +78,7 @@ def test_python_cmdline(testcase: DataDrivenTestCase) -> None:
                 actual_output_content = output_file.read().splitlines()
             normalized_output = normalize_file_output(actual_output_content,
                                                       os.path.abspath(test_temp_dir))
-            if testcase.native_sep and os.path.sep == '\\':
+            if testcase.suite.native_sep and os.path.sep == '\\':
                 normalized_output = [fix_cobertura_filename(line) for line in normalized_output]
             normalized_output = normalize_error_messages(normalized_output)
             assert_string_arrays_equal(expected_content.splitlines(), normalized_output,

mypy/test/testcmdline.py

@@ -55,14 +55,14 @@ class FineGrainedSuite(DataSuite):
     # as skipped, not just elided.
     def should_skip(self, testcase: DataDrivenTestCase) -> bool:
         if self.use_cache:
-            if testcase.name.endswith("-skip-cache"):
+            if testcase.only_when == '-only_when_nocache':
                 return True
             # TODO: In caching mode we currently don't well support
             # starting from cached states with errors in them.
             if testcase.output and testcase.output[0] != '==':
                 return True
         else:
-            if testcase.name.endswith("-skip-nocache"):
+            if testcase.only_when == '-only_when_cache':
                 return True
 
         return False

mypy/test/testfinegrained.py

@@ -237,7 +237,7 @@ a.py:1: error: invalid syntax
 b.py:3: error: Too many arguments for "f"
 a.py:3: error: Too many arguments for "g"
 
-[case testDeleteFileWithBlockingError-skip-cache]
+[case testDeleteFileWithBlockingError-only_when_nocache]
 -- Different cache/no-cache tests because:
 -- Error message ordering differs
 import a
@@ -258,7 +258,7 @@ main:1: error: Cannot find module named 'a'
 main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
 b.py:1: error: Cannot find module named 'a'
 
-[case testDeleteFileWithBlockingError-skip-nocache]
+[case testDeleteFileWithBlockingError-only_when_cache]
 -- Different cache/no-cache tests because:
 -- Error message ordering differs
 import a
@@ -331,7 +331,7 @@ import blocker2
 ==
 a.py:1: error: "int" not callable
 
-[case testFixingBlockingErrorTriggersDeletion1-skip-cache]
+[case testFixingBlockingErrorTriggersDeletion1-only_when_nocache]
 -- Disabled in cache mdode:
 -- Cache mode fails to produce the error in the final step, but this is
 -- a manifestation of a bug that can occur in no-cache mode also.

test-data/unit/fine-grained-blockers.test

@@ -392,7 +392,7 @@ main:1: error: Cannot find module named 'a'
 main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports" flag would help)
 ==
 
-[case testDeletionOfSubmoduleTriggersImportFrom1-skip-cache]
+[case testDeletionOfSubmoduleTriggersImportFrom1-only_when_nocache]
 -- Different cache/no-cache tests because:
 -- missing module error message mismatch
 from p import q
@@ -410,7 +410,7 @@ main:1: note: (Perhaps setting MYPYPATH or using the "--ignore-missing-imports"
 
 -- TODO: Fix this bug. It is a real bug that was been papered over
 -- by the test harness.
-[case testDeletionOfSubmoduleTriggersImportFrom1-skip-nocache-skip]
+[case testDeletionOfSubmoduleTriggersImportFrom1-only_when_cache-skip]
 -- Different cache/no-cache tests because:
 -- missing module error message mismatch
 from p import q
@@ -1019,7 +1019,7 @@ a/b.py:3: error: Revealed type is 'Any'
 ==
 a/b.py:3: error: Unsupported operand types for + ("int" and "str")
 
-[case testDeleteModuleWithinPackageInitIgnored-skip-cache]
+[case testDeleteModuleWithinPackageInitIgnored-only_when_nocache]
 -- Disabled in cache mode because incorrect behavior:
 -- Having deleted files specified on command line seems dodgy, though.
 # cmd: mypy x.py a/b.py
@@ -1217,7 +1217,7 @@ main:2: error: Too few arguments for "foo" of "Foo"
 -- does not appear in the cache, for cache mode. They are run in
 -- cache mode only because stale and rechecked differ heavily between
 -- the modes.
-[case testAddModuleAfterCache1-skip-nocache]
+[case testAddModuleAfterCache1-only_when_cache]
 # cmd: mypy main a.py
 # cmd2: mypy main a.py b.py
 # cmd3: mypy main a.py b.py
@@ -1244,7 +1244,7 @@ def foo(x: int) -> None: pass
 a.py:2: error: Too many arguments for "foo"
 ==
 
-[case testAddModuleAfterCache2-skip-nocache]
+[case testAddModuleAfterCache2-only_when_cache]
 # cmd: mypy main a.py
 # cmd2: mypy main a.py b.py
 # cmd3: mypy main a.py b.py
@@ -1269,7 +1269,7 @@ def foo(x: int) -> None: pass
 a.py:2: error: Too many arguments for "foo"
 ==
 
-[case testAddModuleAfterCache3-skip-nocache]
+[case testAddModuleAfterCache3-only_when_cache]
 # cmd: mypy main a.py
 # cmd2: mypy main a.py b.py c.py d.py e.py f.py g.py
 # cmd3: mypy main a.py b.py c.py d.py e.py f.py g.py
@@ -1300,7 +1300,7 @@ a.py:2: error: Too many arguments for "foo"
 ==
 
 
-[case testAddModuleAfterCache4-skip-nocache]
+[case testAddModuleAfterCache4-only_when_cache]
 # cmd: mypy main a.py
 # cmd2: mypy main a.py b.py
 # cmd3: mypy main a.py b.py
@@ -1319,7 +1319,7 @@ def foo(x: int) -> None: pass
 b.py:2: error: Too many arguments for "foo"
 ==
 
-[case testAddModuleAfterCache5-skip-nocache]
+[case testAddModuleAfterCache5-only_when_cache]
 # cmd: mypy main a.py
 # cmd2: mypy main a.py b.py
 # cmd3: mypy main a.py b.py
@@ -1346,7 +1346,7 @@ def foo(x: int) -> None: pass
 b.py:2: error: Too many arguments for "foo"
 ==
 
-[case testAddModuleAfterCache6-skip-nocache]
+[case testAddModuleAfterCache6-only_when_cache]
 # cmd: mypy main a.py
 # cmd2: mypy main a.py b.py
 # cmd3: mypy main a.py b.py
@@ -1374,7 +1374,7 @@ def foo(x: int) -> None: pass
 a.py:2: error: Too many arguments for "foo"
 ==
 
-[case testRenameAndDeleteModuleAfterCache-skip-nocache]
+[case testRenameAndDeleteModuleAfterCache-only_when_cache]
 import a
 [file a.py]
 from b1 import f
@@ -1394,7 +1394,7 @@ f()
 [out]
 ==
 
-[case testDeleteModuleAfterCache-skip-nocache]
+[case testDeleteModuleAfterCache-only_when_cache]
 import a
 [file a.py]
 from b import f

test-data/unit/fine-grained-modules.test

@@ -36,7 +36,7 @@
 -- incremental runs is not yet supported.
 --
 -- Each test case run without caching and with caching (if the initial run passes),
--- unless it has one a --skip-cache or --skip-nocache suffix. We sometimes
+-- unless it has one a -only_when_cache or -only_when_nocache arguments. We sometimes
 -- skip caching test cases to speed up tests, if the caching variant is not useful.
 -- The caching test case variants get an implicit _cached suffix.
 
@@ -313,7 +313,7 @@ main:5: error: Module has no attribute "B"
 ==
 main:5: error: Module has no attribute "B"
 
-[case testContinueToReportErrorAtTopLevel-skip-cache]
+[case testContinueToReportErrorAtTopLevel-only_when_nocache]
 -- Different cache/no-cache tests because:
 -- Error message ordering differs
 import n
@@ -338,7 +338,7 @@ n.py:2: error: "A" has no attribute "g"
 ==
 n.py:2: error: "A" has no attribute "g"
 
-[case testContinueToReportErrorAtTopLevel-skip-nocache]
+[case testContinueToReportErrorAtTopLevel-only_when_cache]
 -- Different cache/no-cache tests because:
 -- Error message ordering differs
 import n
@@ -4556,7 +4556,7 @@ def g(x: T) -> T:
 ==
 a.py:2: error: Value of type variable "T" of function cannot be "int"
 
-[case testGenericFineCallableBoundDeleted-skip-nocache]
+[case testGenericFineCallableBoundDeleted-only_when_cache]
 # See https://github.com/python/mypy/issues/4783
 import a
 [file a.py]
@@ -5953,7 +5953,7 @@ class C:
 ==
 a.py:4: error: Argument 1 to "func" has incompatible type "B"; expected "P"
 
-[case testTwoProtocolsTwoFilesCrossedUpdateType-skip-cache]
+[case testTwoProtocolsTwoFilesCrossedUpdateType-only_when_nocache]
 # this test and the next one (TwoProtocolsTwoFilesCrossedDeleteAttr) has errors ordered
 # opposite way with and without cache, therefore skip one of each.
 import a
@@ -5991,7 +5991,7 @@ b2.py:7: error: Argument 1 to "f" has incompatible type "D"; expected "P1"
 b2.py:7: note: Following member(s) of "D" have conflicts:
 b2.py:7: note:     x: expected "int", got "str"
 
-[case testTwoProtocolsTwoFilesCrossedDeleteAttr-skip-nocache]
+[case testTwoProtocolsTwoFilesCrossedDeleteAttr-only_when_cache]
 import a
 [file a.py]
 import b1
@@ -6732,7 +6732,7 @@ def g(x: str) -> str: pass
 ==
 main:2: error: Argument 1 to "g" has incompatible type "int"; expected "str"
 
-[case testExtendedUnpacking-skip-cache]
+[case testExtendedUnpacking-only_when_nocache]
 from typing import List
 from a import g
 def f() -> List[int]:
@@ -6752,7 +6752,7 @@ def g() -> Tuple[str, str]: pass
 ==
 main:5: error: Incompatible return value type (got "List[str]", expected "List[int]")
 
-[case testUnpackInExpression1-skip-cache]
+[case testUnpackInExpression1-only_when_nocache]
 from typing import Tuple, List
 from a import t
 
@@ -6776,7 +6776,7 @@ def t() -> Tuple[str]: ...
 main:5: error: Incompatible return value type (got "Tuple[int, str]", expected "Tuple[int, int]")
 main:8: error: List item 1 has incompatible type "Tuple[str]"; expected "int"
 
-[case testUnpackInExpression2-skip-cache]
+[case testUnpackInExpression2-only_when_nocache]
 from typing import Set
 from a import t
 
@@ -6796,7 +6796,7 @@ def t() -> Tuple[str]: pass
 ==
 main:5: error: Argument 2 to <set> has incompatible type "*Tuple[str]"; expected "int"
 
-[case testUnpackInExpression3-skip-cache]
+[case testUnpackInExpression3-only_when_nocache]
 from typing import Dict
 from a import d
 
@@ -6816,7 +6816,7 @@ def d() -> Dict[int, int]: pass
 ==
 main:5: error: Argument 1 to "update" of "dict" has incompatible type "Dict[int, int]"; expected "Mapping[int, str]"
 
-[case testAwaitAndAsyncDef-skip-cache]
+[case testAwaitAndAsyncDef-only_when_nocache]
 from a import g
 
 async def f() -> int:
@@ -6836,7 +6836,7 @@ async def g() -> str:
 ==
 main:4: error: Incompatible return value type (got "str", expected "int")
 
-[case testAwaitAnd__await__-skip-cache]
+[case testAwaitAnd__await__-only_when_nocache]
 from a import C
 
 async def f(c: C) -> int:
@@ -6899,7 +6899,7 @@ main:6: error: Incompatible return value type (got "str", expected "int")
 ==
 main:6: error: Incompatible return value type (got "object", expected "int")
 
-[case testAsyncWith2-skip-cache]
+[case testAsyncWith2-only_when_nocache]
 from a import C
 
 async def f() -> int:
@@ -7005,7 +7005,7 @@ main:7: error: Unsupported operand types for + ("str" and "int")
 ==
 main:5: error: Return type of "m" incompatible with supertype "B"
 
-[case testLiskovFineVariableClean-skip-cache]
+[case testLiskovFineVariableClean-only_when_nocache]
 import b
 class A(b.B):
     x: str
@@ -7019,7 +7019,7 @@ class B:
 ==
 main:3: error: Incompatible types in assignment (expression has type "str", base class "B" defined the type as "int")
 
-[case testLiskovFineVariableCleanDefInMethod-skip-cache]
+[case testLiskovFineVariableCleanDefInMethod-only_when_nocache]
 import b
 class A(b.B):
     def meth(self) -> None:
@@ -7034,7 +7034,7 @@ class B:
 ==
 main:4: error: Incompatible types in assignment (expression has type "str", base class "B" defined the type as "int")
 
-[case testLiskovFineVariableCleanDefInMethodNested-skip-cache]
+[case testLiskovFineVariableCleanDefInMethodNested-only_when_nocache]
 from b import B
 def outer() -> None:
     class A(B):
@@ -7050,7 +7050,7 @@ class B:
 ==
 main:5: error: Incompatible types in assignment (expression has type "str", base class "B" defined the type as "int")
 
-[case testLiskovFineVariableInFunctionClean-skip-cache]
+[case testLiskovFineVariableInFunctionClean-only_when_nocache]
 from b import B
 def outer() -> None:
     class A(B):
@@ -7065,7 +7065,7 @@ class B:
 ==
 main:4: error: Incompatible types in assignment (expression has type "str", base class "B" defined the type as "int")
 
-[case testLiskovFineDecoratorClean-skip-cache]
+[case testLiskovFineDecoratorClean-only_when_nocache]
 import b
 from c import deco
 class A(b.B):