Add --lf, -ff

Author: pkch

mypy/waiter.py

@@ -3,7 +3,7 @@
 This is used for running mypy tests.
 """
 
-from typing import Dict, List, Optional, Set, Tuple, Any
+from typing import Dict, List, Optional, Set, Tuple, Any, Iterable
 
 import os
 from multiprocessing import cpu_count
@@ -113,7 +113,8 @@ class Waiter:
     LOGSIZE = 50
     FULL_LOG_FILENAME = '.runtest_log.json'
 
-    def __init__(self, limit: int = 0, *, verbosity: int = 0, xfail: List[str] = []) -> None:
+    def __init__(self, limit: int = 0, *, verbosity: int = 0, xfail: List[str] = [],
+                 lf: bool = False, ff: bool = False) -> None:
         self.verbosity = verbosity
         self.queue = []  # type: List[LazySubprocess]
         # Index of next task to run in the queue.
@@ -130,12 +131,15 @@ def __init__(self, limit: int = 0, *, verbosity: int = 0, xfail: List[str] = [])
                 # major mistake to count *all* CPUs on the machine.
                 limit = len(sched_getaffinity(0))
         self.limit = limit
+        self.lf = lf
+        self.ff = ff
         assert limit > 0
         self.xfail = set(xfail)
         self._note = None  # type: Noter
         self.times1 = {}  # type: Dict[str, float]
         self.times2 = {}  # type: Dict[str, float]
-        self.new_log = defaultdict(dict)  # type: Dict[str, Dict[str, Any]]
+        self.new_log = defaultdict(dict)  # type: Dict[str, Dict[str, float]]
+        self.sequential_tasks = set()  # type: Set[str]
 
     def load_log_file(self) -> Optional[List[Dict[str, Dict[str, Any]]]]:
         try:
@@ -149,9 +153,13 @@ def load_log_file(self) -> Optional[List[Dict[str, Dict[str, Any]]]]:
             test_log = []
         return test_log
 
-    def add(self, cmd: LazySubprocess) -> int:
+    def add(self, cmd: LazySubprocess, sequential: bool = False) -> int:
         rv = len(self.queue)
+        if cmd.name in (task.name for task in self.queue):
+            sys.exit('Duplicate test name: {}'.format(cmd.name))
         self.queue.append(cmd)
+        if sequential:
+            self.sequential_tasks.add(cmd.name)
         return rv
 
     def _start_next(self) -> None:
@@ -187,8 +195,8 @@ def _poll_current(self) -> Tuple[int, int]:
                 code = cmd.process.poll()
                 if code is not None:
                     cmd.end_time = time.perf_counter()
-                    self.new_log[cmd.name]['exit_code'] = code
-                    self.new_log[cmd.name]['runtime'] = cmd.end_time - cmd.start_time
+                    self.new_log['exit_code'][cmd.name] = code
+                    self.new_log['runtime'][cmd.name] = cmd.end_time - cmd.start_time
                     return pid, code
 
     def _wait_next(self) -> Tuple[List[str], int, int]:
@@ -262,16 +270,43 @@ def run(self) -> int:
             self._note = Noter(len(self.queue))
         print('SUMMARY  %d tasks selected' % len(self.queue))
 
-        if self.limit > 1:
-            logs = self.load_log_file()
-            if logs:
+        def avg(lst: Iterable[float]) -> float:
+            valid_items = [item for item in lst if item is not None]
+            if not valid_items:
                 # we don't know how long a new task takes
                 # better err by putting it in front in case it is slow:
                 # a fast task in front hurts performance less than a slow task in the back
-                default = float('inf')
-                times = {cmd.name: sum(log[cmd.name].get('runtime', default) for log in logs)
-                         / len(logs) for cmd in self.queue}
-                self.queue = sorted(self.queue, key=lambda cmd: times[cmd.name], reverse=True)
+                return float('inf')
+            else:
+                return sum(valid_items) / len(valid_items)
+
+        logs = self.load_log_file()
+        if logs:
+            times = {cmd.name: avg(log['runtime'].get(cmd.name, None) for log in logs)
+                     for cmd in self.queue}
+
+            def sort_function(cmd: LazySubprocess) -> Tuple[Any, int, float]:
+                # longest tasks first
+                runtime = -times[cmd.name]
+                # sequential tasks go first by default
+                sequential = -(cmd.name in self.sequential_tasks)
+                if self.ff:
+                    # failed tasks first with -ff
+                    exit_code = -logs[-1]['exit_code'].get(cmd.name, 0)
+                    if not exit_code:
+                        # avoid interrupting parallel tasks with sequential in between
+                        # => order: seq failed, parallel failed, parallel passed, seq passed
+                        # => among failed tasks, sequential should go before parallel
+                        # => among successful tasks, sequential should go after parallel
+                        sequential = -sequential
+                else:
+                    # ignore exit code without -ff
+                    exit_code = 0
+                return exit_code, sequential, runtime
+            self.queue = sorted(self.queue, key=sort_function)
+            if self.lf:
+                self.queue = [cmd for cmd in self.queue
+                              if logs[-1]['exit_code'].get(cmd.name, 0)]
 
         sys.stdout.flush()
         # Failed tasks.
@@ -280,15 +315,35 @@ def run(self) -> int:
         total_tests = 0
         # Number of failed test cases.
         total_failed_tests = 0
+        running_sequential_task = False
         while self.current or self.next < len(self.queue):
             while len(self.current) < self.limit and self.next < len(self.queue):
+                # only start next task if idle, or current and next tasks are both parallel
+                if running_sequential_task:
+                    break
+                if self.queue[self.next].name in self.sequential_tasks:
+                    if self.current:
+                        break
+                    else:
+                        running_sequential_task = True
                 self._start_next()
             fails, tests, test_fails = self._wait_next()
+            running_sequential_task = False
             all_failures += fails
             total_tests += tests
             total_failed_tests += test_fails
         if self.verbosity == 0:
             self._note.clear()
+
+        if self.new_log:  # don't append empty log, it will corrupt the cache file
+            # log only LOGSIZE most recent tests
+            test_log = (self.load_log_file() + [self.new_log])[:self.LOGSIZE]
+            try:
+                with open(self.FULL_LOG_FILENAME, 'w') as fp:
+                    json.dump(test_log, fp, sort_keys=True, indent=4)
+            except Exception as e:
+                print('cannot save test log file:', e)
+
         if all_failures:
             summary = 'SUMMARY  %d/%d tasks and %d/%d tests failed' % (
                 len(all_failures), len(self.queue), total_failed_tests, total_tests)
@@ -305,16 +360,6 @@ def run(self) -> int:
                 len(self.queue), total_tests))
             print('*** OK ***')
             sys.stdout.flush()
-
-        if self.limit > 1:
-            # log only LOGSIZE most recent tests
-            test_log = (self.load_log_file() + [self.new_log])[:self.LOGSIZE]
-            try:
-                with open(self.FULL_LOG_FILENAME, 'w') as fp:
-                    json.dump(test_log, fp, sort_keys=True, indent=4)
-            except Exception as e:
-                print('cannot save test log file:', e)
-
         return 0
 
 

runtests.py

@@ -42,7 +42,8 @@ def get_versions():  # type: () -> typing.List[str]
 
 class Driver:
 
-    def __init__(self, whitelist: List[str], blacklist: List[str],
+    def __init__(self, *, whitelist: List[str], blacklist: List[str],
+            lf: bool, ff: bool,
             arglist: List[str], pyt_arglist: List[str],
             verbosity: int, parallel_limit: int,
             xfail: List[str], coverage: bool) -> None:
@@ -51,18 +52,13 @@ def __init__(self, whitelist: List[str], blacklist: List[str],
         self.arglist = arglist
         self.pyt_arglist = pyt_arglist
         self.verbosity = verbosity
-        self.waiter = Waiter(verbosity=verbosity, limit=parallel_limit, xfail=xfail)
-        self.sequential = Waiter(verbosity=verbosity, limit=1, xfail=xfail)
+        self.waiter = Waiter(verbosity=verbosity, limit=parallel_limit, xfail=xfail, lf=lf, ff=ff)
         self.versions = get_versions()
         self.cwd = os.getcwd()
         self.mypy = os.path.join(self.cwd, 'scripts', 'mypy')
         self.env = dict(os.environ)
         self.coverage = coverage
 
-    def run(self) -> int:
-        exit_code = self.sequential.run() & self.waiter.run()
-        return exit_code
-
     def prepend_path(self, name: str, paths: List[str]) -> None:
         old_val = self.env.get(name)
         paths = [p for p in paths if isdir(p)]
@@ -115,7 +111,7 @@ def add_pytest(self, name: str, pytest_args: List[str], coverage: bool = False)
         else:
             args = [sys.executable, '-m', 'pytest'] + pytest_args
 
-        self.sequential.add(LazySubprocess(full_name, args, env=self.env))
+        self.waiter.add(LazySubprocess(full_name, args, env=self.env), sequential=True)
 
     def add_python(self, name: str, *args: str, cwd: Optional[str] = None) -> None:
         name = 'run %s' % name
@@ -168,8 +164,7 @@ def add_flake8(self, cwd: Optional[str] = None) -> None:
         self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env))
 
     def list_tasks(self) -> None:
-        for id, task in enumerate(itertools.chain(self.sequential.queue,
-                                                  self.waiter.queue)):
+        for id, task in enumerate(self.waiter.queue):
             print('{id}:{task}'.format(id=id, task=task.name))
 
 
@@ -184,7 +179,8 @@ def add_basic(driver: Driver) -> None:
 
 
 def add_selftypecheck(driver: Driver) -> None:
-    driver.add_mypy_package('package mypy', 'mypy', '--config-file', 'mypy_self_check.ini')
+    driver.add_mypy_package('package mypy nonstrict optional', 'mypy', '--config-file',
+                            'mypy_self_check.ini')
     driver.add_mypy_package('package mypy', 'mypy', '--config-file', 'mypy_strict_optional.ini')
 
 
@@ -305,7 +301,8 @@ def add_samples(driver: Driver) -> None:
 
 
 def usage(status: int) -> None:
-    print('Usage: %s [-h | -v | -q | [-x] FILTER | -a ARG | -p ARG] ... [-- FILTER ...]'
+    print('Usage: %s [-h | -v | -q | --lf | --ff | [-x] FILTER | -a ARG | -p ARG]'
+          '... [-- FILTER ...]'
           % sys.argv[0])
     print()
     print('Run mypy tests. If given no arguments, run all tests.')
@@ -318,6 +315,8 @@ def usage(status: int) -> None:
     print('Options:')
     print('  -h, --help             show this help')
     print('  -v, --verbose          increase driver verbosity')
+    print('  --lf                   rerun only the tests that failed at the last run')
+    print('  --ff                   run all tests but run the last failures first')
     print('  -q, --quiet            decrease driver verbosity')
     print('  -jN                    run N tasks at once (default: one per CPU)')
     print('  -a, --argument ARG     pass an argument to myunit tasks')
@@ -357,6 +356,8 @@ def main() -> None:
     blacklist = []  # type: List[str]
     arglist = []  # type: List[str]
     pyt_arglist = []  # type: List[str]
+    lf = False
+    ff = False
     list_only = False
     coverage = False
 
@@ -383,6 +384,12 @@ def main() -> None:
                 curlist = arglist
             elif a == '-p' or a == '--pytest_arg':
                 curlist = pyt_arglist
+            # will also pass this option to pytest
+            elif a == '--lf':
+                lf = True
+            # will also pass this option to pytest
+            elif a == '--ff':
+                ff = True
             elif a == '-l' or a == '--list':
                 list_only = True
             elif a == '-c' or a == '--coverage':
@@ -400,11 +407,17 @@ def main() -> None:
         sys.exit('-a must be followed by an argument')
     if curlist is pyt_arglist:
         sys.exit('-p must be followed by an argument')
+    if lf and ff:
+        sys.exit('use either --lf or --ff, not both')
     # empty string is a substring of all names
     if not whitelist:
         whitelist.append('')
+    if lf:
+        pyt_arglist.append('--lf')
+    if ff:
+        pyt_arglist.append('--ff')
 
-    driver = Driver(whitelist=whitelist, blacklist=blacklist,
+    driver = Driver(whitelist=whitelist, blacklist=blacklist, lf=lf, ff=ff,
                     arglist=arglist, pyt_arglist=pyt_arglist, verbosity=verbosity,
                     parallel_limit=parallel_limit, xfail=[], coverage=coverage)
 
@@ -429,7 +442,7 @@ def main() -> None:
         driver.list_tasks()
         return
 
-    exit_code = driver.run()
+    exit_code = driver.waiter.run()
     t1 = time.perf_counter()
     print('total runtime:', t1 - t0, 'sec')