Always use the multiprocess module.

This seems to work on freebsd and openbsd these days.

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@303280 91177308-0d34-0410-b5e6-96231b3b80d8

Author: espindola

utils/lit/lit/main.py

@@ -282,15 +282,6 @@ def main_with_tmp(builtinParameters):
     debug_group.add_argument("--show-tests", dest="showTests",
                       help="Show all discovered tests",
                       action="store_true", default=False)
-    debug_group.add_argument("--use-process-pool", dest="executionStrategy",
-                      help="Run tests in parallel with a process pool",
-                      action="store_const", const="PROCESS_POOL")
-    debug_group.add_argument("--use-processes", dest="executionStrategy",
-                      help="Run tests in parallel with processes (not threads)",
-                      action="store_const", const="PROCESSES")
-    debug_group.add_argument("--use-threads", dest="executionStrategy",
-                      help="Run tests in parallel with threads (not processes)",
-                      action="store_const", const="THREADS")
 
     opts = parser.parse_args()
     args = opts.test_paths
@@ -305,9 +296,6 @@ def main_with_tmp(builtinParameters):
     if opts.numThreads is None:
         opts.numThreads = lit.util.detectCPUs()
 
-    if opts.executionStrategy is None:
-        opts.executionStrategy = 'PROCESS_POOL'
-
     if opts.maxFailures == 0:
         parser.error("Setting --max-failures to 0 does not have any effect.")
 
@@ -490,8 +478,7 @@ def main_with_tmp(builtinParameters):
     startTime = time.time()
     display = TestingProgressDisplay(opts, len(run.tests), progressBar)
     try:
-        run.execute_tests(display, opts.numThreads, opts.maxTime,
-                          opts.executionStrategy)
+        run.execute_tests(display, opts.numThreads, opts.maxTime)
     except KeyboardInterrupt:
         sys.exit(2)
     display.finish()

utils/lit/lit/run.py

@@ -13,11 +13,7 @@
 except ImportError:
     win32api = None
 
-try:
-    import multiprocessing
-except ImportError:
-    multiprocessing = None
-
+import multiprocessing
 import lit.Test
 
 def abort_now():
@@ -227,8 +223,7 @@ def __init__(self, lit_config, tests):
     def execute_test(self, test):
         return execute_test(test, self.lit_config, self.parallelism_semaphores)
 
-    def execute_tests(self, display, jobs, max_time=None,
-                      execution_strategy=None):
+    def execute_tests(self, display, jobs, max_time=None):
         """
         execute_tests(display, jobs, [max_time])
 
@@ -249,100 +244,6 @@ def execute_tests(self, display, jobs, max_time=None,
         computed. Tests which were not actually executed (for any reason) will
         be given an UNRESOLVED result.
         """
-
-        if execution_strategy == 'PROCESS_POOL':
-            self.execute_tests_with_mp_pool(display, jobs, max_time)
-            return
-        # FIXME: Standardize on the PROCESS_POOL execution strategy and remove
-        # the other two strategies.
-
-        use_processes = execution_strategy == 'PROCESSES'
-
-        # Choose the appropriate parallel execution implementation.
-        consumer = None
-        if jobs != 1 and use_processes and multiprocessing:
-            try:
-                task_impl = multiprocessing.Process
-                queue_impl = multiprocessing.Queue
-                sem_impl = multiprocessing.Semaphore
-                canceled_flag =  multiprocessing.Value('i', 0)
-                consumer = MultiprocessResultsConsumer(self, display, jobs)
-            except:
-                # multiprocessing fails to initialize with certain OpenBSD and
-                # FreeBSD Python versions: http://bugs.python.org/issue3770
-                # Unfortunately the error raised also varies by platform.
-                self.lit_config.note('failed to initialize multiprocessing')
-                consumer = None
-        if not consumer:
-            task_impl = threading.Thread
-            queue_impl = queue.Queue
-            sem_impl = threading.Semaphore
-            canceled_flag = LockedValue(0)
-            consumer = ThreadResultsConsumer(display)
-
-        self.parallelism_semaphores = {k: sem_impl(v)
-            for k, v in self.lit_config.parallelism_groups.items()}
-
-        # Create the test provider.
-        provider = TestProvider(queue_impl, canceled_flag)
-        handleFailures(provider, consumer, self.lit_config.maxFailures)
-
-        # Putting tasks into the threading or multiprocessing Queue may block,
-        # so do it in a separate thread.
-        # https://docs.python.org/2/library/multiprocessing.html
-        # e.g: On Mac OS X, we will hang if we put 2^15 elements in the queue
-        # without taking any out.
-        queuer = task_impl(target=provider.queue_tests, args=(self.tests, jobs))
-        queuer.start()
-
-        # Install a console-control signal handler on Windows.
-        if win32api is not None:
-            def console_ctrl_handler(type):
-                provider.cancel()
-                return True
-            win32api.SetConsoleCtrlHandler(console_ctrl_handler, True)
-
-        # Install a timeout handler, if requested.
-        if max_time is not None:
-            def timeout_handler():
-                provider.cancel()
-            timeout_timer = threading.Timer(max_time, timeout_handler)
-            timeout_timer.start()
-
-        # If not using multiple tasks, just run the tests directly.
-        if jobs == 1:
-            run_one_tester(self, provider, consumer)
-        else:
-            # Otherwise, execute the tests in parallel
-            self._execute_tests_in_parallel(task_impl, provider, consumer, jobs)
-
-        queuer.join()
-
-        # Cancel the timeout handler.
-        if max_time is not None:
-            timeout_timer.cancel()
-
-        # Update results for any tests which weren't run.
-        for test in self.tests:
-            if test.result is None:
-                test.setResult(lit.Test.Result(lit.Test.UNRESOLVED, '', 0.0))
-
-    def _execute_tests_in_parallel(self, task_impl, provider, consumer, jobs):
-        # Start all of the tasks.
-        tasks = [task_impl(target=run_one_tester,
-                           args=(self, provider, consumer))
-                 for i in range(jobs)]
-        for t in tasks:
-            t.start()
-
-        # Allow the consumer to handle results, if necessary.
-        consumer.handle_results()
-
-        # Wait for all the tasks to complete.
-        for t in tasks:
-            t.join()
-
-    def execute_tests_with_mp_pool(self, display, jobs, max_time=None):
         # Don't do anything if we aren't going to run any tests.
         if not self.tests or jobs == 0:
             return