Skip to content

gh-109276: libregrtest: limit number workers #109288

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 12, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion Lib/test/libregrtest/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,12 @@ def run_tests_sequentially(self, runtests):

save_modules = sys.modules.keys()

msg = "Run tests sequentially"
jobs = runtests.get_jobs()
if jobs is not None:
tests = f'{jobs} tests'
else:
tests = 'tests'
msg = f"Run {tests} sequentially"
if runtests.timeout:
msg += " (timeout: %s)" % format_duration(runtests.timeout)
self.log(msg)
Expand Down
18 changes: 16 additions & 2 deletions Lib/test/libregrtest/run_workers.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from .single import PROGRESS_MIN_TIME
from .utils import (
StrPath, StrJSON, TestName, MS_WINDOWS,
format_duration, print_warning)
format_duration, print_warning, plural)
from .worker import create_worker_process, USE_PROCESS_GROUP

if MS_WINDOWS:
Expand Down Expand Up @@ -401,10 +401,24 @@ def __init__(self, num_workers: int, runtests: RunTests,
self.worker_timeout = None
self.workers = None

jobs = self.runtests.get_jobs()
if jobs is not None:
# Don't spawn more threads than the number of jobs:
# these worker threads would never get anything to do.
self.num_workers = min(self.num_workers, jobs)

def start_workers(self) -> None:
self.workers = [WorkerThread(index, self)
for index in range(1, self.num_workers + 1)]
msg = f"Run tests in parallel using {len(self.workers)} child processes"
jobs = self.runtests.get_jobs()
if jobs is not None:
tests = f'{jobs} tests'
else:
tests = 'tests'
nworkers = len(self.workers)
processes = plural(nworkers, "process", "processes")
msg = (f"Run {tests} in parallel using "
f"{nworkers} worker {processes}")
if self.timeout:
msg += (" (timeout: %s, worker timeout: %s)"
% (format_duration(self.timeout),
Expand Down
7 changes: 7 additions & 0 deletions Lib/test/libregrtest/runtests.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,13 @@ def get_match_tests(self, test_name) -> FilterTuple | None:
else:
return None

def get_jobs(self):
# Number of run_single_test() calls needed to run all tests.
# None means that there is not bound limit (--forever option).
if self.forever:
return None
return len(self.tests)

def iter_tests(self):
if self.forever:
while True:
Expand Down
13 changes: 11 additions & 2 deletions Lib/test/libregrtest/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,11 +70,20 @@ def strip_py_suffix(names: list[str]):
names[idx] = basename


def plural(n, singular, plural=None):
if n == 1:
return singular
elif plural is not None:
return plural
else:
return singular + 's'


def count(n, word):
if n == 1:
return "%d %s" % (n, word)
return f"{n} {word}"
else:
return "%d %ss" % (n, word)
return f"{n} {word}s"


def printlist(x, width=70, indent=4, file=None):
Expand Down