From 23eac1f094049d21dea416ff74008b6647bd95b7 Mon Sep 17 00:00:00 2001 From: NewUserHa <32261870+NewUserHa@users.noreply.github.com> Date: Thu, 18 Jan 2024 07:37:15 +0800 Subject: [PATCH 1/2] update concurrent.features.process get_chunks to use new itertools.batched. --- Lib/concurrent/futures/process.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index ffaffdb8b3d0aa..6f0ff9ab37d2bc 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -193,11 +193,7 @@ def _on_queue_feeder_error(self, e, obj): def _get_chunks(*iterables, chunksize): """ Iterates over zip()ed iterables in chunks. """ it = zip(*iterables) - while True: - chunk = tuple(itertools.islice(it, chunksize)) - if not chunk: - return - yield chunk + return itertools.batched(it, chunksize) def _process_chunk(fn, chunk): From 2872cfa0cf1012079a850bb23ea804eb046060c7 Mon Sep 17 00:00:00 2001 From: NewUserHa <32261870+NewUserHa@users.noreply.github.com> Date: Sat, 27 Jan 2024 03:31:54 +0800 Subject: [PATCH 2/2] Update process.py --- Lib/concurrent/futures/process.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 6f0ff9ab37d2bc..ca843e11eeb83d 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -190,12 +190,6 @@ def _on_queue_feeder_error(self, e, obj): super()._on_queue_feeder_error(e, obj) -def _get_chunks(*iterables, chunksize): - """ Iterates over zip()ed iterables in chunks. """ - it = zip(*iterables) - return itertools.batched(it, chunksize) - - def _process_chunk(fn, chunk): """ Processes a chunk of an iterable passed to map. @@ -843,7 +837,7 @@ def map(self, fn, *iterables, timeout=None, chunksize=1): raise ValueError("chunksize must be >= 1.") results = super().map(partial(_process_chunk, fn), - _get_chunks(*iterables, chunksize=chunksize), + itertools.batched(zip(*iterables), chunksize), timeout=timeout) return _chain_from_iterable_of_lists(results)