File tree Expand file tree Collapse file tree 1 file changed +8
-0
lines changed Expand file tree Collapse file tree 1 file changed +8
-0
lines changed Original file line number Diff line number Diff line change 41
41
class CAT (AbstractSerialisable ):
42
42
"""This is a collection of serialisable model parts.
43
43
"""
44
+ FORCE_SPAWN_MP = True
44
45
45
46
def __init__ (self ,
46
47
cdb : CDB ,
@@ -381,6 +382,13 @@ def _multiprocess(
381
382
batch_iter : Iterator [list [tuple [str , str , bool ]]]
382
383
) -> Iterator [tuple [str , Union [dict , Entities , OnlyCUIEntities ]]]:
383
384
external_processes = n_process - 1
385
+ if self .FORCE_SPAWN_MP :
386
+ import multiprocessing as mp
387
+ logger .info (
388
+ "Forcing multiprocessing start method to 'spawn' "
389
+ "due to known compatibility issues with 'fork' and "
390
+ "libraries using threads or native extensions." )
391
+ mp .set_start_method ("spawn" , force = True )
384
392
with ProcessPoolExecutor (max_workers = external_processes ) as executor :
385
393
yield from self ._mp_one_batch_per_process (
386
394
executor , batch_iter , external_processes )
You can’t perform that action at this time.
0 commit comments