Skip to content
Snippets Groups Projects
Commit 500833ea authored by Loïc Le Guyader's avatar Loïc Le Guyader
Browse files

Merge branch 'FastCCD-parallel-backend' into 'master'

Using joblib instead of multiprocessing

See merge request !61
parents 4cb4f415 3a2d9c7f
No related branches found
No related tags found
1 merge request!61Using joblib instead of multiprocessing
import multiprocessing
from joblib import Parallel, delayed, parallel_backend
from time import strftime
import tempfile
import shutil
......@@ -321,8 +321,10 @@ class FastCCD:
timestamp = strftime('%X')
print(f'start time: {timestamp}')
with multiprocessing.Pool(self.Nworker) as pool:
res = list(tqdm(pool.imap(process_one_module, jobs), total=len(self.h5list)))
with parallel_backend('threading', n_jobs=self.Nworker):
res = Parallel( verbose=20)(
delayed(process_one_module)(job) for job in tqdm(jobs)
)
print('finished:', strftime('%X'))
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment