From 3a2d9c7f2032063eeac968ff86dc51b4057b80fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Le=20Guyader?= <loic.le.guyader@xfel.eu>
Date: Fri, 15 Nov 2019 20:35:22 +0100
Subject: [PATCH] Using joblib instead of multiprocessing
---
FastCCD.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/FastCCD.py b/FastCCD.py
index 6d4b2a1..1f4124c 100644
--- a/FastCCD.py
+++ b/FastCCD.py
@@ -1,4 +1,4 @@
-import multiprocessing
+from joblib import Parallel, delayed, parallel_backend
from time import strftime
import tempfile
import shutil
@@ -321,8 +321,10 @@ class FastCCD:
timestamp = strftime('%X')
print(f'start time: {timestamp}')
- with multiprocessing.Pool(self.Nworker) as pool:
- res = list(tqdm(pool.imap(process_one_module, jobs), total=len(self.h5list)))
+ with parallel_backend('threading', n_jobs=self.Nworker):
+ res = Parallel( verbose=20)(
+ delayed(process_one_module)(job) for job in tqdm(jobs)
+ )
print('finished:', strftime('%X'))
--
GitLab