From 1d0e8315476899f028f1c3f90dfbc22a13e0438c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Lo=C3=AFc=20Le=20Guyader?= <loic.le.guyader@xfel.eu>
Date: Wed, 1 Apr 2020 18:36:20 +0200
Subject: [PATCH] Adds number of DSSC multiprocessing workers
---
DSSC.py | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/DSSC.py b/DSSC.py
index 0e88693..0adf64b 100644
--- a/DSSC.py
+++ b/DSSC.py
@@ -45,6 +45,7 @@ class DSSC:
self.mask = None
self.max_fraction_memory = 0.8
self.filter_mask = None
+ self.Nworker = 6
print('DSSC configuration')
print(f'Topic: {self.topic}')
@@ -308,8 +309,8 @@ class DSSC:
max_GB = psutil.virtual_memory().available/1024**3
print(f'max available memory: {max_GB} GB')
- # max_GB / (8byte * 16modules * 128px * 512px * N_pulses)
- self.chunksize = int(self.max_fraction_memory*max_GB * 1024**3 // (8 * 16 * 128 * 512 * self.fpt))
+ # max_GB / (8byte * Nworker * 128px * 512px * N_pulses)
+ self.chunksize = int(self.max_fraction_memory*max_GB * 1024**3 // (8 * self.Nworker * 128 * 512 * self.fpt))
print('processing', self.chunksize, 'trains per chunk')
@@ -329,9 +330,9 @@ class DSSC:
timestamp = strftime('%X')
print(f'start time: {timestamp}')
- with multiprocessing.Pool(16) as pool:
+ with multiprocessing.Pool(self.Nworker) as pool:
module_data = pool.map(process_one_module, jobs)
-
+
print('finished:', strftime('%X'))
# rearange the multiprocessed data
--
GitLab