Skip to content
Snippets Groups Projects
Commit 1d0e8315 authored by Loïc Le Guyader's avatar Loïc Le Guyader
Browse files

Adds number of DSSC multiprocessing workers

parent 7402ed47
No related branches found
No related tags found
1 merge request!76Extra data and xfel kernel
...@@ -45,6 +45,7 @@ class DSSC: ...@@ -45,6 +45,7 @@ class DSSC:
self.mask = None self.mask = None
self.max_fraction_memory = 0.8 self.max_fraction_memory = 0.8
self.filter_mask = None self.filter_mask = None
self.Nworker = 6
print('DSSC configuration') print('DSSC configuration')
print(f'Topic: {self.topic}') print(f'Topic: {self.topic}')
...@@ -308,8 +309,8 @@ class DSSC: ...@@ -308,8 +309,8 @@ class DSSC:
max_GB = psutil.virtual_memory().available/1024**3 max_GB = psutil.virtual_memory().available/1024**3
print(f'max available memory: {max_GB} GB') print(f'max available memory: {max_GB} GB')
# max_GB / (8byte * 16modules * 128px * 512px * N_pulses) # max_GB / (8byte * Nworker * 128px * 512px * N_pulses)
self.chunksize = int(self.max_fraction_memory*max_GB * 1024**3 // (8 * 16 * 128 * 512 * self.fpt)) self.chunksize = int(self.max_fraction_memory*max_GB * 1024**3 // (8 * self.Nworker * 128 * 512 * self.fpt))
print('processing', self.chunksize, 'trains per chunk') print('processing', self.chunksize, 'trains per chunk')
...@@ -329,9 +330,9 @@ class DSSC: ...@@ -329,9 +330,9 @@ class DSSC:
timestamp = strftime('%X') timestamp = strftime('%X')
print(f'start time: {timestamp}') print(f'start time: {timestamp}')
with multiprocessing.Pool(16) as pool: with multiprocessing.Pool(self.Nworker) as pool:
module_data = pool.map(process_one_module, jobs) module_data = pool.map(process_one_module, jobs)
print('finished:', strftime('%X')) print('finished:', strftime('%X'))
# rearange the multiprocessed data # rearange the multiprocessed data
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment