From 4ebf733f16acba4cffef6ab23622c71de22f8dd0 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Wed, 15 Jun 2022 00:30:55 +0200
Subject: [PATCH] maxtaskperchild multiprocessing pool configuration
---
notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 1bf5f0b35..ab99edd47 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -99,7 +99,7 @@
"n_cores_files = 4 # Number of files to be processed in parallel\n",
"sequences_per_node = 2 # number of sequence files per cluster node if run as SLURM job, set to 0 to not run SLURM parallel\n",
"max_nodes = 8 # Maximum number of SLURM jobs to split correction work into\n",
- "\n",
+ "max_tasks_per_worker = -1 # the number of tasks a correction pool worker process can complete before it will exit and be replaced with a fresh worker process. Leave as -1 to keep worker alive as long as pool.\n",
"\n",
"def balance_sequences(in_folder, run, sequences, sequences_per_node, karabo_da, max_nodes):\n",
" from xfel_calibrate.calibrate import balance_sequences as bs\n",
@@ -622,8 +622,9 @@
"outputs": [],
"source": [
"step_timer.start()\n",
- "\n",
- "with multiprocessing.Pool() as pool:\n",
+ "if max_tasks_per_worker == -1:\n",
+ " max_tasks_per_worker = None\n",
+ "with multiprocessing.Pool(maxtasksperchild=max_tasks_per_worker) as pool:\n",
" step_timer.done_step('Started pool')\n",
" \n",
" for file_batch in batches(file_list, n_cores_files):\n",
--
GitLab