diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb index 1bf5f0b35c86e0d2bb6922e6b33178ca4c22801f..ab99edd4768548e7dd696fa55cb6efa7c1dccd65 100644 --- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb +++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb @@ -99,7 +99,7 @@ "n_cores_files = 4 # Number of files to be processed in parallel\n", "sequences_per_node = 2 # number of sequence files per cluster node if run as SLURM job, set to 0 to not run SLURM parallel\n", "max_nodes = 8 # Maximum number of SLURM jobs to split correction work into\n", - "\n", + "max_tasks_per_worker = -1 # the number of tasks a correction pool worker process can complete before it will exit and be replaced with a fresh worker process. Leave as -1 to keep worker alive as long as pool.\n", "\n", "def balance_sequences(in_folder, run, sequences, sequences_per_node, karabo_da, max_nodes):\n", " from xfel_calibrate.calibrate import balance_sequences as bs\n", @@ -622,8 +622,9 @@ "outputs": [], "source": [ "step_timer.start()\n", - "\n", - "with multiprocessing.Pool() as pool:\n", + "if max_tasks_per_worker == -1:\n", + " max_tasks_per_worker = None\n", + "with multiprocessing.Pool(maxtasksperchild=max_tasks_per_worker) as pool:\n", " step_timer.done_step('Started pool')\n", " \n", " for file_batch in batches(file_list, n_cores_files):\n",