From ba87cc7dfbd914b2169e0d929e9d4687619cd2ec Mon Sep 17 00:00:00 2001
From: Thomas Kluyver <thomas@kluyver.me.uk>
Date: Wed, 12 May 2021 14:07:00 +0100
Subject: [PATCH] Clear up how notebook path is passed into concurrent_run()
---
src/xfel_calibrate/calibrate.py | 22 ++++++++++------------
1 file changed, 10 insertions(+), 12 deletions(-)
diff --git a/src/xfel_calibrate/calibrate.py b/src/xfel_calibrate/calibrate.py
index 2e351242e..ae8b9e3b1 100755
--- a/src/xfel_calibrate/calibrate.py
+++ b/src/xfel_calibrate/calibrate.py
@@ -707,7 +707,7 @@ def remove_duplications(l):
def concurrent_run(
- temp_path, nb, nbname, args, cparm=None, cval=None,
+ temp_path: str, nb, nb_path: Path, args, cparm=None, cval=None,
final_job=False, job_list=[], fmt_args={}, cluster_cores=8,
sequential=False, dep_jids=[],
show_title=True, user_venv: Optional[Path] = None,
@@ -735,9 +735,7 @@ def concurrent_run(
if not show_title:
first_markdown_cell(new_nb).source = ''
set_figure_format(new_nb, args["vector_figs"])
- base_name = nbname.replace(".ipynb", "")
- new_name = "{}__{}__{}.ipynb".format(
- os.path.basename(base_name), cparm, suffix)
+ new_name = f"{nb_path.stem}__{cparm}__{suffix}.ipynb"
nbpath = os.path.join(temp_path, new_name)
nbformat.write(new_nb, nbpath)
@@ -765,7 +763,7 @@ def concurrent_run(
os.path.abspath(nbpath), # path to notebook
python, # path to python to run notebook (& ipcluster)
cluster_profile,
- base_name.upper(),
+ nb_path.stem.upper(),
args["detector"].upper(),
args["type"].upper(),
finalize_script,
@@ -890,7 +888,7 @@ def run():
if args["concurrency_par"] is not None:
concurrency["parameter"] = args["concurrency_par"]
- notebook_path = os.path.join(PKG_DIR, notebook)
+ notebook_path = Path(PKG_DIR, notebook)
nb = nbformat.read(notebook_path, as_version=4)
# extend parameters if needed
@@ -987,10 +985,10 @@ def run():
cluster_cores = concurrency.get("cluster cores", 8)
# Check if there are pre-notebooks
for pre_notebook in pre_notebooks:
- pre_notebook_path = os.path.join(PKG_DIR, pre_notebook)
+ pre_notebook_path = Path(PKG_DIR, pre_notebook)
lead_nb = nbformat.read(pre_notebook_path, as_version=4)
jobid = concurrent_run(run_tmp_path, lead_nb,
- os.path.basename(pre_notebook_path),
+ pre_notebook_path,
args,
job_list=joblist, fmt_args=fmt_args,
cluster_cores=cluster_cores,
@@ -1000,7 +998,7 @@ def run():
if concurrency.get("parameter", None) is None:
jobid = concurrent_run(run_tmp_path, nb,
- os.path.basename(notebook), args,
+ notebook_path, args,
final_job=True, job_list=joblist,
fmt_args=fmt_args,
cluster_cores=cluster_cores,
@@ -1065,7 +1063,7 @@ def run():
final_job = (cnum == len(list(cvals)) - 1 and len(dep_notebooks) == 0)
cval = [cval, ] if not isinstance(cval, list) and cvtype is list else cval
- jobid = concurrent_run(run_tmp_path, nb, notebook, args,
+ jobid = concurrent_run(run_tmp_path, nb, notebook_path, args,
cvar, cval, final_job,
jlist, fmt_args,
cluster_cores=cluster_cores,
@@ -1078,11 +1076,11 @@ def run():
# Run dependent notebooks (e.g. summaries after correction)
for i, dep_notebook in enumerate(dep_notebooks):
- dep_notebook_path = os.path.join(PKG_DIR, dep_notebook)
+ dep_notebook_path = Path(PKG_DIR, dep_notebook)
dep_nb = nbformat.read(dep_notebook_path, as_version=4)
final_job = i == len(dep_notebooks) - 1
jobid = concurrent_run(run_tmp_path, dep_nb,
- os.path.basename(dep_notebook_path),
+ dep_notebook_path,
args,
dep_jids=joblist,
final_job=final_job,
--
GitLab