diff --git a/bin/slurm_calibrate.sh b/bin/slurm_calibrate.sh
index 2d8dfccbf3a596ab23c8fa329b14c05bd224f88f..2483c460d86290544e119145d121cbb1c37058ab 100755
--- a/bin/slurm_calibrate.sh
+++ b/bin/slurm_calibrate.sh
@@ -43,9 +43,14 @@ then
sleep 15
fi
-
echo "Running notebook"
-${python_path} -m princess ${nb_path} --save
+if [ "$caltype" == "CORRECT" ]
+then
+ # calparrot stores and repeats calcat queries
+ ${python_path} -m calparrot -- ${python_path} -m princess ${nb_path} --save
+else
+ ${python_path} -m princess ${nb_path} --save
+fi
# stop the cluster if requested
if [ "${ipcluster_profile}" != "NO_CLUSTER" ]
diff --git a/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
index 2d9290776070ce6b4b873b2740784663a4b7bdb1..31b4eafa961e8686af5a70caad9956f0254d05b8 100644
--- a/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
+++ b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
@@ -167,9 +167,6 @@
"start = perf_counter()\n",
"for k_da in karabo_da:\n",
" pdu = None\n",
- " if k_da in retrieved_constants:\n",
- " print(f\"Constant for {k_da} already in {metadata.filename}, won't query again.\") # noqa\n",
- " continue\n",
" retrieved_constants[k_da] = dict()\n",
" const_mdata = retrieved_constants[k_da][\"constants\"] = dict()\n",
" for calibrations, condition in [\n",
diff --git a/setup.py b/setup.py
index eccf1fd27e90252e915254b83e53ea99e528cb42..192a5d57e1cd7176d9156bafe5b490077488d7d4 100644
--- a/setup.py
+++ b/setup.py
@@ -110,6 +110,7 @@ if "readthedocs.org" not in sys.executable:
install_requires += [
"iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.4.0", # noqa
"XFELDetectorAnalysis @ git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git@2.7.0", # noqa
+ "CalParrot @ git+ssh://git@git.xfel.eu:10022/calibration/calparrot.git@0.1", # noqa
]
setup(
diff --git a/src/xfel_calibrate/calibrate.py b/src/xfel_calibrate/calibrate.py
index 704b18e20c35e9990f0c6901be8b041d2e3287ae..53f0cba19e70b05503835ca946f58c98f22713f1 100755
--- a/src/xfel_calibrate/calibrate.py
+++ b/src/xfel_calibrate/calibrate.py
@@ -385,11 +385,14 @@ class JobArgs:
"""Run this job in a local process, return exit status"""
return call(self.format_cmd(python), cwd=work_dir)
- def submit_job(self, work_dir, python, slurm_opts, after_ok=(), after_any=()):
+ def submit_job(
+ self, work_dir, python, slurm_opts, after_ok=(), after_any=(), env=None
+ ):
"""Submit this job to Slurm, return its job ID"""
cmd = slurm_opts.get_launcher_command(work_dir, after_ok, after_any)
cmd += self.format_cmd(python)
- output = check_output(cmd, cwd=work_dir).decode('utf-8')
+ # sbatch propagates environment variables into the job by default
+ output = check_output(cmd, cwd=work_dir, env=env).decode('utf-8')
return output.partition(';')[0].strip() # job ID
@@ -440,7 +443,7 @@ class JobChain:
'steps': [step.to_dict() for step in self.steps]
}, f, indent=2)
- def submit_jobs(self, slurm_opts: SlurmOptions):
+ def submit_jobs(self, slurm_opts: SlurmOptions, env=None):
"""Submit these jobs to Slurm, return a list of job IDs
Slurm dependencies are used to manage the sequence of jobs.
@@ -453,7 +456,9 @@ class JobChain:
step_job_ids = []
kw = {('after_any' if step.after_error else 'after_ok'): dep_job_ids}
for job_desc in step.jobs:
- jid = job_desc.submit_job(self.work_dir, self.python, slurm_opts, **kw)
+ jid = job_desc.submit_job(
+ self.work_dir, self.python, slurm_opts, env=env, **kw
+ )
step_job_ids.append(jid)
dep_job_ids = step_job_ids
all_job_ids.extend(step_job_ids)
diff --git a/src/xfel_calibrate/repeat.py b/src/xfel_calibrate/repeat.py
index 9992ac59be047e7ea3f042c26fbbbdf665fc44a9..e8d94134f3b8a46384f1272b754fea66c070b80c 100644
--- a/src/xfel_calibrate/repeat.py
+++ b/src/xfel_calibrate/repeat.py
@@ -135,12 +135,12 @@ def main(argv=None):
out_folder = parameters['out-folder']
params_to_set = {'metadata_folder': "."}
if args.out_folder:
- out_folder = parameters['out-folder'] = args.out_folder
+ out_folder = parameters['out-folder'] = os.path.abspath(args.out_folder)
params_to_set['out_folder'] = out_folder
update_notebooks_params(cal_work_dir, params_to_set)
if args.report_to:
- report_to = args.report_to
+ report_to = os.path.abspath(args.report_to)
else: # Default to saving report in output folder
report_to = str(Path(out_folder, f'xfel-calibrate-repeat-{run_uuid}'))
cal_metadata['report-path'] = f'{report_to}.pdf'
@@ -159,12 +159,17 @@ def main(argv=None):
job_chain.run_direct()
joblist = []
else:
+ # The queries to look up constants should all be the same as those
+ # from the previous calibration - tell CalParrot to warn if not.
+ env = os.environ.copy()
+ env['CALPARROT_NEW_QUERY'] = 'warn'
+
joblist = job_chain.submit_jobs(SlurmOptions(
partition=args.slurm_partition,
mem=args.slurm_mem,
- ))
+ ), env=env)
- fmt_args = {'run_path': cal_work_dir,
+ fmt_args = {'cal_work_dir': cal_work_dir,
'out_path': out_folder,
'version': get_pycalib_version(),
'report_to': report_to,