From dbf49924811d7fbc755508ff5f59e9ce5396c8e7 Mon Sep 17 00:00:00 2001
From: Thomas Kluyver <thomas@kluyver.me.uk>
Date: Mon, 28 Nov 2022 18:43:09 +0000
Subject: [PATCH] Warn for new constant lookups while repeating correction
---
setup.py | 2 +-
src/xfel_calibrate/calibrate.py | 13 +++++++++----
src/xfel_calibrate/repeat.py | 7 ++++++-
3 files changed, 16 insertions(+), 6 deletions(-)
diff --git a/setup.py b/setup.py
index 56ad883bb..05fdf727c 100644
--- a/setup.py
+++ b/setup.py
@@ -116,7 +116,7 @@ if "readthedocs.org" not in sys.executable:
install_requires += [
"iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.3.0", # noqa
"XFELDetectorAnalysis @ git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git@2.7.0", # noqa
- "CalParrot @ git+ssh://git@git.xfel.eu:10022/calibration/calparrot.git@b0cfc58ff0f231685630b2a12f566faf243b1035", # noqa
+ "CalParrot @ git+ssh://git@git.xfel.eu:10022/calibration/calparrot.git@6ffb57019fa71f56a194d0ba8e468dc630e1a8c2", # noqa
]
setup(
diff --git a/src/xfel_calibrate/calibrate.py b/src/xfel_calibrate/calibrate.py
index 704b18e20..53f0cba19 100755
--- a/src/xfel_calibrate/calibrate.py
+++ b/src/xfel_calibrate/calibrate.py
@@ -385,11 +385,14 @@ class JobArgs:
"""Run this job in a local process, return exit status"""
return call(self.format_cmd(python), cwd=work_dir)
- def submit_job(self, work_dir, python, slurm_opts, after_ok=(), after_any=()):
+ def submit_job(
+ self, work_dir, python, slurm_opts, after_ok=(), after_any=(), env=None
+ ):
"""Submit this job to Slurm, return its job ID"""
cmd = slurm_opts.get_launcher_command(work_dir, after_ok, after_any)
cmd += self.format_cmd(python)
- output = check_output(cmd, cwd=work_dir).decode('utf-8')
+ # sbatch propagates environment variables into the job by default
+ output = check_output(cmd, cwd=work_dir, env=env).decode('utf-8')
return output.partition(';')[0].strip() # job ID
@@ -440,7 +443,7 @@ class JobChain:
'steps': [step.to_dict() for step in self.steps]
}, f, indent=2)
- def submit_jobs(self, slurm_opts: SlurmOptions):
+ def submit_jobs(self, slurm_opts: SlurmOptions, env=None):
"""Submit these jobs to Slurm, return a list of job IDs
Slurm dependencies are used to manage the sequence of jobs.
@@ -453,7 +456,9 @@ class JobChain:
step_job_ids = []
kw = {('after_any' if step.after_error else 'after_ok'): dep_job_ids}
for job_desc in step.jobs:
- jid = job_desc.submit_job(self.work_dir, self.python, slurm_opts, **kw)
+ jid = job_desc.submit_job(
+ self.work_dir, self.python, slurm_opts, env=env, **kw
+ )
step_job_ids.append(jid)
dep_job_ids = step_job_ids
all_job_ids.extend(step_job_ids)
diff --git a/src/xfel_calibrate/repeat.py b/src/xfel_calibrate/repeat.py
index 990b53756..e8d94134f 100644
--- a/src/xfel_calibrate/repeat.py
+++ b/src/xfel_calibrate/repeat.py
@@ -159,10 +159,15 @@ def main(argv=None):
job_chain.run_direct()
joblist = []
else:
+ # The queries to look up constants should all be the same as those
+ # from the previous calibration - tell CalParrot to warn if not.
+ env = os.environ.copy()
+ env['CALPARROT_NEW_QUERY'] = 'warn'
+
joblist = job_chain.submit_jobs(SlurmOptions(
partition=args.slurm_partition,
mem=args.slurm_mem,
- ))
+ ), env=env)
fmt_args = {'cal_work_dir': cal_work_dir,
'out_path': out_folder,
--
GitLab