Skip to content
Snippets Groups Projects
Commit dbf49924 authored by Thomas Kluyver's avatar Thomas Kluyver
Browse files

Warn for new constant lookups while repeating correction

parent f6e2e0bc
No related branches found
No related tags found
1 merge request!767Store CalCat requests/responses for reproducibility
...@@ -116,7 +116,7 @@ if "readthedocs.org" not in sys.executable: ...@@ -116,7 +116,7 @@ if "readthedocs.org" not in sys.executable:
install_requires += [ install_requires += [
"iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.3.0", # noqa "iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.3.0", # noqa
"XFELDetectorAnalysis @ git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git@2.7.0", # noqa "XFELDetectorAnalysis @ git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git@2.7.0", # noqa
"CalParrot @ git+ssh://git@git.xfel.eu:10022/calibration/calparrot.git@b0cfc58ff0f231685630b2a12f566faf243b1035", # noqa "CalParrot @ git+ssh://git@git.xfel.eu:10022/calibration/calparrot.git@6ffb57019fa71f56a194d0ba8e468dc630e1a8c2", # noqa
] ]
setup( setup(
......
...@@ -385,11 +385,14 @@ class JobArgs: ...@@ -385,11 +385,14 @@ class JobArgs:
"""Run this job in a local process, return exit status""" """Run this job in a local process, return exit status"""
return call(self.format_cmd(python), cwd=work_dir) return call(self.format_cmd(python), cwd=work_dir)
def submit_job(self, work_dir, python, slurm_opts, after_ok=(), after_any=()): def submit_job(
self, work_dir, python, slurm_opts, after_ok=(), after_any=(), env=None
):
"""Submit this job to Slurm, return its job ID""" """Submit this job to Slurm, return its job ID"""
cmd = slurm_opts.get_launcher_command(work_dir, after_ok, after_any) cmd = slurm_opts.get_launcher_command(work_dir, after_ok, after_any)
cmd += self.format_cmd(python) cmd += self.format_cmd(python)
output = check_output(cmd, cwd=work_dir).decode('utf-8') # sbatch propagates environment variables into the job by default
output = check_output(cmd, cwd=work_dir, env=env).decode('utf-8')
return output.partition(';')[0].strip() # job ID return output.partition(';')[0].strip() # job ID
...@@ -440,7 +443,7 @@ class JobChain: ...@@ -440,7 +443,7 @@ class JobChain:
'steps': [step.to_dict() for step in self.steps] 'steps': [step.to_dict() for step in self.steps]
}, f, indent=2) }, f, indent=2)
def submit_jobs(self, slurm_opts: SlurmOptions): def submit_jobs(self, slurm_opts: SlurmOptions, env=None):
"""Submit these jobs to Slurm, return a list of job IDs """Submit these jobs to Slurm, return a list of job IDs
Slurm dependencies are used to manage the sequence of jobs. Slurm dependencies are used to manage the sequence of jobs.
...@@ -453,7 +456,9 @@ class JobChain: ...@@ -453,7 +456,9 @@ class JobChain:
step_job_ids = [] step_job_ids = []
kw = {('after_any' if step.after_error else 'after_ok'): dep_job_ids} kw = {('after_any' if step.after_error else 'after_ok'): dep_job_ids}
for job_desc in step.jobs: for job_desc in step.jobs:
jid = job_desc.submit_job(self.work_dir, self.python, slurm_opts, **kw) jid = job_desc.submit_job(
self.work_dir, self.python, slurm_opts, env=env, **kw
)
step_job_ids.append(jid) step_job_ids.append(jid)
dep_job_ids = step_job_ids dep_job_ids = step_job_ids
all_job_ids.extend(step_job_ids) all_job_ids.extend(step_job_ids)
......
...@@ -159,10 +159,15 @@ def main(argv=None): ...@@ -159,10 +159,15 @@ def main(argv=None):
job_chain.run_direct() job_chain.run_direct()
joblist = [] joblist = []
else: else:
# The queries to look up constants should all be the same as those
# from the previous calibration - tell CalParrot to warn if not.
env = os.environ.copy()
env['CALPARROT_NEW_QUERY'] = 'warn'
joblist = job_chain.submit_jobs(SlurmOptions( joblist = job_chain.submit_jobs(SlurmOptions(
partition=args.slurm_partition, partition=args.slurm_partition,
mem=args.slurm_mem, mem=args.slurm_mem,
)) ), env=env)
fmt_args = {'cal_work_dir': cal_work_dir, fmt_args = {'cal_work_dir': cal_work_dir,
'out_path': out_folder, 'out_path': out_folder,
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment