diff --git a/src/xfel_calibrate/calibrate.py b/src/xfel_calibrate/calibrate.py
index 655bc6cad85193be88d5f9f272d466953b86d5e9..bc43ac42465558005380f4f6b1d3febd78045791 100755
--- a/src/xfel_calibrate/calibrate.py
+++ b/src/xfel_calibrate/calibrate.py
@@ -583,7 +583,11 @@ def run(argv=None):
title = title.rstrip()
- run_uuid = f"t{datetime.now().strftime('%y%m%d_%H%M%S.%f')}"
+ # request_time is in local timezone
+ if args["request_time"] == "Now":
+ request_time = datetime.now()
+ else:
+ request_time = datetime.fromisoformat(args["request_time"])
# check if concurrency parameter is given and we run concurrently
if concurrency_par is not None and not any(
@@ -596,8 +600,7 @@ def run(argv=None):
default_params_by_name = {p.name: p.value for p in nb_details.default_params}
if 'cluster_profile' in default_params_by_name:
if args.get("cluster_profile") == default_params_by_name["cluster_profile"]:
- args['cluster_profile'] = "slurm_prof_{}".format(run_uuid)
-
+ args['cluster_profile'] = f"slurm_prof_{request_time:%y%m%d_%H%M%S.%f}"
# wait on all jobs to run and then finalize the run by creating a report from the notebooks
out_path = Path(default_report_path) / nb_details.detector / nb_details.caltype / datetime.now().isoformat()
@@ -609,26 +612,22 @@ def run(argv=None):
out_path.mkdir(parents=True, exist_ok=True)
- # Use given report name, falling back to notebook title
+ # Use given report name, or automatic unique name if not specified
+ det_name = args.get('karabo_id', nb_details.detector)
+ unique_name = f"{det_name}-{nb_details.caltype}-{request_time:%y%m%d_%H%M%S.%f}"
if args['skip_report']:
report_to = ''
elif args["report_to"] is None:
- report_to = out_path / title.replace(" ", "")
+ report_to = out_path / f"{unique_name}.pdf"
print(f"report_to not specified, will use {report_to}")
else:
- report_to = Path(args["report_to"])
- if report_to.is_dir():
- print(f"report_to is directory, will use title '{title}' for filename")
- report_to = report_to / title.replace(" ", "")
- elif len(report_to.parts) == 1:
- print(f"report_to path contained no path, saving report in '{out_path}'")
- report_to = out_path / report_to
-
- workdir_name = f"slurm_out_{nb_details.detector}_{nb_details.caltype}_{run_uuid}"
+ report_to = Path(args["report_to"]).with_suffix('.pdf').absolute()
+
if report_to:
- cal_work_dir = report_to.parent / workdir_name
+ # Work dir matching report file but without .pdf
+ cal_work_dir = report_to.with_suffix('')
else:
- cal_work_dir = out_path / workdir_name
+ cal_work_dir = out_path / unique_name
cal_work_dir.mkdir(parents=True)
# Write all input parameters to rst file to be included to final report
@@ -660,7 +659,7 @@ def run(argv=None):
parm_subdict[name] = p.value
metadata["pycalibration-version"] = version
- metadata["report-path"] = f"{report_to}.pdf" if report_to \
+ metadata["report-path"] = str(report_to) if report_to \
else '# REPORT SKIPPED #'
metadata['reproducible'] = not args['not_reproducible']
metadata["concurrency"] = {
@@ -689,11 +688,6 @@ def run(argv=None):
folder = get_par_attr(parms, 'in_folder', 'value', '')
- if args["request_time"] == "Now":
- request_time = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
- else:
- request_time = args["request_time"]
-
pre_jobs = []
cluster_cores = concurrency.get("cluster cores", 8)
# Check if there are pre-notebooks
@@ -823,7 +817,7 @@ def run(argv=None):
'author': author,
'report_to': report_to,
'in_folder': folder,
- 'request_time': request_time,
+ 'request_time': request_time.strftime("%Y-%m-%dT%H:%M:%S"),
'submission_time': submission_time,
}
diff --git a/src/xfel_calibrate/finalize.py b/src/xfel_calibrate/finalize.py
index 0df50980e32551a0db3f9b1bf3a8d6548856af28..41b3f9dd4b84f61a666647064959b966ba915543 100644
--- a/src/xfel_calibrate/finalize.py
+++ b/src/xfel_calibrate/finalize.py
@@ -414,7 +414,7 @@ def finalize(joblist, finaljob, cal_work_dir, out_path, version, title, author,
version,
report_to,
)
- det = metadata['calibration-configurations'].get('karabo-id', report_to.name)
+ det = metadata['calibration-configurations'].get('karabo-id', report_to.stem)
else:
try:
det = metadata['calibration-configurations']['karabo-id']
diff --git a/src/xfel_calibrate/nb_args.py b/src/xfel_calibrate/nb_args.py
index e3e1dc5e2adbdd040bb566790064672321a52bd5..93c6eba71ed8b0df2bbb22bcca83d2652a7434d0 100644
--- a/src/xfel_calibrate/nb_args.py
+++ b/src/xfel_calibrate/nb_args.py
@@ -53,8 +53,7 @@ def make_initial_parser(**kwargs):
help="Prepare notebooks but don't run them")
parser.add_argument('--report-to', type=str,
- help='Filename (and optionally path) for output'
- ' report')
+ help='Full path for the PDF report output')
parser.add_argument('--not-reproducible', action='store_true',
help='Disable checks to allow the processing result '
diff --git a/src/xfel_calibrate/repeat.py b/src/xfel_calibrate/repeat.py
index e8d94134f3b8a46384f1272b754fea66c070b80c..3112a960f5f428cce261595347f36c10bdc5739f 100644
--- a/src/xfel_calibrate/repeat.py
+++ b/src/xfel_calibrate/repeat.py
@@ -13,7 +13,6 @@ from cal_tools.tools import CalibrationMetadata
from .calibrate import (
JobChain, SlurmOptions, run_finalize, get_pycalib_version,
)
-from .settings import temp_path
# This function is copied and modified from Python 3.8.10
# Copyright © 2001-2022 Python Software Foundation; All Rights Reserved
@@ -123,30 +122,35 @@ def main(argv=None):
start_time = datetime.now()
run_uuid = f"t{start_time:%y%m%d_%H%M%S}"
- cal_work_dir = Path(temp_path, f'slurm_out_repeat_{run_uuid}')
- copytree_no_metadata(
- args.from_dir, cal_work_dir, ignore=shutil.ignore_patterns('slurm-*.out')
- )
- print(f"New working directory: {cal_work_dir}")
-
- cal_metadata = CalibrationMetadata(cal_work_dir)
- parameters = cal_metadata['calibration-configurations']
+ parameters = CalibrationMetadata(args.from_dir)['calibration-configurations']
+ karabo_id = parameters['karabo-id']
out_folder = parameters['out-folder']
params_to_set = {'metadata_folder': "."}
if args.out_folder:
out_folder = parameters['out-folder'] = os.path.abspath(args.out_folder)
params_to_set['out_folder'] = out_folder
- update_notebooks_params(cal_work_dir, params_to_set)
if args.report_to:
- report_to = os.path.abspath(args.report_to)
+ report_to = Path(args.report_to).with_suffix('.pdf').absolute()
else: # Default to saving report in output folder
- report_to = str(Path(out_folder, f'xfel-calibrate-repeat-{run_uuid}'))
- cal_metadata['report-path'] = f'{report_to}.pdf'
+ report_to = Path(out_folder, f'{karabo_id}-repeat-{run_uuid}.pdf')
+ # Copy working directory to new location
+ cal_work_dir = report_to.with_suffix('')
+ copytree_no_metadata(
+ args.from_dir, cal_work_dir, ignore=shutil.ignore_patterns('slurm-*.out')
+ )
+ print(f"New working directory: {cal_work_dir}")
+
+ # Update metadata YAML file & notebooks with any changes
+ cal_metadata = CalibrationMetadata(cal_work_dir)
+ cal_metadata['calibration-configurations'] = parameters
+ cal_metadata['report-path'] = str(report_to)
cal_metadata.save()
+ update_notebooks_params(cal_work_dir, params_to_set)
+
# finalize & some notebooks expect yaml metadata in the output folder
Path(out_folder).mkdir(parents=True, exist_ok=True)
shutil.copy(cal_work_dir / 'calibration_metadata.yml', out_folder)
diff --git a/tests/test_xfel_calibrate/conftest.py b/tests/test_xfel_calibrate/conftest.py
index a98c410259385d6dc4a7d837b374331681635996..21161c99edb124c607acca3de6a60c10a4fda88e 100644
--- a/tests/test_xfel_calibrate/conftest.py
+++ b/tests/test_xfel_calibrate/conftest.py
@@ -215,7 +215,9 @@ class CalibrateCall:
self.in_folder = in_folder
self.out_folder = out_folder
- self.args = [command, detector, cal_type, '--report-to', str(reports_dir)]
+ self.args = [
+ command, detector, cal_type, '--report-to', str(reports_dir / 'test.pdf')
+ ]
if in_folder:
self.args.extend(["--in-folder", str(self.in_folder)])
if out_folder:
diff --git a/webservice/webservice.py b/webservice/webservice.py
index 9fa488f2308fe1d25ccb9c908b72d545822ed796..34da59a3d9f68f511f78e5383ca263ed67041fd1 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -956,7 +956,7 @@ class ActionsServer:
This will trigger a correction process to be launched for that run in
the given cycle and proposal.
"""
- request_time = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
+ request_time = datetime.now()
try:
runnr = runnr.strip('r')
@@ -970,9 +970,9 @@ class ActionsServer:
return msg.encode()
with time_db_transaction(self.job_db, 'Insert request'):
- cur = self.job_db.execute( # 2
+ cur = self.job_db.execute(
"INSERT INTO requests VALUES (NULL, ?, ?, ?, 'CORRECT', ?)",
- (rid, proposal, int(runnr), request_time)
+ (rid, proposal, int(runnr), request_time.isoformat())
)
req_id = cur.lastrowid
@@ -1105,7 +1105,7 @@ class ActionsServer:
with time_db_transaction(self.job_db, 'Insert request'):
cur = self.job_db.execute(
"INSERT INTO requests VALUES (NULL, ?, ?, ?, 'CORRECT', ?)",
- (rid, proposal, int(runnr), request_time.strftime('%Y-%m-%dT%H:%M:%S'))
+ (rid, proposal, int(runnr), request_time.isoformat())
)
req_id = cur.lastrowid
@@ -1170,7 +1170,7 @@ class ActionsServer:
'--env-cache',
f'/gpfs/exfel/data/scratch/{getuser()}/calib-repeat-envs',
'--report-to',
- f'{reports_dir}/{karabo_id}_RECORRECT_{request_time:%y%m%d_%H%M%S}'
+ f'{reports_dir}/{karabo_id}_RECORRECT_{request_time:%y%m%d_%H%M%S_%f}'
]
with time_db_transaction(self.job_db, 'Insert execution'):
@@ -1216,7 +1216,7 @@ class ActionsServer:
:param runnr: is the run number in integer form, i.e. without leading
"r"
"""
- request_time = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
+ request_time = datetime.now()
try:
pdus, karabo_das, wait_runs = ast.literal_eval(','.join(extra))
@@ -1236,7 +1236,7 @@ class ActionsServer:
with time_db_transaction(self.job_db, 'Insert request'):
cur = self.job_db.execute(
"INSERT INTO requests VALUES (NULL, ?, ?, ?, 'DARK', ?)",
- (rid, proposal, int(wait_runs[-1]), request_time)
+ (rid, proposal, int(wait_runs[-1]), request_time.isoformat())
)
req_id = cur.lastrowid
@@ -1445,9 +1445,9 @@ class ActionsServer:
action=action, instrument=instrument,
cycle=cycle, proposal=proposal,
runs="_".join([f"r{r}" for r in run_nrs]),
- time_stamp=datetime.now().strftime('%y%m%d_%H%M%S'),
+ time_stamp=request_time.strftime('%y%m%d_%H%M%S_%f'),
det_instance=karabo_id,
- request_time=request_time
+ request_time=request_time.isoformat(),
).split()
cmd = parse_config(cmd, dconfig)