diff --git a/src/xfel_calibrate/calibrate.py b/src/xfel_calibrate/calibrate.py
index 655bc6cad85193be88d5f9f272d466953b86d5e9..bd0788a970276d9033bd41f939e62c522eace982 100755
--- a/src/xfel_calibrate/calibrate.py
+++ b/src/xfel_calibrate/calibrate.py
@@ -583,7 +583,11 @@ def run(argv=None):
title = title.rstrip()
- run_uuid = f"t{datetime.now().strftime('%y%m%d_%H%M%S.%f')}"
+ # request_time is in local timezone
+ if args["request_time"] == "Now":
+ request_time = datetime.now()
+ else:
+ request_time = datetime.fromisoformat(args["request_time"])
# check if concurrency parameter is given and we run concurrently
if concurrency_par is not None and not any(
@@ -596,8 +600,7 @@ def run(argv=None):
default_params_by_name = {p.name: p.value for p in nb_details.default_params}
if 'cluster_profile' in default_params_by_name:
if args.get("cluster_profile") == default_params_by_name["cluster_profile"]:
- args['cluster_profile'] = "slurm_prof_{}".format(run_uuid)
-
+ args['cluster_profile'] = f"slurm_prof_{request_time:%y%m%d_%H%M%S.%f}"
# wait on all jobs to run and then finalize the run by creating a report from the notebooks
out_path = Path(default_report_path) / nb_details.detector / nb_details.caltype / datetime.now().isoformat()
@@ -609,26 +612,21 @@ def run(argv=None):
out_path.mkdir(parents=True, exist_ok=True)
- # Use given report name, falling back to notebook title
+ # Use given report name, or automatic unique name if not specified
+ unique_name = f"{args['karabo_id']}-{nb_details.caltype}-{request_time:%y%m%d_%H%M%S.%f}"
if args['skip_report']:
report_to = ''
elif args["report_to"] is None:
- report_to = out_path / title.replace(" ", "")
+ report_to = out_path / f"{unique_name}.pdf"
print(f"report_to not specified, will use {report_to}")
else:
- report_to = Path(args["report_to"])
- if report_to.is_dir():
- print(f"report_to is directory, will use title '{title}' for filename")
- report_to = report_to / title.replace(" ", "")
- elif len(report_to.parts) == 1:
- print(f"report_to path contained no path, saving report in '{out_path}'")
- report_to = out_path / report_to
-
- workdir_name = f"slurm_out_{nb_details.detector}_{nb_details.caltype}_{run_uuid}"
+ report_to = Path(args["report_to"]).with_suffix('.pdf').absolute()
+
if report_to:
- cal_work_dir = report_to.parent / workdir_name
+ # Work dir matching report file but without .pdf
+ cal_work_dir = report_to.with_suffix('')
else:
- cal_work_dir = out_path / workdir_name
+ cal_work_dir = out_path / unique_name
cal_work_dir.mkdir(parents=True)
# Write all input parameters to rst file to be included to final report
@@ -660,7 +658,7 @@ def run(argv=None):
parm_subdict[name] = p.value
metadata["pycalibration-version"] = version
- metadata["report-path"] = f"{report_to}.pdf" if report_to \
+ metadata["report-path"] = str(report_to) if report_to \
else '# REPORT SKIPPED #'
metadata['reproducible'] = not args['not_reproducible']
metadata["concurrency"] = {
@@ -689,11 +687,6 @@ def run(argv=None):
folder = get_par_attr(parms, 'in_folder', 'value', '')
- if args["request_time"] == "Now":
- request_time = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
- else:
- request_time = args["request_time"]
-
pre_jobs = []
cluster_cores = concurrency.get("cluster cores", 8)
# Check if there are pre-notebooks
@@ -823,7 +816,7 @@ def run(argv=None):
'author': author,
'report_to': report_to,
'in_folder': folder,
- 'request_time': request_time,
+ 'request_time': request_time.strftime("%Y-%m-%dT%H:%M:%S"),
'submission_time': submission_time,
}
diff --git a/src/xfel_calibrate/finalize.py b/src/xfel_calibrate/finalize.py
index 0df50980e32551a0db3f9b1bf3a8d6548856af28..41b3f9dd4b84f61a666647064959b966ba915543 100644
--- a/src/xfel_calibrate/finalize.py
+++ b/src/xfel_calibrate/finalize.py
@@ -414,7 +414,7 @@ def finalize(joblist, finaljob, cal_work_dir, out_path, version, title, author,
version,
report_to,
)
- det = metadata['calibration-configurations'].get('karabo-id', report_to.name)
+ det = metadata['calibration-configurations'].get('karabo-id', report_to.stem)
else:
try:
det = metadata['calibration-configurations']['karabo-id']
diff --git a/src/xfel_calibrate/nb_args.py b/src/xfel_calibrate/nb_args.py
index e3e1dc5e2adbdd040bb566790064672321a52bd5..93c6eba71ed8b0df2bbb22bcca83d2652a7434d0 100644
--- a/src/xfel_calibrate/nb_args.py
+++ b/src/xfel_calibrate/nb_args.py
@@ -53,8 +53,7 @@ def make_initial_parser(**kwargs):
help="Prepare notebooks but don't run them")
parser.add_argument('--report-to', type=str,
- help='Filename (and optionally path) for output'
- ' report')
+ help='Full path for the PDF report output')
parser.add_argument('--not-reproducible', action='store_true',
help='Disable checks to allow the processing result '
diff --git a/src/xfel_calibrate/repeat.py b/src/xfel_calibrate/repeat.py
index e8d94134f3b8a46384f1272b754fea66c070b80c..3112a960f5f428cce261595347f36c10bdc5739f 100644
--- a/src/xfel_calibrate/repeat.py
+++ b/src/xfel_calibrate/repeat.py
@@ -13,7 +13,6 @@ from cal_tools.tools import CalibrationMetadata
from .calibrate import (
JobChain, SlurmOptions, run_finalize, get_pycalib_version,
)
-from .settings import temp_path
# This function is copied and modified from Python 3.8.10
# Copyright © 2001-2022 Python Software Foundation; All Rights Reserved
@@ -123,30 +122,35 @@ def main(argv=None):
start_time = datetime.now()
run_uuid = f"t{start_time:%y%m%d_%H%M%S}"
- cal_work_dir = Path(temp_path, f'slurm_out_repeat_{run_uuid}')
- copytree_no_metadata(
- args.from_dir, cal_work_dir, ignore=shutil.ignore_patterns('slurm-*.out')
- )
- print(f"New working directory: {cal_work_dir}")
-
- cal_metadata = CalibrationMetadata(cal_work_dir)
- parameters = cal_metadata['calibration-configurations']
+ parameters = CalibrationMetadata(args.from_dir)['calibration-configurations']
+ karabo_id = parameters['karabo-id']
out_folder = parameters['out-folder']
params_to_set = {'metadata_folder': "."}
if args.out_folder:
out_folder = parameters['out-folder'] = os.path.abspath(args.out_folder)
params_to_set['out_folder'] = out_folder
- update_notebooks_params(cal_work_dir, params_to_set)
if args.report_to:
- report_to = os.path.abspath(args.report_to)
+ report_to = Path(args.report_to).with_suffix('.pdf').absolute()
else: # Default to saving report in output folder
- report_to = str(Path(out_folder, f'xfel-calibrate-repeat-{run_uuid}'))
- cal_metadata['report-path'] = f'{report_to}.pdf'
+ report_to = Path(out_folder, f'{karabo_id}-repeat-{run_uuid}.pdf')
+ # Copy working directory to new location
+ cal_work_dir = report_to.with_suffix('')
+ copytree_no_metadata(
+ args.from_dir, cal_work_dir, ignore=shutil.ignore_patterns('slurm-*.out')
+ )
+ print(f"New working directory: {cal_work_dir}")
+
+ # Update metadata YAML file & notebooks with any changes
+ cal_metadata = CalibrationMetadata(cal_work_dir)
+ cal_metadata['calibration-configurations'] = parameters
+ cal_metadata['report-path'] = str(report_to)
cal_metadata.save()
+ update_notebooks_params(cal_work_dir, params_to_set)
+
# finalize & some notebooks expect yaml metadata in the output folder
Path(out_folder).mkdir(parents=True, exist_ok=True)
shutil.copy(cal_work_dir / 'calibration_metadata.yml', out_folder)