Saving metadata in both folders, adding report path

The file metadata.yml is now primarily generated (or updated) in the output
directory and finalize.py will put a copy (historical snapshot) in the
temporary (slurm) folder.

xfel_calibrate/calibrate.py

@@ -743,11 +743,11 @@ def make_par_table(parms, run_tmp_path):
         finfile.write(textwrap.dedent(tmpl.render(p=col_type, lines=l_parms)))
 
 
-def make_pipeline_yaml(parms, version, run_tmp_path):
-    f_name = "{}/metadata.yml".format(run_tmp_path)
+def make_pipeline_yaml(parms, version, report_path, output_dir):
+    yaml_fn = f"{output_dir}/metadata.yml"
     # add to existing metadata file if it exists
-    if os.path.exists(f_name):
-        with open(f_name, "r") as fd:
+    if os.path.exists(yaml_fn):
+        with open(yaml_fn, "r") as fd:
             metadata = yaml.safe_load(fd)
     else:
         metadata = {}
@@ -758,8 +758,9 @@ def make_pipeline_yaml(parms, version, run_tmp_path):
         parm_subdict[name] = p.value
 
     metadata["pycalibration-version"] = version
+    metadata["report-path"] = f"{report_path}.pdf"
 
-    with open(f_name, "w") as fd:
+    with open(yaml_fn, "w") as fd:
         yaml.safe_dump(metadata, fd)
 
 
@@ -828,28 +829,39 @@ def run():
     # Write all input parameters to rst file to be included to final report
     parms = parameter_values(parms, **args)
     make_par_table(parms, run_tmp_path)
-    # Write input parameters to metadata yaml file, too
-    make_pipeline_yaml(parms, version, run_tmp_path)
     # And save the invocation of this script itself
     save_executed_command(run_tmp_path, version)
 
     # wait on all jobs to run and then finalize the run by creating a report from the notebooks
     out_path = os.path.join(
-        report_path, detector.upper(), caltype.upper(), datetime.now().isoformat()
+        default_report_path, detector.upper(), caltype.upper(), datetime.now().isoformat()
     )
     if try_report_to_output:
         if "out_folder" in args:
             out_path = os.path.abspath(args["out_folder"])
         else:
-            print("No 'out_folder' defined as argument, outputting to '{}' instead.".format(
-                out_path))
+            print(f"No 'out_folder' given, outputting to '{out_path}' instead.")
 
     os.makedirs(out_path, exist_ok=True)
 
-    report_to = title.replace(" ", "")
-    if args["report_to"] is not None:
+    # Use given report name, falling back to notebook title
+    if args["report_to"] is None:
+        report_to = title.replace(" ", "")
+    else:
         report_to = args["report_to"]
 
+    report_path, report_name = os.path.split(report_to)
+    if not report_path:
+        print(f"No path found in 'report_to', saving under '{out_path}'")
+        report_to = f"{out_path}/{report_to}"
+
+    if not report_name:
+        print(f"No title found in 'report_to', using title '{title}'")
+        report_to = f"{report_to}/{title.replace(' ', '')}"
+
+    # Write metadata about calibration job to output folder
+    make_pipeline_yaml(parms, version, report_to, out_path)
+
     folder = get_par_attr(parms, 'in_folder', 'value', '')
 
     if args["request_time"] == "Now":

xfel_calibrate/finalize.py

@@ -320,26 +320,12 @@ def make_report(run_path: str, tmp_path: str, out_path: str, project: str,
     for dtmp in temp_dirs:
         rmtree(f'{dtmp}/')
 
-    # Archiving files in slurm_tmp (will move temp dir)
-    # First merging output metadata into initial (temp) metadata
+    # Archiving files in slurm_tmp
     output_metadata_fn = f"{out_path}/metadata.yml"
     temp_metadata_fn = f"{tmp_path}/metadata.yml"
 
-    if isfile(temp_metadata_fn):
-        with open(temp_metadata_fn, "r") as fd:
-            metadata = yaml.safe_load(fd)
-    else:
-        metadata = {}
-
     if isfile(output_metadata_fn):
-        with open(output_metadata_fn, "r") as fd:
-            metadata_run = yaml.safe_load(fd)
-        # keep in mind: update is not recursive merge
-        metadata.update(metadata_run)
-        remove(output_metadata_fn)
-
-    with open(temp_metadata_fn, "w") as fd:
-        yaml.safe_dump(metadata, fd)
+        copy(output_metadata_fn, temp_metadata_fn)
 
     # Moving temporary files to out-folder after successful execution
     # This helps in keeping elements needed for re-producibility.

xfel_calibrate/settings.py

@@ -8,7 +8,7 @@ temp_path = os.path.abspath("temp/")
 python_path = sys.executable
 
 # Path to store reports in
-report_path = "{}/calibration_reports/".format(os.getcwd())
+default_report_path = "{}/calibration_reports/".format(os.getcwd())
 
 # Also try to output the report to an out_folder defined by the notebook
 try_report_to_output = True