diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index a9471792d01bca6b8a7742bd104b5613d5c7562b..1bf5f0b35c86e0d2bb6922e6b33178ca4c22801f 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -19,6 +19,7 @@
"source": [
"in_folder = \"/gpfs/exfel/exp/SPB/202131/p900230/raw\" # the folder to read data from, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/agipd_resolve_conf\" # the folder to output to, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
"train_ids = [-1] # train IDs to correct, set to -1 for all, range allowed\n",
@@ -512,7 +513,7 @@
"# Retrieve calibration constants to RAM\n",
"agipd_corr.allocate_constants(modules, (3, mem_cells_db, 512, 128))\n",
"\n",
- "metadata = cal_tools.tools.CalibrationMetadata(out_folder)\n",
+ "metadata = cal_tools.tools.CalibrationMetadata(metadata_folder or out_folder)\n",
"# NOTE: this notebook will not overwrite calibration metadata file\n",
"const_yaml = metadata.get(\"retrieved-constants\", {})\n",
"\n",
@@ -528,6 +529,7 @@
" # check if there is a yaml file in out_folder that has the device constants.\n",
" if k_da in const_yaml:\n",
" when = agipd_corr.initialize_from_yaml(k_da, const_yaml, mod)\n",
+ " print(f\"Found constants for {k_da} in calibration_metadata.yml\")\n",
" else:\n",
" # TODO: replace with proper retrieval (as done in pre-correction)\n",
" when = agipd_corr.initialize_from_db(\n",
@@ -544,6 +546,7 @@
" module_idx=mod,\n",
" only_dark=False,\n",
" )\n",
+ " print(f\"Queried CalCat for {k_da}\")\n",
" except Exception as e:\n",
" err = f\"Error: {e}\\nError traceback: {traceback.format_exc()}\"\n",
" when = None\n",
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
index ef845a4d0cfcc0cda1c3cf581b3fc7896023e503..5de75ff6e72aa42ae279044455f624b17745d50e 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
@@ -15,6 +15,7 @@
"source": [
"run = 11 # runs to process, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_Corr\" # path to output to, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"karabo_id = \"SPB_DET_AGIPD1M-1\" # karabo karabo_id\n",
"modules = [-1]\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators"
@@ -51,7 +52,7 @@
"outputs": [],
"source": [
"out_folder = Path(out_folder)\n",
- "metadata = CalibrationMetadata(out_folder)\n",
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
"const_dict = metadata.setdefault(\"retrieved-constants\", {})\n",
"time_dict = const_dict.setdefault(\"time-summary\", {})\n",
"\n",
@@ -135,7 +136,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.7"
+ "version": "3.8.12"
}
},
"nbformat": 4,
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 8051980183b2de5b3bea11a3051f36dcc2238edf..63e5187fbac35c5bb4bc95ccfa5c1e07784a8fd2 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -19,6 +19,7 @@
"source": [
"in_folder = \"/gpfs/exfel/exp/SPB/202030/p900119/raw\" # the folder to read data from, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_\" # the folder to output to, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
"run = 80 # runs to process, required\n",
@@ -112,7 +113,7 @@
"# slopes_ff_from_files left as str for now\n",
"in_folder = Path(in_folder)\n",
"out_folder = Path(out_folder)\n",
- "metadata = tools.CalibrationMetadata(out_folder)"
+ "metadata = tools.CalibrationMetadata(metadata_folder or out_folder)"
]
},
{
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index cdb7b42b8c142b13762a542dd515d60393e68036..ac724c8b43af2e310f428cc43e229d265f43a1ce 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -21,6 +21,7 @@
"source": [
"in_folder = \"/gpfs/exfel/d/raw/CALLAB/202031/p900113\" # path to input data, required\n",
"out_folder = \"\" # path to output to, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"modules = [-1] # list of modules to evaluate, RANGE ALLOWED\n",
"run_high = 9985 # run number in which high gain data was recorded, required\n",
"run_med = 9984 # run number in which medium gain data was recorded, required\n",
@@ -163,7 +164,7 @@
"# Read report path and create file location tuple to add with the injection\n",
"file_loc = f\"proposal:{prop} runs:{run_low} {run_med} {run_high}\"\n",
"\n",
- "report = get_report(out_folder)\n",
+ "report = get_report(metadata_folder)\n",
"cal_db_interface = get_random_db_interface(cal_db_interface)\n",
"print(f'Calibration database interface: {cal_db_interface}')\n",
"\n",
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
index 8854d0f5c6d0ca5791829d12ecd84c775bbbd77a..98bb6ce9ecf4769c14efef552a6c7fd0936e3c65 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
@@ -16,6 +16,7 @@
"source": [
"in_folder = \"\" # in this notebook, in_folder is not used as the data source is in the destination folder\n",
"out_folder = \"\" # the folder to output to, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"hist_file_template = \"hists_m{:02d}_sum.h5\"\n",
"proc_folder = \"\" # Path to corrected image data used to create histograms and validation plots\n",
"raw_folder = \"/gpfs/exfel/exp/MID/202030/p900137/raw\" # folder of raw data. This is used to save information of source data of generated constants, required\n",
@@ -246,7 +247,7 @@
"proposal = list(filter(None, raw_folder.strip('/').split('/')))[-2]\n",
"file_loc = f'Proposal: {proposal}, Run: {run}'\n",
"\n",
- "report = get_report(out_folder)"
+ "report = get_report(metadata_folder)"
]
},
{
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
index 37df92a7cccac843bfe155e3fad432fb37ced4cf..38f726532c34b23e707e098a6fe08db72bc3613b 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
@@ -39,6 +39,7 @@
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
"in_folder = '/gpfs/exfel/exp/SPB/202130/p900188/raw/' # path to input data, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/jsztuk/test/pc\" # path to output to, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"runs = [92, 93, 94, 95, 96, 97, 98, 99] # runs to use, required, range allowed\n",
"n_sequences = 5 # number of sequence files, starting for 0 to evaluate\n",
"\n",
@@ -1450,7 +1451,7 @@
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
"file_loc = proposal + ' ' + ' '.join(list(map(str,runs)))\n",
"\n",
- "report = get_report(out_folder)"
+ "report = get_report(metadata_folder)"
]
},
{
diff --git a/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb b/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
index 048af2e7d23db6e5245d902dcafbb97bbdd5f08f..e31a85bc9b3d756ae4dd359b5ee5e45b6bb55fd8 100644
--- a/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
+++ b/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
@@ -22,6 +22,7 @@
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
"in_folder = \"/gpfs/exfel/exp/SQS/202131/p900210/raw\" # path to input data, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/samartse/data/DSSC\" # path to output to, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"sequences = [0] # sequence files to evaluate.\n",
"modules = [-1] # modules to run for\n",
"run = 20 #run number in which data was recorded, required\n",
@@ -162,7 +163,7 @@
"print(f\"Bias voltage: {bias_voltage}V\")\n",
"file_loc = f'proposal:{prop} runs:{[ v for v in offset_runs.values()][0]}'\n",
"\n",
- "report = get_report(out_folder)"
+ "report = get_report(metadata_folder)"
]
},
{
diff --git a/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb b/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
index 045b9c3d053dfeb8a744b4d47a0fe524ab41ab8c..34e718eb421e68ddcfac313a8a8bd10d1f34a4ab 100644
--- a/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
+++ b/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
@@ -22,6 +22,7 @@
"cluster_profile = \"noDB\" # ipcluster profile to use\n",
"in_folder = \"/gpfs/exfel/exp/SCS/201930/p900074/raw\" # input folder, required\n",
"out_folder = '/gpfs/exfel/data/scratch/ahmedk/test/fastccd' # output folder, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"sequence = 0 # sequence file to use\n",
"run = 351 # which run to read data from, required\n",
"\n",
@@ -158,7 +159,7 @@
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
"file_loc = 'proposal:{} runs:{}'.format(proposal, run)\n",
"\n",
- "report = get_report(out_folder)"
+ "report = get_report(metadata_folder)"
]
},
{
diff --git a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
index 5139fd9aea156fa79f0171bf6e63f09eca532755..cf8e436c38fe30c18407ac8b601b5cc61d514e8a 100644
--- a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
@@ -19,6 +19,7 @@
"source": [
"in_folder = '/gpfs/exfel/exp/SPB/202130/p900204/raw/' # folder under which runs are located, required\n",
"out_folder = '/gpfs/exfel/data/scratch/ahmedk/test/remove' # path to place reports at, required\n",
+ "metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"run_high = 141 # run number for G0 dark run, required\n",
"run_med = 142 # run number for G1 dark run, required\n",
"run_low = 143 # run number for G2 dark run, required\n",
@@ -138,7 +139,7 @@
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
"file_loc = f\"proposal:{proposal} runs:{run_high} {run_med} {run_low}\"\n",
"\n",
- "report = get_report(out_folder)\n",
+ "report = get_report(metadata_folder)\n",
"\n",
"step_timer = step_timing.StepTimer()"
]
diff --git a/notebooks/LPD/LPDChar_Darks_NBC.ipynb b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
index 2b5f78efda66c69a3026cfc4ef57c293f3979edc..008620e36e2ea2c14a77897ffc2ea5554232c924 100644
--- a/notebooks/LPD/LPDChar_Darks_NBC.ipynb
+++ b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
@@ -25,6 +25,7 @@
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
"in_folder = \"/gpfs/exfel/exp/FXE/202030/p900121/raw\" # path to input data, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/LPD/\" # path to output to, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"sequence = 0 # sequence files to evaluate\n",
"modules = [-1] # list of modules to evaluate, RANGE ALLOWED\n",
"run_high = 120 # run number in which high gain data was recorded, required\n",
@@ -354,7 +355,7 @@
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
"file_loc = 'proposal:{} runs:{} {} {}'.format(proposal, run_low, run_med, run_high)\n",
"\n",
- "report = get_report(out_folder)"
+ "report = get_report(metadata_folder)"
]
},
{
diff --git a/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb b/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
index 3c2a3f70eea7f04da425a1243e2a27e972dea821..23d2b6d02894b48f1e06d08afd82e8eb7a6c2cd9 100644
--- a/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
@@ -23,6 +23,7 @@
"source": [
"in_folder = '/gpfs/exfel/exp/HED/202230/p900247/raw' # input folder, required\n",
"out_folder = '' # output folder, required\n",
+ "metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"sequence = 0 # sequence file to use\n",
"run = 45 # which run to read data from, required\n",
"\n",
@@ -114,7 +115,7 @@
"# Read report path and create file location tuple to add with the injection\n",
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
"file_loc = f'proposal:{proposal} runs:{run}'\n",
- "report = get_report(out_folder)\n",
+ "report = get_report(metadata_folder)\n",
"\n",
"ped_dir = os.path.join(in_folder, f\"r{run:04d}\")\n",
"fp_name = path_template.format(run, karabo_da[0]).format(sequence)\n",
diff --git a/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb b/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb
index 7929498d2fdbe58afcb6d873e810cbae36e30c97..f8a341d476d779759aedd45fc31809ea6c4b026a 100644
--- a/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb
+++ b/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb
@@ -22,6 +22,7 @@
"cluster_profile = \"noDB\" # ipcluster profile to use\n",
"in_folder = '/gpfs/exfel/exp/HED/201922/p002550/raw' # input folder, required\n",
"out_folder = '/gpfs/exfel/data/scratch/ahmedk/test/epix10' # output folder, required\n",
+ "metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"sequence = 0 # sequence file to use\n",
"run = 55 # which run to read data from, required\n",
"\n",
@@ -110,7 +111,7 @@
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
"file_loc = 'proposal:{} runs:{}'.format(proposal, run)\n",
"\n",
- "report = get_report(out_folder)\n",
+ "report = get_report(metadata_folder)\n",
"\n",
"x = 356 # rows of the ePix10K\n",
"y = 384 # columns of the ePix10K\n",
diff --git a/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb b/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
index e204cf3f605f20c7342638395402119b7f8e7864..3583a7f377e8ed64e56cb15cf7dcd39820940312 100644
--- a/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
+++ b/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
@@ -11,6 +11,7 @@
"# Summary for processed of dark calibration constants and a comparison with previous injected constants.\n",
"\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/fixed_gain/SPB_summary_fix2\" # path to output to, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"karabo_id = \"SPB_DET_AGIPD1M-1\" # detector instance\n",
"gain_names = ['High gain', 'Medium gain', 'Low gain'] # a list of gain names to be used in plotting\n",
"threshold_names = ['HG-MG threshold', 'MG_LG threshold'] # a list of gain names to be used in plotting"
@@ -186,7 +187,7 @@
"outputs": [],
"source": [
"out_folder = Path(out_folder)\n",
- "metadata = CalibrationMetadata(out_folder)\n",
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
"mod_mapping = metadata.setdefault(\"modules-mapping\", {})\n",
"old_constant_metadata = {}\n",
"for fn in out_folder.glob(\"module_metadata_*.yml\"):\n",
@@ -615,7 +616,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.7"
+ "version": "3.8.12"
}
},
"nbformat": 4,
diff --git a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
index f3e8d5cf8d7bdb397b27b50d376eedab3058c9ff..0c01998595d5f92ef061d7c8c9b415b595ec24e7 100644
--- a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
+++ b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
@@ -27,6 +27,7 @@
"source": [
"in_folder = \"/gpfs/exfel/exp/SQS/202031/p900166/raw\" # input folder, required\n",
"out_folder = '/gpfs/exfel/data/scratch/ahmedk/test/remove' # output folder, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"run = 339 # which run to read data from, required\n",
"\n",
"# Data files parameters.\n",
@@ -117,7 +118,7 @@
"file_loc = f'Proposal: {proposal}, Run: {run}'\n",
"print(\"File Location:\", file_loc)\n",
"\n",
- "report = get_report(out_folder)\n",
+ "report = get_report(metadata_folder)\n",
"print(\"Report Path:\", report)"
]
},
diff --git a/notebooks/pnCCD/Characterize_pnCCD_Gain.ipynb b/notebooks/pnCCD/Characterize_pnCCD_Gain.ipynb
index 9c8b3f53ad60aa38089ec9f687e86ffd62228d72..637f1766cff1562018d53134a134a18b383e2016 100644
--- a/notebooks/pnCCD/Characterize_pnCCD_Gain.ipynb
+++ b/notebooks/pnCCD/Characterize_pnCCD_Gain.ipynb
@@ -29,6 +29,7 @@
"cluster_profile = \"noDB\" # ipcluster profile to use\n",
"in_folder = \"/gpfs/exfel/exp/SQS/202031/p900166/raw\" # input folder for the raw data\n",
"out_folder = '/gpfs/exfel/data/scratch/setoodeh/Test' # output folder\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"run = 347 # which run to read data from\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"\n",
@@ -151,7 +152,7 @@
"file_loc =f'Proposal: {proposal}, Run: {run}'\n",
"print(file_loc)\n",
"\n",
- "report = get_report(out_folder)\n",
+ "report = get_report(metadata_folder)\n",
"print(\"report_path:\", report)\n",
"\n",
"# Paths to the data:\n",
diff --git a/src/xfel_calibrate/calibrate.py b/src/xfel_calibrate/calibrate.py
index b0cc47655858d49218bd484da032f1426d376932..526024fc8e831eb75f84b574cc581733d527808a 100755
--- a/src/xfel_calibrate/calibrate.py
+++ b/src/xfel_calibrate/calibrate.py
@@ -356,7 +356,9 @@ def prepare_job(
cluster_profile = 'NO_CLUSTER'
params = parameter_values(parms, **args)
- params = parameter_values(params, cluster_profile=cluster_profile)
+ params = parameter_values(
+ params, cluster_profile=cluster_profile, metadata_folder="."
+ )
new_nb = replace_definitions(nb, params, execute=False, lang='python')
if not show_title:
title_cell = first_markdown_cell(new_nb)
@@ -657,7 +659,7 @@ def run(argv=None):
python_exe = python_path
# Write metadata about calibration job to output folder
- metadata = cal_tools.tools.CalibrationMetadata(out_path, new=True)
+ metadata = cal_tools.tools.CalibrationMetadata(run_tmp_path, new=True)
parm_subdict = metadata.setdefault("calibration-configurations", {})
for p in parms:
diff --git a/src/xfel_calibrate/finalize.py b/src/xfel_calibrate/finalize.py
index f50e717c60ca715f59156fdaea99333517624f76..5a8d5f087a1e592a0372e0de5c1231b564400529 100644
--- a/src/xfel_calibrate/finalize.py
+++ b/src/xfel_calibrate/finalize.py
@@ -8,6 +8,7 @@ from os.path import isdir, isfile, splitext
from pathlib import Path
from shutil import copy, copytree, move, rmtree
from subprocess import CalledProcessError, check_call, check_output
+from tempfile import TemporaryDirectory
from textwrap import dedent
from time import sleep
from typing import Dict, List
@@ -340,12 +341,6 @@ def make_report(run_path: Path, tmp_path: Path, project: str,
print(f"Removing temporary subdir: {tmp_subdir}")
rmtree(tmp_subdir)
- # Moving temporary files to out-folder after successful execution
- # This helps in keeping elements needed for reproducibility.
- slurm_archive_dir = report_dir / f"slurm_out_{report_name}"
- print(f"Moving temporary files to final location: {slurm_archive_dir}")
- move(str(tmp_path), str(slurm_archive_dir))
-
def make_titlepage(sphinx_path, project, data_path, version):
"""
@@ -396,11 +391,12 @@ def tex_escape(text):
def finalize(joblist, finaljob, run_path, out_path, version, title, author, report_to, data_path='Unknown',
request_time='', submission_time=''):
run_path = Path(run_path)
+ out_path = Path(out_path)
# Archiving files in slurm_tmp
if finaljob:
joblist.append(str(finaljob))
- metadata = cal_tools.tools.CalibrationMetadata(out_path)
+ metadata = cal_tools.tools.CalibrationMetadata(run_path)
job_time_fmt = 'JobID,Start,End,Elapsed,Suspended,State'.split(',')
job_time_summary = get_job_info(joblist, job_time_fmt)
@@ -420,9 +416,10 @@ def finalize(joblist, finaljob, run_path, out_path, version, title, author, repo
}
)
metadata.save()
- metadata.save_copy(run_path)
+
if report_to:
+ report_to = Path(report_to)
prepare_plots(run_path)
sphinx_path = combine_report(run_path, title)
@@ -433,10 +430,33 @@ def finalize(joblist, finaljob, run_path, out_path, version, title, author, repo
title,
author,
version,
- Path(report_to),
+ report_to,
)
+ # Store the contents of the Slurm working directory next to the report.
+ # This contains elements needed for reproducibility.
+ slurm_archive_dir = report_to.parent / f"slurm_out_{report_to.name}"
else:
- # If there is no report location, simply copy the slurm_out_
+ # If there is no report location, simply move the slurm_out_
# directory to the output.
slurm_archive_dir = Path(out_path) / f"slurm_out_{run_path.name}"
- move(str(run_path), str(slurm_archive_dir))
+
+ print(f"Moving temporary files to final location: {slurm_archive_dir}")
+ move(run_path, slurm_archive_dir)
+
+ md_path = slurm_archive_dir / "calibration_metadata.yml"
+ # Notebooks should have a karabo_id parameter, which we'll use to make a
+ # unique name like calibration_metadata_MID_DET_AGIPD1M-1.yml in the output
+ # folder. In case they don't, fall back to a name like the report.
+ # To replace any existing symlink, we create it in a temp folder & rename.
+ det = metadata['calibration-configurations'].get('karabo-id', report_to.name)
+ with TemporaryDirectory(dir=out_path) as td:
+ tlp = Path(td, f"calibration_metadata_{det}.yml")
+ tlp.symlink_to(md_path)
+ tlp.replace(out_path / f"calibration_metadata_{det}.yml")
+
+ # For continuity, we'll also expose it as $out/calibration_metadata.yml.
+ # This is only useful if there's 1 calibration run per output folder,
+ # otherwise the last one to create the link wins.
+ tlp = Path(td, f"calibration_metadata.yml")
+ tlp.symlink_to(md_path)
+ tlp.replace(out_path / "calibration_metadata.yml")
diff --git a/src/xfel_calibrate/nb_args.py b/src/xfel_calibrate/nb_args.py
index e768d9f63a39b59865c28f4c5b0d6b39582edeaa..e3e1dc5e2adbdd040bb566790064672321a52bd5 100644
--- a/src/xfel_calibrate/nb_args.py
+++ b/src/xfel_calibrate/nb_args.py
@@ -162,6 +162,9 @@ def add_args_from_nb(parms, parser, cvar=None, no_required=False):
"""
for p in parms:
+ if p.name == 'metadata_folder':
+ continue # Comes from xfel-calibrate machinery, can't be supplied
+
helpstr = ("Default: %(default)s" if not p.comment
else "{}. Default: %(default)s".format(p.comment.replace("#", " ").strip()))
required = (p.comment is not None
diff --git a/src/xfel_calibrate/repeat.py b/src/xfel_calibrate/repeat.py
index 12d4b30057022f657d9d44faaba7a121d5d8dcb0..06e091f16553042a7cf3bf1c17d964dc24b8541e 100644
--- a/src/xfel_calibrate/repeat.py
+++ b/src/xfel_calibrate/repeat.py
@@ -58,11 +58,13 @@ def main(argv=None):
report_path = cal_metadata['report-path']
out_folder = prev_parameters['out-folder']
+ params_to_set = {'metadata_folder': "."}
if args.out_folder:
report_path = new_report_path(out_folder, report_path, args.out_folder)
out_folder = prev_parameters['out-folder'] = args.out_folder
- update_notebooks_params(working_dir, {'out_folder': out_folder})
+ params_to_set['out_folder'] = out_folder
cal_metadata.save()
+ update_notebooks_params(working_dir, params_to_set)
# finalize & some notebooks expect yaml metadata in the output folder
Path(out_folder).mkdir(parents=True, exist_ok=True)