diff --git a/notebooks/LPD/LPDChar_Darks_NBC.ipynb b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
index de76f651d6b3f713b3e96102a8dd793eb850a3d9..bae9e2fb0649c67f53e73471c0ad9b32bef40ef6 100644
--- a/notebooks/LPD/LPDChar_Darks_NBC.ipynb
+++ b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
@@ -33,6 +33,7 @@
"karabo_id = \"FXE_DET_LPD1M-1\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
"source_name = \"{}/DET/{}CH0:xtdf\" # Source name for raw detector data - filled with karabo_id & module number\n",
+ "ctrl_src_template = \"{}/COMP/FEM_MDL_COMP\" # Control device source name template.\n",
"\n",
"use_dir_creation_date = True # use the creation date of the directory for database time derivation\n",
"cal_db_interface = \"tcp://max-exfl-cal001:8015#8025\" # the database interface to use\n",
@@ -97,7 +98,10 @@
"from extra_data import RunDirectory\n",
"\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.lpdlib import make_cell_order_condition\n",
+ "from cal_tools.lpdlib import (\n",
+ " make_cell_order_condition,\n",
+ " sort_dark_runs_by_gain,\n",
+ ")\n",
"from cal_tools.plotting import (\n",
" create_constant_overview,\n",
" plot_badpix_3d,\n",
@@ -114,6 +118,7 @@
" map_gain_stages,\n",
" module_index_to_qm,\n",
" parse_runs,\n",
+ " raw_data_location_string,\n",
" reorder_axes,\n",
" run_prop_seq_from_path,\n",
" save_const_to_h5,\n",
@@ -131,7 +136,12 @@
"max_cells = mem_cells\n",
"cells = np.arange(max_cells)\n",
"gain_names = ['High', 'Medium', 'Low']\n",
- " \n",
+ "# Check dark runs order and sort if needed.\n",
+ "run_nums = sort_dark_runs_by_gain(\n",
+ " raw_folder=in_folder,\n",
+ " runs=[run_high, run_med, run_low],\n",
+ " ctrl_src=ctrl_src_template.format(karabo_id),\n",
+ ")\n",
"if karabo_da[0] == '-1':\n",
" if modules[0] == -1:\n",
" modules = list(range(16))\n",
@@ -143,7 +153,7 @@
"\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run_high)\n",
+ " creation_time = get_dir_creation_date(in_folder, run_nums[0])\n",
" print(\"Using {} as creation time\".format(creation_time))\n",
"\n",
"if inject_cell_order not in {'auto', 'always', 'never'}:\n",
@@ -157,7 +167,7 @@
"print('CalDB Interface {}'.format(cal_db_interface))\n",
"print(\"Proposal: {}\".format(prop))\n",
"print(\"Memory cells: {}/{}\".format(mem_cells, max_cells))\n",
- "print(\"Runs: {}, {}, {}\".format(run_high, run_med, run_low))\n",
+ "print(\"Runs: {}\".format(run_nums))\n",
"print(\"Using DB: {}\".format(db_output))\n",
"print(\"Input: {}\".format(in_folder))\n",
"print(\"Output: {}\".format(out_folder))\n",
@@ -275,7 +285,7 @@
"\n",
"\n",
"inp = []\n",
- "for gg, run_num in enumerate([run_high, run_med, run_low]):\n",
+ "for gg, run_num in enumerate(run_nums):\n",
" run_path = Path(in_folder, f\"r{run_num:04d}\")\n",
" for i in modules:\n",
" inp.append((run_path, i, gg))\n",
@@ -334,7 +344,7 @@
"source": [
"# Read report path and create file location tuple to add with the injection\n",
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
- "file_loc = 'proposal:{} runs:{} {} {}'.format(proposal, run_low, run_med, run_high)\n",
+ "file_loc = raw_data_location_string(proposal, run_nums)\n",
"\n",
"report = get_report(metadata_folder)"
]
diff --git a/notebooks/LPDMini/LPD_Mini_Char_Darks_NBC.ipynb b/notebooks/LPDMini/LPD_Mini_Char_Darks_NBC.ipynb
index a875f52a4c458590bbaae4c4fe3d08035628eab1..f3ccaa445187d055eac71d04cee66cebc6df8554 100644
--- a/notebooks/LPDMini/LPD_Mini_Char_Darks_NBC.ipynb
+++ b/notebooks/LPDMini/LPD_Mini_Char_Darks_NBC.ipynb
@@ -88,13 +88,17 @@
"\n",
"from cal_tools.calcat_interface import CalCatApi\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.lpdlib import make_cell_order_condition\n",
+ "from cal_tools.lpdlib import (\n",
+ " make_cell_order_condition,\n",
+ " sort_dark_runs_by_gain,\n",
+ ")\n",
"from cal_tools.plotting import plot_badpix_3d\n",
"from cal_tools.restful_config import calibration_client\n",
"from cal_tools.tools import (\n",
" calcat_creation_time,\n",
" get_from_db,\n",
" get_report,\n",
+ " raw_data_location_string,\n",
" run_prop_seq_from_path,\n",
" save_const_to_h5,\n",
" send_to_db,\n",
@@ -110,28 +114,35 @@
"mem_cells = 512\n",
"gain_names = ['High', 'Medium', 'Low']\n",
"const_shape = (mem_cells, 32, 256, 3) # cells, slow_scan, fast_scan, gain\n",
- " \n",
+ "\n",
+ "source_name = source_name.format(karabo_id)\n",
+ "control_source_name = control_source_name.format(karabo_id)\n",
+ "\n",
+ "# Check dark runs order and sort if needed.\n",
+ "run_nums = sort_dark_runs_by_gain(\n",
+ " raw_folder=in_folder,\n",
+ " runs=[run_high, run_med, run_low],\n",
+ " ctrl_src=control_source_name.format(karabo_id),\n",
+ ")\n",
+ "\n",
"gain_runs = {}\n",
"if capacitor_setting == 5:\n",
- " gain_runs[\"high_5pf\"] = run_high\n",
- " gain_runs[\"med_5pf\"] = run_med\n",
- " gain_runs[\"low_5pf\"] = run_low\n",
+ " gain_runs[\"high_5pf\"] = run_nums[0]\n",
+ " gain_runs[\"med_5pf\"] = run_nums[1]\n",
+ " gain_runs[\"low_5pf\"] = run_nums[2]\n",
"elif capacitor_setting == 50:\n",
- " gain_runs[\"high_50pf\"] = run_high\n",
- " gain_runs[\"med_50pf\"] = run_med\n",
- " gain_runs[\"low_50pf\"] = run_low\n",
+ " gain_runs[\"high_50pf\"] = run_nums[0]\n",
+ " gain_runs[\"med_50pf\"] = run_nums[1]\n",
+ " gain_runs[\"low_50pf\"] = run_nums[2]\n",
"\n",
"capacitor_settings = [capacitor_setting]\n",
"capacitor_settings = ['{}pf'.format(c) for c in capacitor_settings]\n",
"\n",
- "creation_time = calcat_creation_time(in_folder, run_high, creation_time)\n",
+ "creation_time = calcat_creation_time(in_folder, run_nums[0], creation_time)\n",
"print(f\"Using {creation_time} as creation time\")\n",
"\n",
- "source_name = source_name.format(karabo_id)\n",
- "control_source_name = control_source_name.format(karabo_id)\n",
- "\n",
"if -1 in {bias_voltage_0, bias_voltage_1}:\n",
- " run_data = RunDirectory(os.path.join(in_folder, f\"r{run_high:04d}\"))\n",
+ " run_data = RunDirectory(os.path.join(in_folder, f\"r{run_nums[0]:04d}\"))\n",
" if bias_voltage_0 == -1:\n",
" bias_voltage_0 = run_data[control_source_name, 'sensorBiasVoltage0'].as_single_value(atol=5.)\n",
" if bias_voltage_1 == -1:\n",
@@ -143,7 +154,7 @@
"print(f'CalDB Interface {cal_db_interface}')\n",
"print(f\"Proposal: {prop}\")\n",
"print(f\"Memory cells: {mem_cells}\")\n",
- "print(f\"Runs: {run_high}, {run_med}, {run_low}\")\n",
+ "print(f\"Runs: {run_nums}\")\n",
"print(f\"Using DB: {db_output}\")\n",
"print(f\"Input: {in_folder}\")\n",
"print(f\"Output: {out_folder}\")\n",
@@ -331,7 +342,7 @@
"source": [
"# Read report path and create file location tuple to add with the injection\n",
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
- "file_loc = 'proposal:{} runs:{} {} {}'.format(proposal, run_low, run_med, run_high)\n",
+ "file_loc = raw_data_location_string(proposal, run_nums)\n",
"\n",
"report = get_report(metadata_folder)"
]
diff --git a/src/cal_tools/lpdlib.py b/src/cal_tools/lpdlib.py
index b61652b33005bf31e1838103c9d5fb59c2f889c9..deb38f44b613e96570a2c10cd003ff54a23a6804 100644
--- a/src/cal_tools/lpdlib.py
+++ b/src/cal_tools/lpdlib.py
@@ -1,9 +1,12 @@
import copy
+from logging import warning
+from pathlib import Path
from typing import List, Optional, Tuple
from warnings import warn
import h5py
import numpy as np
+from extra_data import RunDirectory
from iCalibrationDB import Conditions, Constants, Detectors
from cal_tools.enums import BadPixels
@@ -807,3 +810,41 @@ def make_cell_order_condition(use_param, cellid_pattern) -> Optional[str]:
use = (use_param == 'always')
return (",".join([str(c) for c in cellid_pattern]) + ",") if use else None
+
+
+def sort_dark_runs_by_gain(raw_folder, runs, ctrl_src):
+ """Check gain factors [100, 10, 1] to decide,
+ if provided dark runs are in the correct order.
+
+ Args:
+ raw_folder (Union[str, Path]): The raw data path.
+ runs (list): A list of 3 runs.
+ ctrl_src (str): The CTRL source to check `RUN/.../femAsicGain`.
+ Return:
+ (list): Sorted dark runs.
+ """
+ run_to_gain = dict()
+ expected_gain_factors = [100, 10, 1]
+ assert len(set(runs)) == 3, f"A list of {len(runs)} runs is provided, three different dark runs are expected." # noqa
+
+ for r in runs:
+ run_to_gain[r] = RunDirectory(
+ Path(raw_folder) / f"r{r:04d}").get_run_value(
+ ctrl_src, "femAsicGain")
+
+ if len(set(run_to_gain.values())) < 3:
+ raise ValueError(
+ f"Incorrect gain factors for the provided dark runs: {run_to_gain}."
+ f" The expected gain factors for these runs are {expected_gain_factors}.")
+
+ sorted_run_to_gain = dict(sorted(
+ run_to_gain.items(),
+ key=lambda x: expected_gain_factors.index(x[1])))
+
+ sorted_runs = list(sorted_run_to_gain.keys())
+ if list(run_to_gain.values()) != expected_gain_factors:
+ warning(
+ "Dark runs were incorrectly sorted and "
+ f"have now been corrected to: {sorted_runs}.")
+
+ return sorted_runs
diff --git a/src/cal_tools/tools.py b/src/cal_tools/tools.py
index 88ab28d1e03c23264bb77c1c9a0e6b707a1535a1..d9e5e5f9e420a41dfff546b38e95a5fbd81c7bf8 100644
--- a/src/cal_tools/tools.py
+++ b/src/cal_tools/tools.py
@@ -1044,3 +1044,24 @@ def reorder_axes(a, from_order, to_order):
from_order = list(from_order)
order = tuple([from_order.index(lbl) for lbl in to_order])
return a.transpose(order)
+
+
+def raw_data_location_string(proposal: str, runs: List[int]):
+ """Create RAW data location string to inject as a
+ metadata along with the calibration parameters.
+
+ Args:
+ proposal (string): The proposal number including the preceding `p`.
+ e.g. p900203
+ runs (list): A list of the run numbers
+
+ Returns:
+ str: The string for raw data_location.
+ e.g. `proposal p900203 runs: 9008, 9009, 90010`
+ """
+ if not isinstance(proposal, str) or proposal[0] != "p":
+ raise ValueError(
+ "Invalid proposal format. The proposal should be a string with"
+ " a preceding 'p'. Example: 'p900203'")
+
+ return f"proposal:{proposal} runs:{' '.join(map(str, runs))}"
diff --git a/tests/test_cal_tools.py b/tests/test_cal_tools.py
index 3eb962b0ec1a12c29efd03584b77d5340cd277cf..577a82a4f57122b6153cd4ef92e5f548a424f1de 100644
--- a/tests/test_cal_tools.py
+++ b/tests/test_cal_tools.py
@@ -18,6 +18,7 @@ from cal_tools.tools import (
get_pdu_from_db,
map_seq_files,
module_index_to_qm,
+ raw_data_location_string,
recursive_update,
send_to_db,
write_constants_fragment,
@@ -567,3 +568,16 @@ def test_reorder_axes():
to_order = ('gain', 'fast_scan', 'slow_scan', 'cells')
assert reorder_axes(a, from_order, to_order).shape == (3, 256, 32, 10)
+
+
+def test_raw_data_location_string():
+ raw_data_loc = raw_data_location_string("p900203", [9008, 9009, 9010])
+ assert raw_data_loc == "proposal:p900203 runs:9008 9009 9010"
+
+
+def test_raise_raw_data_location_string():
+ with pytest.raises(ValueError):
+ raw_data_location_string(900203, [9008, 9009, 9010])
+
+ with pytest.raises(ValueError):
+ raw_data_location_string("900203", [9008, 9009, 9010])
diff --git a/tests/test_lpdlib.py b/tests/test_lpdlib.py
new file mode 100644
index 0000000000000000000000000000000000000000..09a5d5530b228b514ceaa849f52f40b1b03a8bc0
--- /dev/null
+++ b/tests/test_lpdlib.py
@@ -0,0 +1,29 @@
+import pytest
+
+from cal_tools.lpdlib import sort_dark_runs_by_gain
+
+
+@pytest.mark.requires_gpfs
+def test_sort_dark_runs_by_gain():
+ # TODO: update used raw data to test data when available.
+ raw_folder = "/gpfs/exfel/exp/CALLAB/202130/p900203/raw"
+ ctrl_src = "FXE_DET_LPD1M-1/COMP/FEM_MDL_COMP"
+ runs = [9008, 9010, 9009]
+ sorted_runs = sort_dark_runs_by_gain(raw_folder, runs, ctrl_src)
+ assert sorted_runs == [9008, 9009, 9010]
+
+
+@pytest.mark.requires_gpfs
+def test_raise_sort_dark_runs_by_gain():
+ # TODO: update used raw data to test data when available.
+ raw_folder = "/gpfs/exfel/exp/CALLAB/202130/p900203/raw"
+ ctrl_src = "FXE_DET_LPD1M-1/COMP/FEM_MDL_COMP"
+ runs = [9008, 9009]
+ with pytest.raises(AssertionError): # Only two runs provided
+ sort_dark_runs_by_gain(raw_folder, runs, ctrl_src)
+
+ runs = [9008, 9008, 9009] # Same two runs provided
+ with pytest.raises(AssertionError):
+ sort_dark_runs_by_gain(raw_folder, runs, ctrl_src)
+
+ #TODO: Add a ValueError check when more LPD dark runs added in p900203