diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 2bd52013ec6a1cd7a7acb855c444dca9990b5e3b..36d88640f1fb95559d49b23eb2e5ccd114342da9 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -40,6 +40,7 @@
"cal_db_timeout = 3000000 # timeout on caldb requests\"\n",
"local_output = True # output constants locally\n",
"db_output = False # output constants to database\n",
+ "sort_runs = True # Sort the selected dark runs. This flag is added for old data (e.g. 900174 r0011).\n",
"\n",
"mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
"bias_voltage = 0 # bias voltage, set to 0 to use stored value in slow data.\n",
@@ -95,7 +96,7 @@
"from collections import OrderedDict\n",
"from datetime import timedelta\n",
"from pathlib import Path\n",
- "from typing import List, Tuple\n",
+ "from typing import Tuple\n",
"\n",
"import matplotlib\n",
"import numpy as np\n",
@@ -141,16 +142,8 @@
"# insert control device if format string (does nothing otherwise)\n",
"ctrl_src = ctrl_source_template.format(karabo_id_control)\n",
"\n",
- "runs_dict = OrderedDict()\n",
"run_numbers = [run_high, run_med, run_low]\n",
"\n",
- "for gain_idx, (run_name, run_number) in enumerate(zip([\"high\", \"med\", \"low\"], run_numbers)):\n",
- " runs_dict[run_name] = {\n",
- " \"number\": run_number,\n",
- " \"gain\": gain_idx,\n",
- " \"dc\": RunDirectory(f'{in_folder}/r{run_number:04d}/')\n",
- " }\n",
- "\n",
"creation_time=None\n",
"if use_dir_creation_date:\n",
" creation_time = get_dir_creation_date(in_folder, run_high)\n",
@@ -205,35 +198,35 @@
"Path(out_folder).mkdir(parents=True, exist_ok=True)\n",
"\n",
"mod_image_size = []\n",
- "for run_dict in runs_dict.values():\n",
+ "for run in run_numbers:\n",
" missing_modules = [] # modules with no images within a run.\n",
" n_trains_list = [] # list of the number of trains for each module within a run.\n",
" # This is important in case of no slurm parallelization over modules is done.\n",
" # (e.g. running notebook interactively)\n",
" for m in modules:\n",
" # validate that there are trains for the selected modules and run.\n",
- " dc = run_dict[\"dc\"].select(\n",
+ " dc = RunDirectory(f'{in_folder}/r{run:04d}/').select(\n",
" instrument_src.format(m), \"*\", require_all=True)\n",
" n_trains = len(dc.train_ids)\n",
"\n",
" if n_trains == 0:\n",
- " print(f\"WARNING: No images for module AGIPD{m:02d}, run {run_dict['number']}.\")\n",
+ " print(f\"WARNING: No images for module AGIPD{m:02d}, run {run}.\")\n",
" missing_modules.append(m)\n",
" # Raise a warning if the module has less trains than expected.\n",
" elif n_trains < min_trains:\n",
- " print(f\"WARNING: AGIPD{m:02d}, run {run_dict['number']} \"\n",
+ " print(f\"WARNING: AGIPD{m:02d}, run {run} \"\n",
" f\"has trains less than minimum trains: {min_trains}.\")\n",
" else:\n",
" print(f\"Processing {max_trains if max_trains < n_trains else n_trains} \"\n",
- " f\"for AGIPD{m:02d}, run {run_dict['number']} \")\n",
+ " f\"for AGIPD{m:02d}, run {run} \")\n",
"\n",
" n_trains_list.append(n_trains)\n",
" mod_image_size.append(np.product(dc[instrument_src.format(m), \"image.data\"].shape) * 2 / 1e9)\n",
"\n",
" if max(n_trains_list) == 0:\n",
- " raise ValueError(f\"No images to process for run: {run_dict['number']}\")\n",
+ " raise ValueError(f\"No images to process for run: {run}\")\n",
" elif max(n_trains_list) < min_trains:\n",
- " raise ValueError(f\"{run_dict['number']} has less than minimum trains: {min_trains}\")\n",
+ " raise ValueError(f\"{run} has less than minimum trains: {min_trains}\")\n",
"\n",
"# Update modules and karabo_da lists based on available modules to processes.\n",
"modules = [m for m in modules if m not in missing_modules]\n",
@@ -264,10 +257,13 @@
"\n",
"agipd_ctrl_dark = AgipdCtrlRuns(\n",
" raw_folder=in_folder,\n",
- " runs=[r[\"number\"] for r in runs_dict.values()],\n",
+ " runs=run_numbers,\n",
" image_src=instrument_src_mod,\n",
" ctrl_src=ctrl_src,\n",
+ " sort_dark_runs_enabled=sort_runs\n",
")\n",
+ "# Update run_numbers list in case it was sorted.\n",
+ "run_numbers = agipd_ctrl_dark.runs\n",
"if mem_cells == 0:\n",
" mem_cells = agipd_ctrl_dark.get_memory_cells()\n",
"\n",
@@ -278,7 +274,6 @@
" bias_voltage = agipd_ctrl_dark.get_bias_voltage(karabo_id_control)\n",
"\n",
"fixed_gain_mode = False\n",
- "agipd_ctrl_dark.validate_gain_modes()\n",
"if gain_mode == -1:\n",
" gain_mode = agipd_ctrl_dark.gain_modes\n",
" fixed_gain_mode = agipd_ctrl_dark.fixed_gain_mode()\n",
@@ -386,16 +381,16 @@
"print(f\"Will use {parallel_num_procs} processes with {parallel_num_threads} threads each\")\n",
"\n",
"def characterize_module(\n",
- " channel: int, runs_dict: dict,\n",
+ " channel: int, gain_run: Tuple[int, int],\n",
") -> Tuple[int, int, np.array, np.array, np.array, np.array, np.array]:\n",
"\n",
+ " gain_index, run = gain_run\n",
" # Select the corresponding module channel.\n",
" instrument_src_mod = instrument_src.format(channel)\n",
"\n",
- " run_dc = runs_dict[\"dc\"].select(instrument_src_mod, require_all=True)\n",
+ " run_dc = RunDirectory(f'{in_folder}/r{run:04d}/').select(instrument_src_mod, require_all=True)\n",
" if max_trains != 0:\n",
" run_dc = run_dc.select_trains(np.s_[:max_trains])\n",
- " gain_index = runs_dict[\"gain\"]\n",
"\n",
" # Read module's image and cellId data.\n",
" im = run_dc[instrument_src_mod, \"image.data\"].ndarray()\n",
@@ -478,16 +473,16 @@
"source": [
"with multiprocessing.Pool(processes=parallel_num_procs) as pool:\n",
" results = pool.starmap(\n",
- " characterize_module, itertools.product(modules, list(runs_dict.values())))\n",
+ " characterize_module, itertools.product(modules, list(enumerate(run_numbers))))\n",
"\n",
"# mapped values for processing 2 modules example:\n",
- "# [\n",
- "# 0, {\"gain\": 0, \"run_number\": <run-high>, \"dc\": <high-dc>},\n",
- "# 0, {\"gain\": 1, \"run_number\": <run-med>, \"dc\": <med-dc>},\n",
- "# 0, {\"gain\": 2, \"run_number\": <run-low>, \"dc\": <low-dc>},\n",
- "# 1, {\"gain\": 0, \"run_number\": <run-high>, \"dc\": <high-dc>},\n",
- "# 1, {\"gain\": 1, \"run_number\": <run-med>, \"dc\": <med-dc>},\n",
- "# 1, {\"gain\": 2, \"run_number\": <run-low>, \"dc\": <low-dc>},\n",
+ "# [(0, (0, 9013))\n",
+ "# 0, (0, run-high),\n",
+ "# 0, (1, run-med),\n",
+ "# 0, (2, run-low),\n",
+ "# 1, (0, run-high),\n",
+ "# 1, (1, run-med),\n",
+ "# 1, (2, run-low),,\n",
"# ]"
]
},
diff --git a/src/cal_tools/agipdlib.py b/src/cal_tools/agipdlib.py
index d743044f16e7015e7382164d13806b5770d1d171..6707c0d9bd8eeaa1ef04d5694912cb1cdfbc64c5 100644
--- a/src/cal_tools/agipdlib.py
+++ b/src/cal_tools/agipdlib.py
@@ -308,6 +308,7 @@ class AgipdCtrlRuns:
runs: List[int]
image_src: str
ctrl_src: str
+ sort_dark_runs_enabled: bool = False
adaptive_gain_modes = [AgipdGainMode.ADAPTIVE_GAIN] * 3
fixed_gain_modes = [
@@ -325,6 +326,8 @@ class AgipdCtrlRuns:
ctrl_src=self.ctrl_src,
) for r in self.runs]
self.gain_modes = self.get_gain_modes()
+ if self.sort_dark_runs_enabled:
+ self.sort_dark_runs()
def _validate_same_value(self, name, values):
if len(set(values)) != 1:
@@ -334,29 +337,47 @@ class AgipdCtrlRuns:
f" with values of {values}, respectively.")
def sort_dark_runs(self):
- assert len(self.run_ctrls) == 3, f"AGIPD dark runs are expected to be 3. {len(self.run_ctrls)} runs are given." # noqa
- # TODO: complete sorting
-
- def validate_gain_modes(self):
- """Validate the runs' gain modes arrangement.
+ """Order dark runs based on run patterns for Adaptive mode
+ or gain modes for Fixed mode.
+ """
+ assert len(self.runs) == 3, f"AGIPD dark runs are expected to be 3. {len(self.runs)} runs are given." # noqa
+ # Expected patterns:
+ # XRay: 0, DarkHG: 1, DarkMG: 2, DarkLG: 3, PC: 4 and CS: 5.
+ sort_by = None
+ sort_values = []
+ if self.gain_modes == self.adaptive_gain_modes: # Adaptive gain # sort by patterns
+ # Patterns -> DarkHG: 1, DarkMG: 2, DarkLG: 3
+ if "AGIPD1M" in self.ctrl_src:
+ sort_by = "patternTypeIndex"
+ elif "AGIPD500K" in self.ctrl_src:
+ sort_by = "expTypeIndex"
+
+ for c in self.run_ctrls:
+ sort_values.append(
+ c.run_dc[self.ctrl_src, sort_by].as_single_value())
- Raises:
- ValueError: Runs are a mix of adaptive and fixed gains
- ValueError: Unexpected gain modes for the dark runs"""
- if self.gain_modes in [
- self.adaptive_gain_modes,
- self.fixed_gain_modes
- ]:
- return # all good.
# Check if a mix of adaptive and fixed gain runs.
- if any(gm == AgipdGainMode.ADAPTIVE_GAIN for gm in self.gain_modes):
+ elif any(gm == AgipdGainMode.ADAPTIVE_GAIN for gm in self.gain_modes):
raise ValueError(
f"Given runs {self.runs} have a mix of ADAPTIVE and "
f"FIXED gain modes: {self.gain_modes}.")
- else: # Unsorted fixed gain runs.
- raise ValueError(
- "Wrong arrangement of given dark runs. Given runs' "
- f"gain_modes are {self.gain_modes} for runs: {self.runs}.")
+ else: # Fixed gain: Patterns is X-Ray: 0 for all runs.
+ sort_by = "gainModeIndex"
+ sort_values = [int(gm) for gm in self.gain_modes]
+
+ zipped_lists = zip(sort_values, self.runs, self.run_ctrls)
+
+ # Sort the lists based on the patterns
+ sorted_zipped_lists = sorted(zipped_lists, key=lambda item: item[0])
+ _, sorted_runs, sorted_run_ctrls = zip(*sorted_zipped_lists)
+ if sorted_runs != self.runs:
+ Warning("Given dark runs are unsorted. Runs will be sorted from"
+ f" {self.runs} with {sort_by}:"
+ f" {sort_values} to {sorted_runs}.")
+ # Update run_ctrls and runs order
+ self.runs = list(sorted_runs)
+ self.run_ctrls = list(sorted_run_ctrls)
+ self.gain_modes = self.get_gain_modes()
def fixed_gain_mode(self):
"""Check if runs are in fixed gain mode.
diff --git a/tests/test_agipdlib.py b/tests/test_agipdlib.py
index 17f29f655170df2a4a26ccfcdf5952c006c63370..89ef4054df964fb3fc4b8a44f32b267dd66ac702 100644
--- a/tests/test_agipdlib.py
+++ b/tests/test_agipdlib.py
@@ -203,16 +203,12 @@ def test_get_gain_mode(mock_agipd1m_run):
assert gain_mode == 0
-"""
-Testing `AgipdCtrlRuns`
+"""Testing `AgipdCtrlRuns`"""
-Runs used:
-FIXED: /gpfs/exfel/exp/CALLAB/202130/p900203/raw/[9011,9012,9013]
-ADAPTIVE: /gpfs/exfel/exp/CALLAB/202130/p900203/raw/[9015,9016,9017]
-"""
TEST_RAW_FOLDER = "/gpfs/exfel/exp/CALLAB/202130/p900203/raw/"
SPB_FIXED_RUNS = [9011, 9012, 9013]
SPB_ADAPTIVE_RUNS = [9015, 9016, 9017]
+
FIXED_CTRL_RUNS = AgipdCtrlRuns(
raw_folder=TEST_RAW_FOLDER,
runs=SPB_FIXED_RUNS,
@@ -294,21 +290,39 @@ def test_raise_fixed_gain_mode():
@pytest.mark.requires_gpfs
-def test_raise_validate_gain_modes():
- adaptive_fixed_ctrls = AgipdCtrlRuns(
+@pytest.mark.parametrize(
+ "runs,expected",
+ [
+ ([9013, 9011, 9012], [9011, 9012, 9013]),
+ ([9017, 9016, 9015], [9015, 9016, 9017]),
+ ],
+)
+def test_sort_dark_runs(runs, expected):
+ runs_ctrls = AgipdCtrlRuns(
raw_folder=TEST_RAW_FOLDER,
- runs=[9011, 9016, 9017],
+ runs=runs,
image_src=SPB_AGIPD_INST_SRC,
ctrl_src=CTRL_SRC,
)
+ runs_ctrls.sort_dark_runs()
+ assert runs_ctrls.runs == expected
+
+
+def test_raise_sort_dark_runs():
with pytest.raises(ValueError):
- adaptive_fixed_ctrls.validate_gain_modes()
+ adaptive_fixed_ctrls = AgipdCtrlRuns(
+ raw_folder=TEST_RAW_FOLDER,
+ runs=[9011, 9016, 9017],
+ image_src=SPB_AGIPD_INST_SRC,
+ ctrl_src=CTRL_SRC,
+ sort_dark_runs_enabled=True
+ )
- unsorted_fixed_gain_ctrls = AgipdCtrlRuns(
+ adaptive_fixed_ctrls = AgipdCtrlRuns(
raw_folder=TEST_RAW_FOLDER,
- runs=[9013, 9011, 9012],
+ runs=[9011, 9016, 9017],
image_src=SPB_AGIPD_INST_SRC,
ctrl_src=CTRL_SRC,
)
with pytest.raises(ValueError):
- unsorted_fixed_gain_ctrls.validate_gain_modes()
+ adaptive_fixed_ctrls.sort_dark_runs()
diff --git a/tests/test_reference_runs/callab_tests.py b/tests/test_reference_runs/callab_tests.py
index 9e41d4c7768ca1c3e9f55703d0f6a9ae13290ed2..f16a1eba6ca7968690680e3f95d7eeecddac3dca 100644
--- a/tests/test_reference_runs/callab_tests.py
+++ b/tests/test_reference_runs/callab_tests.py
@@ -25,9 +25,10 @@ automated_test_config = {
"out-folder": "{}/{}/{}",
# "/gpfs/exfel/exp/SPB/202131/p900215/raw"
"in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
- "run-high": "9011", # Original run: "91"
+ # Unsorted dark runs
+ "run-high": "9013", # Original run "93"
"run-med": "9012", # Original run: "92"
- "run-low": "9013", # Original run "93"
+ "run-low": "9011", # Original run: "91"
"karabo-id-control": "SPB_IRU_AGIPD1M1",
"karabo-id": "SPB_DET_AGIPD1M-1",
"ctrl-source-template": "{}/MDL/FPGA_COMP",
@@ -173,9 +174,10 @@ automated_test_config = {
"out-folder": "{}/{}/{}",
# "/gpfs/exfel/exp/HED/202131/p900228/raw"
"in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ # Unsorted dark runs
"run-high": "9023", # Original run: "25",
- "run-med": "9024", # Original run: "26",
- "run-low": "9025", # Original run: "27",
+ "run-med": "9025", # Original run: "27",
+ "run-low": "9024", # Original run: "26",
"karabo-id-control": "HED_EXP_AGIPD500K2G",
"karabo-id": "HED_DET_AGIPD500K2G",
"ctrl-source-template": "{}/MDL/FPGA_COMP",