diff --git a/notebooks/LPD/LPD_Correct_Fast.ipynb b/notebooks/LPD/LPD_Correct_Fast.ipynb
index cf6bac5aaf10d37fb8e763ace7026b2627f1d437..04d35400aafeaa9197d79ff6a11439d697c84df0 100644
--- a/notebooks/LPD/LPD_Correct_Fast.ipynb
+++ b/notebooks/LPD/LPD_Correct_Fast.ipynb
@@ -21,13 +21,13 @@
"outputs": [],
"source": [
"# Input parameters\n",
- "in_folder = \"/gpfs/exfel/exp/FXE/202201/p003073/raw/\" # the folder to read data from, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/schmidtp/random/LPD_test\" # the folder to output to, required\n",
+ "in_folder = \"/gpfs/exfel/exp/FXE/202401/p005436/raw/\" # the folder to read data from, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/kluyvert/lpd-corr-p5436-r167\" # the folder to output to, required\n",
"metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate.\n",
"sequences = [-1] # Sequences to correct, use [-1] for all\n",
"modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty\n",
"karabo_da = [''] # Data aggregators names to correct, use [''] for all\n",
- "run = 10 # run to process, required\n",
+ "run = 167 # run to process, required\n",
"\n",
"# Source parameters\n",
"karabo_id = 'FXE_DET_LPD1M-1' # Karabo domain for detector.\n",
@@ -104,14 +104,13 @@
"import pasha as psh\n",
"from extra_data.components import LPD1M\n",
"\n",
+ "from cal_tools.calcat_interface2 import CalibrationData, LPDConditions\n",
"import cal_tools.restful_config as rest_cfg\n",
- "from cal_tools.calcat_interface import CalCatError, LPD_CalibrationData\n",
"from cal_tools.lpdalgs import correct_lpd_frames\n",
"from cal_tools.lpdlib import get_mem_cell_pattern, make_cell_order_condition\n",
"from cal_tools.tools import (\n",
- " CalibrationMetadata,\n",
" calcat_creation_time,\n",
- " write_constants_fragment,\n",
+ " write_constants_fragment_extracal,\n",
")\n",
"from cal_tools.files import DataFile"
]
@@ -238,6 +237,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
+ "scrolled": false,
"tags": []
},
"outputs": [],
@@ -253,30 +253,39 @@
" )\n",
"print(\"Memory cells order:\", cell_ids_pattern_s)\n",
"\n",
- "lpd_cal = LPD_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " modules=karabo_da,\n",
+ "conditions = LPDConditions(\n",
" sensor_bias_voltage=bias_voltage,\n",
" memory_cells=mem_cells,\n",
" feedback_capacitor=capacitor,\n",
" source_energy=photon_energy,\n",
" memory_cell_order=cell_ids_pattern_s,\n",
" category=category,\n",
- " event_at=creation_time,\n",
- " client=rest_cfg.calibration_client(),\n",
- " caldb_root=Path(cal_db_root),\n",
")\n",
"\n",
- "lpd_metadata = lpd_cal.metadata([\"Offset\", \"BadPixelsDark\"])\n",
- "try:\n",
- " illum_metadata = lpd_cal.metadata(lpd_cal.illuminated_calibrations)\n",
- " for key, value in illum_metadata.items():\n",
- " lpd_metadata.setdefault(key, {}).update(value)\n",
- "except CalCatError as e: # TODO: replace when API errors are improved.\n",
- " warning(f\"CalCatError: {e}\")\n",
+ "expected_constants = {'Offset', 'BadPixelsDark'}\n",
+ "if rel_gain:\n",
+ " expected_constants.add('RelativeGain')\n",
+ "if ff_map:\n",
+ " expected_constants.update(['FFMap', 'BadPixelsFF'])\n",
+ "if gain_amp_map:\n",
+ " expected_constants.add('GainAmpMap')\n",
+ "\n",
+ "lpd_consts = CalibrationData.from_condition(\n",
+ " conditions,\n",
+ " calibrations=expected_constants,\n",
+ " detector_name=karabo_id,\n",
+ " event_at=creation_time,\n",
+ " client=rest_cfg.extra_calibration_client(),\n",
+ ").select_modules(\n",
+ " aggregator_names=karabo_da\n",
+ ").require_calibrations(\n",
+ " ['Offset']\n",
+ ")\n",
"\n",
"total_time = perf_counter() - start\n",
- "print(f'Looking up constants {total_time:.1f}s')"
+ "print(f'Looking up constants {total_time:.1f}s')\n",
+ "\n",
+ "lpd_consts.summary_table()"
]
},
{
@@ -286,18 +295,18 @@
"outputs": [],
"source": [
"# Validate the constants availability and raise/warn accordingly.\n",
- "for mod, calibrations in lpd_metadata.items():\n",
- " missing_offset = {\"Offset\"} - set(calibrations)\n",
- " warn_missing_constants = {\n",
- " \"BadPixelsDark\", \"BadPixelsFF\", \"GainAmpMap\",\n",
- " \"FFMap\", \"RelativeGain\"} - set(calibrations)\n",
- " if missing_offset:\n",
+ "if not lpd_consts.aggregator_names: # Offset was required above\n",
+ " raise Exception(\"Could not find offset constants for any modules, will not correct data.\")\n",
+ " \n",
+ "for mod in karabo_da.copy():\n",
+ " if mod not in lpd_consts[\"Offset\"].aggregator_names:\n",
" warning(f\"Offset constant is not available to correct {mod}.\")\n",
" karabo_da.remove(mod)\n",
- " if warn_missing_constants:\n",
- " warning(f\"Constants {warn_missing_constants} were not retrieved for {mod}.\")\n",
- "if not karabo_da: # Offsets are missing for all modules.\n",
- " raise Exception(\"Could not find offset constants for any modules, will not correct data.\")\n",
+ " \n",
+ " missing_constants = {c for c in expected_constants\n",
+ " if (c not in lpd_consts) or (mod not in lpd_consts[c].aggregator_names)}\n",
+ " if missing_constants:\n",
+ " warning(f\"Constants {sorted(missing_constants)} were not retrieved for {mod}.\")\n",
"\n",
"# Remove skipped correction modules from data_to_process\n",
"data_to_process = [(mod, in_f, out_f) for mod, in_f, out_f in data_to_process if mod in karabo_da]"
@@ -310,14 +319,22 @@
"outputs": [],
"source": [
"# write constants metadata to fragment YAML\n",
- "write_constants_fragment(\n",
+ "write_constants_fragment_extracal(\n",
" out_folder=(metadata_folder or out_folder),\n",
- " det_metadata=lpd_metadata,\n",
- " caldb_root=lpd_cal.caldb_root,\n",
+ " calib_data=lpd_consts,\n",
+ " caldb_root=cal_db_root,\n",
")\n",
"\n",
"# Load constants data for all constants\n",
- "const_data = lpd_cal.ndarray_map(metadata=lpd_metadata)"
+ "start = perf_counter()\n",
+ "const_data = {kda: {} for kda in lpd_consts.aggregator_names}\n",
+ "for cname, multimodconst in lpd_consts.items():\n",
+ " arr = multimodconst.ndarray(cal_db_root, parallel=8)\n",
+ " for i, kda in enumerate(multimodconst.aggregator_names):\n",
+ " const_data[kda][cname] = arr[i]\n",
+ "\n",
+ "total_time = perf_counter() - start\n",
+ "print(f'Loading constants {total_time:.1f}s')"
]
},
{
@@ -367,17 +384,17 @@
" else:\n",
" ccv_masks[aggregator] = np.zeros(ccv_shape, dtype=np.uint32)\n",
" \n",
- " if rel_gain and 'RelativeGain' in consts:\n",
+ " if 'RelativeGain' in consts:\n",
" ccv_gains[aggregator] *= _prepare_data('RelativeGain', np.float32)\n",
" \n",
- " if ff_map and 'FFMap' in consts:\n",
+ " if 'FFMap' in consts:\n",
" ccv_gains[aggregator] *= _prepare_data('FFMap', np.float32)\n",
" \n",
" if 'BadPixelsFF' in consts:\n",
" np.bitwise_or(ccv_masks[aggregator], _prepare_data('BadPixelsFF', np.uint32),\n",
" out=ccv_masks[aggregator])\n",
" \n",
- " if gain_amp_map and 'GainAmpMap' in consts:\n",
+ " if 'GainAmpMap' in consts:\n",
" ccv_gains[aggregator] *= _prepare_data('GainAmpMap', np.float32)\n",
" \n",
" print('.', end='', flush=True)\n",
@@ -652,9 +669,9 @@
],
"metadata": {
"kernelspec": {
- "display_name": "pycal",
+ "display_name": "Offline Cal",
"language": "python",
- "name": "pycal"
+ "name": "offline-cal"
},
"language_info": {
"codemirror_mode": {
@@ -666,7 +683,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.11"
+ "version": "3.11.8"
}
},
"nbformat": 4,
diff --git a/src/cal_tools/calcat_interface2.py b/src/cal_tools/calcat_interface2.py
index e5e1bc26711cd58f06c8654abfcd60ec6ab6b3a6..464ff85c576b69ef4a7a7435f9a9cfe48b76fc95 100644
--- a/src/cal_tools/calcat_interface2.py
+++ b/src/cal_tools/calcat_interface2.py
@@ -756,7 +756,7 @@ class CalibrationData(Mapping):
modules = self.qm_names
else:
raise ValueError(
- f"{module_naming=} (must be 'aggregator', 'modnum' or 'qm'"
+ f"{module_naming=} (must be 'aggregator', 'modnum' or 'qm')"
)
cal_groups = [
@@ -766,7 +766,7 @@ class CalibrationData(Mapping):
tables = []
# Loop over groups of calibrations.
for cal_group in cal_groups:
- table = ["Modules"] + cal_group
+ table = [["Modules"] + cal_group]
# Loop over calibrations and modules to form the next rows.
for mod in modules:
@@ -790,7 +790,7 @@ class CalibrationData(Mapping):
except KeyError:
mod_consts.append(f"{c_time} ({singleconst.ccv_id})")
- table.append([mod] + mod_consts)
+ table.append([str(mod)] + mod_consts)
tables.append(table)
diff --git a/src/cal_tools/tools.py b/src/cal_tools/tools.py
index 0161fb8922db89a638ac1c8e70ca27e88da2882d..48c780dd43dc7c4c783dd0b6646218686a522faa 100644
--- a/src/cal_tools/tools.py
+++ b/src/cal_tools/tools.py
@@ -970,6 +970,8 @@ def write_constants_fragment(
):
"""Record calibration constants metadata to a fragment file.
+ This variant works with the calcat_interface module in this package.
+
Args:
out_folder (Path): The output folder to store the fragment file.
det_metadata (dict): A dictionary with the desired detector metadata.
@@ -993,6 +995,40 @@ def write_constants_fragment(
CalibrationMetadata(out_folder).add_fragment(metadata)
+def write_constants_fragment_extracal(
+ out_folder: Path,
+ calib_data: 'extra.calibration.CalibrationData',
+ caldb_root: Path,
+):
+ """Record calibration constants metadata to a fragment file.
+
+ This variant works with the extra.calibration API for found constants.
+
+ Args:
+ out_folder (Path): The output folder to store the fragment file.
+ calib_data (extra.calibration.CalibrationData): Retrieved constant details
+ caldb_root (Path): The calibration database root path for constant files.
+ """
+ retrieved_consts = {}
+ for kda, pdu in zip(calib_data.aggregator_names, calib_data.pdu_names):
+ retrieved_consts[kda] = {
+ "physical-name": pdu,
+ "constants": {},
+ }
+
+ for cname, multimodconst in calib_data.items():
+ for kda in multimodconst.aggregator_names:
+ const = multimodconst[kda]
+ retrieved_consts[kda]["constants"][cname] = {
+ "path": str(caldb_root / const.path),
+ "dataset": const.dataset,
+ "creation-time": const.metadata("begin_validity_at"),
+ "ccv_id": const.ccv_id
+ }
+
+ CalibrationMetadata(out_folder).add_fragment({"retrieved-constants": retrieved_consts})
+
+
def write_compressed_frames(
arr: np.ndarray,
ofile: h5py.File,