From 92f842c05efeff72bc5951a04b059ef737b205e1 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Mon, 25 Nov 2024 15:30:05 +0100
Subject: [PATCH] fix: Skip exposure_timeout for gain constant retrieval
---
...Jungfrau_Gain_Correct_and_Verify_NBC.ipynb | 142 +++++++++++-------
src/cal_tools/calcat_interface2.py | 9 +-
2 files changed, 89 insertions(+), 62 deletions(-)
diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
index 88d03d80d..094be7934 100644
--- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
@@ -37,6 +37,7 @@
"cal_db_interface = \"tcp://max-exfl-cal001:8017#8025\" # the database interface to use\n",
"cal_db_timeout = 180000 # timeout on caldb requests\n",
"creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
+ "cal_db_root = '/gpfs/exfel/d/cal/caldb_store' # The calibration database root path to access constant files. For example accessing constants from the test database.\n",
"\n",
"# Parameters affecting corrected data.\n",
"relative_gain = True # do relative gain correction.\n",
@@ -94,7 +95,7 @@
"from matplotlib.colors import LogNorm\n",
"\n",
"import cal_tools.restful_config as rest_cfg\n",
- "from cal_tools.calcat_interface import JUNGFRAU_CalibrationData\n",
+ "from cal_tools.calcat_interface2 import CalibrationData, JUNGFRAUConditions\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.files import DataFile\n",
"from cal_tools.jungfrau.jungfraulib import JungfrauCtrl\n",
@@ -104,7 +105,8 @@
" calcat_creation_time,\n",
" exit_notebook,\n",
" map_seq_files,\n",
- " write_constants_fragment,\n",
+ " pdus_by_detector_id,\n",
+ " write_constants_fragment_extracal,\n",
")\n",
"\n",
"warnings.filterwarnings('ignore')\n",
@@ -119,6 +121,8 @@
"metadata": {},
"outputs": [],
"source": [
+ "step_timer = StepTimer()\n",
+ "\n",
"in_folder = Path(in_folder)\n",
"out_folder = Path(out_folder)\n",
"run_folder = in_folder / f'r{run:04d}'\n",
@@ -213,27 +217,52 @@
"metadata": {},
"outputs": [],
"source": [
+ "# For plotting\n",
+ "cc = rest_cfg.extra_calibration_client()\n",
+ "det_id = cc.detector_by_identifier(karabo_id)['id']\n",
+ "pdus = pdus_by_detector_id(cc, det_id, snapshot_at=creation_time)\n",
+ "\n",
+ "da_to_pdu = dict()\n",
+ "for pdu in pdus:\n",
+ " if pdu['karabo_da'] in karabo_da: # exclude unselected das\n",
+ " da_to_pdu[pdu['karabo_da']] = pdu['physical_name']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "expected_constants = [\"Offset10Hz\", \"BadPixelsDark10Hz\"]\n",
+ "if relative_gain:\n",
+ " expected_constants += [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"]\n",
+ "\n",
+ "\n",
"def jungfrau_cal_mdata(gm):\n",
- " jf_cal = JUNGFRAU_CalibrationData(\n",
- " detector_name=karabo_id,\n",
+ " conditions = JUNGFRAUConditions(\n",
" sensor_bias_voltage=bias_voltage,\n",
- " event_at=creation_time,\n",
- " modules=karabo_da,\n",
" memory_cells=memory_cells,\n",
" integration_time=integration_time,\n",
" exposure_timeout=exposure_timeout,\n",
" gain_setting=gain_setting,\n",
" gain_mode=gm,\n",
- " client=rest_cfg.calibration_client(),\n",
" )\n",
+ " \n",
+ " jf_consts = CalibrationData.from_condition(\n",
+ " conditions,\n",
+ " calibrations=expected_constants,\n",
+ " detector_name=karabo_id,\n",
+ " event_at=creation_time,\n",
+ " client=rest_cfg.extra_calibration_client(),\n",
+ " ).select_modules(\n",
+ " aggregator_names=karabo_da\n",
+ " ).require_calibrations(\n",
+ " ['Offset10Hz']\n",
+ " )\n",
+ " jf_consts.summary_table()\n",
"\n",
- " constant_names = [\"Offset10Hz\", \"BadPixelsDark10Hz\"]\n",
- " if relative_gain:\n",
- " constant_names += [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"]\n",
- " jf_metadata = jf_cal.metadata(calibrations=constant_names) \n",
- " # Display retrieved calibration constants timestamps\n",
- " jf_cal.display_markdown_retrieved_constants(metadata=jf_metadata)\n",
- " return jf_cal, jf_metadata\n",
+ " return jf_consts, jf_consts.aggregator_names\n",
"\n",
"def force_fixed_gain_constants():\n",
" \"\"\"JF corrections in burst mode are only supported when\n",
@@ -250,31 +279,31 @@
" from cal_tools.calcat_interface import CalCatError\n",
"\n",
" try:\n",
- " jf_cal, jf_metadata = jungfrau_cal_mdata(gm=1)\n",
- " except CalCatError as e:\n",
+ " jungfrau_consts = jungfrau_cal_mdata(gm=1)\n",
+ " except CalCatError as e: # TODO: update with CalibrationError class\n",
" warning(\n",
" \"No fixed gain constants found. \"\n",
" \"Looking for dynamic gain constant. \"\n",
" f\"(CalCatError: {e}.\")\n",
" else:\n",
- " return jf_cal, jf_metadata\n",
+ " return jungfrau_consts\n",
"\n",
" # In case of CALCATError exception look for dynamic gain constants\n",
- " jf_cal, jf_metadata = jungfrau_cal_mdata(gm=0)\n",
+ " jungfrau_consts = jungfrau_cal_mdata(gm=0)\n",
"\n",
- " for mod, ccvs in jf_metadata.items():\n",
- " offset = ccvs.get(\"Offset10Hz\")\n",
- " if not offset: # This module wont be corrected later after validating constants.\n",
- " continue\n",
- " time_difference = creation_time - datetime.fromisoformat(offset[\"begin_validity_at\"])\n",
+ " das_with_offset = []\n",
+ " for mod in jungfrau_consts.aggregator_names:\n",
+ " offset_creation_time = jungfrau_consts[\"Offset10Hz\", mod].metadata(\"begin_validity_at\")\n",
+ " time_difference = creation_time - datetime.fromisoformat(offset_creation_time)\n",
" if abs(time_difference.days) > 3:\n",
- " warning(\n",
+ " warnings.warn(\n",
" f\"No dynamic gain constant retrieved for {mod} with at least\"\n",
" \" 3 days time difference with the RAW data creation date.\"\n",
" \" Please make sure there are available constants.\")\n",
- " jf_metadata[mod].pop(\"Offset10Hz\")\n",
+ " else:\n",
+ " das_with_offset.append(mod)\n",
"\n",
- " return jf_cal, jf_metadata"
+ " return jungfrau_consts, das_with_offset"
]
},
{
@@ -293,9 +322,9 @@
"outputs": [],
"source": [
"if force_fixed_gain_constants_flag:\n",
- " jf_cal, jf_metadata = force_fixed_gain_constants()\n",
+ " jungfrau_consts, das_to_correct = force_fixed_gain_constants()\n",
"else:\n",
- " jf_cal, jf_metadata = jungfrau_cal_mdata(gain_mode)"
+ " jungfrau_consts, das_to_correct = jungfrau_cal_mdata(gain_mode)"
]
},
{
@@ -304,24 +333,23 @@
"metadata": {},
"outputs": [],
"source": [
- "# Validate the constants availability and raise/warn correspondingly. \n",
- "for mod in karabo_da[:]:\n",
- " calibrations = jf_metadata.get(mod, {})\n",
+ "# Validate the constants availability and raise/warn accordingly.\n",
+ "das_missing_offset = list(set(karabo_da) - set(das_to_correct))\n",
+ "warnings.warn(f\"Offset constant is not available to correct {das_missing_offset}\")\n",
"\n",
- " missing_dark_constants = {\"Offset10Hz\", \"BadPixelsDark10Hz\"} - set(calibrations)\n",
- " missing_gain_constants = {\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"} - set(calibrations)\n",
- "\n",
- " if missing_dark_constants:\n",
- " warning(\n",
- " f\"Dark constants {missing_dark_constants} are not available to correct {mod}.\"\n",
- " f\" Module {mod} won't be corrected.\")\n",
- " karabo_da.remove(mod)\n",
- "\n",
- " if relative_gain and missing_gain_constants:\n",
- " warning(f\"Gain constants {missing_gain_constants} were not retrieved for {mod}.\"\n",
- " \" No Relative gain correction for this module\")\n",
- "if not karabo_da: # Dark constants are missing for all modules.\n",
- " raise ValueError(\"Dark constants are missing for all modules.\")"
+ "if not das_to_correct: # No Offsets for all modules.\n",
+ " # TODO: Move this typical error to a CalibrationError class\n",
+ " raise Exception(\"Could not find offset constants for any modules, will not correct data.\")\n",
+ " \n",
+ "for mod in karabo_da.copy():\n",
+ " if mod not in das_to_correct:\n",
+ " warnings.warn(f\"Offset constant is not available to correct {mod}.\")\n",
+ " \n",
+ " missing_constants = {c for c in expected_constants\n",
+ " if (c not in jungfrau_consts) or (mod not in jungfrau_consts[c].aggregator_names)}\n",
+ " if missing_constants:\n",
+ " # TODO: Move this typical warning to a CalibrationWarning class\n",
+ " warnings.warn(f\"Constants {sorted(missing_constants)} were not retrieved for {mod}.\")"
]
},
{
@@ -331,20 +359,20 @@
"outputs": [],
"source": [
"# Record constant details in YAML metadata\n",
- "write_constants_fragment(\n",
+ "write_constants_fragment_extracal(\n",
" out_folder=(metadata_folder or out_folder),\n",
- " det_metadata=jf_metadata,\n",
- " caldb_root=jf_cal.caldb_root)\n",
+ " calib_data=jungfrau_consts,\n",
+ " caldb_root=cal_db_root)\n",
"\n",
+ "# Load constants data for all constants\n",
+ "step_timer.start()\n",
+ "const_data = {kda: {} for kda in jungfrau_consts.aggregator_names}\n",
+ "for cname, multimodconst in jungfrau_consts.items():\n",
+ " arr = multimodconst.ndarray(parallel=8)\n",
+ " for i, kda in enumerate(multimodconst.aggregator_names):\n",
+ " const_data[kda][cname] = arr[i]\n",
"\n",
- "# load constants arrays after storing fragment YAML file\n",
- "# and validating constants availability.\n",
- "const_data = jf_cal.ndarray_map(metadata=jf_metadata)\n",
- "\n",
- "# For plotting\n",
- "da_to_pdu = {}\n",
- "for mod_info in jf_cal.physical_detector_units.values():\n",
- " da_to_pdu[mod_info[\"karabo_da\"]] = mod_info[\"physical_name\"]"
+ "step_timer.done_step(f'Loading constants')"
]
},
{
@@ -546,8 +574,6 @@
"metadata": {},
"outputs": [],
"source": [
- "step_timer = StepTimer()\n",
- "\n",
"n_cpus = multiprocessing.cpu_count()\n",
"context = psh.context.ProcessContext(num_workers=n_cpus)\n",
"print(f\"Using {n_cpus} workers for correction.\")"
diff --git a/src/cal_tools/calcat_interface2.py b/src/cal_tools/calcat_interface2.py
index d662f2672..d120b472a 100644
--- a/src/cal_tools/calcat_interface2.py
+++ b/src/cal_tools/calcat_interface2.py
@@ -1106,15 +1106,16 @@ class JUNGFRAUConditions(ConditionsBase):
"Pixels X",
"Pixels Y",
"Integration Time",
- "Exposure timeout",
"Sensor temperature",
"Gain Setting",
"Gain mode",
]
+ _dark_params = _params + ["Exposure timeout"]
+
calibration_types = {
- "Offset10Hz": _params,
- "Noise10Hz": _params,
- "BadPixelsDark10Hz": _params,
+ "Offset10Hz": _dark_params,
+ "Noise10Hz": _dark_params,
+ "BadPixelsDark10Hz": _dark_params,
"RelativeGain10Hz": _params,
"BadPixelsFF10Hz": _params,
}
--
GitLab