From 38494902985718959232d1f559c184747ee941cd Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Tue, 21 Feb 2023 14:19:26 +0100
Subject: [PATCH] fix validation and error only for lut and fix constant keys
---
.../Gotthard2/Correction_Gotthard2_NBC.ipynb | 56 ++++++++++++-------
...retrieve_constants_precorrection_NBC.ipynb | 37 ++++++------
2 files changed, 55 insertions(+), 38 deletions(-)
diff --git a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
index 550efc3df..639594abd 100644
--- a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
+++ b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
@@ -237,9 +237,9 @@
" const_data[mod] = dict()\n",
" # load constants temporarily using defined local paths.\n",
" with h5py.File(constants_file, \"r\") as cfile:\n",
- " const_data[mod][\"LUT\"] = cfile[\"LUT\"][()]\n",
- " const_data[mod][\"Offset\"] = cfile[\"offset_map\"][()].astype(np.float32)\n",
- " const_data[mod][\"RelativeGain\"] = cfile[\"gain_map\"][()].astype(np.float32)\n",
+ " const_data[mod][\"LUTGotthard2\"] = cfile[\"LUT\"][()]\n",
+ " const_data[mod][\"OffsetGotthard2\"] = cfile[\"offset_map\"][()].astype(np.float32)\n",
+ " const_data[mod][\"RelativeGainGotthard2\"] = cfile[\"gain_map\"][()].astype(np.float32)\n",
" const_data[mod][\"Mask\"] = cfile[\"bpix_ff\"][()].astype(np.uint32)\n",
"else:\n",
" if const_yaml:\n",
@@ -248,7 +248,6 @@
" const_data[mod] = dict()\n",
" for cname, mdata in const_yaml[mod][\"constants\"].items():\n",
" const_data[mod][cname] = dict()\n",
- " print(mdata)\n",
" if mdata[\"creation-time\"]:\n",
" with h5py.File(mdata[\"path\"], \"r\") as cf:\n",
" const_data[mod][cname] = np.copy(\n",
@@ -261,28 +260,37 @@
" constant_names += [\"RelativeGainGotthard2\", \"BadPixelsFFGotthard2\"]\n",
"\n",
" # Retrieve metadata for all pnccd constants.\n",
- "\n",
" const_data = g2_cal.ndarray_map(constant_names)\n",
"\n",
" # Validate the constants availability and raise/warn correspondingly.\n",
" for mod, calibrations in const_data.items():\n",
- " missing_dark_constants = set(\n",
- " c for c in [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"] if c not in calibrations.keys())\n",
- " missing_gain_constants = set(\n",
- " c for c in [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"] if gain_correction and c not in calibrations.keys()) # noqa\n",
- " if missing_dark_constants:\n",
- " raise KeyError(\n",
- " f\"Dark constants {missing_dark_constants} are not available for correction. Mod: {mod}\")\n",
+ " missing_dark_constants = {\"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"} - set(calibrations)\n",
+ " missing_gain_constants = {\"BadPixelsFFGotthard2\", \"RelativeGainGotthard2\"} - set(calibrations)\n",
+ "\n",
+ " if \"LUTGotthard2\" not in calibrations.keys():\n",
+ " warning(f\"LUTGotthard2 is not available to correct {mod}\")\n",
+ " karabo_da.remove(mod)\n",
+ "\n",
+ " if offset_correction and missing_dark_constants:\n",
+ " warning(f\"Dark constants {missing_dark_constants} are not available to correct {mod}.\")\n",
+ " if \"OffsetGotthard2\" in missing_dark_constants:\n",
+ " warning(\"Offset correction is disabled for all modules.\")\n",
+ " offset_correction = False\n",
"\n",
" if gain_correction and missing_gain_constants:\n",
- " warning(f\"Gain constants {missing_gain_constants} are not retrieved. Mod {mod}\")\n",
- " gain_correction = False\n",
+ " warning(f\"Gain constants {missing_gain_constants} are not retrieved for mod {mod}\")\n",
+ " if \"RelativeGainGotthard2\" in missing_gain_constants:\n",
+ " warning(\"Gain correction is disabled for all modules.\")\n",
+ " gain_correction = False\n",
"\n",
" # Create the mask array.\n",
- " bpix = const_data[mod][\"BadPixelsDark\"]\n",
- " if const_data[mod].get(\"BadPixelsFF\") is not None:\n",
- " bpix |= const_data[mod][\"BadPixelsFF\"]\n",
- " const_data[mod][\"Mask\"] = bpix"
+ " bpix = const_data[mod].get(\"BadPixelsDarkGotthard2\", np.zeros((1280, 2, 3), dtype=np.uint32))\n",
+ " if const_data[mod].get(\"BadPixelsFFGotthard2\") is not None:\n",
+ " bpix |= const_data[mod][\"BadPixelsFFGotthard2\"]\n",
+ " const_data[mod][\"Mask\"] = bpix\n",
+ "\n",
+ "if not karabo_da:\n",
+ " raise ValueError(\"LUTGotthard2 constants are not available for all modules.\")"
]
},
{
@@ -304,13 +312,19 @@
"source": [
"def correct_train(wid, index, d):\n",
" g = gain[index]\n",
- " gotthard2algs.convert_to_10bit(d, const_data[mod][\"LUT\"], data_corr[index, ...])\n",
+ " gotthard2algs.convert_to_10bit(d, const_data[mod][\"LUTGotthard2\"], data_corr[index, ...])\n",
" gotthard2algs.correct_train(\n",
" data_corr[index, ...],\n",
" mask[index, ...],\n",
" g,\n",
- " const_data[mod][\"Offset\"],\n",
- " const_data[mod][\"RelativeGain\"].astype(np.float32, copy=False),\n",
+ " const_data[mod].get(\n",
+ " \"OffsetGotthard2\",\n",
+ " np.zeros((1280, 2, 3), dtype=np.float32)\n",
+ " ),\n",
+ " const_data[mod].get(\n",
+ " \"RelativeGainGotthard2\",\n",
+ " np.ones((1280, 2, 3), dtype=np.float32)\n",
+ " ).astype(np.float32, copy=False), # Old gain constants are not float32.\n",
" const_data[mod][\"Mask\"],\n",
" apply_offset=offset_correction,\n",
" apply_gain=gain_correction,\n",
diff --git a/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb b/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
index 3a50b46c7..5dbdd63a6 100644
--- a/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
+++ b/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
@@ -67,12 +67,10 @@
"from pathlib import Path\n",
"\n",
"import cal_tools.restful_config as rest_cfg\n",
- "from calibration_client import CalibrationClient\n",
"from cal_tools.calcat_interface import GOTTHARD2_CalibrationData\n",
"from cal_tools.gotthard2 import gotthard2lib\n",
"from cal_tools.tools import (\n",
" calcat_creation_time,\n",
- " get_from_db,\n",
" CalibrationMetadata,\n",
")"
]
@@ -156,20 +154,27 @@
" constant_names += [\"RelativeGainGotthard2\", \"BadPixelsFFGotthard2\"]\n",
"\n",
"# Retrieve metadata for all pnccd constants.\n",
- "\n",
"g2_metadata = g2_cal.metadata(constant_names)\n",
+ "\n",
+ "missing_lut_modules = set()\n",
"# Validate the constants availability and raise/warn correspondingly.\n",
"for mod, ccv_dict in g2_metadata.items():\n",
- " missing_dark_constants = set(\n",
- " c for c in [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"] if c not in ccv_dict.keys())\n",
- " missing_gain_constants = set(\n",
- " c for c in [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"] if gain_correction and c not in ccv_dict.keys()) # noqa\n",
- " if missing_dark_constants:\n",
- " raise KeyError(\n",
- " f\"Dark constants {missing_dark_constants} are not available for correction. Mod: {mod}\")\n",
- " \n",
+ "\n",
+ " missing_dark_constants = {\"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"} - set(ccv_dict)\n",
+ " missing_gain_constants = {\"BadPixelsFFGotthard2\", \"RelativeGainGotthard2\"} - set(ccv_dict)\n",
+ "\n",
+ " if \"LUTGotthard2\" not in ccv_dict.keys():\n",
+ " warning(f\"LUTGotthard2 is not available to correct {mod}\")\n",
+ " missing_lut_modules.add(mod)\n",
+ "\n",
+ " if offset_correction and missing_dark_constants:\n",
+ " warning(f\"Dark constants {missing_dark_constants} are not available to correct {mod}\")\n",
+ "\n",
" if gain_correction and missing_gain_constants:\n",
- " warning(f\"Gain constants {missing_gain_constants} are not retrieved. Mod {mod}\")\n",
+ " warning(f\"Gain constants {missing_gain_constants} are not retrieved for {mod}\")\n",
+ "\n",
+ "if missing_lut_modules == set(karabo_da):\n",
+ " raise ValueError(\"LUTGotthard2 constants are not available for all modules.\")\n",
"\n",
"# Add constants metadata in retrieved_constants dict.\n",
"for mod, ccv_dict in g2_metadata.items():\n",
@@ -180,6 +185,7 @@
" \"path\": str(g2_cal.caldb_root / ccv_metadata[\"path\"]),\n",
" \"dataset\": ccv_metadata[\"dataset\"],\n",
" \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
+ " \"ccv_id\": ccv_metadata[\"ccv_id\"],\n",
" }\n",
" mod_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n",
" \n",
@@ -199,18 +205,15 @@
" module_constants = retrieved_constants[mod]\n",
" print(f\"Module: {mod}:\")\n",
" for cname, mdata in module_constants[\"constants\"].items():\n",
- " if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
- " mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
" print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
"\n",
- " for cname in [\"Offset\", \"BadPixelsDark\", \"RelativeGain\", \"BadPixelsFF\"]:\n",
+ " for cname in [\"OffsetGotthard2\", \"BadPixelsDarkGotthard2\", \"RelativeGainGotthard2\", \"BadPixelsFFGotthard2\"]:\n",
" if cname in module_constants[\"constants\"]:\n",
" module_timestamps[cname] = module_constants[\"constants\"][cname][\"creation-time\"]\n",
" else:\n",
" module_timestamps[cname] = \"NA\"\n",
"\n",
- "time_summary = retrieved_constants.setdefault(\"time-summary\", {})\n",
- "time_summary[\"SAll\"] = timestamps\n",
+ "retrieved_constants[\"time-summary\"] = timestamps\n",
"\n",
"metadata.save()"
]
--
GitLab