From c35466306c163dd5fb7a1c43a00f68c0e450cc78 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Thu, 23 Feb 2023 09:12:13 +0100
Subject: [PATCH] move const functions to enable correction of modules with no
dark constants
---
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 221 ++++++++++++------
...IPD_Retrieve_Constants_Precorrection.ipynb | 76 ++++--
src/cal_tools/agipdlib.py | 179 +-------------
3 files changed, 210 insertions(+), 266 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index ce74db574..62646f280 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -130,13 +130,13 @@
"from datetime import timedelta\n",
"from logging import warning\n",
"from pathlib import Path\n",
- "from time import perf_counter\n",
"\n",
"import tabulate\n",
"from dateutil import parser\n",
"from IPython.display import Latex, Markdown, display\n",
"\n",
"warnings.filterwarnings('ignore')\n",
+ "import h5py\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"import yaml\n",
@@ -155,7 +155,6 @@
"sns.set_style(\"ticks\")\n",
"\n",
"import cal_tools.restful_config as rest_cfg\n",
- "from calibration_client import CalibrationClient\n",
"from cal_tools import agipdalgs as calgs\n",
"from cal_tools.agipdlib import (\n",
" AgipdCorrections,\n",
@@ -164,6 +163,10 @@
" LitFrameSelection,\n",
")\n",
"from cal_tools.ana_tools import get_range\n",
+ "from cal_tools.calcat_interface import (\n",
+ " AGIPD_CalibrationData,\n",
+ " CalCatError,\n",
+ ")\n",
"from cal_tools.enums import AgipdGainMode, BadPixels\n",
"from cal_tools.step_timing import StepTimer\n",
"from cal_tools.tools import (\n",
@@ -182,7 +185,9 @@
"source": [
"in_folder = Path(in_folder)\n",
"out_folder = Path(out_folder)\n",
- "run_folder = in_folder / f'r{run:04d}'"
+ "run_folder = in_folder / f'r{run:04d}'\n",
+ "\n",
+ "step_timer = StepTimer()"
]
},
{
@@ -328,37 +333,6 @@
" print(f'Processing all valid trains')"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# set everything up filewise\n",
- "mapped_files, _, total_sequences, _, _ = map_modules_from_folder(\n",
- " str(in_folder), run, path_template, karabo_da, sequences\n",
- ")\n",
- "file_list = []\n",
- "\n",
- "# ToDo: Split table over pages\n",
- "print(f\"Processing a total of {total_sequences} sequence files in chunks of {n_cores_files}\")\n",
- "table = []\n",
- "ti = 0\n",
- "for k, files in mapped_files.items():\n",
- " i = 0\n",
- " for f in list(files.queue):\n",
- " file_list.append(f)\n",
- " if i == 0:\n",
- " table.append((ti, k, i, f))\n",
- " else:\n",
- " table.append((ti, \"\", i, f))\n",
- " i += 1\n",
- " ti += 1\n",
- "md = display(Latex(tabulate.tabulate(table, tablefmt='latex',\n",
- " headers=[\"#\", \"module\", \"# module\", \"file\"])))\n",
- "file_list = sorted(file_list, key=lambda name: name[-10:])"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -572,6 +546,57 @@
"# NOTE: this notebook will not overwrite calibration metadata file\n",
"const_yaml = metadata.get(\"retrieved-constants\", {})\n",
"\n",
+ "dark_constants = {\"Offset\", \"Noise\", \"BadPixelsDark\"}\n",
+ "if not gain_mode:\n",
+ " dark_constants.add(\"ThresholdsDark\")\n",
+ "\n",
+ "gain_constants = set()\n",
+ "if any(agipd_corr.pc_bools):\n",
+ " gain_constants |= {\"SlopesPC\", \"BadPixelsPC\"}\n",
+ "if agipd_corr.corr_bools.get(\"xray_corr\"):\n",
+ " gain_constants |= {\"SlopesFF\", \"BadPixelsFF\"}\n",
+ "\n",
+ "all_constants = dark_constants.copy()\n",
+ "all_constants |= gain_constants\n",
+ "\n",
+ "print(f'Preparing constants (FF: {agipd_corr.corr_bools.get(\"xray_corr\", False)}, PC: {any(agipd_corr.pc_bools)}, '\n",
+ " f'BLC: {any(agipd_corr.blc_bools)})')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def check_constants_availability(\n",
+ " const_data: dict, karabo_da: str, \n",
+ ") -> bool : \n",
+ " \"\"\"\n",
+ " Validate the constants availability and raise/warn correspondingly.\n",
+ " \"\"\"\n",
+ " missing_dark_constants = dark_constants - set(const_data)\n",
+ " missing_gain_constants = gain_constants - set(const_data)\n",
+ "\n",
+ " if missing_gain_constants:\n",
+ " warning(\n",
+ " f\"Gain constants {missing_gain_constants} are\"\n",
+ " f\" not retrieved for {karabo_da}\")\n",
+ "\n",
+ " if missing_dark_constants:\n",
+ " warning(f\"Dark constants {missing_dark_constants} are not available\"\n",
+ " f\" to correct {karabo_da}\")\n",
+ " return False\n",
+ " else:\n",
+ " return True"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
"def retrieve_constants(mod):\n",
" \"\"\"\n",
" Retrieve calibration constants and load them to shared memory\n",
@@ -579,35 +604,70 @@
" Metadata for constants is taken from yml file or retrieved from the DB\n",
" \"\"\"\n",
" k_da = module_index_to_karabo_da[mod]\n",
- " # check if there is a yaml file in out_folder that has the device constants.\n",
+ " # string of the device name.\n",
+ " cons_data = dict()\n",
+ " when = dict()\n",
+ " variant = dict()\n",
" if k_da in const_yaml:\n",
- " when = agipd_corr.initialize_from_yaml(k_da, const_yaml, mod)\n",
- " print(f\"Found constants for {k_da} in calibration_metadata.yml\")\n",
- " else:\n",
- " when = agipd_corr.initialize_from_db(\n",
- " karabo_id=karabo_id,\n",
- " karabo_da=k_da,\n",
- " creation_time=creation_time,\n",
- " memory_cells=mem_cells_db,\n",
- " bias_voltage=bias_voltage,\n",
- " photon_energy=9.2,\n",
- " gain_setting=gain_setting,\n",
- " acquisition_rate=acq_rate,\n",
+ " db_module = const_yaml[k_da][\"physical-name\"]\n",
+ " for cname, mdata in const_yaml[k_da][\"constants\"].items():\n",
+ " base_key = f\"{db_module}/{cname}/0\"\n",
+ " when[cname] = mdata[\"creation-time\"]\n",
+ " if when[cname]:\n",
+ " with h5py.File(mdata[\"path\"], \"r\") as cf:\n",
+ " cons_data[cname] = cf[f\"{base_key}/data\"][()]\n",
+ " # Set variant to 0 if the attribute is missing\n",
+ " # as for old constants.\n",
+ " if \"variant\" in cf[base_key].attrs.keys():\n",
+ " variant[cname] = cf[base_key].attrs[\"variant\"]\n",
+ " else:\n",
+ " variant[cname] = 0\n",
+ " else: # Contact the database for module's constants.\n",
+ " agipd_cal = AGIPD_CalibrationData(\n",
+ " detector_name=karabo_id,\n",
+ " modules=[k_da],\n",
+ " sensor_bias_voltage=bias_voltage,\n",
+ " memory_cells=memory_cells,\n",
+ " acquisition_rate=acquisition_rate,\n",
" integration_time=integration_time,\n",
+ " source_energy=photon_energy,\n",
" gain_mode=gain_mode,\n",
- " module_idx=mod,\n",
- " client=rest_cfg.calibration_client(),\n",
+ " gain_setting=gain_setting,\n",
+ " event_at=creation_time,\n",
+ " client=client,\n",
" )\n",
+ "\n",
+ " for cname in all_constants:\n",
+ " if cname in gain_constants:\n",
+ " agipd_cal.gain_mode = None\n",
+ " else:\n",
+ " agipd_cal.gain_mode = gain_mode\n",
+ " try:\n",
+ " agipd_md = agipd_cal.metadata([cname])[k_da]\n",
+ " except CalCatError as e:\n",
+ " pass # a validation of missing constants is done later.\n",
+ " for cname in all_constants:\n",
+ " if agipd_md.get(cname):\n",
+ " when[cname] = agipd_md[cname][\"begin_validity_at\"]\n",
+ " dataset = agipd_md[cname][\"dataset\"]\n",
+ " with h5py.File(agipd_cal.caldb_root / agipd_md[cname][\"path\"], \"r\") as cf: # noqa\n",
+ " cons_data[cname] = np.copy(cf[f\"{dataset}/data\"])\n",
+ " variant[cname] = cf[dataset].attrs[\"variant\"] if cf[dataset].attrs.keys() else 0 # noqa\n",
" print(f\"Queried CalCat for {k_da}\")\n",
- " return mod, when, k_da\n",
"\n",
+ " mod_const_validated = check_constants_availability(cons_data, k_da)\n",
"\n",
- "print(f'Preparing constants (FF: {agipd_corr.corr_bools.get(\"xray_corr\", False)}, PC: {any(agipd_corr.pc_bools)}, '\n",
- " f'BLC: {any(agipd_corr.blc_bools)})')\n",
- "ts = perf_counter()\n",
+ " if mod_const_validated:\n",
+ " agipd_corr.init_constants(cons_data, when, mod, variant)\n",
+ " return mod, when, k_da\n",
+ " else:\n",
+ " print(\"Missing dark constants, hence no correction for this module.\")\n",
+ " return None, None, k_da\n",
+ "\n",
+ "step_timer.start()\n",
"with multiprocessing.Pool(processes=len(modules)) as pool:\n",
" const_out = pool.map(retrieve_constants, modules)\n",
- "print(f\"Constants were loaded in {perf_counter()-ts:.01f}s\")"
+ "step_timer.done_step(f'Constants were loaded in ')"
]
},
{
@@ -622,6 +682,11 @@
"timestamps = {}\n",
"\n",
"for i, (modno, when, k_da) in enumerate(const_out):\n",
+ "\n",
+ " if modno is None:\n",
+ " karabo_da.remove(k_da)\n",
+ " continue\n",
+ "\n",
" qm = module_index_to_qm(modno)\n",
"\n",
" if k_da not in const_yaml:\n",
@@ -646,14 +711,47 @@
" else:\n",
" module_timestamps[key] = \"NA\"\n",
" timestamps[qm] = module_timestamps\n",
+ " \n",
+ "if not karabo_da: # Dark constants are missing for all modules.\n",
+ " raise ValueError(\"Dark constants are missing for all modules.\")\n",
"\n",
"seq = sequences[0] if sequences else 0\n",
- "\n",
"if timestamps:\n",
" with open(f\"{out_folder}/retrieved_constants_s{seq}.yml\",\"w\") as fd:\n",
" yaml.safe_dump({\"time-summary\": {f\"S{seq}\": timestamps}}, fd)"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# set everything up filewise\n",
+ "mapped_files, _, total_sequences, _, _ = map_modules_from_folder(\n",
+ " str(in_folder), run, path_template, karabo_da, sequences\n",
+ ")\n",
+ "file_list = []\n",
+ "\n",
+ "# ToDo: Split table over pages\n",
+ "print(f\"Processing a total of {total_sequences} sequence files in chunks of {n_cores_files}\")\n",
+ "table = []\n",
+ "ti = 0\n",
+ "for k, files in mapped_files.items():\n",
+ " i = 0\n",
+ " for f in list(files.queue):\n",
+ " file_list.append(f)\n",
+ " if i == 0:\n",
+ " table.append((ti, k, i, f))\n",
+ " else:\n",
+ " table.append((ti, \"\", i, f))\n",
+ " i += 1\n",
+ " ti += 1\n",
+ "md = display(Latex(tabulate.tabulate(table, tablefmt='latex',\n",
+ " headers=[\"#\", \"module\", \"# module\", \"file\"])))\n",
+ "file_list = sorted(file_list, key=lambda name: name[-10:])"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -699,15 +797,6 @@
" yield i_proc, i * n_img // n_chunks, (i+1) * n_img // n_chunks"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "step_timer = StepTimer()"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -1028,9 +1117,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"pulse_range = [np.min(pulseId[pulseId>=0]), np.max(pulseId[pulseId>=0])]\n",
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index fe2255505..46b2dad4a 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -90,7 +90,6 @@
"from extra_data import RunDirectory\n",
"\n",
"import cal_tools.restful_config as rest_cfg\n",
- "from calibration_client import CalibrationClient\n",
"from cal_tools.agipdlib import AgipdCtrl, SnowResolution\n",
"from cal_tools.calcat_interface import AGIPD_CalibrationData, CalCatError\n",
"from cal_tools.step_timing import StepTimer\n",
@@ -270,6 +269,26 @@
"acq_rate = all_acq_rates.pop()"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dark_constants = {\"Offset\", \"Noise\", \"BadPixelsDark\"}\n",
+ "if not gain_mode:\n",
+ " dark_constants.add(\"ThresholdsDark\")\n",
+ "\n",
+ "gain_constants = set()\n",
+ "if pc_bools:\n",
+ " gain_constants |= {\"SlopesPC\", \"BadPixelsPC\"}\n",
+ "if xray_gain:\n",
+ " gain_constants |= {\"SlopesFF\", \"BadPixelsFF\"}\n",
+ "\n",
+ "all_constants = dark_constants.copy()\n",
+ "all_constants |= gain_constants"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -292,22 +311,11 @@
" client=rest_cfg.calibration_client(),\n",
")\n",
"\n",
- "dark_constants = [\"Offset\", \"Noise\", \"BadPixelsDark\"] if gain_mode else [\"ThresholdsDark\", \"Offset\", \"Noise\", \"BadPixelsDark\"]\n",
- "\n",
- "gain_constants = []\n",
- "if pc_bools:\n",
- " gain_constants += [\"SlopesPC\", \"BadPixelsPC\"]\n",
- "if xray_gain:\n",
- " gain_constants += [\"SlopesFF\", \"BadPixelsFF\"]\n",
- "\n",
- "constants = dark_constants\n",
- "constants += gain_constants\n",
- "\n",
"# The main metadata dict for all modules and retrieved CCVs.\n",
"agipd_metadata = {mod:{} for mod in agipd_cal.modules}\n",
"\n",
"# This is done to alter the condition for gain constants and avoid using gain_mode.\n",
- "for cname in constants:\n",
+ "for cname in all_constants:\n",
" if cname in gain_constants:\n",
" agipd_cal.gain_mode = None\n",
" else:\n",
@@ -317,21 +325,42 @@
" for mod, ccv_entry in a.items():\n",
" agipd_metadata[mod].update(ccv_entry)\n",
" except CalCatError as e:\n",
- " pass # a validation of missing constants is done later.\n",
- "\n",
+ " pass # Validating missing constants is done later."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "missing_dark_modules = set()\n",
"# Validate the constants availability and raise/warn correspondingly. \n",
"for mod, ccv_dict in agipd_metadata.items():\n",
- " missing_dark_constants = set(\n",
- " c for c in dark_constants if c not in ccv_dict.keys())\n",
- " missing_gain_constants = set(\n",
- " c for c in gain_constants if c not in ccv_dict.keys())\n",
+ "\n",
+ " missing_dark_constants = dark_constants - set(ccv_dict)\n",
" if missing_dark_constants:\n",
- " raise KeyError(\n",
- " f\"Dark constants {missing_dark_constants} are not available for correction. Module: {mod}\") # noqa\n",
+ " warning(\n",
+ " f\"Dark constants {missing_dark_constants} are not available to correct {mod}\") # noqa\n",
+ " missing_dark_modules.add(mod)\n",
+ "\n",
+ " missing_gain_constants = gain_constants - set(ccv_dict)\n",
" if missing_gain_constants:\n",
" warning(\n",
- " f\"Gain constants {missing_gain_constants} were not retrieved. Module: {mod}\")\n",
+ " f\"Gain constants {missing_gain_constants} were not retrieved for {mod}\")\n",
"\n",
+ "if missing_dark_modules == set(karabo_da):\n",
+ " raise ValueError(\n",
+ " \"Dark constants are not available for all modules.\"\n",
+ " \"No correction can be performed.\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
"for mod, ccv_dict in agipd_metadata.items():\n",
" mdata_dict = {\"constants\": dict()}\n",
" for cname, ccv_metadata in ccv_dict.items():\n",
@@ -346,6 +375,7 @@
" \"path\": str(agipd_cal.caldb_root / ccv_metadata[\"path\"]),\n",
" \"dataset\": ccv_metadata[\"dataset\"],\n",
" \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
+ " \"ccv_id\": ccv_metadata[\"ccv_id\"],\n",
" }\n",
" mdata_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n",
" retrieved_constants[mod] = mdata_dict"
@@ -368,8 +398,6 @@
"\n",
" print(f\"{module_name}:\")\n",
" for cname, mdata in module_constants[\"constants\"].items():\n",
- " if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
- " mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
" print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
"\n",
" for cname in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
diff --git a/src/cal_tools/agipdlib.py b/src/cal_tools/agipdlib.py
index 72ff97b49..66a5c2613 100644
--- a/src/cal_tools/agipdlib.py
+++ b/src/cal_tools/agipdlib.py
@@ -1190,12 +1190,12 @@ class AgipdCorrections:
bpixels = cons_data["BadPixelsDark"].astype(np.uint32)
if self.corr_bools.get("xray_corr"):
- if when["BadPixelsFF"]:
+ if when.get("BadPixelsFF"):
bpixels |= cons_data["BadPixelsFF"].astype(np.uint32)[...,
:bpixels.shape[2], # noqa
None]
- if when["SlopesFF"]: # Checking if constant was retrieved
+ if when.get("SlopesFF"): # Checking if constant was retrieved
slopesFF = cons_data["SlopesFF"]
# This could be used for backward compatibility
@@ -1235,7 +1235,7 @@ class AgipdCorrections:
# add additional bad pixel information
if any(self.pc_bools):
- if when["BadPixelsPC"]:
+ if when.get("BadPixelsPC"):
bppc = np.moveaxis(cons_data["BadPixelsPC"].astype(np.uint32),
0, 2)
bpixels |= bppc[..., :bpixels.shape[2], None]
@@ -1243,7 +1243,7 @@ class AgipdCorrections:
# calculate relative gain from the constants
rel_gain = np.ones((128, 512, self.max_cells, 3), np.float32)
- if when["SlopesPC"]:
+ if when.get("SlopesPC"):
slopesPC = cons_data["SlopesPC"].astype(np.float32, copy=False)
# This will handle some historical data in a different format
@@ -1337,177 +1337,6 @@ class AgipdCorrections:
return
- def initialize_from_yaml(
- self, karabo_da: str, const_yaml: Dict[str, Any], module_idx: int
- ) -> Dict[str, Any]:
- """Initialize calibration constants from a yaml file
-
- :param karabo_da: a karabo data aggregator
- :param const_yaml: from the "retrieved-constants" part of a yaml file
- from pre-notebook, which consists of metadata of either the constant
- file path or the empty constant shape, and the creation-time of the
- retrieved constants
- :param module_idx: Index of module
- :return when: dict of retrieved constants with their creation-time
- """
-
- # string of the device name.
- cons_data = dict()
- when = dict()
- variant = dict()
- db_module = const_yaml[karabo_da]["physical-name"]
- for cname, mdata in const_yaml[karabo_da]["constants"].items():
- base_key = f"{db_module}/{cname}/0"
- when[cname] = mdata["creation-time"]
- if when[cname]:
- with h5py.File(mdata["path"], "r") as cf:
- cons_data[cname] = np.copy(cf[f"{base_key}/data"])
- # Set variant to 0 if the attribute is missing
- # as for old constants.
- if "variant" in cf[base_key].attrs.keys():
- variant[cname] = cf[base_key].attrs["variant"]
- else:
- variant[cname] = 0
- else:
- # Create empty constant using the list elements
- cons_data[cname] = getattr(np, mdata["path"][0])(mdata["path"][1]) # noqa
-
- self.init_constants(cons_data, when, module_idx, variant)
-
- return when
-
- def initialize_from_db(
- self, karabo_id: str,
- karabo_da: str,
- module_idx: int,
- creation_time: datetime,
- memory_cells: int,
- bias_voltage: int,
- gain_setting: float,
- gain_mode: int,
- acquisition_rate: float,
- integration_time: int,
- photon_energy: float = 9.2,
- client=None,
- ):
- """ Initialize calibration constants from the calibration database
-
- :param karabo_id: karabo detector identifier.
- :param karabo_da: karabo module data aggregator name.
- :param module_idx: module index to save retrieved CCV in sharedmem.
- :param creation_time: time for desired calibration constant version.
- :param memory_cells: number of memory cells used for CCV conditions.
- :param bias_voltage: bias voltage used for CCV conditions.
- :param gain_setting: gain setting used for CCV conditions.
- :param gain_mode: gain mode used for CCV conditions.
- :param acquisition_rate: acquistion rate used for CCV conditions.
- :param integration_time: integration time used for CCV conditions.
- :param photon_energy: photon energy value used for CCV conditions.
- :param client: calibration_client object for CalibrationData objects.
-
- Loading the following constants based on the
- correction configuration:
-
- Dark Image Derived
- ------------------
-
- * Constants.AGIPD.Offset
- * Constants.AGIPD.Noise
- * Constants.AGIPD.BadPixelsDark
- * Constants.AGIPD.ThresholdsDark
-
- Pulse Capacitor Derived
- -----------------------
-
- * Constants.AGIPD.SlopesPC
- * Constants.AGIPD.BadPixelsPC
-
- Flat-Field Derived
-
- * Constants.AGIPD.SlopesFF
- * Constants.AGIPD.BadPixelsFF
- """
- from cal_tools.calcat_interface import (
- AGIPD_CalibrationData,
- CalCatError,
- )
-
- when = {}
- cons_data = {}
- variant = {}
-
- agipd_cal = AGIPD_CalibrationData(
- detector_name=karabo_id,
- modules=[karabo_da],
- sensor_bias_voltage=bias_voltage,
- memory_cells=memory_cells,
- acquisition_rate=acquisition_rate,
- integration_time=integration_time,
- source_energy=photon_energy,
- gain_mode=gain_mode,
- gain_setting=gain_setting,
- event_at=creation_time,
- client=client,
- )
-
- dark_constants = [
- "Offset",
- "Noise",
- "BadPixelsDark"
- ] if self.gain_mode else [
- "ThresholdsDark",
- "Offset",
- "Noise",
- "BadPixelsDark"
- ]
-
- gain_constants = []
- if any(self.pc_bools):
- gain_constants += ["SlopesPC", "BadPixelsPC"]
- if self.corr_bools.get("xray_corr"):
- gain_constants += ["SlopesFF", "BadPixelsFF"]
- metadata = {karabo_da: {}}
- constants = dark_constants
- constants += gain_constants
- for cname in constants:
- if cname in gain_constants:
- agipd_cal.gain_mode = None
- else:
- agipd_cal.gain_mode = gain_mode
- try:
- metadata[karabo_da].update(
- agipd_cal.metadata([cname])[karabo_da])
- except CalCatError as e:
- pass # a validation of missing constants is done later.
-
- # Validate the constants availability and raise/warn correspondingly.
- missing_dark_constants = set(
- c for c in dark_constants if c not in metadata[karabo_da].keys())
- missing_gain_constants = set(
- c for c in gain_constants if c not in metadata[karabo_da].keys())
- if missing_dark_constants:
- raise KeyError(
- f"Dark constants {missing_dark_constants} are not available"
- f" for correction. Module: {karabo_da}")
- if missing_gain_constants:
- warning(
- f"Gain constants {missing_gain_constants} were"
- f" not retrieved. Module: {karabo_da}")
-
- for cname in constants:
- if metadata[karabo_da].get(cname):
- when[cname] = metadata[karabo_da][cname]["begin_validity_at"]
- dataset = metadata[karabo_da][cname]["dataset"]
- with h5py.File(agipd_cal.caldb_root / metadata[karabo_da][cname]["path"], "r") as cf: # noqa
- cons_data[cname] = np.copy(cf[f"{dataset}/data"])
- variant[cname] = cf[dataset].attrs["variant"] if cf[dataset].attrs.keys() else 0 # noqa
- else:
- when[cname] = None
-
- self.init_constants(cons_data, when, module_idx, variant)
-
- return when
-
def allocate_constants(self, modules, constant_shape):
"""
Allocate memory for correction constants
--
GitLab