diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 5201f84d61c2837f4254de7a761152fd012fabb0..eee1cf0950b0f5ce15fcf87ec7fca01279081e50 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -164,7 +164,6 @@
")\n",
"from cal_tools.ana_tools import get_range\n",
"from cal_tools.enums import AgipdGainMode, BadPixels\n",
- "from cal_tools.restful_config import restful_config\n",
"from cal_tools.step_timing import StepTimer\n",
"from cal_tools.tools import (\n",
" CalibrationMetadata,\n",
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index dc6bf84a22374919cf26560c34116f024ae26ded..31d42f0087a46c8b1002c0591351219f22980ed1 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -93,7 +93,6 @@
"from calibration_client import CalibrationClient\n",
"from cal_tools.agipdlib import AgipdCtrl, SnowResolution\n",
"from cal_tools.calcat_interface import AGIPD_CalibrationData, CalCatError\n",
- "from cal_tools.restful_config import restful_config\n",
"from cal_tools.step_timing import StepTimer\n",
"from cal_tools.tools import CalibrationMetadata, calcat_creation_time, module_index_to_qm\n",
"from cal_tools.enums import AgipdGainMode"
@@ -307,10 +306,12 @@
"for cname in constants:\n",
" if cname in gain_constants:\n",
" agipd_cal.gain_mode = None\n",
+ " else:\n",
+ " agipd_cal.gain_mode = gain_mode\n",
" try:\n",
" a = agipd_cal.metadata([cname])\n",
" for mod, ccv_entry in a.items():\n",
- " agipd_metadata[mod].update(a[mod])\n",
+ " agipd_metadata[mod].update(ccv_entry)\n",
" except CalCatError as e:\n",
" pass # a validation of missing constants is done later.\n",
"\n",
diff --git a/src/cal_tools/agipdlib.py b/src/cal_tools/agipdlib.py
index 13781f7cd78cfb9197bfbf0f5352d18aea735dc0..0225bafa65d56dc4f54431f5f639224f2d442083 100644
--- a/src/cal_tools/agipdlib.py
+++ b/src/cal_tools/agipdlib.py
@@ -1409,7 +1409,10 @@ class AgipdCorrections:
* Constants.AGIPD.SlopesFF
* Constants.AGIPD.BadPixelsFF
"""
- from cal_tools.calcat_interface import AGIPD_CalibrationData
+ from cal_tools.calcat_interface import (
+ AGIPD_CalibrationData,
+ CalCatError,
+ )
when = {}
cons_data = {}
@@ -1445,24 +1448,33 @@ class AgipdCorrections:
gain_constants += ["SlopesPC", "BadPixelsPC"]
if self.corr_bools.get("xray_corr"):
gain_constants += ["SlopesFF", "BadPixelsFF"]
-
+ metadata = {karabo_da: {}}
constants = dark_constants
constants += gain_constants
- metadata = agipd_cal.metadata(constants)
+ for cname in constants:
+ if cname in gain_constants:
+ agipd_cal.gain_mode = None
+ else:
+ agipd_cal.gain_mode = gain_mode
+ try:
+ metadata[karabo_da].update(
+ agipd_cal.metadata([cname])[karabo_da])
+ except CalCatError as e:
+ pass # a validation of missing constants is done later.
+
# Validate the constants availability and raise/warn correspondingly.
- for mod, ccv_dict in metadata.items():
- missing_dark_constants = set(
- c for c in dark_constants if c not in ccv_dict.keys())
- missing_gain_constants = set(
- c for c in gain_constants if c not in ccv_dict.keys())
- if missing_dark_constants:
- raise KeyError(
- f"Dark constants {missing_dark_constants} are not available"
- f" for correction. Module: {mod}")
- if missing_gain_constants:
- warning(
- f"Gain constants {missing_gain_constants} were"
- f" not retrieved. Module: {mod}")
+ missing_dark_constants = set(
+ c for c in dark_constants if c not in metadata[karabo_da].keys())
+ missing_gain_constants = set(
+ c for c in gain_constants if c not in metadata[karabo_da].keys())
+ if missing_dark_constants:
+ warning(
+ f"Dark constants {missing_dark_constants} are not available"
+ f" for correction. Module: {karabo_da}")
+ if missing_gain_constants:
+ warning(
+ f"Gain constants {missing_gain_constants} were"
+ f" not retrieved. Module: {karabo_da}")
for cname in constants:
if metadata[karabo_da].get(cname):
diff --git a/src/cal_tools/calcat_interface.py b/src/cal_tools/calcat_interface.py
index b51df9aa6c5734c8275ba83d0bf854ca282f4d37..b4843d4df6257b5f015cfc93e859c6f9bca2d8d1 100644
--- a/src/cal_tools/calcat_interface.py
+++ b/src/cal_tools/calcat_interface.py
@@ -106,7 +106,7 @@ class CalCatApi(metaclass=ClientWrapper):
"""Parse different ways to specify time to CalCat."""
if isinstance(dt, datetime):
- return dt.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%S%Z")
+ return dt.astimezone(timezone.utc).isoformat()
elif isinstance(dt, date):
return cls.format_time(datetime.combine(dt, time()))
@@ -145,13 +145,13 @@ class CalCatApi(metaclass=ClientWrapper):
def physical_detector_units(
self,
detector_id,
- snapshot_at,
- module_naming="da",
+ pdu_snapshot_at,
+ module_naming="da"
):
"""Physical detector unit metadata."""
resp_pdus = PhysicalDetectorUnit.get_all_by_detector(
- self.client, detector_id, self.format_time(snapshot_at)
+ self.client, detector_id, self.format_time(pdu_snapshot_at)
)
if not resp_pdus["success"]:
@@ -181,6 +181,7 @@ class CalCatApi(metaclass=ClientWrapper):
else:
raise ValueError(f"{module_naming} is unknown!")
+
@lru_cache()
def calibration_id(self, calibration_name):
"""ID for a calibration in CalCat."""
@@ -212,7 +213,7 @@ class CalCatApi(metaclass=ClientWrapper):
condition,
modules=None,
event_at=None,
- snapshot_at=None,
+ pdu_snapshot_at=None,
metadata=None,
module_naming="da",
):
@@ -233,7 +234,7 @@ class CalCatApi(metaclass=ClientWrapper):
or None for all (default).
event_at (datetime, date, str or None): Time at which the
CCVs should have been valid or None for now (default).
- snapshot_at (datetime, date, str or None): Time of database
+ pdu_snapshot_at (datetime, date, str or None): Time of database
state to look at or None for now (default).
metadata (dict or None): Mapping to fill for results or
None for a new dictionary (default).
@@ -251,7 +252,7 @@ class CalCatApi(metaclass=ClientWrapper):
passed.
"""
event_at = self.format_time(event_at)
- snapshot_at = self.format_time(snapshot_at)
+ pdu_snapshot_at = self.format_time(pdu_snapshot_at)
if metadata is None:
metadata = CCVMetadata()
@@ -270,15 +271,22 @@ class CalCatApi(metaclass=ClientWrapper):
}
calibration_ids = list(cal_id_map.keys())
+ # Map aggregator to the selected module name.
+ da_to_modname = {
+ data['karabo_da']: mod_name for mod_name, data in
+ self.physical_detector_units(
+ self.detector(detector_name)['id'],
+ pdu_snapshot_at,
+ module_naming=module_naming
+ ).items()
+ if not modules or mod_name in modules
+ }
# The API call supports a single module or all modules, as the
# performance increase is only minor in between. Hence, all
# modules are queried if more than one is selected and filtered
# afterwards, if necessary.
- karabo_da = (
- next(iter(modules))
- if modules is not None and len(modules) == 1
- else "",
- ) # noqa
+ karabo_da = next(iter(da_to_modname)) if len(da_to_modname) == 1 else '',
+
resp_versions = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions( # noqa
self.client,
detector_name,
@@ -286,7 +294,7 @@ class CalCatApi(metaclass=ClientWrapper):
self.format_cond(condition),
karabo_da=karabo_da,
event_at=event_at,
- snapshot_at=snapshot_at,
+ pdu_snapshot_at=pdu_snapshot_at,
)
if not resp_versions["success"]:
@@ -294,17 +302,7 @@ class CalCatApi(metaclass=ClientWrapper):
for ccv in resp_versions["data"]:
try:
- if module_naming == "da":
- mod = ccv["physical_detector_unit"]["karabo_da"]
- # Can be used for AGIPD, LPD, and DSSC.
- elif module_naming == "qm":
- mod = module_index_to_qm(
- int(ccv["physical_detector_unit"]["karabo_da"][-2:])
- )
- elif module_naming == "modno":
- mod = int(ccv["physical_detector_unit"]["karabo_da"][-2:])
- else:
- raise ValueError(f"{module_naming} is unknown!")
+ mod = da_to_modname[ccv['physical_detector_unit']['karabo_da']]
except KeyError:
# Not included in our modules
continue
@@ -376,7 +374,6 @@ class CalibrationData:
modules=None,
client=None,
event_at=None,
- snapshot_at=None,
module_naming="da",
):
"""Initialize a new CalibrationData object.
@@ -393,8 +390,6 @@ class CalibrationData:
communication, global one by default.
event_at (datetime, date, str or None): Default time at which the
CCVs should have been valid, now if omitted
- snapshot_at (datetime, date, str or None): Default time of
- database state to look at, now if omitted.
module_naming (str or None): Expected module name convention to be
used as metadata dict keys. Expected values are:
`da`: data aggregator name is used. Default.
@@ -409,10 +404,11 @@ class CalibrationData:
self.detector_name = detector_name
self.modules = modules
self.event_at = event_at
- self.snapshot_at = snapshot_at
+ self.pdu_snapshot_at = event_at
self.module_naming = module_naming
if client is None:
+
client = (
self.__class__.default_client
or self.__class__.new_anonymous_client()
@@ -525,7 +521,7 @@ class CalibrationData:
@property
def physical_detector_units(self):
return self._api.physical_detector_units(
- self.detector["id"], self.snapshot_at, self.module_naming
+ self.detector["id"], self.pdu_snapshot_at, self.module_naming
)
@property
@@ -540,7 +536,7 @@ class CalibrationData:
"modules",
"client",
"event_at",
- "snapshot_at",
+ "pdu_snapshot_at",
} | {self._simplify_parameter_name(name) for name in self.parameters}
kwargs = {key: getattr(self, key) for key in keys}
@@ -552,7 +548,7 @@ class CalibrationData:
self,
calibrations=None,
event_at=None,
- snapshot_at=None,
+ pdu_snapshot_at=None,
):
"""Query CCV metadata for calibrations, conditions and time.
@@ -562,7 +558,7 @@ class CalibrationData:
event_at (datetime, date, str or None): Time at which the
CCVs should have been valid, now or default value passed at
initialization time if omitted.
- snapshot_at (datetime, date, str or None): Time of database
+ pdu_snapshot_at (datetime, date, str or None): Time of database
state to look at, now or default value passed at
initialization time if omitted.
@@ -577,7 +573,7 @@ class CalibrationData:
self.condition,
self.modules,
event_at or self.event_at,
- snapshot_at or self.snapshot_at,
+ pdu_snapshot_at or self.pdu_snapshot_at,
metadata,
module_naming=self.module_naming,
)
@@ -587,7 +583,7 @@ class CalibrationData:
self,
module,
calibration,
- metadata,
+ metadata=None,
):
"""Load CCV data as ndarray.
@@ -600,6 +596,8 @@ class CalibrationData:
Returns:
(ndarray): CCV data
"""
+ import numpy as np
+
if self.caldb_root is None:
raise RuntimeError("calibration database store unavailable")
@@ -609,9 +607,12 @@ class CalibrationData:
if metadata is None:
metadata = self.metadata([calibration])
- return self._load_ccv_data(metadata, module, calibration)
+ row = metadata[module][calibration]
+
+ with h5py.File(self.caldb_root / row['path'], 'r') as f:
+ return np.asarray(f[row['dataset'] + '/data'])
- def _allocate_const_arrays(self, metadata, const_load_mp, const_data):
+ def _allocate_constant_arrays(self, metadata, const_load_mp, const_data):
for mod, ccv_entry in metadata.items():
const_data[mod] = {}
@@ -625,8 +626,23 @@ class CalibrationData:
shape=shape, dtype=dtype
)
- def load_constants_data(self, metadata):
- def load_constant_dataset(wid, index, mod):
+ def load_constants_from_metadata(self, metadata):
+ """Load the data for all constants in metadata object.
+
+ Args:
+ metadata (CCVMetadata, optional): CCV metadata to load
+ constant data for, may be None to query metadata.
+ Returns:
+ (Dict): A dictionary of constant data.
+ {module: {calibration: ndarray}}.
+ """
+ def _load_constant_dataset(wid, index, mod):
+ """Load constant dataset from the CCVMetadata `metadata` into
+ a shared allocated array.
+
+ Args:
+ mod (str): module key in `metadata` object
+ """
for cname, mdata in metadata[mod].items():
with h5py.File(self.caldb_root / mdata["path"], "r") as cf:
cf[f"{mdata['dataset']}/data"].read_direct(
@@ -635,8 +651,8 @@ class CalibrationData:
const_data = dict()
const_load_mp = psh.ProcessContext(num_workers=24)
- self._allocate_const_arrays(metadata, const_load_mp, const_data)
- const_load_mp.map(load_constant_dataset, list(metadata.keys()))
+ self._allocate_constant_arrays(metadata, const_load_mp, const_data)
+ const_load_mp.map(_load_constant_dataset, list(metadata.keys()))
return const_data
@@ -663,42 +679,15 @@ class CalibrationData:
if metadata is None:
metadata = self.metadata(calibrations)
- return self.load_constants_data(metadata)
-
- def data_map(
- self,
- calibrations=None,
- metadata=None,
- ):
- """Load all CCV data in a nested map of ndarrays.
-
- Args:
- calibrations (Iterable of str, optional): Calibration constants
- or None for all available (default).
- metadata (CCVMetadata, optional): CCV metadata to load constant
- for or None to query metadata automatically (default).
- Returns:
- (dict of dict of ndarray): CCV data by module number and
- calibration constant name.
- {module: {calibration: ndarray}}
- """
- if self.caldb_root is None:
- raise RuntimeError("calibration database store unavailable")
-
- if metadata is None:
- metadata = self.metadata(calibrations)
-
- return self.load_constants_data(metadata)
+ return self.load_constants_from_metadata(metadata)
def _build_condition(self, parameters):
cond = dict()
for db_name in parameters:
value = getattr(self, self._simplify_parameter_name(db_name), None)
-
if value is not None:
cond[db_name] = value
-
return cond
@classmethod
@@ -842,7 +831,7 @@ class SplitConditionCalibrationData(CalibrationData):
self,
calibrations=None,
event_at=None,
- snapshot_at=None,
+ pdu_snapshot_at=None,
):
"""Query CCV metadata for calibrations, conditions and time.
@@ -852,7 +841,7 @@ class SplitConditionCalibrationData(CalibrationData):
event_at (datetime, date, str or None): Time at which the
CCVs should have been valid, now or default value passed at
initialization time if omitted.
- snapshot_at (datetime, date, str or None): Time of database
+ pdu_snapshot_at (datetime, date, str or None): Time of database
state to look at, now or default value passed at
initialization time if omitted.
@@ -878,7 +867,7 @@ class SplitConditionCalibrationData(CalibrationData):
self.dark_condition,
self.modules,
event_at or self.event_at,
- snapshot_at or self.snapshot_at,
+ pdu_snapshot_at or self.pdu_snapshot_at,
metadata,
module_naming=self.module_naming,
)
@@ -892,7 +881,7 @@ class SplitConditionCalibrationData(CalibrationData):
self.illuminated_condition,
self.modules,
event_at or self.event_at,
- snapshot_at or self.snapshot_at,
+ pdu_snapshot_at or self.pdu_snapshot_at,
metadata,
module_naming=self.module_naming,
)
@@ -937,7 +926,6 @@ class AGIPD_CalibrationData(SplitConditionCalibrationData):
modules=None,
client=None,
event_at=None,
- snapshot_at=None,
gain_setting=None,
gain_mode=None,
module_naming="da",
@@ -951,7 +939,6 @@ class AGIPD_CalibrationData(SplitConditionCalibrationData):
modules,
client,
event_at,
- snapshot_at,
module_naming,
)
@@ -967,12 +954,15 @@ class AGIPD_CalibrationData(SplitConditionCalibrationData):
def _build_condition(self, parameters):
cond = super()._build_condition(parameters)
-
# Fix-up some database quirks.
- if int(cond.get("Gain mode", -1)) == 0:
+ gain_mode = cond.get("Gain mode", None)
+
+ if gain_mode is not None and int(gain_mode) == 0:
del cond["Gain mode"]
+ elif gain_mode is not None:
+ cond["Gain mode"] = 1
- if int(cond.get("Integration time", -1)) == 12:
+ if int(cond.get("Integration time", None)) == 12:
del cond["Integration time"]
return cond
@@ -1015,7 +1005,6 @@ class LPD_CalibrationData(SplitConditionCalibrationData):
modules=None,
client=None,
event_at=None,
- snapshot_at=None,
module_naming="da",
):
super().__init__(
@@ -1023,7 +1012,6 @@ class LPD_CalibrationData(SplitConditionCalibrationData):
modules,
client,
event_at,
- snapshot_at,
module_naming,
)
@@ -1068,7 +1056,6 @@ class DSSC_CalibrationData(CalibrationData):
modules=None,
client=None,
event_at=None,
- snapshot_at=None,
module_naming="da",
):
super().__init__(
@@ -1076,7 +1063,6 @@ class DSSC_CalibrationData(CalibrationData):
modules,
client,
event_at,
- snapshot_at,
module_naming,
)
@@ -1124,7 +1110,6 @@ class JUNGFRAU_CalibrationData(CalibrationData):
modules=None,
client=None,
event_at=None,
- snapshot_at=None,
module_naming="da",
):
super().__init__(
@@ -1132,7 +1117,6 @@ class JUNGFRAU_CalibrationData(CalibrationData):
modules,
client,
event_at,
- snapshot_at,
module_naming,
)
@@ -1155,15 +1139,20 @@ class JUNGFRAU_CalibrationData(CalibrationData):
return cond
-class PNCCD_CalibrationData(CalibrationData):
- calibrations = {
+class PNCCD_CalibrationData(SplitConditionCalibrationData):
+ """Calibration data for the pnCCD detector."""
+
+ dark_calibrations = {
"OffsetCCD",
"BadPixelsDarkCCD",
"NoiseCCD",
+ }
+ illuminated_calibrations = {
"RelativeGainCCD",
"CTECCD",
}
- parameters = [
+
+ dark_parameters = [
"Sensor Bias Voltage",
"Memory cells",
"Pixels X",
@@ -1173,6 +1162,8 @@ class PNCCD_CalibrationData(CalibrationData):
"Gain Setting",
]
+ illuminated_parameters = dark_parameters + ["Source energy"]
+
def __init__(
self,
detector_name,
@@ -1180,20 +1171,20 @@ class PNCCD_CalibrationData(CalibrationData):
integration_time,
sensor_temperature,
gain_setting,
+ source_energy=9.2,
pixels_x=1024,
pixels_y=1024,
+ modules=None,
client=None,
event_at=None,
- snapshot_at=None,
module_naming="da",
):
# Ignore modules for this detector.
super().__init__(
detector_name,
- None,
+ modules,
client,
event_at,
- snapshot_at,
module_naming,
)
@@ -1204,9 +1195,11 @@ class PNCCD_CalibrationData(CalibrationData):
self.integration_time = integration_time
self.sensor_temperature = sensor_temperature
self.gain_setting = gain_setting
-
+ self.source_energy = source_energy
class EPIX100_CalibrationData(SplitConditionCalibrationData):
+ """Calibration data for the ePix100 detector."""
+
dark_calibrations = {
"OffsetEPix100",
"NoiseEPix100",
@@ -1237,18 +1230,17 @@ class EPIX100_CalibrationData(SplitConditionCalibrationData):
pixels_x=708,
pixels_y=768,
source_energy=9.2,
+ modules=None,
client=None,
event_at=None,
- snapshot_at=None,
module_naming="da",
):
# Ignore modules for this detector.
super().__init__(
detector_name,
- None,
+ modules,
client,
event_at,
- snapshot_at,
module_naming,
)
@@ -1263,6 +1255,8 @@ class EPIX100_CalibrationData(SplitConditionCalibrationData):
class GOTTHARD2_CalibrationData(CalibrationData):
+ """Calibration data for the Gotthard II detector."""
+
calibrations = {
"LUTGotthard2" "OffsetGotthard2",
"NoiseGotthard2",
@@ -1286,18 +1280,17 @@ class GOTTHARD2_CalibrationData(CalibrationData):
exposure_period,
acquisition_rate,
single_photon,
+ modules=None,
client=None,
event_at=None,
- snapshot_at=None,
module_naming="da",
):
# Ignore modules for this detector.
super().__init__(
detector_name,
- None,
+ modules,
client,
event_at,
- snapshot_at,
module_naming,
)