diff --git a/src/cal_tools/agipdlib.py b/src/cal_tools/agipdlib.py
index a39ca0ea0ab304fad6d35c8f2be7b25bd0193317..4b4d1e3eb2cb3072566550f09939c0f948c58867 100644
--- a/src/cal_tools/agipdlib.py
+++ b/src/cal_tools/agipdlib.py
@@ -24,7 +24,7 @@ from cal_tools.agipdutils import (
)
from cal_tools.enums import AgipdGainMode, BadPixels, SnowResolution
from cal_tools.h5_copy_except import h5_copy_except_paths
-from cal_tools.tools import get_constant_from_db_and_time
+from cal_tools.tools import get_from_db
class AgipdCtrl:
def __init__(
@@ -1014,16 +1014,21 @@ class AgipdCorrections:
when = {}
cons_data = {}
for cname, cval in const_dict.items():
- condition = \
- getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])
- cons_data[cname], when[cname] = \
- get_constant_from_db_and_time(karabo_id, karabo_da,
- getattr(Constants.AGIPD, cname)(), # noqa
- condition,
- getattr(np, cval[0])(cval[1]),
- cal_db_interface,
- creation_time,
- print_once=0)
+ condition = getattr(
+ Conditions, cval[2][0]).AGIPD(**cval[2][1])
+ cons_data[cname] = {}
+ cons_data[cname]['data'], md = get_from_db(
+ karabo_id, karabo_da,
+ getattr(Constants.AGIPD, cname)(),
+ condition,
+ getattr(np, cval[0])(cval[1]),
+ cal_db_interface,
+ creation_time,
+ print_once=0,
+ )
+ cons_data[cname]["version"] = md.calibration_constant_version.ccv_tag
+ when[cname] = md.calibration_constant_version.begin_at
+
return cons_data, when
def init_constants(self, cons_data, when, module_idx):
@@ -1075,11 +1080,11 @@ class AgipdCorrections:
# assuming this method runs in parallel.
calgs_opts = dict(num_threads=os.cpu_count() // len(self.offset))
- calgs.transpose_constant(self.offset[module_idx], cons_data['Offset'], **calgs_opts)
- calgs.transpose_constant(self.noise[module_idx], cons_data['Noise'], **calgs_opts)
+ calgs.transpose_constant(self.offset[module_idx], cons_data['data']['Offset'], **calgs_opts)
+ calgs.transpose_constant(self.noise[module_idx], cons_data['data']['Noise'], **calgs_opts)
if self.gain_mode is AgipdGainMode.ADAPTIVE_GAIN:
calgs.transpose_constant(self.thresholds[module_idx],
- cons_data['ThresholdsDark'][..., :3],
+ cons_data['data']['ThresholdsDark'][..., :3],
**calgs_opts)
if self.corr_bools.get("low_medium_gap"):
@@ -1087,17 +1092,17 @@ class AgipdCorrections:
t1 = self.thresholds[module_idx][1]
t1[t1 <= 1.05 * t0] = np.iinfo(np.uint16).max
- bpixels = cons_data["BadPixelsDark"].astype(np.uint32)
+ bpixels = cons_data["data"]["BadPixelsDark"].astype(np.uint32)
if self.corr_bools.get("xray_corr"):
if when["BadPixelsFF"]:
- bpixels |= cons_data["BadPixelsFF"].astype(np.uint32)[...,
+ bpixels |= cons_data["data"]["BadPixelsFF"].astype(np.uint32)[...,
:bpixels.shape[2], # noqa
None]
if when["SlopesFF"]: # Checking if constant was retrieved
- slopesFF = cons_data["SlopesFF"]
+ slopesFF = cons_data["data"]["SlopesFF"]
# This could be used for backward compatibility
# for very old SlopesFF constants
if len(slopesFF.shape) == 4:
@@ -1134,7 +1139,7 @@ class AgipdCorrections:
# add additional bad pixel information
if any(self.pc_bools):
if when["BadPixelsPC"]:
- bppc = np.moveaxis(cons_data["BadPixelsPC"].astype(np.uint32),
+ bppc = np.moveaxis(cons_data["data"]["BadPixelsPC"].astype(np.uint32),
0, 2)
bpixels |= bppc[..., :bpixels.shape[2], None]
@@ -1142,7 +1147,7 @@ class AgipdCorrections:
rel_gain = np.ones((128, 512, self.max_cells, 3), np.float32)
if when["SlopesPC"]:
- slopesPC = cons_data["SlopesPC"].astype(np.float32, copy=False)
+ slopesPC = cons_data["data"]["SlopesPC"].astype(np.float32, copy=False)
# This will handle some historical data in a different format
# constant dimension injected first
@@ -1243,13 +1248,20 @@ class AgipdCorrections:
when = dict()
db_module = const_yaml[karabo_da]["physical-detector-unit"]
for cname, mdata in const_yaml[karabo_da]["constants"].items():
+ cons_data[cname] = {}
when[cname] = mdata["creation-time"]
if when[cname]:
with h5py.File(mdata["file-path"], "r") as cf:
- cons_data[cname] = np.copy(cf[f"{db_module}/{cname}/0/data"]) # noqa
+ cons_data[cname]["data"] = np.copy(
+ cf[f"{db_module}/{cname}/0/data"])
+ # TODO: choose an acceptable name.
+ if f"{db_module}/{cname}/0/version" in cf.keys():
+ cons_data[cname]["version"] = cf[
+ f"{db_module}/{cname}/0/version"]
else:
# Create empty constant using the list elements
- cons_data[cname] = getattr(np, mdata["file-path"][0])(mdata["file-path"][1]) # noqa
+ cons_data[cname]["data"] = getattr(
+ np, mdata["file-path"][0])(mdata["file-path"][1])
self.init_constants(cons_data, when, module_idx)
diff --git a/src/cal_tools/tools.py b/src/cal_tools/tools.py
index 71fdba355ccbe047d958069b6d5567aeb437274a..1746c6f0879c32565876a70f488ee543bf2ec915 100644
--- a/src/cal_tools/tools.py
+++ b/src/cal_tools/tools.py
@@ -603,7 +603,9 @@ def get_from_db(karabo_id: str, karabo_da: str,
)
with h5py.File(Path(hdf5path, filename), "r") as f:
arr = f[f"{h5path}/data"][()]
+ version = f[f"{h5path}/version"][()]
metadata.calibration_constant.data = arr
+ metadata.calibration_constant_version.ccv_tag = version
if verbosity > 0:
if constant.name not in already_printed or verbosity > 1:
@@ -625,7 +627,8 @@ def send_to_db(db_module: str, karabo_id: str, constant, condition,
creation_time: Optional[datetime.datetime] = None,
timeout: int = 30000,
ntries: int = 7,
- doraise: bool = False):
+ doraise: bool = False,
+ ccv_tag: Optional[int] = None):
"""Return calibration constants and metadata requested from CalDB
:param db_module: database module (PDU/Physical Detector Unit)
@@ -634,14 +637,16 @@ def send_to_db(db_module: str, karabo_id: str, constant, condition,
:param condition: Calibration condition
:param file_loc: Location of raw data.
:param report_path: xfel-calbrate report path to inject along with
- the calibration constant versions to the database.
+ the calibration constant versions to the database.
:param cal_db_interface: Interface string, e.g. "tcp://max-exfl016:8015"
:param creation_time: Latest time for constant to be created
:param timeout: Timeout for zmq request
:param ntries: number of tries to contact the database,
- ntries is set to 7 so that if the timeout started
- at 30s last timeout will be ~ 1h.
+ ntries is set to 7 so that if the timeout started
+ at 30s last timeout will be ~ 1h.
:param doraise: if True raise errors during communication with DB
+ :param ccv_tag: A calibration constant version tag to save in the
+ h5 file along with the constant data.
"""
success = False
@@ -670,6 +675,7 @@ def send_to_db(db_module: str, karabo_id: str, constant, condition,
metadata.calibration_constant_version.device_name = db_module
metadata.calibration_constant_version.karabo_da = None
metadata.calibration_constant_version.raw_data_location = file_loc
+ metadata.calibration_constant_version.ccv_tag = ccv_tag
if constant.data is None:
raise ValueError(
"There is no data available to "