diff --git a/setup.py b/setup.py
index f8a3aea1a8b9e485360590716d7d80336a12cebb..f490a0e02c2ea7524702f2a1543c446b5081df7a 100644
--- a/setup.py
+++ b/setup.py
@@ -94,7 +94,7 @@ install_requires = [
if "readthedocs.org" not in sys.executable:
install_requires += [
- "iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.1.0", # noqa
+ "iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.2.0", # noqa
"XFELDetectorAnalysis @ git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git@2.7.0", # noqa
]
diff --git a/src/cal_tools/agipdlib.py b/src/cal_tools/agipdlib.py
index a39ca0ea0ab304fad6d35c8f2be7b25bd0193317..5f0c1d137502f99df99d018cb68356206fd5b2c3 100644
--- a/src/cal_tools/agipdlib.py
+++ b/src/cal_tools/agipdlib.py
@@ -1,6 +1,7 @@
import os
import posixpath
import zlib
+from datetime import datetime
from multiprocessing.pool import ThreadPool
from typing import Any, Dict, List, Optional, Tuple
@@ -24,7 +25,7 @@ from cal_tools.agipdutils import (
)
from cal_tools.enums import AgipdGainMode, BadPixels, SnowResolution
from cal_tools.h5_copy_except import h5_copy_except_paths
-from cal_tools.tools import get_constant_from_db_and_time
+from cal_tools.tools import get_from_db
class AgipdCtrl:
def __init__(
@@ -103,7 +104,7 @@ class AgipdCtrl:
def get_gain_setting(
self,
- creation_time: "datetime.datetime",
+ creation_time: datetime,
) -> Optional[int]:
"""Retrieve Gain setting.
@@ -987,46 +988,7 @@ class AgipdCorrections:
data=cntsv,
fletcher32=True)
- def retrieve_constant_and_time(self, karabo_id: str, karabo_da: str,
- const_dict: Dict[str, Any],
- cal_db_interface: str,
- creation_time: str
- ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
- """A wrapper around get_constant_from_db_and_time to get
- constant image and creation time using a dictionary of
- the names of the required constant from the data base
- and their parameter operating conditions.
-
- :param karabo_id: karabo identifier
- :param karabo_da: karabo data aggregator
- :param const_dict: (Dict) A dictionary with constants to retrieve
- and their operating conditions.
- e.g.{ "Offset": ["zeros", (128, 512, memory_cells, 3), "Dark"]
- "Dark-cond": {Condition-name}: condition-value }
- :param cal_db_interface: (Str) The port interface for
- calibrationDBRemote
- :param creation_time: (Str) Raw data creation time
- :return:
- cons_data: (np.array) Constant data
- when: (Dict) A dictionary with all retrieved constants
- and their creation-time. {"Constant-Name": "creation-time"}
- """
- when = {}
- cons_data = {}
- for cname, cval in const_dict.items():
- condition = \
- getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])
- cons_data[cname], when[cname] = \
- get_constant_from_db_and_time(karabo_id, karabo_da,
- getattr(Constants.AGIPD, cname)(), # noqa
- condition,
- getattr(np, cval[0])(cval[1]),
- cal_db_interface,
- creation_time,
- print_once=0)
- return cons_data, when
-
- def init_constants(self, cons_data, when, module_idx):
+ def init_constants(self, cons_data, when, module_idx, variant):
"""
For CI derived gain, a mean multiplication factor of 4.48 compared
to medium gain is used, as no reliable CI data for all memory cells
@@ -1068,6 +1030,7 @@ class AgipdCorrections:
:param when: A dictionary for the creation time
of each retrieved constant.
:param module_idx: A module_idx index
+ :param variant: A dictionary for the variant of each retrieved CCV.
:return:
"""
@@ -1241,23 +1204,31 @@ class AgipdCorrections:
# string of the device name.
cons_data = dict()
when = dict()
+ variant = dict()
db_module = const_yaml[karabo_da]["physical-detector-unit"]
for cname, mdata in const_yaml[karabo_da]["constants"].items():
+ base_key = f"{db_module}/{cname}/0"
when[cname] = mdata["creation-time"]
if when[cname]:
with h5py.File(mdata["file-path"], "r") as cf:
- cons_data[cname] = np.copy(cf[f"{db_module}/{cname}/0/data"]) # noqa
+ cons_data[cname] = np.copy(cf[f"{base_key}/data"])
+ # Set variant to 0 if the attribute is missing
+ # as for old constants.
+ if "variant" in cf[base_key].attrs.keys():
+ variant[cname] = cf[base_key].attrs["variant"]
+ else:
+ variant[cname] = 0
else:
# Create empty constant using the list elements
cons_data[cname] = getattr(np, mdata["file-path"][0])(mdata["file-path"][1]) # noqa
- self.init_constants(cons_data, when, module_idx)
+ self.init_constants(cons_data, when, module_idx, variant)
return when
def initialize_from_db(self, karabo_id: str, karabo_da: str,
cal_db_interface: str,
- creation_time: 'datetime.datetime',
+ creation_time: datetime,
memory_cells: float, bias_voltage: int,
photon_energy: float, gain_setting: float,
acquisition_rate: float, integration_time: int,
@@ -1330,11 +1301,32 @@ class AgipdCorrections:
integration_time=integration_time
)
- cons_data, when = self.retrieve_constant_and_time(
- karabo_id, karabo_da, const_dict, cal_db_interface, creation_time
- )
+ when = {}
+ cons_data = {}
+ variant = {}
+
+ for cname, cval in const_dict.items():
+ condition = getattr(
+ Conditions, cval[2][0]).AGIPD(**cval[2][1])
+ cdata, md = get_from_db(
+ karabo_id=karabo_id,
+ karabo_da=karabo_da,
+ constant=getattr(Constants.AGIPD, cname)(),
+ condition=condition,
+ empty_constant=getattr(np, cval[0])(cval[1]),
+ cal_db_interface=cal_db_interface,
+ creation_time=creation_time,
+ verbosity=0,
+ )
+ cons_data[cname] = cdata
+ variant[cname] = md.calibration_constant_version.variant
+
+ when[cname] = None
+ # Read the CCV begin at if constant was retrieved successfully.
+ if md and md.comm_db_success:
+ when[cname] = md.calibration_constant_version.begin_at
- self.init_constants(cons_data, when, module_idx)
+ self.init_constants(cons_data, when, module_idx, variant)
return when
diff --git a/src/cal_tools/tools.py b/src/cal_tools/tools.py
index 71fdba355ccbe047d958069b6d5567aeb437274a..9a20cfe1dc5009ff96439b6d66fb7b1f8cc29d12 100644
--- a/src/cal_tools/tools.py
+++ b/src/cal_tools/tools.py
@@ -456,12 +456,11 @@ def get_pdu_from_db(karabo_id: str, karabo_da: Union[str, list],
metadata = _init_metadata(constant, condition, None)
- # A random interface is chosen if there is # for address range.
- this_interface = get_random_db_interface(cal_db_interface)
# CalibrationDBRemote expects a string.
if snapshot_at is not None and hasattr(snapshot_at, 'isoformat'):
snapshot_at = snapshot_at.isoformat()
+ # A random interface is chosen if there is # for address range.
db_interface = get_random_db_interface(cal_db_interface)
pdu_dicts = metadata.retrieve_pdus_for_detector(receiver=db_interface,
@@ -492,7 +491,7 @@ def get_pdu_from_db(karabo_id: str, karabo_da: Union[str, list],
if i is None:
db_modules.append(None)
else:
- db_modules.append(pdu_dicts[i]["pdu_physical_name"])
+ db_modules.append(pdu_dicts[i]["pdu_physical_name"])
return db_modules
@@ -602,8 +601,10 @@ def get_from_db(karabo_id: str, karabo_da: str,
f" and data_set_name is {h5path}."
)
with h5py.File(Path(hdf5path, filename), "r") as f:
- arr = f[f"{h5path}/data"][()]
- metadata.calibration_constant.data = arr
+ metadata.calibration_constant.data = f[f"{h5path}/data"][()] # noqa
+ # The variant attribute is missing for old constants.
+ if "variant" in f[h5path].attrs.keys():
+ metadata.calibration_constant_version.variant = f[h5path].attrs["variant"] # noqa
if verbosity > 0:
if constant.name not in already_printed or verbosity > 1:
@@ -625,8 +626,9 @@ def send_to_db(db_module: str, karabo_id: str, constant, condition,
creation_time: Optional[datetime.datetime] = None,
timeout: int = 30000,
ntries: int = 7,
- doraise: bool = False):
- """Return calibration constants and metadata requested from CalDB
+ doraise: bool = False,
+ variant: int = 0):
+ """Send new calibration constants and metadata requested to CalDB
:param db_module: database module (PDU/Physical Detector Unit)
:param karabo_id: karabo identifier
@@ -634,14 +636,16 @@ def send_to_db(db_module: str, karabo_id: str, constant, condition,
:param condition: Calibration condition
:param file_loc: Location of raw data.
:param report_path: xfel-calbrate report path to inject along with
- the calibration constant versions to the database.
+ the calibration constant versions to the database.
:param cal_db_interface: Interface string, e.g. "tcp://max-exfl016:8015"
:param creation_time: Latest time for constant to be created
:param timeout: Timeout for zmq request
:param ntries: number of tries to contact the database,
- ntries is set to 7 so that if the timeout started
- at 30s last timeout will be ~ 1h.
+ ntries is set to 7 so that if the timeout started
+ at 30s last timeout will be ~ 1h.
:param doraise: if True raise errors during communication with DB
+ :param variant: A calibration constant version variant attribute
+ for the constant file.
"""
success = False
@@ -670,6 +674,7 @@ def send_to_db(db_module: str, karabo_id: str, constant, condition,
metadata.calibration_constant_version.device_name = db_module
metadata.calibration_constant_version.karabo_da = None
metadata.calibration_constant_version.raw_data_location = file_loc
+ metadata.calibration_constant_version.variant = variant
if constant.data is None:
raise ValueError(
"There is no data available to "