New calcat interface

The MR is based on the work done on this MR in EXtra-data: https://github.com/European-XFEL/EXtra-data/pull/262

New CALCAT interface to replace calibrationDBRemote, iCalibrationDB, and cal_tools.get_from_db for retrieving constants from CALCAT.

Related to https://git.xfel.eu/calibration/planning/-/issues/150

Description

How Has This Been Tested?

• Has been tested along with all the correction notebooks and detectors available. Reference runs

Relevant Documents (optional)

Types of changes

• New feature (non-breaking change which adds functionality)

Checklist:

• Update snapshot parameter after calibration client is updated.
• I updated the documentation accordingly.
• I added tests where appropriate.

Reviewers

@kluyvert @schmidtp

src/cal_tools/calcat_interface.py

@@ -106,7 +106,7 @@ class CalCatApi(metaclass=ClientWrapper):
         """Parse different ways to specify time to CalCat."""
 
         if isinstance(dt, datetime):
-            return dt.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%S%Z")
+            return dt.astimezone(timezone.utc).isoformat()
         elif isinstance(dt, date):
             return cls.format_time(datetime.combine(dt, time()))
 
@@ -145,13 +145,13 @@ class CalCatApi(metaclass=ClientWrapper):
     def physical_detector_units(
         self,
         detector_id,
-        snapshot_at,
-        module_naming="da",
+        pdu_snapshot_at,
+        module_naming="da"
     ):
         """Physical detector unit metadata."""
 
         resp_pdus = PhysicalDetectorUnit.get_all_by_detector(
-            self.client, detector_id, self.format_time(snapshot_at)
+            self.client, detector_id, self.format_time(pdu_snapshot_at)
         )
 
         if not resp_pdus["success"]:
@@ -181,6 +181,7 @@ class CalCatApi(metaclass=ClientWrapper):
         else:
             raise ValueError(f"{module_naming} is unknown!")
 
+
     @lru_cache()
     def calibration_id(self, calibration_name):
         """ID for a calibration in CalCat."""
@@ -212,7 +213,7 @@ class CalCatApi(metaclass=ClientWrapper):
         condition,
         modules=None,
         event_at=None,
-        snapshot_at=None,
+        pdu_snapshot_at=None,
         metadata=None,
         module_naming="da",
     ):
@@ -233,7 +234,7 @@ class CalCatApi(metaclass=ClientWrapper):
                 or None for all (default).
             event_at (datetime, date, str or None): Time at which the
                 CCVs should have been valid or None for now (default).
-            snapshot_at (datetime, date, str or None): Time of database
+            pdu_snapshot_at (datetime, date, str or None): Time of database
                 state to look at or None for now (default).
             metadata (dict or None): Mapping to fill for results or
                 None for a new dictionary (default).
@@ -251,7 +252,7 @@ class CalCatApi(metaclass=ClientWrapper):
                 passed.
         """
         event_at = self.format_time(event_at)
-        snapshot_at = self.format_time(snapshot_at)
+        pdu_snapshot_at = self.format_time(pdu_snapshot_at)
 
         if metadata is None:
             metadata = CCVMetadata()
@@ -270,15 +271,22 @@ class CalCatApi(metaclass=ClientWrapper):
         }
         calibration_ids = list(cal_id_map.keys())
 
+        # Map aggregator to the selected module name.
+        da_to_modname = {
+            data['karabo_da']: mod_name for mod_name, data in
+            self.physical_detector_units(
+                self.detector(detector_name)['id'],
+                pdu_snapshot_at,
+                module_naming=module_naming
+            ).items()
+            if not modules or mod_name in modules
+        }
         # The API call supports a single module or all modules, as the
         # performance increase is only minor in between. Hence, all
         # modules are queried if more than one is selected and filtered
         # afterwards, if necessary.
-        karabo_da = (
-            next(iter(modules))
-            if modules is not None and len(modules) == 1
-            else "",
-        )  # noqa
+        karabo_da = next(iter(da_to_modname)) if len(da_to_modname) == 1 else '',
+
         resp_versions = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(  # noqa
             self.client,
             detector_name,
@@ -286,7 +294,7 @@ class CalCatApi(metaclass=ClientWrapper):
             self.format_cond(condition),
             karabo_da=karabo_da,
             event_at=event_at,
-            snapshot_at=snapshot_at,
+            pdu_snapshot_at=pdu_snapshot_at,
         )
 
         if not resp_versions["success"]:
@@ -294,17 +302,8 @@ class CalCatApi(metaclass=ClientWrapper):
 
         for ccv in resp_versions["data"]:
             try:
-                if module_naming == "da":
-                    mod = ccv["physical_detector_unit"]["karabo_da"]
-                # Can be used for AGIPD, LPD, and DSSC.
-                elif module_naming == "qm":
-                    mod = module_index_to_qm(
-                        int(ccv["physical_detector_unit"]["karabo_da"][-2:])
-                    )
-                elif module_naming == "modno":
-                    mod = int(ccv["physical_detector_unit"]["karabo_da"][-2:])
-                else:
-                    raise ValueError(f"{module_naming} is unknown!")
+                mod = da_to_modname[ccv['physical_detector_unit']['karabo_da']]
+                print(mod)
             except KeyError:
                 # Not included in our modules
                 continue
@@ -376,7 +375,6 @@ class CalibrationData:
         modules=None,
         client=None,
         event_at=None,
-        snapshot_at=None,
         module_naming="da",
     ):
         """Initialize a new CalibrationData object.
@@ -393,8 +391,6 @@ class CalibrationData:
                 communication, global one by default.
             event_at (datetime, date, str or None): Default time at which the
                 CCVs should have been valid, now if omitted
-            snapshot_at (datetime, date, str or None): Default time of
-                database state to look at, now if omitted.
             module_naming (str or None): Expected module name convention to be
                 used as metadata dict keys. Expected values are:
                 `da`: data aggregator name is used. Default.
@@ -409,10 +405,11 @@ class CalibrationData:
         self.detector_name = detector_name
         self.modules = modules
         self.event_at = event_at
-        self.snapshot_at = snapshot_at
+        self.pdu_snapshot_at = event_at
         self.module_naming = module_naming
 
         if client is None:
+
             client = (
                 self.__class__.default_client
                 or self.__class__.new_anonymous_client()
@@ -525,7 +522,7 @@ class CalibrationData:
     @property
     def physical_detector_units(self):
         return self._api.physical_detector_units(
-            self.detector["id"], self.snapshot_at, self.module_naming
+            self.detector["id"], self.pdu_snapshot_at, self.module_naming
         )
 
     @property
@@ -540,7 +537,7 @@ class CalibrationData:
             "modules",
             "client",
             "event_at",
-            "snapshot_at",
+            "pdu_snapshot_at",
         } | {self._simplify_parameter_name(name) for name in self.parameters}
 
         kwargs = {key: getattr(self, key) for key in keys}
@@ -552,7 +549,7 @@ class CalibrationData:
         self,
         calibrations=None,
         event_at=None,
-        snapshot_at=None,
+        pdu_snapshot_at=None,
     ):
         """Query CCV metadata for calibrations, conditions and time.
 
@@ -562,7 +559,7 @@ class CalibrationData:
             event_at (datetime, date, str or None): Time at which the
                 CCVs should have been valid, now or default value passed at
                 initialization time if omitted.
-            snapshot_at (datetime, date, str or None): Time of database
+            pdu_snapshot_at (datetime, date, str or None): Time of database
                 state to look at, now or default value passed at
                 initialization time if omitted.
 
@@ -577,7 +574,7 @@ class CalibrationData:
             self.condition,
             self.modules,
             event_at or self.event_at,
-            snapshot_at or self.snapshot_at,
+            pdu_snapshot_at or self.pdu_snapshot_at,
             metadata,
             module_naming=self.module_naming,
         )
@@ -587,7 +584,7 @@ class CalibrationData:
         self,
         module,
         calibration,
-        metadata,
+        metadata=None,
     ):
         """Load CCV data as ndarray.
 
@@ -600,6 +597,8 @@ class CalibrationData:
         Returns:
             (ndarray): CCV data
         """
+        import numpy as np
+
         if self.caldb_root is None:
             raise RuntimeError("calibration database store unavailable")
 
@@ -609,7 +608,10 @@ class CalibrationData:
         if metadata is None:
             metadata = self.metadata([calibration])
 
-        return self._load_ccv_data(metadata, module, calibration)
+        row = metadata[module][calibration]
+
+        with h5py.File(self.caldb_root / row['path'], 'r') as f:
+            return np.asarray(f[row['dataset'] + '/data'])
 
     def _allocate_constant_arrays(self, metadata, const_load_mp, const_data):
 
@@ -832,7 +834,7 @@ class SplitConditionCalibrationData(CalibrationData):
         self,
         calibrations=None,
         event_at=None,
-        snapshot_at=None,
+        pdu_snapshot_at=None,
     ):
         """Query CCV metadata for calibrations, conditions and time.
 
@@ -842,7 +844,7 @@ class SplitConditionCalibrationData(CalibrationData):
             event_at (datetime, date, str or None): Time at which the
                 CCVs should have been valid, now or default value passed at
                 initialization time if omitted.
-            snapshot_at (datetime, date, str or None): Time of database
+            pdu_snapshot_at (datetime, date, str or None): Time of database
                 state to look at, now or default value passed at
                 initialization time if omitted.
 
@@ -868,7 +870,7 @@ class SplitConditionCalibrationData(CalibrationData):
                 self.dark_condition,
                 self.modules,
                 event_at or self.event_at,
-                snapshot_at or self.snapshot_at,
+                pdu_snapshot_at or self.pdu_snapshot_at,
                 metadata,
                 module_naming=self.module_naming,
             )
@@ -882,7 +884,7 @@ class SplitConditionCalibrationData(CalibrationData):
                 self.illuminated_condition,
                 self.modules,
                 event_at or self.event_at,
-                snapshot_at or self.snapshot_at,
+                pdu_snapshot_at or self.pdu_snapshot_at,
                 metadata,
                 module_naming=self.module_naming,
             )
@@ -927,7 +929,6 @@ class AGIPD_CalibrationData(SplitConditionCalibrationData):
         modules=None,
         client=None,
         event_at=None,
-        snapshot_at=None,
         gain_setting=None,
         gain_mode=None,
         module_naming="da",
@@ -941,7 +942,6 @@ class AGIPD_CalibrationData(SplitConditionCalibrationData):
             modules,
             client,
             event_at,
-            snapshot_at,
             module_naming,
         )
 
@@ -1005,7 +1005,6 @@ class LPD_CalibrationData(SplitConditionCalibrationData):
         modules=None,
         client=None,
         event_at=None,
-        snapshot_at=None,
         module_naming="da",
     ):
         super().__init__(
@@ -1013,7 +1012,6 @@ class LPD_CalibrationData(SplitConditionCalibrationData):
             modules,
             client,
             event_at,
-            snapshot_at,
             module_naming,
         )
 
@@ -1058,7 +1056,6 @@ class DSSC_CalibrationData(CalibrationData):
         modules=None,
         client=None,
         event_at=None,
-        snapshot_at=None,
         module_naming="da",
     ):
         super().__init__(
@@ -1066,7 +1063,6 @@ class DSSC_CalibrationData(CalibrationData):
             modules,
             client,
             event_at,
-            snapshot_at,
             module_naming,
         )
 
@@ -1114,7 +1110,6 @@ class JUNGFRAU_CalibrationData(CalibrationData):
         modules=None,
         client=None,
         event_at=None,
-        snapshot_at=None,
         module_naming="da",
     ):
         super().__init__(
@@ -1122,7 +1117,6 @@ class JUNGFRAU_CalibrationData(CalibrationData):
             modules,
             client,
             event_at,
-            snapshot_at,
             module_naming,
         )
 
@@ -1172,18 +1166,17 @@ class PNCCD_CalibrationData(CalibrationData):
         gain_setting,
         pixels_x=1024,
         pixels_y=1024,
+        modules=None,
         client=None,
         event_at=None,
-        snapshot_at=None,
         module_naming="da",
     ):
         # Ignore modules for this detector.
         super().__init__(
             detector_name,
-            None,
+            modules,
             client,
             event_at,
-            snapshot_at,
             module_naming,
         )
 
@@ -1227,18 +1220,17 @@ class EPIX100_CalibrationData(SplitConditionCalibrationData):
         pixels_x=708,
         pixels_y=768,
         source_energy=9.2,
+        modules=None,
         client=None,
         event_at=None,
-        snapshot_at=None,
         module_naming="da",
     ):
         # Ignore modules for this detector.
         super().__init__(
             detector_name,
-            None,
+            modules,
             client,
             event_at,
-            snapshot_at,
             module_naming,
         )
 
@@ -1276,18 +1268,17 @@ class GOTTHARD2_CalibrationData(CalibrationData):
         exposure_period,
         acquisition_rate,
         single_photon,
+        modules=None,
         client=None,
         event_at=None,
-        snapshot_at=None,
         module_naming="da",
     ):
         # Ignore modules for this detector.
         super().__init__(
             detector_name,
-            None,
+            modules,
             client,
             event_at,
-            snapshot_at,
             module_naming,
         )