New calcat interface

The MR is based on the work done on this MR in EXtra-data: https://github.com/European-XFEL/EXtra-data/pull/262

New CALCAT interface to replace calibrationDBRemote, iCalibrationDB, and cal_tools.get_from_db for retrieving constants from CALCAT.

Related to https://git.xfel.eu/calibration/planning/-/issues/150

Description

How Has This Been Tested?

• Has been tested along with all the correction notebooks and detectors available. Reference runs

Relevant Documents (optional)

Types of changes

• New feature (non-breaking change which adds functionality)

Checklist:

• Update snapshot parameter after calibration client is updated.
• I updated the documentation accordingly.
• I added tests where appropriate.

Reviewers

@kluyvert @schmidtp

src/cal_tools/calcat_interface.py

@@ -106,7 +106,7 @@ class CalCatApi(metaclass=ClientWrapper):
         """Parse different ways to specify time to CalCat."""
 
         if isinstance(dt, datetime):
-            return dt.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%S%Z")
+            return dt.astimezone(timezone.utc).isoformat()
         elif isinstance(dt, date):
             return cls.format_time(datetime.combine(dt, time()))
 
@@ -375,7 +375,6 @@ class CalibrationData:
         modules=None,
         client=None,
         event_at=None,
-        pdu_snapshot_at=None,
         module_naming="da",
     ):
         """Initialize a new CalibrationData object.
@@ -392,8 +391,6 @@ class CalibrationData:
                 communication, global one by default.
             event_at (datetime, date, str or None): Default time at which the
                 CCVs should have been valid, now if omitted
-            pdu_snapshot_at (datetime, date, str or None): Default time of
-                database state to look at, now if omitted.
             module_naming (str or None): Expected module name convention to be
                 used as metadata dict keys. Expected values are:
                 `da`: data aggregator name is used. Default.
@@ -408,7 +405,7 @@ class CalibrationData:
         self.detector_name = detector_name
         self.modules = modules
         self.event_at = event_at
-        self.pdu_snapshot_at = pdu_snapshot_at
+        self.pdu_snapshot_at = event_at
         self.module_naming = module_naming
 
         if client is None:
@@ -528,8 +525,6 @@ class CalibrationData:
             self.detector["id"], self.pdu_snapshot_at, self.module_naming
         )
 
-
-
     @property
     def condition(self):
         return self._build_condition(self.parameters)
@@ -589,7 +584,7 @@ class CalibrationData:
         self,
         module,
         calibration,
-        metadata,
+        metadata=None,
     ):
         """Load CCV data as ndarray.
 
@@ -602,6 +597,8 @@ class CalibrationData:
         Returns:
             (ndarray): CCV data
         """
+        import numpy as np
+
         if self.caldb_root is None:
             raise RuntimeError("calibration database store unavailable")
 
@@ -611,7 +608,10 @@ class CalibrationData:
         if metadata is None:
             metadata = self.metadata([calibration])
 
-        return self._load_ccv_data(metadata, module, calibration)
+        row = metadata[module][calibration]
+
+        with h5py.File(self.caldb_root / row['path'], 'r') as f:
+            return np.asarray(f[row['dataset'] + '/data'])
 
     def _allocate_constant_arrays(self, metadata, const_load_mp, const_data):
 
@@ -929,7 +929,6 @@ class AGIPD_CalibrationData(SplitConditionCalibrationData):
         modules=None,
         client=None,
         event_at=None,
-        pdu_snapshot_at=None,
         gain_setting=None,
         gain_mode=None,
         module_naming="da",
@@ -943,7 +942,6 @@ class AGIPD_CalibrationData(SplitConditionCalibrationData):
             modules,
             client,
             event_at,
-            pdu_snapshot_at,
             module_naming,
         )
 
@@ -1007,7 +1005,6 @@ class LPD_CalibrationData(SplitConditionCalibrationData):
         modules=None,
         client=None,
         event_at=None,
-        pdu_snapshot_at=None,
         module_naming="da",
     ):
         super().__init__(
@@ -1015,7 +1012,6 @@ class LPD_CalibrationData(SplitConditionCalibrationData):
             modules,
             client,
             event_at,
-            pdu_snapshot_at,
             module_naming,
         )
 
@@ -1060,7 +1056,6 @@ class DSSC_CalibrationData(CalibrationData):
         modules=None,
         client=None,
         event_at=None,
-        pdu_snapshot_at=None,
         module_naming="da",
     ):
         super().__init__(
@@ -1068,7 +1063,6 @@ class DSSC_CalibrationData(CalibrationData):
             modules,
             client,
             event_at,
-            pdu_snapshot_at,
             module_naming,
         )
 
@@ -1116,7 +1110,6 @@ class JUNGFRAU_CalibrationData(CalibrationData):
         modules=None,
         client=None,
         event_at=None,
-        pdu_snapshot_at=None,
         module_naming="da",
     ):
         super().__init__(
@@ -1124,7 +1117,6 @@ class JUNGFRAU_CalibrationData(CalibrationData):
             modules,
             client,
             event_at,
-            pdu_snapshot_at,
             module_naming,
         )
 
@@ -1177,7 +1169,6 @@ class PNCCD_CalibrationData(CalibrationData):
         modules=None,
         client=None,
         event_at=None,
-        pdu_snapshot_at=None,
         module_naming="da",
     ):
         # Ignore modules for this detector.
@@ -1186,7 +1177,6 @@ class PNCCD_CalibrationData(CalibrationData):
             modules,
             client,
             event_at,
-            pdu_snapshot_at,
             module_naming,
         )
 
@@ -1233,7 +1223,6 @@ class EPIX100_CalibrationData(SplitConditionCalibrationData):
         modules=None,
         client=None,
         event_at=None,
-        pdu_snapshot_at=None,
         module_naming="da",
     ):
         # Ignore modules for this detector.
@@ -1242,7 +1231,6 @@ class EPIX100_CalibrationData(SplitConditionCalibrationData):
             modules,
             client,
             event_at,
-            pdu_snapshot_at,
             module_naming,
         )
 
@@ -1283,7 +1271,6 @@ class GOTTHARD2_CalibrationData(CalibrationData):
         modules=None,
         client=None,
         event_at=None,
-        pdu_snapshot_at=None,
         module_naming="da",
     ):
         # Ignore modules for this detector.
@@ -1292,7 +1279,6 @@ class GOTTHARD2_CalibrationData(CalibrationData):
             modules,
             client,
             event_at,
-            pdu_snapshot_at,
             module_naming,
         )