Revised CalCat API

This is based on our discussion last week, but it's still a WIP. In particular, I haven't implemented actually loading the constants yet.

I've tweaked the design slightly: rather than having CalibrationData subclasses for each detector type, you use CalibrationData.from_conditions() with a detector specific object like AGIPDConditions. Jump to the tests to see what this looks like.

Questions:

• How to deal with missing modules?
  • A module may be missing for one type of constant but not for another
  • How do we even tell if modules are missing - should we query the PDUs so we know how many modules to expect?
• How to deal with missing calibration types (e.g. SlopesPC)
  • Especially if one group (e.g. illuminated) not found - this currently raises an error, even if we have other constants we could return
• Naming:
  • 'Calibrations' vs 'calibration' types vs 'constant types' (e.g. Offset, SlopesPC
  • 'Modules' vs 'aggregators'
• How many of SingleConstantVersion fields do we actually want?
• Should .select_modules() and .select_calibrations() take *args or a list?

cc @schmidtp @ahmedk @hammerd

src/cal_tools/calcat_interface2.py

 import json
 import re
 from collections.abc import Mapping
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from datetime import date, datetime, time, timezone
 from functools import lru_cache
 from pathlib import Path
@@ -15,6 +15,10 @@ import requests
 from oauth2_xfel_client import Oauth2ClientBackend
 
 
+# Default address to connect to, only available internally
+CALCAT_PROXY_URL = "http://exflcalproxy.desy.de:8080/"
+
+
 class ModuleNameError(KeyError):
     def __init__(self, name):
         self.name = name
@@ -23,7 +27,7 @@ class ModuleNameError(KeyError):
         return f"No module named {self.name!r}"
 
 
-class APIError(requests.HTTPError):
+class CalCatAPIError(requests.HTTPError):
     """Used when the response includes error details as JSON"""
 
 
@@ -57,6 +61,11 @@ class CalCatAPIClient:
             return dt.astimezone(timezone.utc).isoformat()
         elif isinstance(dt, date):
             return cls.format_time(datetime.combine(dt, time()))
+        elif not isinstance(dt, str):
+            raise TypeError(
+                f"Timestamp parameter ({dt!r}) must be a string, datetime or "
+                f"date object"
+            )
 
         return dt
 
@@ -78,7 +87,7 @@ class CalCatAPIClient:
             except Exception:
                 resp.raise_for_status()
             else:
-                raise APIError(
+                raise CalCatAPIError(
                     f"Error {resp.status_code} from API: "
                     f"{d.get('info', 'missing details')}"
                 )
@@ -153,7 +162,7 @@ global_client = None
 def get_client():
     global global_client
     if global_client is None:
-        setup_client("http://exflcalproxy:8080/", None, None, None)
+        setup_client(CALCAT_PROXY_URL, None, None, None)
     return global_client
 
 
@@ -188,13 +197,27 @@ def setup_client(
         user_email=user_email,
     )
 
+    # Check we can connect to exflcalproxy
+    if oauth_client is None and base_url == CALCAT_PROXY_URL:
+        try:
+            # timeout=(connect_timeout, read_timeout)
+            global_client.get_request("me", timeout=(1, 5))
+        except requests.ConnectionError as e:
+            raise RuntimeError(
+                "Could not connect to calibration catalog proxy. This proxy allows "
+                "unauthenticated access inside the XFEL/DESY network. To look up "
+                "calibration constants from outside, you will need to create an Oauth "
+                "client ID & secret in the CalCat web interface. You will still not "
+                "be able to load constants without the constant store folder."
+            ) from e
 
-_default_caldb_root = ...
+
+_default_caldb_root = None
 
 
 def _get_default_caldb_root():
     global _default_caldb_root
-    if _default_caldb_root is ...:
+    if _default_caldb_root is None:
         onc_path = Path("/common/cal/caldb_store")
         maxwell_path = Path("/gpfs/exfel/d/cal/caldb_store")
         if onc_path.is_dir():
@@ -202,7 +225,10 @@ def _get_default_caldb_root():
         elif maxwell_path.is_dir():
             _default_caldb_root = maxwell_path
         else:
-            _default_caldb_root = None
+            raise RuntimeError(
+                f"Neither {onc_path} nor {maxwell_path} was found. If the caldb_store "
+                "directory is at another location, pass its path as caldb_root."
+            )
 
     return _default_caldb_root
 
@@ -214,33 +240,18 @@ class SingleConstant:
     CalCat calls this a calibration constant version (CCV).
     """
 
-    id: int
-    version_name: str
-    constant_id: int
-    constant_name: str
-    condition_id: int
     path: Path
     dataset: str
-    begin_validity_at: datetime
-    end_validity_at: datetime
-    raw_data_location: str
-    physical_name: str  # PDU name
+    ccv_id: Optional[int]
+    pdu_name: Optional[str]
 
     @classmethod
     def from_response(cls, ccv: dict) -> "SingleConstant":
-        const = ccv["calibration_constant"]
         return cls(
-            id=ccv["id"],
-            version_name=ccv["name"],
-            constant_id=const["id"],
-            constant_name=const["name"],
-            condition_id=const["condition_id"],
             path=Path(ccv["path_to_file"]) / ccv["file_name"],
             dataset=ccv["data_set_name"],
-            begin_validity_at=ccv["begin_validity_at"],
-            end_validity_at=ccv["end_validity_at"],
-            raw_data_location=ccv["raw_data_location"],
-            physical_name=ccv["physical_detector_unit"]["physical_name"],
+            ccv_id=ccv["id"],
+            pdu_name=ccv["physical_detector_unit"]["physical_name"],
         )
 
     def dataset_obj(self, caldb_root=None) -> h5py.Dataset:
@@ -283,12 +294,45 @@ def prepare_selection(
 
 
 @dataclass
-class MultiModuleConstant:
+class MultiModuleConstant(Mapping):
     """A group of similar constants for several modules of one detector"""
 
     constants: Dict[str, SingleConstant]  # Keys e.g. 'LPD00'
     module_details: List[Dict]
     detector_name: str  # e.g. 'HED_DET_AGIPD500K2G'
+    calibration_name: str
+
+    def __repr__(self):
+        return (
+            f"<MultiModuleConstant: {self.calibration_name} for "
+            f"{len(self.constants)} modules of {self.detector_name}>"
+        )
+
+    def __iter__(self):
+        return iter(self.constants)
+
+    def __len__(self):
+        return len(self.constants)
+
+    def __getitem__(self, key):
+        if key in (None, ""):
+            raise KeyError(key)
+
+        candidate_kdas = set()
+        if key in self.constants:  # Karabo DA name, e.g. 'LPD00'
+            candidate_kdas.add(key)
+
+        for m in self.module_details:
+            names = (m["module_number"], m["virtual_device_name"], m["physical_name"])
+            if key in names and m["karabo_da"] in self.constants:
+                candidate_kdas.add(m["karabo_da"])
+
+        if not candidate_kdas:
+            raise KeyError(key)
+        elif len(candidate_kdas) > 1:
+            raise KeyError(f"Ambiguous key: {key} matched {candidate_kdas}")
+
+        return self.constants[candidate_kdas.pop()]
 
     def select_modules(
         self, module_nums=None, *, aggregator_names=None, qm_names=None
@@ -298,7 +342,7 @@ class MultiModuleConstant:
         )
         d = {aggr: scv for (aggr, scv) in self.constants.items() if aggr in aggs}
         mods = [m for m in self.module_details if m["karabo_da"] in d]
-        return MultiModuleConstant(d, mods, self.detector_name)
+        return replace(self, constants=d, module_details=mods)
 
     # These properties label only the modules we have constants for, which may
     # be a subset of what's in module_details
@@ -367,7 +411,7 @@ class MultiModuleConstant:
         # Dimension labels
         dims = ["module"] + ["dim_%d" % i for i in range(ndarr.ndim - 1)]
         coords = {"module": modules}
-        name = self.constants[self.aggregator_names[0]].constant_name
+        name = self.calibration_name
 
         return xarray.DataArray(ndarr, dims=dims, coords=coords, name=name)
 
@@ -491,6 +535,10 @@ class CalibrationData(Mapping):
             kda = pdu["karabo_da"] = pdu.pop("karabo_da_at_ccv_begin_at")
             det_id = pdu["detector_id"] = pdu.pop("detector_id_at_ccv_begin_at")
             pdu["virtual_device_name"] = pdu.pop("virtual_device_name_at_ccv_begin_at")
+            if pdu.get("module_number_at_ccv_begin_at") is not None:
+                pdu["module_number"] = pdu.pop("module_number_at_ccv_begin_at")
+            else:
+                pdu["module_number"] = int(re.findall(r"\d+", kda)[-1])
 
             det_ids.add(det_id)
             if kda in pdus:
@@ -517,9 +565,15 @@ class CalibrationData(Mapping):
         return cls(constant_groups, module_details, det_name)
 
     def __getitem__(self, key) -> MultiModuleConstant:
-        return MultiModuleConstant(
-            self.constant_groups[key], self.module_details, self.detector_name
-        )
+        if isinstance(key, str):
+            return MultiModuleConstant(
+                self.constant_groups[key], self.module_details, self.detector_name, key
+            )
+        elif isinstance(key, tuple) and len(key) == 2:
+            cal_type, module = key
+            return self[cal_type][module]
+        else:
+            raise TypeError(f"Key should be string or 2-tuple (got {key!r})")
 
     def __iter__(self):
         return iter(self.constant_groups)
@@ -527,6 +581,9 @@ class CalibrationData(Mapping):
     def __len__(self):
         return len(self.constant_groups)
 
+    def __contains__(self, item):
+        return item in self.constant_groups
+
     def __repr__(self):
         return (
             f"<CalibrationData: {', '.join(sorted(self.constant_groups))} "