Revised CalCat API

This is based on our discussion last week, but it's still a WIP. In particular, I haven't implemented actually loading the constants yet.

I've tweaked the design slightly: rather than having CalibrationData subclasses for each detector type, you use CalibrationData.from_conditions() with a detector specific object like AGIPDConditions. Jump to the tests to see what this looks like.

Questions:

• How to deal with missing modules?
  • A module may be missing for one type of constant but not for another
  • How do we even tell if modules are missing - should we query the PDUs so we know how many modules to expect?
• How to deal with missing calibration types (e.g. SlopesPC)
  • Especially if one group (e.g. illuminated) not found - this currently raises an error, even if we have other constants we could return
• Naming:
  • 'Calibrations' vs 'calibration' types vs 'constant types' (e.g. Offset, SlopesPC
  • 'Modules' vs 'aggregators'
• How many of SingleConstantVersion fields do we actually want?
• Should .select_modules() and .select_calibrations() take *args or a list?

cc @schmidtp @ahmedk @hammerd

src/cal_tools/calcat_interface2.py

 import json
 import re
 from collections.abc import Mapping
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 from datetime import date, datetime, time, timezone
 from functools import lru_cache
 from pathlib import Path
@@ -15,6 +15,10 @@ import requests
 from oauth2_xfel_client import Oauth2ClientBackend
 
 
+# Default address to connect to, only available internally
+CALCAT_PROXY_URL = "http://exflcalproxy.desy.de:8080/"
+
+
 class ModuleNameError(KeyError):
     def __init__(self, name):
         self.name = name
@@ -23,7 +27,7 @@ class ModuleNameError(KeyError):
         return f"No module named {self.name!r}"
 
 
-class APIError(requests.HTTPError):
+class CalCatAPIError(requests.HTTPError):
     """Used when the response includes error details as JSON"""
 
 
@@ -83,7 +87,7 @@ class CalCatAPIClient:
             except Exception:
                 resp.raise_for_status()
             else:
-                raise APIError(
+                raise CalCatAPIError(
                     f"Error {resp.status_code} from API: "
                     f"{d.get('info', 'missing details')}"
                 )
@@ -158,7 +162,7 @@ global_client = None
 def get_client():
     global global_client
     if global_client is None:
-        setup_client("http://exflcalproxy:8080/", None, None, None)
+        setup_client(CALCAT_PROXY_URL, None, None, None)
     return global_client
 
 
@@ -193,13 +197,27 @@ def setup_client(
         user_email=user_email,
     )
 
+    # Check we can connect to exflcalproxy
+    if oauth_client is None and base_url == CALCAT_PROXY_URL:
+        try:
+            # timeout=(connect_timeout, read_timeout)
+            global_client.get_request("me", timeout=(1, 5))
+        except requests.ConnectionError as e:
+            raise RuntimeError(
+                "Could not connect to calibration catalog proxy. This proxy allows "
+                "unauthenticated access inside the XFEL/DESY network. To look up "
+                "calibration constants from outside, you will need to create an Oauth "
+                "client ID & secret in the CalCat web interface. You will still not "
+                "be able to load constants without the constant store folder."
+            ) from e
+
 
-_default_caldb_root = ...
+_default_caldb_root = None
 
 
 def _get_default_caldb_root():
     global _default_caldb_root
-    if _default_caldb_root is ...:
+    if _default_caldb_root is None:
         onc_path = Path("/common/cal/caldb_store")
         maxwell_path = Path("/gpfs/exfel/d/cal/caldb_store")
         if onc_path.is_dir():
@@ -207,7 +225,10 @@ def _get_default_caldb_root():
         elif maxwell_path.is_dir():
             _default_caldb_root = maxwell_path
         else:
-            _default_caldb_root = None
+            raise RuntimeError(
+                f"Neither {onc_path} nor {maxwell_path} was found. If the caldb_store "
+                "directory is at another location, pass its path as caldb_root."
+            )
 
     return _default_caldb_root
 
@@ -288,12 +309,45 @@ def prepare_selection(
 
 
 @dataclass
-class MultiModuleConstant:
+class MultiModuleConstant(Mapping):
     """A group of similar constants for several modules of one detector"""
 
     constants: Dict[str, SingleConstant]  # Keys e.g. 'LPD00'
     module_details: List[Dict]
     detector_name: str  # e.g. 'HED_DET_AGIPD500K2G'
+    calibration_name: str
+
+    def __repr__(self):
+        return (
+            f"<MultiModuleConstant: {self.calibration_name} for "
+            f"{len(self.constants)} modules of {self.detector_name}>"
+        )
+
+    def __iter__(self):
+        return iter(self.constants)
+
+    def __len__(self):
+        return len(self.constants)
+
+    def __getitem__(self, key):
+        if key in (None, ""):
+            raise KeyError(key)
+
+        candidate_kdas = set()
+        if key in self.constants:  # Karabo DA name, e.g. 'LPD00'
+            candidate_kdas.add(key)
+
+        for m in self.module_details:
+            names = (m["module_number"], m["virtual_device_name"], m["physical_name"])
+            if key in names and m["karabo_da"] in self.constants:
+                candidate_kdas.add([m["karabo_da"]])
+
+        if not candidate_kdas:
+            raise KeyError(key)
+        elif len(candidate_kdas) > 1:
+            raise KeyError(f"Ambiguous key: {key} matched {candidate_kdas}")
+
+        return self.constants[candidate_kdas.pop()]
 
     def select_modules(
         self, module_nums=None, *, aggregator_names=None, qm_names=None
@@ -303,7 +357,7 @@ class MultiModuleConstant:
         )
         d = {aggr: scv for (aggr, scv) in self.constants.items() if aggr in aggs}
         mods = [m for m in self.module_details if m["karabo_da"] in d]
-        return MultiModuleConstant(d, mods, self.detector_name)
+        return replace(self, constants=d, module_details=mods)
 
     # These properties label only the modules we have constants for, which may
     # be a subset of what's in module_details
@@ -522,9 +576,15 @@ class CalibrationData(Mapping):
         return cls(constant_groups, module_details, det_name)
 
     def __getitem__(self, key) -> MultiModuleConstant:
-        return MultiModuleConstant(
-            self.constant_groups[key], self.module_details, self.detector_name
-        )
+        if isinstance(key, str):
+            return MultiModuleConstant(
+                self.constant_groups[key], self.module_details, self.detector_name, key
+            )
+        elif isinstance(key, tuple) and len(key) == 2:
+            cal_type, module = key
+            return self[cal_type][module]
+        else:
+            raise TypeError(f"Key should be string or 2-tuple (got {key!r})")
 
     def __iter__(self):
         return iter(self.constant_groups)
@@ -532,6 +592,9 @@ class CalibrationData(Mapping):
     def __len__(self):
         return len(self.constant_groups)
 
+    def __contains__(self, item):
+        return item in self.constant_groups
+
     def __repr__(self):
         return (
             f"<CalibrationData: {', '.join(sorted(self.constant_groups))} "