Revised CalCat API

This is based on our discussion last week, but it's still a WIP. In particular, I haven't implemented actually loading the constants yet.

I've tweaked the design slightly: rather than having CalibrationData subclasses for each detector type, you use CalibrationData.from_conditions() with a detector specific object like AGIPDConditions. Jump to the tests to see what this looks like.

Questions:

• How to deal with missing modules?
  • A module may be missing for one type of constant but not for another
  • How do we even tell if modules are missing - should we query the PDUs so we know how many modules to expect?
• How to deal with missing calibration types (e.g. SlopesPC)
  • Especially if one group (e.g. illuminated) not found - this currently raises an error, even if we have other constants we could return
• Naming:
  • 'Calibrations' vs 'calibration' types vs 'constant types' (e.g. Offset, SlopesPC
  • 'Modules' vs 'aggregators'
• How many of SingleConstantVersion fields do we actually want?
• Should .select_modules() and .select_calibrations() take *args or a list?

cc @schmidtp @ahmedk @hammerd

src/cal_tools/calcat_interface2.py

@@ -10,7 +10,6 @@ from urllib.parse import urljoin
 from warnings import warn
 
 import h5py
-import numpy as np
 import pasha as psh
 import requests
 from oauth2_xfel_client import Oauth2ClientBackend
@@ -226,7 +225,8 @@ def prepare_selection(
     n_specified = sum([module_nums is not None, aggs is not None, qm_names is not None])
     if n_specified > 1:
         raise TypeError(
-            "select_modules() accepts only one of module_nums, aggregator_names & qm_names"
+            "select_modules() accepts only one of module_nums, aggregator_names "
+            "& qm_names"
         )
 
     if module_nums is not None:
@@ -326,7 +326,7 @@ class ModulesConstantVersions:
 
 
 @lru_cache()
-def detector(identifier, client=None):
+def detector_by_name(identifier, client=None):
     client = client or get_client()
     res = client.get("detectors", {"identifier": identifier})
     if not res:
@@ -336,6 +336,14 @@ def detector(identifier, client=None):
     return res[0]
 
 
+@lru_cache()
+def detector_id_to_name(det_id: int, client=None):
+    """Convert a numeric detector ID to a name like 'FXE_DET_LPD1M-1'"""
+    client = client or get_client()
+    res = client.get(f"detectors/{det_id}")
+    return res["identifier"]  # "name" & "karabo_name" appear to be equivalent
+
+
 @lru_cache()
 def calibration_id(name, client=None):
     """ID for a calibration in CalCat."""
@@ -359,9 +367,10 @@ def calibration_name(cal_id, client=None):
 class CalibrationData(Mapping):
     """Collected constants for a given detector"""
 
-    def __init__(self, constant_groups, module_details):
+    def __init__(self, constant_groups, module_details, detector_name):
         self.constant_groups = constant_groups
         self.module_details = module_details
+        self.detector_name = detector_name
 
     @staticmethod
     def _format_cond(condition):
@@ -402,7 +411,7 @@ class CalibrationData(Mapping):
 
         client = client or get_client()
 
-        detector_id = detector(detector_name)["id"]
+        detector_id = detector_by_name(detector_name, client)["id"]
         pdus = client.get(
             "physical_detector_units/get_all_by_detector",
             {
@@ -448,7 +457,7 @@ class CalibrationData(Mapping):
             const_type: ModulesConstantVersions(d, module_details)
             for const_type, d in constant_groups.items()
         }
-        return cls(mcvs, module_details)
+        return cls(mcvs, module_details, detector_name)
 
     @classmethod
     def from_report(
@@ -494,13 +503,14 @@ class CalibrationData(Mapping):
 
         if len(det_ids) > 1:
             raise Exception(f"Found multiple detector IDs in report: {det_ids}")
+        det_name = detector_id_to_name(det_ids.pop(), client)
 
         module_details = sorted(pdus.values(), key=lambda d: d["karabo_da"])
         mcvs = {
             const_type: ModulesConstantVersions(d, module_details)
             for const_type, d in constant_groups.items()
         }
-        return cls(mcvs, module_details)
+        return cls(mcvs, module_details, det_name)
 
     def __getitem__(self, key) -> ModulesConstantVersions:
         return self.constant_groups[key]
@@ -566,6 +576,12 @@ class CalibrationData(Mapping):
         return type(self)(mcvs, self.module_details)
 
     def merge(self, *others: "CalibrationData") -> "CalibrationData":
+        det_names = set(cd.detector_name for cd in (self,) + others)
+        if len(det_names) > 1:
+            raise Exception("Cannot merge calibration data for different "
+                            f"detectors: " + ", ".join(sorted(det_names)))
+        det_name = det_names.pop()
+
         cal_types = set(self.constant_groups)
         aggregators = set(self.aggregator_names)
         pdus_d = {m["karabo_da"]: m for m in self.module_details}
@@ -597,7 +613,7 @@ class CalibrationData(Mapping):
                     d.update(caldata[cal_type].constants)
             mcvs[cal_type] = ModulesConstantVersions(d, module_details)
 
-        return type(self)(mcvs, module_details)
+        return type(self)(mcvs, module_details, det_name)
 
 
 class ConditionsBase: