diff --git a/src/calng/AgipdCorrection.py b/src/calng/AgipdCorrection.py
index 22b505813fe70169b0c1b993e748db8132648e26..d6f525b00ec127e66abf72ccadede8a7321d9c83 100644
--- a/src/calng/AgipdCorrection.py
+++ b/src/calng/AgipdCorrection.py
@@ -193,7 +193,7 @@ class AgipdCorrection(BaseCorrection):
self._update_bad_pixel_selection()
self.updateState(State.ON)
- self.KARABO_SLOT(self.loadMostRecentConstantsWrap)
+ self.KARABO_SLOT(self.loadMostRecentConstants)
def process_input(self, data, metadata):
"""Registered for dataInput, handles all processing and sending"""
@@ -316,31 +316,16 @@ class AgipdCorrection(BaseCorrection):
"performance.lastProcessingDuration", time_spent * 1000
)
- def loadMostRecentConstantsWrap(self):
+ def loadMostRecentConstants(self):
self.flush_constants()
+ self.calibration_constant_manager.flush_constants()
for constant in AgipdConstants:
- try:
- metadata, data = self.calibration_constant_manager.get_constant_version(
- constant
- )
- except Exception as e:
- self.log.WARN(f"Failed getting {constant} with {e}")
- else:
- self.log.INFO(f"Retrieved something for {constant}")
- print(data)
- self._load_constant_to_gpu(constant, (metadata, data))
- """
- threads = [
self.calibration_constant_manager.get_constant_version_and_call_me_back(
constant, self._load_constant_to_gpu
)
- for constant in AgipdConstants
- ]
- """
- def _load_constant_to_gpu(self, constant, found):
+ def _load_constant_to_gpu(self, constant, constant_data):
# TODO: encode correction / constant dependencies in a clever way
- metadata, constant_data = found
if constant is AgipdConstants.ThresholdsDark:
if self.gain_mode is not AgipdGainMode.ADAPTIVE_GAIN:
self.log.INFO("Loaded ThresholdsDark ignored due to fixed gain mode")
@@ -381,6 +366,8 @@ class AgipdCorrection(BaseCorrection):
self.set("corrections.enabled.badPixels", True)
self.set("corrections.preview.badPixels", True)
+ self._update_correction_flags()
+
def _update_pulse_filter(self, filter_string):
"""Called whenever the pulse filter changes, typically followed by
_update_shapes"""
diff --git a/src/calng/base_correction.py b/src/calng/base_correction.py
index 14632431b39dcbb7aff3f752a50d0b63af47d0f9..4dfa87265f35aa1d41e752d419f0c9da1d3422a3 100644
--- a/src/calng/base_correction.py
+++ b/src/calng/base_correction.py
@@ -205,7 +205,12 @@ class BaseCorrection(PythonDevice):
.commit(),
)
- (SLOT_ELEMENT(expected).key("loadMostRecentConstantsWrap").commit())
+ (
+ SLOT_ELEMENT(expected)
+ .key("loadMostRecentConstants")
+ .displayedName("Load most recent constants")
+ .commit()
+ )
preview_schema = Schema()
(
@@ -490,8 +495,9 @@ class BaseCorrection(PythonDevice):
elif scene_name == "constants":
payload["data"] = scenes.correction_device_constants_scene(
device_id=self.getInstanceId(),
- node_prefix="constantParameters",
schema=self.getFullSchema(),
+ param_prefix="constantParameters",
+ status_prefix="foundConstants",
)
else:
payload["success"] = False
@@ -554,8 +560,8 @@ class BaseCorrection(PythonDevice):
enabled &= self._correction_flag_class.NONE
self._correction_flag_enabled = enabled
self._correction_flag_preview = preview
- self.log.INFO(f"Corrections for dataOutput: {str(enabled)}")
- self.log.INFO(f"Corrections for preview: {str(preview)}")
+ self.log.DEBUG(f"Corrections for dataOutput: {str(enabled)}")
+ self.log.DEBUG(f"Corrections for preview: {str(preview)}")
def _update_output_schema(self, data):
"""Updates the schema of dataOutput based on data we want to send
diff --git a/src/calng/calcat_utils.py b/src/calng/calcat_utils.py
index a24fea6ed743f34f8e19362da77876e4b07aec90..9f002407cb6cad68416727a8092d74ede14a89f8 100644
--- a/src/calng/calcat_utils.py
+++ b/src/calng/calcat_utils.py
@@ -18,7 +18,13 @@ from calibration_client.modules import (
Parameter,
PhysicalDetectorUnit,
)
-from karabo.bound import DOUBLE_ELEMENT, NODE_ELEMENT, STRING_ELEMENT, UINT32_ELEMENT
+from karabo.bound import (
+ BOOL_ELEMENT,
+ DOUBLE_ELEMENT,
+ NODE_ELEMENT,
+ STRING_ELEMENT,
+ UINT32_ELEMENT,
+)
from . import utils
@@ -57,6 +63,39 @@ def _check_resp(resp, exception=Exception):
raise exception(resp["info"])
+def _add_status_schema_from_enum(schema, prefix, enum_class):
+ for constant in enum_class:
+ constant_node = f"{prefix}.{constant.name}"
+ (
+ NODE_ELEMENT(schema).key(constant_node).commit(),
+ BOOL_ELEMENT(schema)
+ .key(f"{constant_node}.found")
+ .readOnly()
+ .initialValue(False)
+ .commit(),
+ STRING_ELEMENT(schema)
+ .key(f"{constant_node}.createdAt")
+ .readOnly()
+ .initialValue("")
+ .commit(),
+ STRING_ELEMENT(schema)
+ .key(f"{constant_node}.calibrationId")
+ .readOnly()
+ .initialValue("")
+ .commit(),
+ STRING_ELEMENT(schema)
+ .key(f"{constant_node}.conditionId")
+ .readOnly()
+ .initialValue("")
+ .commit(),
+ STRING_ELEMENT(schema)
+ .key(f"{constant_node}.constantId")
+ .readOnly()
+ .initialValue("")
+ .commit(),
+ )
+
+
class DetectorStandin(typing.NamedTuple):
detector_name: str
modno_to_source: dict
@@ -90,25 +129,44 @@ class BaseCalcatFriend:
_constants_need_conditions = None # subclass should set
@staticmethod
- def add_schema(schema, detector_type, prefix="constantParameters"):
+ def add_schema(
+ schema,
+ detector_type,
+ param_prefix="constantParameters",
+ status_prefix="foundConstants",
+ ):
"""Add elements needed by this object to device's schema (expectedSchema)
- All elements added to schema go under prefix which should end with name of
+ All elements added to schema go under prefixes which should end with name of
node which does not exist yet. To change default values and add more fields,
extend this method in subclass.
- The values set on the device schema can handily be queried via _get. This
- should only be done in helper methods generating "conditions"; see for example
- AgipdCalcatFriend.dark_condition.
+ The param_prefix node will hold all the parameters needed to build constant
+ condition dicts for querying CalCat. These values are set either directly on
+ the device or via manager and this class gets them from the device using helper
+ function _get_param. See for example AgipdCalcatFriend.dark_condition.
+
+ The status_prefix node is used to report information about what was found in
+ CalCat. This class will update the values on the device using the helper
+ function _set_status. This should not need to happen in subclass methods.
"""
- (NODE_ELEMENT(schema).key(prefix).commit(),)
+ (
+ NODE_ELEMENT(schema)
+ .key(param_prefix)
+ .displayedName("Constant retrieval parameters")
+ .commit(),
+ NODE_ELEMENT(schema)
+ .key(status_prefix)
+ .displayedName("Constants retrieved")
+ .commit(),
+ )
# Parameters which any detector would probably have (extend this in subclass)
# TODO: probably switch to floating point for everything, including mem cells
(
STRING_ELEMENT(schema)
- .key(f"{prefix}.detectorType")
+ .key(f"{param_prefix}.detectorType")
.displayedName("Detector type name")
.description(
"Name of detector type in CalCat; typically has suffix '-Type'"
@@ -117,33 +175,33 @@ class BaseCalcatFriend:
.initialValue(detector_type)
.commit(),
STRING_ELEMENT(schema)
- .key(f"{prefix}.detectorName")
+ .key(f"{param_prefix}.detectorName")
.assignmentOptional()
.defaultValue("")
.commit(),
STRING_ELEMENT(schema)
- .key(f"{prefix}.karaboDa")
+ .key(f"{param_prefix}.karaboDa")
.assignmentOptional()
.defaultValue("")
.commit(),
UINT32_ELEMENT(schema)
- .key(f"{prefix}.memoryCells")
+ .key(f"{param_prefix}.memoryCells")
.assignmentOptional()
.defaultValue(352)
.reconfigurable()
.commit(),
UINT32_ELEMENT(schema)
- .key(f"{prefix}.pixelsX")
+ .key(f"{param_prefix}.pixelsX")
.assignmentOptional()
.defaultValue(512)
.commit(),
UINT32_ELEMENT(schema)
- .key(f"{prefix}.pixelsY")
+ .key(f"{param_prefix}.pixelsY")
.assignmentOptional()
.defaultValue(512)
.commit(),
DOUBLE_ELEMENT(schema)
- .key(f"{prefix}.biasVoltage")
+ .key(f"{param_prefix}.biasVoltage")
.assignmentOptional()
.defaultValue(300)
.reconfigurable()
@@ -154,10 +212,12 @@ class BaseCalcatFriend:
self,
device,
secrets_fn: pathlib.Path,
- prefix="constantParameters",
+ param_prefix="constantParameters",
+ status_prefix="foundConstants",
):
self.device = device
- self.prefix = prefix
+ self.param_prefix = param_prefix
+ self.status_prefix = status_prefix
if not secrets_fn.is_file():
self.device.log.WARN(f"Missing CalCat secrets file (expected {secrets_fn})")
@@ -183,19 +243,23 @@ class BaseCalcatFriend:
)
self.device.log.INFO("CalCat connection established")
- def _get(self, key):
+ def _get_param(self, key):
"""Helper to get value from attached device schema"""
- return self.device.get(f"{self.prefix}.{key}")
+ return self.device.get(f"{self.param_prefix}.{key}")
+
+ def _set_status(self, key, value):
+ """Helper to update information about found constants on device"""
+ self.device.set(f"{self.status_prefix}.{key}", value)
@functools.cached_property
def detector_id(self):
- resp = Detector.get_by_identifier(self.client, self._get("detectorName"))
+ resp = Detector.get_by_identifier(self.client, self._get_param("detectorName"))
_check_resp(resp, DetectorNotFound)
return resp["data"]["id"]
@functools.cached_property
def detector_type_id(self):
- resp = DetectorType.get_by_name(self.client, self._get("detectorType"))
+ resp = DetectorType.get_by_name(self.client, self._get_param("detectorType"))
_check_resp(resp, DetectorNotFound)
return resp["data"]["id"]
@@ -252,21 +316,29 @@ class BaseCalcatFriend:
def get_constant_version(self, constant, snapshot_at=None):
# TODO: support snapshot
- # TODO: support creation time
- karabo_da = self._get("karaboDa")
- self.device.log.DEBUG(f"Going looking for {constant} for {karabo_da}")
+ # TODO: catch exceptions, give warnings appropriately
+ karabo_da = self._get_param("karaboDa")
+ self.device.log.DEBUG(f"Will look for {constant} for {karabo_da}")
+
if karabo_da not in self._karabo_da_to_float_uuid:
raise ModuleNotFound(f"Module map did not include {karabo_da}")
+
if isinstance(constant, str):
constant = self._constant_enum_class[constant]
+
calibration_id = self.calibration_id(constant.name)
+ self._set_status(f"{constant.name}.calibrationId", calibration_id)
+
condition = self._constants_need_conditions[constant]()
condition_id = self.condition_id(
self._karabo_da_to_float_uuid[karabo_da], condition
)
+ self._set_status(f"{constant.name}.conditionId", condition_id)
+
constant_id = self.constant_id(
calibration_id=calibration_id, condition_id=condition_id
)
+ self._set_status(f"{constant.name}.constantId", constant_id)
resp = CalibrationConstantVersion.get_by_uk(
self.client,
@@ -279,14 +351,17 @@ class BaseCalcatFriend:
timestamp = resp["data"][
"begin_at"
] # TODO: check which key we like (also has begin_validity_at)
+ self._set_status(f"{constant.name}.createdAt", timestamp)
+
file_path = (
self.caldb_store / resp["data"]["path_to_file"] / resp["data"]["file_name"]
)
- # TODO: return time stamps and such, those are useful
# TODO: handle FileNotFoundError if we are led astray
with h5py.File(file_path, "r") as fd:
- ary = np.array(fd[resp["data"]["data_set_name"]]["data"])
- return timestamp, ary
+ constant_data = np.array(fd[resp["data"]["data_set_name"]]["data"])
+ self._set_status(f"{constant.name}.found", True)
+
+ return constant_data
def get_constant_version_and_call_me_back(
self, constant, callback, snapshot_at=None
@@ -294,7 +369,6 @@ class BaseCalcatFriend:
"""WIP. Callback function will get constant name, constant data, and hopefully
soon also some metadata or whatever.
"""
- # TODO: thread safe caching throughout this class
# TODO: do we want to use asyncio / "modern" async?
def aux():
data = self.get_constant_version(constant, snapshot_at)
@@ -304,6 +378,11 @@ class BaseCalcatFriend:
thread.start()
return thread
+ def flush_constants(self):
+ for constant in self._constant_enum_class:
+ self._set_status(f"{constant.name}.createdAt", "")
+ self._set_status(f"{constant.name}.found", False)
+
class AgipdConstants(enum.Enum):
SlopesFF = enum.auto()
@@ -318,12 +397,8 @@ class AgipdConstants(enum.Enum):
class AgipdCalcatFriend(BaseCalcatFriend):
_constant_enum_class = AgipdConstants
- def __init__(
- self,
- device,
- prefix="constantParameters",
- ):
- super().__init__(device, prefix)
+ def __init__(self, device, *args, **kwargs):
+ super().__init__(device, *args, **kwargs)
self._constants_need_conditions = {
AgipdConstants.ThresholdsDark: self.dark_condition,
AgipdConstants.Offset: self.dark_condition,
@@ -335,62 +410,66 @@ class AgipdCalcatFriend(BaseCalcatFriend):
}
@staticmethod
- def add_schema(schema, prefix="constantParameters"):
+ def add_schema(
+ schema, param_prefix="constantParameters", status_prefix="foundConstants"
+ ):
super(AgipdCalcatFriend, AgipdCalcatFriend).add_schema(
- schema, "AGIPD-Type", prefix
+ schema, "AGIPD-Type", param_prefix, status_prefix
)
(
DOUBLE_ELEMENT(schema)
- .key(f"{prefix}.acquisitionRate")
+ .key(f"{param_prefix}.acquisitionRate")
.assignmentOptional()
.defaultValue(1.1)
.reconfigurable()
.commit(),
DOUBLE_ELEMENT(schema)
- .key(f"{prefix}.gainSetting")
+ .key(f"{param_prefix}.gainSetting")
.assignmentOptional()
.defaultValue(0)
.reconfigurable()
.commit(),
DOUBLE_ELEMENT(schema)
- .key(f"{prefix}.photonEnergy")
+ .key(f"{param_prefix}.photonEnergy")
.assignmentOptional()
.defaultValue(9.2)
.reconfigurable()
.commit(),
DOUBLE_ELEMENT(schema)
- .key(f"{prefix}.gainMode")
+ .key(f"{param_prefix}.gainMode")
.assignmentOptional()
.defaultValue(0)
.reconfigurable()
.commit(),
DOUBLE_ELEMENT(schema)
- .key(f"{prefix}.integrationtime")
+ .key(f"{param_prefix}.integrationtime")
.assignmentOptional()
.defaultValue(12)
.reconfigurable()
.commit(),
)
+ _add_status_schema_from_enum(schema, status_prefix, AgipdConstants)
+
def dark_condition(self):
res = OperatingConditions()
- res["Pixels X"] = self._get("pixelsX")
- res["Pixels Y"] = self._get("pixelsY")
- res["Memory cells"] = self._get("memoryCells")
- res["Acquisition rate"] = self._get("acquisitionRate")
- res["Sensor Bias Voltage"] = self._get("biasVoltage")
+ res["Pixels X"] = self._get_param("pixelsX")
+ res["Pixels Y"] = self._get_param("pixelsY")
+ res["Memory cells"] = self._get_param("memoryCells")
+ res["Acquisition rate"] = self._get_param("acquisitionRate")
+ res["Sensor Bias Voltage"] = self._get_param("biasVoltage")
return res
def illuminated_condition(self):
res = OperatingConditions()
- res["Memory cells"] = self._get("memoryCells")
- res["Sensor Bias Voltage"] = self._get("biasVoltage")
- res["Pixels X"] = self._get("pixelsX")
- res["Pixels Y"] = self._get("pixelsY")
- res["Source Energy"] = self._get("photonEnergy")
- res["Acquisition rate"] = self._get("acquisitionRate")
- res["Gain Setting"] = self._get("gainSetting")
+ res["Memory cells"] = self._get_param("memoryCells")
+ res["Sensor Bias Voltage"] = self._get_param("biasVoltage")
+ res["Pixels X"] = self._get_param("pixelsX")
+ res["Pixels Y"] = self._get_param("pixelsY")
+ res["Source Energy"] = self._get_param("photonEnergy")
+ res["Acquisition rate"] = self._get_param("acquisitionRate")
+ res["Gain Setting"] = self._get_param("gainSetting")
return res
@@ -401,46 +480,46 @@ class DsscConstants(enum.Enum):
class DsscCalcatFriend(BaseCalcatFriend):
_constant_enum_class = DsscConstants
- def __init__(
- self,
- device,
- prefix="constantParameters",
- ):
- super().__init__(device, prefix)
+ def __init__(self, device, *args, **kwargs):
+ super().__init__(device, *args, **kwargs)
self._constants_need_conditions = {
DsscConstants.Offset: self.dark_condition,
}
@staticmethod
- def add_schema(schema, prefix="constantParameters"):
+ def add_schema(
+ schema, param_prefix="constantParameters", status_prefix="foundConstants"
+ ):
super(DsscCalcatFriend, DsscCalcatFriend).add_schema(
- schema, "DSSC-Type", prefix
+ schema, "DSSC-Type", param_prefix, status_prefix
)
(
DOUBLE_ELEMENT(schema)
- .key(f"{prefix}.pulseIdChecksum")
+ .key(f"{param_prefix}.pulseIdChecksum")
.assignmentOptional()
.defaultValue(2.8866323107820637e-36)
.commit(),
DOUBLE_ELEMENT(schema)
- .key(f"{prefix}.acquisitionRate")
+ .key(f"{param_prefix}.acquisitionRate")
.assignmentOptional()
.defaultValue(4.5)
.commit(),
DOUBLE_ELEMENT(schema)
- .key(f"{prefix}.encodedGain")
+ .key(f"{param_prefix}.encodedGain")
.assignmentOptional()
.defaultValue(67328)
.commit(),
)
+ _add_status_schema_from_enum(schema, status_prefix, DsscConstants)
+
def dark_condition(self):
res = OperatingConditions()
- res["Memory cells"] = self._get("memoryCells")
- res["Sensor Bias Voltage"] = self._get("biasVoltage")
- res["Pixels X"] = self._get("pixelsX")
- res["Pixels Y"] = self._get("pixelsY")
- # res["Pulse id checksum"] = self._get("pulseIdChecksum")
- # res["Acquisition rate"] = self._get("acquisitionRate")
- # res["Encoded gain"] = self._get("encodedGain")
+ res["Memory cells"] = self._get_param("memoryCells")
+ res["Sensor Bias Voltage"] = self._get_param("biasVoltage")
+ res["Pixels X"] = self._get_param("pixelsX")
+ res["Pixels Y"] = self._get_param("pixelsY")
+ # res["Pulse id checksum"] = self._get_param("pulseIdChecksum")
+ # res["Acquisition rate"] = self._get_param("acquisitionRate")
+ # res["Encoded gain"] = self._get_param("encodedGain")
return res
diff --git a/src/calng/scenes.py b/src/calng/scenes.py
index f12ab19e4a0f9e718fbd848c066aef781ffbe40b..aae1e48015077a9e4de6df38a826dd362a8047c5 100644
--- a/src/calng/scenes.py
+++ b/src/calng/scenes.py
@@ -1,5 +1,6 @@
from karabo.bound import Types
from karabo.common.scenemodel.api import (
+ DisplayCommandModel,
DisplayLabelModel,
DisplayStateColorModel,
DisplayTextLogModel,
@@ -15,7 +16,7 @@ PADDING = 10
RECONFIGURABLE = 4 # TODO: look up proper enum
-def correction_device_constants_scene(device_id, node_prefix, schema):
+def correction_device_constants_scene(device_id, schema, param_prefix, status_prefix):
# TODO: are there layout models in scene model somewhere? like gridbox?
subscenes = []
max_height = 0
@@ -31,8 +32,8 @@ def correction_device_constants_scene(device_id, node_prefix, schema):
)
)
row_offset += BASE_INC
- for key in schema.getKeys(node_prefix):
- key_path = f"{node_prefix}.{key}"
+ for key in schema.getKeys(param_prefix):
+ key_path = f"{param_prefix}.{key}"
subscenes.append(
LabelModel(
text=schema.getDisplayedName(key_path)
@@ -46,7 +47,7 @@ def correction_device_constants_scene(device_id, node_prefix, schema):
)
subscenes.append(
DisplayLabelModel(
- keys=[f"{device_id}.{node_prefix}.{key}"],
+ keys=[f"{device_id}.{param_prefix}.{key}"],
width=7 * BASE_INC,
height=BASE_INC,
x=col_offset + 5 * BASE_INC,
@@ -58,7 +59,7 @@ def correction_device_constants_scene(device_id, node_prefix, schema):
if value_type in (Types.DOUBLE, Types.FLOAT):
subscenes.append(
DoubleLineEditModel(
- keys=[f"{device_id}.{node_prefix}.{key}"],
+ keys=[f"{device_id}.{param_prefix}.{key}"],
width=7 * BASE_INC,
height=BASE_INC,
x=col_offset + 12 * BASE_INC,
@@ -68,7 +69,7 @@ def correction_device_constants_scene(device_id, node_prefix, schema):
elif value_type in (Types.INT32, Types.UINT32, Types.INT64, Types.UINT64):
subscenes.append(
IntLineEditModel(
- keys=[f"{device_id}.{node_prefix}.{key}"],
+ keys=[f"{device_id}.{param_prefix}.{key}"],
width=7 * BASE_INC,
height=BASE_INC,
x=col_offset + 12 * BASE_INC,
@@ -93,20 +94,63 @@ def correction_device_constants_scene(device_id, node_prefix, schema):
row_offset = PADDING
col_offset += 19 * BASE_INC # width of first column
col_offset += PADDING # and padding
+ subscenes.append(
+ DisplayCommandModel(
+ keys=[f"{device_id}.loadMostRecentConstants"],
+ x=col_offset,
+ y=row_offset,
+ width=10 * BASE_INC,
+ height=BASE_INC,
+ )
+ )
+ row_offset += BASE_INC
subscenes.append(
LabelModel(
text="Constants found and loaded",
width=10 * BASE_INC,
+ height=BASE_INC,
x=col_offset,
y=row_offset,
)
)
row_offset += BASE_INC
+ for constant_name in schema.getKeys(status_prefix):
+ constant_path = f"{status_prefix}.{constant_name}"
+ subscenes.append(
+ LabelModel(
+ text=constant_name,
+ width=10 * BASE_INC,
+ height=BASE_INC,
+ x=col_offset,
+ y=row_offset,
+ )
+ )
+ row_offset += BASE_INC
+ for field in ("found", "createdAt"):
+ subscenes.append(
+ LabelModel(
+ text=field, # TODO: displayedName
+ width=5 * BASE_INC,
+ height=BASE_INC,
+ x=col_offset,
+ y=row_offset,
+ )
+ )
+ subscenes.append(
+ DisplayLabelModel(
+ keys=[f"{device_id}.{constant_path}.{field}"],
+ width=7 * BASE_INC,
+ height=BASE_INC,
+ x=col_offset + 5 * BASE_INC,
+ y=row_offset,
+ )
+ )
+ row_offset += BASE_INC
max_height = max(max_height, row_offset)
scene = SceneModel(
height=max_height + 2 * PADDING,
- width=col_offset + 10 * BASE_INC + PADDING,
+ width=col_offset + 12 * BASE_INC + PADDING,
children=subscenes,
)
return write_scene(scene)