From a39751afd012bded93bb21fb3ed0a3f83c22c656 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 6 Jan 2022 16:20:47 +0100
Subject: [PATCH] Add prototype assembler and manual geometry devices
Copied over from my hacky utils; have used these in my development Karabo
environment for a while. Devices here (and corresponding changes to manager)
sketch how I interpret our plan for the next step in online calibration
topology; manual geometry devices to be later supplanted by or supplemented with
something using encoders or such.
---
src/calng/CalibrationManager.py | 70 ++---------
src/calng/ManualAgipdGeometry.py | 26 +++++
src/calng/ManualDsscGeometry.py | 26 +++++
src/calng/SimpleAssembler.py | 186 ++++++++++++++++++++++++++++++
src/calng/manual_geometry_base.py | 161 ++++++++++++++++++++++++++
5 files changed, 407 insertions(+), 62 deletions(-)
create mode 100644 src/calng/ManualAgipdGeometry.py
create mode 100644 src/calng/ManualDsscGeometry.py
create mode 100644 src/calng/SimpleAssembler.py
create mode 100644 src/calng/manual_geometry_base.py
diff --git a/src/calng/CalibrationManager.py b/src/calng/CalibrationManager.py
index bd519b39..59f485f0 100644
--- a/src/calng/CalibrationManager.py
+++ b/src/calng/CalibrationManager.py
@@ -1194,78 +1194,24 @@ class CalibrationManager(DeviceClientBase, Device):
background(_activate_bridge(bridge_device_id))
- # Instantiate preview layer matchers and assemblers.
+ # Instantiate preview assemblers.
+ geometry_device_id = self.geometryDevice.value
for layer, output_pipeline, server in self.previewLayers.value:
- # Preview matcher.
- matcher_device_id = device_id_templates['previewMatcher'].format(
+ assembler_device_id = device_id_templates['assembler'].format(
layer=layer)
config = Hash()
- config['channels'] = [
- f'{device_id}:{output_pipeline}'
- for device_id in correct_device_id_by_module.values()]
config['fastSources'] = [
Hash('fsSelect', True,
'fsSource',
f'{input_source_by_module[virtual_id]}')
for (virtual_id, device_id)
in correct_device_id_by_module.items()]
- config['pathToStack'] = 'data.adc'
-
- if not await self._instantiate_device(
- server, class_ids['previewMatcher'], matcher_device_id, config
- ):
- return
-
- # Preview assembler.
- assembler_device_id = device_id_templates['assembler'].format(
- layer=layer)
-
- config = Hash()
- config['input.connectedOutputChannels'] = [
- f'{matcher_device_id}:output']
- config['modules'] = [
- Hash('source', input_source_by_module.get('Q1M1', ''),
- 'offX', 474, 'offY', 612, 'rot', 90),
- Hash('source', input_source_by_module.get('Q1M2', ''),
- 'offX', 316, 'offY', 612, 'rot', 90),
- Hash('source', input_source_by_module.get('Q1M3', ''),
- 'offX', 158, 'offY', 612, 'rot', 90),
- Hash('source', input_source_by_module.get('Q1M4', ''),
- 'offX', 0, 'offY', 612, 'rot', 90),
- Hash('source', input_source_by_module.get('Q2M1', ''),
- 'offX', 1136, 'offY', 612, 'rot', 90),
- Hash('source', input_source_by_module.get('Q2M2', ''),
- 'offX', 978, 'offY', 612, 'rot', 90),
- Hash('source', input_source_by_module.get('Q2M3', ''),
- 'offX', 820, 'offY', 612, 'rot', 90),
- Hash('source', input_source_by_module.get('Q2M4', ''),
- 'offX', 662, 'offY', 612, 'rot', 90),
- Hash('source', input_source_by_module.get('Q3M1', ''),
- 'offX', 712, 'offY', 0, 'rot', 270),
- Hash('source', input_source_by_module.get('Q3M2', ''),
- 'offX', 870, 'offY', 0, 'rot', 270),
- Hash('source', input_source_by_module.get('Q3M3', ''),
- 'offX', 1028, 'offY', 0, 'rot', 270),
- Hash('source', input_source_by_module.get('Q3M4', ''),
- 'offX', 1186, 'offY', 0, 'rot', 270),
- Hash('source', input_source_by_module.get('Q4M1', ''),
- 'offX', 50, 'offY', 0, 'rot', 270),
- Hash('source', input_source_by_module.get('Q4M2', ''),
- 'offX', 208, 'offY', 0, 'rot', 270),
- Hash('source', input_source_by_module.get('Q4M3', ''),
- 'offX', 366, 'offY', 0, 'rot', 270),
- Hash('source', input_source_by_module.get('Q4M4', ''),
- 'offX', 524, 'offY', 0, 'rot', 270),
- ]
- config['pathsToCombine'] = ['data.adc']
- config['trainIdPath'] = 'image.trainId'
- config['pulseIdPath'] = 'image.pulseId'
- config['preview.enablePreview'] = True
- config['preview.pathToPreview'] = 'data.adc'
- config['preview.downSample'] = 2
- config['badpixelPath'] = 'image.bad_pixels'
- config['rotated90Grad'] = True
+ config['channels'] = [
+ f'{device_id}:{output_pipeline}'
+ for device_id in correct_device_id_by_module.values()]
+ config['geometryInput.connectedOutputChannels'] = [
+ f'{geometry_device_id}:geometryOutput']
if not await self._instantiate_device(
server, class_ids['assembler'], assembler_device_id, config
diff --git a/src/calng/ManualAgipdGeometry.py b/src/calng/ManualAgipdGeometry.py
new file mode 100644
index 00000000..b01e96bc
--- /dev/null
+++ b/src/calng/ManualAgipdGeometry.py
@@ -0,0 +1,26 @@
+import extra_geom
+from karabo.bound import KARABO_CLASSINFO
+
+from ._version import version as deviceVersion
+from .manual_geometry_base import ManualGeometryBase
+
+
+@KARABO_CLASSINFO("ManualAgipdGeometry", deviceVersion)
+class ManualAgipdGeometry(ManualGeometryBase):
+ geometry_class = extra_geom.AGIPD_1MGeometry
+
+ @staticmethod
+ def expectedParameters(expected):
+ super(ManualAgipdGeometry, ManualAgipdGeometry).expectedParameters(expected)
+
+ expected.setDefaultValue("quadrantCorners.Q1.x", -525)
+ expected.setDefaultValue("quadrantCorners.Q1.y", 625)
+
+ expected.setDefaultValue("quadrantCorners.Q2.x", -550)
+ expected.setDefaultValue("quadrantCorners.Q2.y", -10)
+
+ expected.setDefaultValue("quadrantCorners.Q3.x", 520)
+ expected.setDefaultValue("quadrantCorners.Q3.y", -160)
+
+ expected.setDefaultValue("quadrantCorners.Q4.x", 542.5)
+ expected.setDefaultValue("quadrantCorners.Q4.y", 475)
diff --git a/src/calng/ManualDsscGeometry.py b/src/calng/ManualDsscGeometry.py
new file mode 100644
index 00000000..27f951d1
--- /dev/null
+++ b/src/calng/ManualDsscGeometry.py
@@ -0,0 +1,26 @@
+import extra_geom
+from karabo.bound import KARABO_CLASSINFO
+
+from ._version import version as deviceVersion
+from .manual_geometry_base import ManualGeometryBase
+
+
+@KARABO_CLASSINFO("ManualDsscGeometry", deviceVersion)
+class ManualDsscGeometry(ManualGeometryBase):
+ geometry_class = extra_geom.DSSC_1MGeometry
+
+ @staticmethod
+ def expectedParameters(expected):
+ super(ManualDsscGeometry, ManualDsscGeometry).expectedParameters(expected)
+
+ expected.setDefaultValue("quadrantCorners.Q1.x", -130)
+ expected.setDefaultValue("quadrantCorners.Q1.y", 5)
+
+ expected.setDefaultValue("quadrantCorners.Q2.x", -130)
+ expected.setDefaultValue("quadrantCorners.Q2.y", -125)
+
+ expected.setDefaultValue("quadrantCorners.Q3.x", 5)
+ expected.setDefaultValue("quadrantCorners.Q3.y", -125)
+
+ expected.setDefaultValue("quadrantCorners.Q4.x", 5)
+ expected.setDefaultValue("quadrantCorners.Q4.y", 5)
diff --git a/src/calng/SimpleAssembler.py b/src/calng/SimpleAssembler.py
new file mode 100644
index 00000000..9f7d7523
--- /dev/null
+++ b/src/calng/SimpleAssembler.py
@@ -0,0 +1,186 @@
+import functools
+import pickle
+import re
+
+import numpy as np
+from karabo.bound import (
+ FLOAT_ELEMENT,
+ IMAGEDATA_ELEMENT,
+ INPUT_CHANNEL,
+ KARABO_CLASSINFO,
+ OUTPUT_CHANNEL,
+ OVERWRITE_ELEMENT,
+ STRING_ELEMENT,
+ UINT64_ELEMENT,
+ ChannelMetaData,
+ Dims,
+ Encoding,
+ Epochstamp,
+ Hash,
+ ImageData,
+ MetricPrefix,
+ Schema,
+ Timestamp,
+ Trainstamp,
+ Unit,
+)
+from karabo.common.api import KARABO_SCHEMA_DISPLAY_TYPE_SCENES as DT_SCENES
+from TrainMatcher import TrainMatcher, scenes as trainmatcher_scenes
+
+from . import scenes
+from ._version import version as deviceVersion
+from calng import utils
+
+
+preview_schema = Schema()
+(
+ IMAGEDATA_ELEMENT(preview_schema).key("image").commit(),
+ UINT64_ELEMENT(preview_schema).key("trainId").readOnly().commit(),
+)
+
+xtdf_source_re = re.compile(r".*\/DET\/(\d+)CH0:xtdf")
+
+# TODO: merge scene with TrainMatcher's nice overview
+@KARABO_CLASSINFO("SimpleAssembler", deviceVersion)
+class SimpleAssembler(TrainMatcher.TrainMatcher):
+ @staticmethod
+ def expectedParameters(expected):
+ (
+ OVERWRITE_ELEMENT(expected)
+ .key("availableScenes")
+ # TODO: add assemblerOverview scene
+ .setNewDefaultValue(["scene", "assemblerOverview"])
+ .commit(),
+ FLOAT_ELEMENT(expected)
+ .key("processingTime")
+ .unit(Unit.SECOND)
+ .metricPrefix(MetricPrefix.MILLI)
+ .readOnly()
+ .initialValue(0)
+ .warnHigh(500)
+ .info("Cannot keep up with GUI limit")
+ .needsAcknowledging(False)
+ .commit(),
+ FLOAT_ELEMENT(expected)
+ .key("timeOfFlight")
+ .unit(Unit.SECOND)
+ .metricPrefix(MetricPrefix.MILLI)
+ .readOnly()
+ .initialValue(0)
+ .warnHigh(1000)
+ .info("Time of flight exceeding 1 s")
+ .needsAcknowledging(False)
+ .commit(),
+ STRING_ELEMENT(expected)
+ .key("pathToStack")
+ .assignmentOptional()
+ .defaultValue("image.data")
+ .commit(),
+ INPUT_CHANNEL(expected)
+ .key("geometryInput")
+ .displayedName("Geometry input")
+ .commit(),
+ OUTPUT_CHANNEL(expected) # can OVERWRITE_ELEMENT even do this?
+ .key("output")
+ .dataSchema(preview_schema)
+ .commit(),
+ )
+
+ def initialization(self):
+ super().initialization()
+
+ # TODO: match inside device, fill multiple independent buffers
+
+ self._path_to_stack = self.get("pathToStack")
+ self.geometry = None
+ self.input_buffer = None
+
+ self.KARABO_ON_DATA("geometryInput", self.receive_geometry)
+ self.KARABO_SLOT(self.requestScene)
+
+ self.start()
+
+ def requestScene(self, params):
+ # TODO: unify with TrainMatcher overview
+ scene_name = params.get("name", default="")
+ if scene_name == "assemblerOverview":
+ payload = Hash("name", scene_name, "success", True)
+ payload["data"] = scenes.simple_assembler_overview(
+ device_id=self.getInstanceId(),
+ geometry_device_id=self.get("geometryInput.connectedOutputChannels")[
+ 0
+ ].split(":")[0],
+ )
+ self.reply(
+ Hash(
+ "type",
+ "deviceScene",
+ "origin",
+ self.getInstanceId(),
+ "payload",
+ payload,
+ )
+ )
+ else:
+ return super().requestScene(params)
+
+ def receive_geometry(self, data, metadata):
+ self.log.INFO("Received a new geometry")
+ self.geometry = pickle.loads(data.get("pickledGeometry"))
+ self.input_buffer = np.zeros(self.geometry.expected_data_shape)
+
+ def _send(self, train_id, sources):
+ # TODO: adapt to appropriate hook for new TrainMatcher (no _send)
+ if self.geometry is None:
+ self.log.WARN("Have not received a geometry yet")
+ return
+
+ timestamp = Timestamp(Epochstamp(), Trainstamp(train_id))
+
+ module_indices_unfilled = set(range(self.input_buffer.shape[0]))
+ for source, (data, metadata) in sources.items():
+ # TODO: handle failure to "parse" source, get data out
+ module_index = self._source_to_index(source)
+ self.input_buffer[module_index] = np.squeeze(data.get(self._path_to_stack))
+ module_indices_unfilled.discard(module_index)
+
+ for unfilled_module in module_indices_unfilled:
+ self.input_buffer[module_index].fill(0)
+ # TODO: configurable treatment of missing modules
+
+ # TODO: reusable output buffer to save on allocation
+ assembled, _ = self.geometry.position_modules_fast(self.input_buffer)
+
+ # TODO: optionally include control data
+ out_hash = Hash(
+ "image",
+ ImageData(
+ # TODO: get around this being mirrored...
+ (assembled[::-1, ::-1]).astype(np.int32),
+ Dims(*assembled.shape),
+ Encoding.GRAY,
+ ),
+ "trainId",
+ train_id,
+ )
+ channel = self.signalSlotable.getOutputChannel("output")
+ channel.write(out_hash, ChannelMetaData(self.getInstanceId(), timestamp))
+ channel.update()
+ self.rate_out.update()
+
+ def _update_rate(self):
+ self._buffered_status_update.set(
+ "processingTime", self._processing_time_ema.get()
+ )
+ self._buffered_status_update.set("timeOfFlight", self._time_of_flight_ema.get())
+ self._buffered_status_update.set("rate", self._rate_tracker.get())
+ self.set(self._buffered_status_update)
+
+ @functools.lru_cache
+ def _source_to_index(self, source):
+ # note: cache means warning only shows up once (also not performance-critical)
+ match = xtdf_source_re.match(source)
+ if match is None:
+ self.log.WARN(f"Couldn't figure out index for source {source}")
+ return 0
+ return int(match.group(1))
diff --git a/src/calng/manual_geometry_base.py b/src/calng/manual_geometry_base.py
new file mode 100644
index 00000000..e7629f3a
--- /dev/null
+++ b/src/calng/manual_geometry_base.py
@@ -0,0 +1,161 @@
+import pickle
+
+from karabo.bound import (
+ DOUBLE_ELEMENT,
+ KARABO_CLASSINFO,
+ IMAGEDATA_ELEMENT,
+ NODE_ELEMENT,
+ OUTPUT_CHANNEL,
+ SLOT_ELEMENT,
+ VECTOR_CHAR_ELEMENT,
+ VECTOR_STRING_ELEMENT,
+ Encoding,
+ Hash,
+ ImageData,
+ PythonDevice,
+ Schema,
+ State,
+)
+from karabo.common.api import KARABO_SCHEMA_DISPLAY_TYPE_SCENES as DT_SCENES
+import numpy as np
+from matplotlib.backends.backend_agg import FigureCanvasAgg
+import matplotlib.pyplot as plt
+
+from . import scenes
+from ._version import version as deviceVersion
+
+
+geometry_schema = Schema()
+(
+ VECTOR_CHAR_ELEMENT(geometry_schema)
+ .key("pickledGeometry")
+ .displayedName("Pickled geometry")
+ .assignmentOptional()
+ .defaultValue([])
+ .commit()
+)
+
+preview_schema = Schema()
+(IMAGEDATA_ELEMENT(preview_schema).key("overview").commit())
+
+
+@KARABO_CLASSINFO("ManualGeometryBase", deviceVersion)
+class ManualGeometryBase(PythonDevice):
+ @staticmethod
+ def expectedParameters(expected):
+ # "mandatory" for geometry serving device
+ (
+ OUTPUT_CHANNEL(expected)
+ .key("geometryOutput")
+ .dataSchema(geometry_schema)
+ .commit(),
+ SLOT_ELEMENT(expected).key("pleaseSendYourGeometry").commit(),
+ SLOT_ELEMENT(expected).key("reloadScenes").commit(),
+ OUTPUT_CHANNEL(expected)
+ .key("previewOutput")
+ .dataSchema(preview_schema)
+ .commit(),
+ IMAGEDATA_ELEMENT(expected).key("layoutPreview").commit(),
+ )
+
+ # configuring this one manually
+ # subclasses should set better defaults
+ (NODE_ELEMENT(expected).key("quadrantCorners").commit(),)
+ for q in range(1, 5):
+ (
+ NODE_ELEMENT(expected).key(f"quadrantCorners.Q{q}").commit(),
+ DOUBLE_ELEMENT(expected)
+ .key(f"quadrantCorners.Q{q}.x")
+ .assignmentOptional()
+ .defaultValue(0)
+ .reconfigurable()
+ .commit(),
+ DOUBLE_ELEMENT(expected)
+ .key(f"quadrantCorners.Q{q}.y")
+ .assignmentOptional()
+ .defaultValue(0)
+ .reconfigurable()
+ .commit(),
+ )
+
+ # scenes are fun
+ # TODO: add the scene
+ (
+ VECTOR_STRING_ELEMENT(expected)
+ .key("availableScenes")
+ .setSpecialDisplayType(DT_SCENES)
+ .readOnly()
+ .initialValue(["overview"])
+ .commit(),
+ )
+
+ def __init__(self, config):
+ super().__init__(config)
+
+ self.KARABO_SLOT(self.pleaseSendYourGeometry)
+ self.KARABO_SLOT(self.requestScene)
+ self.KARABO_SLOT(self.reloadScenes)
+ self.update_geom()
+ plt.switch_backend("agg")
+ self.updateState(State.ON)
+
+ def requestScene(self, params):
+ payload = Hash()
+ scene_name = params.get("name", default="")
+ payload["name"] = scene_name
+ payload["success"] = True
+ if scene_name == "overview":
+ payload["data"] = scenes.manual_geometry_overview(
+ device_id=self.getInstanceId()
+ )
+ else:
+ payload["success"] = False
+ response = Hash()
+ response["type"] = "deviceScene"
+ response["origin"] = self.getInstanceId()
+ response["payload"] = payload
+ self.reply(response)
+
+ def update_geom(self):
+ self.quadrant_corners = tuple(
+ (self.get(f"quadrantCorners.Q{q}.x"), self.get(f"quadrantCorners.Q{q}.y"))
+ for q in range(1, 5)
+ )
+ self.geom = self.geometry_class.from_quad_positions(self.quadrant_corners)
+ self.pickled = pickle.dumps(self.geom)
+ # TODO: send to anyone who asks? make slot for that?
+ self.writeChannel("geometryOutput", Hash("pickledGeometry", self.pickled))
+
+ def reloadScenes(self):
+ global scenes
+ import importlib
+
+ scenes = importlib.reload(scenes)
+
+ def pleaseSendYourGeometry(self):
+ self.writeChannel("geometryOutput", Hash("pickledGeometry", self.pickled))
+ axis = self.geom.inspect()
+ axis.figure.tight_layout(pad=0)
+ axis.figure.set_facecolor("none")
+ # axis.figure.set_size_inches(6, 6)
+ # axis.figure.set_dpi(300)
+ canvas = FigureCanvasAgg(axis.figure)
+ canvas.draw()
+ image_buffer = np.frombuffer(canvas.buffer_rgba(), dtype=np.uint8).reshape(
+ canvas.get_width_height()[::-1] + (4,)
+ )
+ self.set(
+ "layoutPreview",
+ ImageData(image_buffer, encoding=Encoding.RGBA, bitsPerPixel=3 * 8),
+ )
+ # self.writeChannel("previewOutput", Hash("overview", ImageData()))
+
+ def preReconfigure(self, config):
+ self._prereconfigure_update_hash = config
+
+ def postReconfigure(self):
+ if any(
+ path.startswith("quadrantCorners")
+ for path in self._prereconfigure_update_hash.getPaths()
+ ):
+ self.update_geom()
--
GitLab