diff --git a/setup.py b/setup.py index 4aea697e6bf938b7f4bc85822bcb067d4b9d24d9..9edc6a54f1f970fda125e53e268f470079648dec 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,11 @@ setup(name='calng', 'DsscCorrection = calng.DsscCorrection:DsscCorrection', 'JungfrauCorrection = calng.JungfrauCorrection:JungfrauCorrection', 'ModuleStacker = calng.ModuleStacker:ModuleStacker', + 'ManualAgipdGeometry = calng.ManualAgipdGeometry:ManualAgipdGeometry', + 'ManualDsscGeometry = calng.ManualDsscGeometry:ManualDsscGeometry', + 'ManualJungfrauGeometry = calng.ManualJungfrauGeometry:ManualJungfrauGeometry', 'ShmemToZMQ = calng.ShmemToZMQ:ShmemToZMQ', + 'SimpleAssembler = calng.SimpleAssembler:SimpleAssembler', ], 'karabo.middlelayer_device': [ diff --git a/src/calng/CalibrationManager.py b/src/calng/CalibrationManager.py index bd519b395bdd93340064383b8214f31e0978d16e..07f1277266c948308b66d1fd86856e5421cb5cf1 100644 --- a/src/calng/CalibrationManager.py +++ b/src/calng/CalibrationManager.py @@ -82,14 +82,6 @@ class ClassIdsNode(Configurable): accessMode=AccessMode.INITONLY, assignment=Assignment.MANDATORY) - previewMatcherClass = String( - displayedName='Preview matcher class', - description='Device class to use for matching the output of a preview ' - 'layer.', - defaultValue='ModuleStacker', - accessMode=AccessMode.INITONLY, - assignment=Assignment.MANDATORY) - assemblerClass = String( displayedName='Assembler class', description='Device class to use for assembling the matched output of ' @@ -125,14 +117,6 @@ class DeviceIdsNode(Configurable): accessMode=AccessMode.INITONLY, assignment=Assignment.MANDATORY) - previewMatcherSuffix = String( - displayedName='Preview matcher suffix', - description='Suffix for preview layer matching device IDs. The ' - 'formatting placeholder \'layer\' may be used.', - defaultValue='MATCH_{layer}', - accessMode=AccessMode.INITONLY, - assignment=Assignment.MANDATORY) - assemblerSuffix = String( displayedName='Assembler suffix', description='Suffix for assembler device IDs. The formatting ' @@ -408,6 +392,13 @@ class CalibrationManager(DeviceClientBase, Device): self.deviceServers = value self._servers_changed = True + imageDataPath = String( + displayedName='Image data path', + description='Path in DAQ hash to actual image data, used for preview', + accessMode=AccessMode.RECONFIGURABLE, + assignment=Assignment.OPTIONAL, + defaultValue='image.data') + geometryDevice = String( displayedName='Geometry device', description='[NYI] Device ID for a geometry device defining the ' @@ -1055,7 +1046,7 @@ class CalibrationManager(DeviceClientBase, Device): device_id_templates = {} class_args = (self.detectorType.value.lower().capitalize(),) - for role in ['correction', 'groupMatcher', 'bridge', 'previewMatcher', + for role in ['correction', 'groupMatcher', 'bridge', 'assembler']: class_ids[role] = getattr( self.classIds, f'{role}Class').value.format(*class_args) @@ -1194,78 +1185,26 @@ class CalibrationManager(DeviceClientBase, Device): background(_activate_bridge(bridge_device_id)) - # Instantiate preview layer matchers and assemblers. + # Instantiate preview layer assemblers. + geometry_device_id = self.geometryDevice.value for layer, output_pipeline, server in self.previewLayers.value: - # Preview matcher. - matcher_device_id = device_id_templates['previewMatcher'].format( + assembler_device_id = device_id_templates['assembler'].format( layer=layer) config = Hash() - config['channels'] = [ - f'{device_id}:{output_pipeline}' - for device_id in correct_device_id_by_module.values()] + # TODO: put _image_data_path in corr dev schema, get from there + config['pathToStack'] = self.imageDataPath.value config['fastSources'] = [ Hash('fsSelect', True, 'fsSource', f'{input_source_by_module[virtual_id]}') for (virtual_id, device_id) in correct_device_id_by_module.items()] - config['pathToStack'] = 'data.adc' - - if not await self._instantiate_device( - server, class_ids['previewMatcher'], matcher_device_id, config - ): - return - - # Preview assembler. - assembler_device_id = device_id_templates['assembler'].format( - layer=layer) - - config = Hash() - config['input.connectedOutputChannels'] = [ - f'{matcher_device_id}:output'] - config['modules'] = [ - Hash('source', input_source_by_module.get('Q1M1', ''), - 'offX', 474, 'offY', 612, 'rot', 90), - Hash('source', input_source_by_module.get('Q1M2', ''), - 'offX', 316, 'offY', 612, 'rot', 90), - Hash('source', input_source_by_module.get('Q1M3', ''), - 'offX', 158, 'offY', 612, 'rot', 90), - Hash('source', input_source_by_module.get('Q1M4', ''), - 'offX', 0, 'offY', 612, 'rot', 90), - Hash('source', input_source_by_module.get('Q2M1', ''), - 'offX', 1136, 'offY', 612, 'rot', 90), - Hash('source', input_source_by_module.get('Q2M2', ''), - 'offX', 978, 'offY', 612, 'rot', 90), - Hash('source', input_source_by_module.get('Q2M3', ''), - 'offX', 820, 'offY', 612, 'rot', 90), - Hash('source', input_source_by_module.get('Q2M4', ''), - 'offX', 662, 'offY', 612, 'rot', 90), - Hash('source', input_source_by_module.get('Q3M1', ''), - 'offX', 712, 'offY', 0, 'rot', 270), - Hash('source', input_source_by_module.get('Q3M2', ''), - 'offX', 870, 'offY', 0, 'rot', 270), - Hash('source', input_source_by_module.get('Q3M3', ''), - 'offX', 1028, 'offY', 0, 'rot', 270), - Hash('source', input_source_by_module.get('Q3M4', ''), - 'offX', 1186, 'offY', 0, 'rot', 270), - Hash('source', input_source_by_module.get('Q4M1', ''), - 'offX', 50, 'offY', 0, 'rot', 270), - Hash('source', input_source_by_module.get('Q4M2', ''), - 'offX', 208, 'offY', 0, 'rot', 270), - Hash('source', input_source_by_module.get('Q4M3', ''), - 'offX', 366, 'offY', 0, 'rot', 270), - Hash('source', input_source_by_module.get('Q4M4', ''), - 'offX', 524, 'offY', 0, 'rot', 270), - ] - config['pathsToCombine'] = ['data.adc'] - config['trainIdPath'] = 'image.trainId' - config['pulseIdPath'] = 'image.pulseId' - config['preview.enablePreview'] = True - config['preview.pathToPreview'] = 'data.adc' - config['preview.downSample'] = 2 - config['badpixelPath'] = 'image.bad_pixels' - config['rotated90Grad'] = True + config['channels'] = [ + f'{device_id}:{output_pipeline}' + for device_id in correct_device_id_by_module.values()] + config['geometryInput.connectedOutputChannels'] = [ + f'{geometry_device_id}:geometryOutput'] if not await self._instantiate_device( server, class_ids['assembler'], assembler_device_id, config diff --git a/src/calng/ManualAgipdGeometry.py b/src/calng/ManualAgipdGeometry.py new file mode 100644 index 0000000000000000000000000000000000000000..20428d6bdbfe38709305c6fc90820445e25e14f2 --- /dev/null +++ b/src/calng/ManualAgipdGeometry.py @@ -0,0 +1,26 @@ +import extra_geom +from karabo.bound import KARABO_CLASSINFO + +from ._version import version as deviceVersion +from .manual_geometry_base import ManualQuadrantsGeometryBase + + +@KARABO_CLASSINFO("ManualAgipdGeometry", deviceVersion) +class ManualAgipdGeometry(ManualQuadrantsGeometryBase): + geometry_class = extra_geom.AGIPD_1MGeometry + + @staticmethod + def expectedParameters(expected): + super(ManualAgipdGeometry, ManualAgipdGeometry).expectedParameters(expected) + + expected.setDefaultValue("quadrantCorners.Q1.x", -525) + expected.setDefaultValue("quadrantCorners.Q1.y", 625) + + expected.setDefaultValue("quadrantCorners.Q2.x", -550) + expected.setDefaultValue("quadrantCorners.Q2.y", -10) + + expected.setDefaultValue("quadrantCorners.Q3.x", 520) + expected.setDefaultValue("quadrantCorners.Q3.y", -160) + + expected.setDefaultValue("quadrantCorners.Q4.x", 542.5) + expected.setDefaultValue("quadrantCorners.Q4.y", 475) diff --git a/src/calng/ManualDsscGeometry.py b/src/calng/ManualDsscGeometry.py new file mode 100644 index 0000000000000000000000000000000000000000..5b3d9b2e4d2394bf67b78c7fed2cc084d384612f --- /dev/null +++ b/src/calng/ManualDsscGeometry.py @@ -0,0 +1,26 @@ +import extra_geom +from karabo.bound import KARABO_CLASSINFO + +from ._version import version as deviceVersion +from .manual_geometry_base import ManualQuadrantsGeometryBase + + +@KARABO_CLASSINFO("ManualDsscGeometry", deviceVersion) +class ManualDsscGeometry(ManualQuadrantsGeometryBase): + geometry_class = extra_geom.DSSC_1MGeometry + + @staticmethod + def expectedParameters(expected): + super(ManualDsscGeometry, ManualDsscGeometry).expectedParameters(expected) + + expected.setDefaultValue("quadrantCorners.Q1.x", -130) + expected.setDefaultValue("quadrantCorners.Q1.y", 5) + + expected.setDefaultValue("quadrantCorners.Q2.x", -130) + expected.setDefaultValue("quadrantCorners.Q2.y", -125) + + expected.setDefaultValue("quadrantCorners.Q3.x", 5) + expected.setDefaultValue("quadrantCorners.Q3.y", -125) + + expected.setDefaultValue("quadrantCorners.Q4.x", 5) + expected.setDefaultValue("quadrantCorners.Q4.y", 5) diff --git a/src/calng/ManualJungfrauGeometry.py b/src/calng/ManualJungfrauGeometry.py new file mode 100644 index 0000000000000000000000000000000000000000..57f7da914e77cb7aee28ffd078fe64c13231c259 --- /dev/null +++ b/src/calng/ManualJungfrauGeometry.py @@ -0,0 +1,28 @@ +import extra_geom +from karabo.bound import KARABO_CLASSINFO, OVERWRITE_ELEMENT, Hash + +from ._version import version as deviceVersion +from .manual_geometry_base import ManualModulesGeometryBase + + +@KARABO_CLASSINFO("ManualJungfrauGeometry", deviceVersion) +class ManualJungfrauGeometry(ManualModulesGeometryBase): + geometry_class = extra_geom.JUNGFRAUGeometry + + @staticmethod + def expectedParameters(expected): + # TODO: come up with some sweet defaults (this is two modules from docs 4M) + ( + OVERWRITE_ELEMENT(expected) + .key("modules") + .setNewDefaultValue( + [ + Hash( + "posX", 95, "posY", 564, "orientationX", -1, "orientationY", -1 + ), + Hash( + "posX", 95, "posY", 17, "orientationX", -1, "orientationY", -1 + ), + ] + ) + ) diff --git a/src/calng/SimpleAssembler.py b/src/calng/SimpleAssembler.py new file mode 100644 index 0000000000000000000000000000000000000000..dbd191fb6968b08dba54641457c557f5c3ed37aa --- /dev/null +++ b/src/calng/SimpleAssembler.py @@ -0,0 +1,199 @@ +import functools +import pickle +import re + +import numpy as np +from calng import utils +from karabo.bound import ( + FLOAT_ELEMENT, + IMAGEDATA_ELEMENT, + INPUT_CHANNEL, + KARABO_CLASSINFO, + OUTPUT_CHANNEL, + OVERWRITE_ELEMENT, + STRING_ELEMENT, + UINT64_ELEMENT, + ChannelMetaData, + Dims, + Encoding, + Epochstamp, + Hash, + ImageData, + MetricPrefix, + Schema, + Timestamp, + Trainstamp, + Unit, +) +from karabo.common.api import KARABO_SCHEMA_DISPLAY_TYPE_SCENES as DT_SCENES +from TrainMatcher import TrainMatcher +from TrainMatcher import scenes as trainmatcher_scenes + +from . import scenes +from ._version import version as deviceVersion + +preview_schema = Schema() +( + IMAGEDATA_ELEMENT(preview_schema).key("image").commit(), + + UINT64_ELEMENT(preview_schema).key("trainId").readOnly().commit(), +) + +xtdf_source_re = re.compile(r".*\/DET\/(\d+)CH0:xtdf") +daq_source_re = re.compile(r".*\/DET\/.*?(\d+):daqOutput") + +# TODO: merge scene with TrainMatcher's nice overview +@KARABO_CLASSINFO("SimpleAssembler", deviceVersion) +class SimpleAssembler(TrainMatcher.TrainMatcher): + @staticmethod + def expectedParameters(expected): + ( + OVERWRITE_ELEMENT(expected) + .key("availableScenes") + .setNewDefaultValue(["scene", "assemblerOverview"]) + .commit(), + + FLOAT_ELEMENT(expected) + .key("processingTime") + .unit(Unit.SECOND) + .metricPrefix(MetricPrefix.MILLI) + .readOnly() + .initialValue(0) + .warnHigh(500) + .info("Cannot keep up with GUI limit") + .needsAcknowledging(False) + .commit(), + + FLOAT_ELEMENT(expected) + .key("timeOfFlight") + .unit(Unit.SECOND) + .metricPrefix(MetricPrefix.MILLI) + .readOnly() + .initialValue(0) + .warnHigh(1000) + .info("Time of flight exceeding 1 s") + .needsAcknowledging(False) + .commit(), + + STRING_ELEMENT(expected) + .key("pathToStack") + .assignmentOptional() + .defaultValue("image.data") + .commit(), + + INPUT_CHANNEL(expected) + .key("geometryInput") + .displayedName("Geometry input") + .commit(), + + OUTPUT_CHANNEL(expected) # can OVERWRITE_ELEMENT even do this? + .key("output") + .dataSchema(preview_schema) + .commit(), + ) + + def initialization(self): + super().initialization() + + # TODO: match inside device, fill multiple independent buffers + + self._path_to_stack = self.get("pathToStack") + self.geometry = None + self.input_buffer = None + + self.KARABO_ON_DATA("geometryInput", self.receive_geometry) + self.KARABO_SLOT(self.requestScene) + + self.start() + + def requestScene(self, params): + # TODO: unify with TrainMatcher overview + scene_name = params.get("name", default="") + if scene_name == "assemblerOverview": + payload = Hash("name", scene_name, "success", True) + payload["data"] = scenes.simple_assembler_overview( + device_id=self.getInstanceId(), + geometry_device_id=self.get("geometryInput.connectedOutputChannels")[ + 0 + ].split(":")[0], + ) + self.reply( + Hash( + "type", + "deviceScene", + "origin", + self.getInstanceId(), + "payload", + payload, + ) + ) + else: + return super().requestScene(params) + + def receive_geometry(self, data, metadata): + self.log.INFO("Received a new geometry") + self.geometry = pickle.loads(data.get("pickledGeometry")) + self.input_buffer = np.zeros(self.geometry.expected_data_shape) + + def _send(self, train_id, sources): + # TODO: adapt to appropriate hook for new TrainMatcher (no _send) + if self.geometry is None: + self.log.WARN("Have not received a geometry yet") + return + + timestamp = Timestamp(Epochstamp(), Trainstamp(train_id)) + + module_indices_unfilled = set(range(self.input_buffer.shape[0])) + for source, (data, metadata) in sources.items(): + # TODO: handle failure to "parse" source, get data out + module_index = self._source_to_index(source) + self.input_buffer[module_index] = np.squeeze(data.get(self._path_to_stack)) + module_indices_unfilled.discard(module_index) + + for unfilled_module in module_indices_unfilled: + self.input_buffer[unfilled_module].fill(0) + # TODO: configurable treatment of missing modules + + # TODO: reusable output buffer to save on allocation + assembled, _ = self.geometry.position_modules_fast(self.input_buffer) + + # TODO: optionally include control data + out_hash = Hash( + "image", + ImageData( + # TODO: get around this being mirrored... + (assembled[::-1, ::-1]).astype(np.int32), + Dims(*assembled.shape), + Encoding.GRAY, + ), + "trainId", + train_id, + ) + channel = self.signalSlotable.getOutputChannel("output") + channel.write(out_hash, ChannelMetaData(self.getInstanceId(), timestamp)) + channel.update() + self.rate_out.update() + + def _update_rate(self): + self._buffered_status_update.set( + "processingTime", self._processing_time_ema.get() + ) + self._buffered_status_update.set("timeOfFlight", self._time_of_flight_ema.get()) + self._buffered_status_update.set("rate", self._rate_tracker.get()) + self.set(self._buffered_status_update) + + @functools.lru_cache + def _source_to_index(self, source): + # note: cache means warning only shows up once (also not performance-critical) + # TODO: allow user to inspect, modify the mapping + + match = xtdf_source_re.match(source) + if match is not None: + return int(match.group(1)) + + match = daq_source_re.match(source) + if match is not None: + return int(match.group(1)) - 1 + + self.log.WARN(f"Couldn't figure out index for source {source}") + return 0 diff --git a/src/calng/manual_geometry_base.py b/src/calng/manual_geometry_base.py new file mode 100644 index 0000000000000000000000000000000000000000..2d2827b8e39c35eb3fe2b5e558d942d1e6d00207 --- /dev/null +++ b/src/calng/manual_geometry_base.py @@ -0,0 +1,227 @@ +import pickle + +import matplotlib.pyplot as plt +import numpy as np +from karabo.bound import ( + DOUBLE_ELEMENT, + IMAGEDATA_ELEMENT, + INT32_ELEMENT, + KARABO_CLASSINFO, + NODE_ELEMENT, + OUTPUT_CHANNEL, + SLOT_ELEMENT, + TABLE_ELEMENT, + VECTOR_CHAR_ELEMENT, + VECTOR_STRING_ELEMENT, + Encoding, + Hash, + ImageData, + PythonDevice, + Schema, + State, +) +from karabo.common.api import KARABO_SCHEMA_DISPLAY_TYPE_SCENES as DT_SCENES +from matplotlib.backends.backend_agg import FigureCanvasAgg + +from . import scenes +from ._version import version as deviceVersion + +geometry_schema = Schema() +( + VECTOR_CHAR_ELEMENT(geometry_schema) + .key("pickledGeometry") + .displayedName("Pickled geometry") + .assignmentOptional() + .defaultValue([]) + .commit() +) + +preview_schema = Schema() +(IMAGEDATA_ELEMENT(preview_schema).key("overview").commit()) + +ModuleColumn = Schema() +( + DOUBLE_ELEMENT(ModuleColumn) + .key("posX") + .assignmentOptional() + .defaultValue(95) + .reconfigurable() + .commit(), + + DOUBLE_ELEMENT(ModuleColumn) + .key("posY") + .assignmentOptional() + .defaultValue(564) + .reconfigurable() + .commit(), + + INT32_ELEMENT(ModuleColumn) + .key("orientationX") + .assignmentOptional() + .defaultValue(-1) + .reconfigurable() + .commit(), + + INT32_ELEMENT(ModuleColumn) + .key("orientationY") + .assignmentOptional() + .defaultValue(-1) + .reconfigurable() + .commit(), +) + + +@KARABO_CLASSINFO("ManualGeometryBase", deviceVersion) +class ManualGeometryBase(PythonDevice): + @staticmethod + def expectedParameters(expected): + # "mandatory" for geometry serving device + ( + OUTPUT_CHANNEL(expected) + .key("geometryOutput") + .dataSchema(geometry_schema) + .commit(), + + SLOT_ELEMENT(expected).key("pleaseSendYourGeometry").commit(), + + SLOT_ELEMENT(expected).key("reloadScenes").commit(), + + OUTPUT_CHANNEL(expected) + .key("previewOutput") + .dataSchema(preview_schema) + .commit(), + + IMAGEDATA_ELEMENT(expected).key("layoutPreview").commit(), + ) + + # scenes are fun + ( + VECTOR_STRING_ELEMENT(expected) + .key("availableScenes") + .setSpecialDisplayType(DT_SCENES) + .readOnly() + .initialValue(["overview"]) + .commit(), + ) + + def update_geom(self): + raise NotImplementedError() + + def __init__(self, config): + super().__init__(config) + + self.KARABO_SLOT(self.pleaseSendYourGeometry) + self.KARABO_SLOT(self.requestScene) + self.KARABO_SLOT(self.reloadScenes) + self.update_geom() + plt.switch_backend("agg") + self.updateState(State.ON) + + def requestScene(self, params): + payload = Hash() + scene_name = params.get("name", default="") + payload["name"] = scene_name + payload["success"] = True + if scene_name == "overview": + payload["data"] = scenes.manual_geometry_overview( + device_id=self.getInstanceId() + ) + else: + payload["success"] = False + response = Hash() + response["type"] = "deviceScene" + response["origin"] = self.getInstanceId() + response["payload"] = payload + self.reply(response) + + def pleaseSendYourGeometry(self): + self.writeChannel("geometryOutput", Hash("pickledGeometry", self.pickled)) + axis = self.geom.inspect() + axis.figure.tight_layout(pad=0) + axis.figure.set_facecolor("none") + # axis.figure.set_size_inches(6, 6) + # axis.figure.set_dpi(300) + canvas = FigureCanvasAgg(axis.figure) + canvas.draw() + image_buffer = np.frombuffer(canvas.buffer_rgba(), dtype=np.uint8).reshape( + canvas.get_width_height()[::-1] + (4,) + ) + self.set( + "layoutPreview", + ImageData(image_buffer, encoding=Encoding.RGBA, bitsPerPixel=3 * 8), + ) + + def preReconfigure(self, config): + self._prereconfigure_update_hash = config + + def postReconfigure(self): + if any( + path.startswith("quadrantCorners") + for path in self._prereconfigure_update_hash.getPaths() + ): + self.update_geom() + del self._prereconfigure_update_hash + + +@KARABO_CLASSINFO("ManualQuadrantsGeometryBase", deviceVersion) +class ManualQuadrantsGeometryBase(ManualGeometryBase): + @staticmethod + def expectedParameters(expected): + # note: subclasses should set better defaults + (NODE_ELEMENT(expected).key("quadrantCorners").commit(),) + for q in range(1, 5): + ( + NODE_ELEMENT(expected).key(f"quadrantCorners.Q{q}").commit(), + DOUBLE_ELEMENT(expected) + .key(f"quadrantCorners.Q{q}.x") + .assignmentOptional() + .defaultValue(0) + .reconfigurable() + .commit(), + + DOUBLE_ELEMENT(expected) + .key(f"quadrantCorners.Q{q}.y") + .assignmentOptional() + .defaultValue(0) + .reconfigurable() + .commit(), + ) + + def update_geom(self): + self.quadrant_corners = tuple( + (self.get(f"quadrantCorners.Q{q}.x"), self.get(f"quadrantCorners.Q{q}.y")) + for q in range(1, 5) + ) + self.geom = self.geometry_class.from_quad_positions(self.quadrant_corners) + self.pickled = pickle.dumps(self.geom) + # TODO: send to anyone who asks? make slot for that? send on connect? + self.writeChannel("geometryOutput", Hash("pickledGeometry", self.pickled)) + + +@KARABO_CLASSINFO("ManualModulesGeometryBase", deviceVersion) +class ManualModulesGeometryBase(ManualGeometryBase): + @staticmethod + def expectedParameters(expected): + ( + TABLE_ELEMENT(expected) + .key("modules") + .setColumns(ModuleColumn) + .assignmentOptional() + .defaultValue([]) + .reconfigurable() + .commit(), + ) + + def update_geom(self): + modules = self.get("modules") + module_pos = [(module.get("posX"), module.get("posY")) for module in modules] + orientations = [ + (module.get("orientationX"), module.get("orientationY")) + for module in modules + ] + self.geom = self.geometry_class.from_module_positions( + module_pos, orientations=orientations + ) + self.pickled = pickle.dumps(self.geom) + # TODO: send to anyone who asks? make slot for that? + self.writeChannel("geometryOutput", Hash("pickledGeometry", self.pickled))