Skip to content
Snippets Groups Projects
Commit 0db2b1d5 authored by David Hammer's avatar David Hammer
Browse files

Merge branch 'add-simple-assembler-geometry' into deployed/spb-jungfrau

parents ede1decd 06b1dfe6
No related branches found
No related tags found
1 merge request!12Snapshot: field test deployed version as of end of run 202201
...@@ -28,7 +28,11 @@ setup(name='calng', ...@@ -28,7 +28,11 @@ setup(name='calng',
'DsscCorrection = calng.DsscCorrection:DsscCorrection', 'DsscCorrection = calng.DsscCorrection:DsscCorrection',
'JungfrauCorrection = calng.JungfrauCorrection:JungfrauCorrection', 'JungfrauCorrection = calng.JungfrauCorrection:JungfrauCorrection',
'ModuleStacker = calng.ModuleStacker:ModuleStacker', 'ModuleStacker = calng.ModuleStacker:ModuleStacker',
'ManualAgipdGeometry = calng.ManualAgipdGeometry:ManualAgipdGeometry',
'ManualDsscGeometry = calng.ManualDsscGeometry:ManualDsscGeometry',
'ManualJungfrauGeometry = calng.ManualJungfrauGeometry:ManualJungfrauGeometry',
'ShmemToZMQ = calng.ShmemToZMQ:ShmemToZMQ', 'ShmemToZMQ = calng.ShmemToZMQ:ShmemToZMQ',
'SimpleAssembler = calng.SimpleAssembler:SimpleAssembler',
], ],
'karabo.middlelayer_device': [ 'karabo.middlelayer_device': [
......
...@@ -82,14 +82,6 @@ class ClassIdsNode(Configurable): ...@@ -82,14 +82,6 @@ class ClassIdsNode(Configurable):
accessMode=AccessMode.INITONLY, accessMode=AccessMode.INITONLY,
assignment=Assignment.MANDATORY) assignment=Assignment.MANDATORY)
previewMatcherClass = String(
displayedName='Preview matcher class',
description='Device class to use for matching the output of a preview '
'layer.',
defaultValue='ModuleStacker',
accessMode=AccessMode.INITONLY,
assignment=Assignment.MANDATORY)
assemblerClass = String( assemblerClass = String(
displayedName='Assembler class', displayedName='Assembler class',
description='Device class to use for assembling the matched output of ' description='Device class to use for assembling the matched output of '
...@@ -125,14 +117,6 @@ class DeviceIdsNode(Configurable): ...@@ -125,14 +117,6 @@ class DeviceIdsNode(Configurable):
accessMode=AccessMode.INITONLY, accessMode=AccessMode.INITONLY,
assignment=Assignment.MANDATORY) assignment=Assignment.MANDATORY)
previewMatcherSuffix = String(
displayedName='Preview matcher suffix',
description='Suffix for preview layer matching device IDs. The '
'formatting placeholder \'layer\' may be used.',
defaultValue='MATCH_{layer}',
accessMode=AccessMode.INITONLY,
assignment=Assignment.MANDATORY)
assemblerSuffix = String( assemblerSuffix = String(
displayedName='Assembler suffix', displayedName='Assembler suffix',
description='Suffix for assembler device IDs. The formatting ' description='Suffix for assembler device IDs. The formatting '
...@@ -408,6 +392,13 @@ class CalibrationManager(DeviceClientBase, Device): ...@@ -408,6 +392,13 @@ class CalibrationManager(DeviceClientBase, Device):
self.deviceServers = value self.deviceServers = value
self._servers_changed = True self._servers_changed = True
imageDataPath = String(
displayedName='Image data path',
description='Path in DAQ hash to actual image data, used for preview',
accessMode=AccessMode.RECONFIGURABLE,
assignment=Assignment.OPTIONAL,
defaultValue='image.data')
geometryDevice = String( geometryDevice = String(
displayedName='Geometry device', displayedName='Geometry device',
description='[NYI] Device ID for a geometry device defining the ' description='[NYI] Device ID for a geometry device defining the '
...@@ -1055,7 +1046,7 @@ class CalibrationManager(DeviceClientBase, Device): ...@@ -1055,7 +1046,7 @@ class CalibrationManager(DeviceClientBase, Device):
device_id_templates = {} device_id_templates = {}
class_args = (self.detectorType.value.lower().capitalize(),) class_args = (self.detectorType.value.lower().capitalize(),)
for role in ['correction', 'groupMatcher', 'bridge', 'previewMatcher', for role in ['correction', 'groupMatcher', 'bridge',
'assembler']: 'assembler']:
class_ids[role] = getattr( class_ids[role] = getattr(
self.classIds, f'{role}Class').value.format(*class_args) self.classIds, f'{role}Class').value.format(*class_args)
...@@ -1194,78 +1185,26 @@ class CalibrationManager(DeviceClientBase, Device): ...@@ -1194,78 +1185,26 @@ class CalibrationManager(DeviceClientBase, Device):
background(_activate_bridge(bridge_device_id)) background(_activate_bridge(bridge_device_id))
# Instantiate preview layer matchers and assemblers. # Instantiate preview layer assemblers.
geometry_device_id = self.geometryDevice.value
for layer, output_pipeline, server in self.previewLayers.value: for layer, output_pipeline, server in self.previewLayers.value:
# Preview matcher. assembler_device_id = device_id_templates['assembler'].format(
matcher_device_id = device_id_templates['previewMatcher'].format(
layer=layer) layer=layer)
config = Hash() config = Hash()
config['channels'] = [ # TODO: put _image_data_path in corr dev schema, get from there
f'{device_id}:{output_pipeline}' config['pathToStack'] = self.imageDataPath.value
for device_id in correct_device_id_by_module.values()]
config['fastSources'] = [ config['fastSources'] = [
Hash('fsSelect', True, Hash('fsSelect', True,
'fsSource', 'fsSource',
f'{input_source_by_module[virtual_id]}') f'{input_source_by_module[virtual_id]}')
for (virtual_id, device_id) for (virtual_id, device_id)
in correct_device_id_by_module.items()] in correct_device_id_by_module.items()]
config['pathToStack'] = 'data.adc' config['channels'] = [
f'{device_id}:{output_pipeline}'
if not await self._instantiate_device( for device_id in correct_device_id_by_module.values()]
server, class_ids['previewMatcher'], matcher_device_id, config config['geometryInput.connectedOutputChannels'] = [
): f'{geometry_device_id}:geometryOutput']
return
# Preview assembler.
assembler_device_id = device_id_templates['assembler'].format(
layer=layer)
config = Hash()
config['input.connectedOutputChannels'] = [
f'{matcher_device_id}:output']
config['modules'] = [
Hash('source', input_source_by_module.get('Q1M1', ''),
'offX', 474, 'offY', 612, 'rot', 90),
Hash('source', input_source_by_module.get('Q1M2', ''),
'offX', 316, 'offY', 612, 'rot', 90),
Hash('source', input_source_by_module.get('Q1M3', ''),
'offX', 158, 'offY', 612, 'rot', 90),
Hash('source', input_source_by_module.get('Q1M4', ''),
'offX', 0, 'offY', 612, 'rot', 90),
Hash('source', input_source_by_module.get('Q2M1', ''),
'offX', 1136, 'offY', 612, 'rot', 90),
Hash('source', input_source_by_module.get('Q2M2', ''),
'offX', 978, 'offY', 612, 'rot', 90),
Hash('source', input_source_by_module.get('Q2M3', ''),
'offX', 820, 'offY', 612, 'rot', 90),
Hash('source', input_source_by_module.get('Q2M4', ''),
'offX', 662, 'offY', 612, 'rot', 90),
Hash('source', input_source_by_module.get('Q3M1', ''),
'offX', 712, 'offY', 0, 'rot', 270),
Hash('source', input_source_by_module.get('Q3M2', ''),
'offX', 870, 'offY', 0, 'rot', 270),
Hash('source', input_source_by_module.get('Q3M3', ''),
'offX', 1028, 'offY', 0, 'rot', 270),
Hash('source', input_source_by_module.get('Q3M4', ''),
'offX', 1186, 'offY', 0, 'rot', 270),
Hash('source', input_source_by_module.get('Q4M1', ''),
'offX', 50, 'offY', 0, 'rot', 270),
Hash('source', input_source_by_module.get('Q4M2', ''),
'offX', 208, 'offY', 0, 'rot', 270),
Hash('source', input_source_by_module.get('Q4M3', ''),
'offX', 366, 'offY', 0, 'rot', 270),
Hash('source', input_source_by_module.get('Q4M4', ''),
'offX', 524, 'offY', 0, 'rot', 270),
]
config['pathsToCombine'] = ['data.adc']
config['trainIdPath'] = 'image.trainId'
config['pulseIdPath'] = 'image.pulseId'
config['preview.enablePreview'] = True
config['preview.pathToPreview'] = 'data.adc'
config['preview.downSample'] = 2
config['badpixelPath'] = 'image.bad_pixels'
config['rotated90Grad'] = True
if not await self._instantiate_device( if not await self._instantiate_device(
server, class_ids['assembler'], assembler_device_id, config server, class_ids['assembler'], assembler_device_id, config
......
import extra_geom
from karabo.bound import KARABO_CLASSINFO
from ._version import version as deviceVersion
from .manual_geometry_base import ManualQuadrantsGeometryBase
@KARABO_CLASSINFO("ManualAgipdGeometry", deviceVersion)
class ManualAgipdGeometry(ManualQuadrantsGeometryBase):
geometry_class = extra_geom.AGIPD_1MGeometry
@staticmethod
def expectedParameters(expected):
super(ManualAgipdGeometry, ManualAgipdGeometry).expectedParameters(expected)
expected.setDefaultValue("quadrantCorners.Q1.x", -525)
expected.setDefaultValue("quadrantCorners.Q1.y", 625)
expected.setDefaultValue("quadrantCorners.Q2.x", -550)
expected.setDefaultValue("quadrantCorners.Q2.y", -10)
expected.setDefaultValue("quadrantCorners.Q3.x", 520)
expected.setDefaultValue("quadrantCorners.Q3.y", -160)
expected.setDefaultValue("quadrantCorners.Q4.x", 542.5)
expected.setDefaultValue("quadrantCorners.Q4.y", 475)
import extra_geom
from karabo.bound import KARABO_CLASSINFO
from ._version import version as deviceVersion
from .manual_geometry_base import ManualQuadrantsGeometryBase
@KARABO_CLASSINFO("ManualDsscGeometry", deviceVersion)
class ManualDsscGeometry(ManualQuadrantsGeometryBase):
geometry_class = extra_geom.DSSC_1MGeometry
@staticmethod
def expectedParameters(expected):
super(ManualDsscGeometry, ManualDsscGeometry).expectedParameters(expected)
expected.setDefaultValue("quadrantCorners.Q1.x", -130)
expected.setDefaultValue("quadrantCorners.Q1.y", 5)
expected.setDefaultValue("quadrantCorners.Q2.x", -130)
expected.setDefaultValue("quadrantCorners.Q2.y", -125)
expected.setDefaultValue("quadrantCorners.Q3.x", 5)
expected.setDefaultValue("quadrantCorners.Q3.y", -125)
expected.setDefaultValue("quadrantCorners.Q4.x", 5)
expected.setDefaultValue("quadrantCorners.Q4.y", 5)
import extra_geom
from karabo.bound import KARABO_CLASSINFO, OVERWRITE_ELEMENT, Hash
from ._version import version as deviceVersion
from .manual_geometry_base import ManualModulesGeometryBase
@KARABO_CLASSINFO("ManualJungfrauGeometry", deviceVersion)
class ManualJungfrauGeometry(ManualModulesGeometryBase):
geometry_class = extra_geom.JUNGFRAUGeometry
@staticmethod
def expectedParameters(expected):
# TODO: come up with some sweet defaults (this is two modules from docs 4M)
(
OVERWRITE_ELEMENT(expected)
.key("modules")
.setNewDefaultValue(
[
Hash(
"posX", 95, "posY", 564, "orientationX", -1, "orientationY", -1
),
Hash(
"posX", 95, "posY", 17, "orientationX", -1, "orientationY", -1
),
]
)
)
import functools
import pickle
import re
import numpy as np
from calng import utils
from karabo.bound import (
FLOAT_ELEMENT,
IMAGEDATA_ELEMENT,
INPUT_CHANNEL,
KARABO_CLASSINFO,
OUTPUT_CHANNEL,
OVERWRITE_ELEMENT,
STRING_ELEMENT,
UINT64_ELEMENT,
ChannelMetaData,
Dims,
Encoding,
Epochstamp,
Hash,
ImageData,
MetricPrefix,
Schema,
Timestamp,
Trainstamp,
Unit,
)
from karabo.common.api import KARABO_SCHEMA_DISPLAY_TYPE_SCENES as DT_SCENES
from TrainMatcher import TrainMatcher
from TrainMatcher import scenes as trainmatcher_scenes
from . import scenes
from ._version import version as deviceVersion
preview_schema = Schema()
(
IMAGEDATA_ELEMENT(preview_schema).key("image").commit(),
UINT64_ELEMENT(preview_schema).key("trainId").readOnly().commit(),
)
xtdf_source_re = re.compile(r".*\/DET\/(\d+)CH0:xtdf")
daq_source_re = re.compile(r".*\/DET\/.*?(\d+):daqOutput")
# TODO: merge scene with TrainMatcher's nice overview
@KARABO_CLASSINFO("SimpleAssembler", deviceVersion)
class SimpleAssembler(TrainMatcher.TrainMatcher):
@staticmethod
def expectedParameters(expected):
(
OVERWRITE_ELEMENT(expected)
.key("availableScenes")
.setNewDefaultValue(["scene", "assemblerOverview"])
.commit(),
FLOAT_ELEMENT(expected)
.key("processingTime")
.unit(Unit.SECOND)
.metricPrefix(MetricPrefix.MILLI)
.readOnly()
.initialValue(0)
.warnHigh(500)
.info("Cannot keep up with GUI limit")
.needsAcknowledging(False)
.commit(),
FLOAT_ELEMENT(expected)
.key("timeOfFlight")
.unit(Unit.SECOND)
.metricPrefix(MetricPrefix.MILLI)
.readOnly()
.initialValue(0)
.warnHigh(1000)
.info("Time of flight exceeding 1 s")
.needsAcknowledging(False)
.commit(),
STRING_ELEMENT(expected)
.key("pathToStack")
.assignmentOptional()
.defaultValue("image.data")
.commit(),
INPUT_CHANNEL(expected)
.key("geometryInput")
.displayedName("Geometry input")
.commit(),
OUTPUT_CHANNEL(expected) # can OVERWRITE_ELEMENT even do this?
.key("output")
.dataSchema(preview_schema)
.commit(),
)
def initialization(self):
super().initialization()
# TODO: match inside device, fill multiple independent buffers
self._path_to_stack = self.get("pathToStack")
self.geometry = None
self.input_buffer = None
self.KARABO_ON_DATA("geometryInput", self.receive_geometry)
self.KARABO_SLOT(self.requestScene)
self.start()
def requestScene(self, params):
# TODO: unify with TrainMatcher overview
scene_name = params.get("name", default="")
if scene_name == "assemblerOverview":
payload = Hash("name", scene_name, "success", True)
payload["data"] = scenes.simple_assembler_overview(
device_id=self.getInstanceId(),
geometry_device_id=self.get("geometryInput.connectedOutputChannels")[
0
].split(":")[0],
)
self.reply(
Hash(
"type",
"deviceScene",
"origin",
self.getInstanceId(),
"payload",
payload,
)
)
else:
return super().requestScene(params)
def receive_geometry(self, data, metadata):
self.log.INFO("Received a new geometry")
self.geometry = pickle.loads(data.get("pickledGeometry"))
self.input_buffer = np.zeros(self.geometry.expected_data_shape)
def _send(self, train_id, sources):
# TODO: adapt to appropriate hook for new TrainMatcher (no _send)
if self.geometry is None:
self.log.WARN("Have not received a geometry yet")
return
timestamp = Timestamp(Epochstamp(), Trainstamp(train_id))
module_indices_unfilled = set(range(self.input_buffer.shape[0]))
for source, (data, metadata) in sources.items():
# TODO: handle failure to "parse" source, get data out
module_index = self._source_to_index(source)
self.input_buffer[module_index] = np.squeeze(data.get(self._path_to_stack))
module_indices_unfilled.discard(module_index)
for unfilled_module in module_indices_unfilled:
self.input_buffer[unfilled_module].fill(0)
# TODO: configurable treatment of missing modules
# TODO: reusable output buffer to save on allocation
assembled, _ = self.geometry.position_modules_fast(self.input_buffer)
# TODO: optionally include control data
out_hash = Hash(
"image",
ImageData(
# TODO: get around this being mirrored...
(assembled[::-1, ::-1]).astype(np.int32),
Dims(*assembled.shape),
Encoding.GRAY,
),
"trainId",
train_id,
)
channel = self.signalSlotable.getOutputChannel("output")
channel.write(out_hash, ChannelMetaData(self.getInstanceId(), timestamp))
channel.update()
self.rate_out.update()
def _update_rate(self):
self._buffered_status_update.set(
"processingTime", self._processing_time_ema.get()
)
self._buffered_status_update.set("timeOfFlight", self._time_of_flight_ema.get())
self._buffered_status_update.set("rate", self._rate_tracker.get())
self.set(self._buffered_status_update)
@functools.lru_cache
def _source_to_index(self, source):
# note: cache means warning only shows up once (also not performance-critical)
# TODO: allow user to inspect, modify the mapping
match = xtdf_source_re.match(source)
if match is not None:
return int(match.group(1))
match = daq_source_re.match(source)
if match is not None:
return int(match.group(1)) - 1
self.log.WARN(f"Couldn't figure out index for source {source}")
return 0
import pickle
import matplotlib.pyplot as plt
import numpy as np
from karabo.bound import (
DOUBLE_ELEMENT,
IMAGEDATA_ELEMENT,
INT32_ELEMENT,
KARABO_CLASSINFO,
NODE_ELEMENT,
OUTPUT_CHANNEL,
SLOT_ELEMENT,
TABLE_ELEMENT,
VECTOR_CHAR_ELEMENT,
VECTOR_STRING_ELEMENT,
Encoding,
Hash,
ImageData,
PythonDevice,
Schema,
State,
)
from karabo.common.api import KARABO_SCHEMA_DISPLAY_TYPE_SCENES as DT_SCENES
from matplotlib.backends.backend_agg import FigureCanvasAgg
from . import scenes
from ._version import version as deviceVersion
geometry_schema = Schema()
(
VECTOR_CHAR_ELEMENT(geometry_schema)
.key("pickledGeometry")
.displayedName("Pickled geometry")
.assignmentOptional()
.defaultValue([])
.commit()
)
preview_schema = Schema()
(IMAGEDATA_ELEMENT(preview_schema).key("overview").commit())
ModuleColumn = Schema()
(
DOUBLE_ELEMENT(ModuleColumn)
.key("posX")
.assignmentOptional()
.defaultValue(95)
.reconfigurable()
.commit(),
DOUBLE_ELEMENT(ModuleColumn)
.key("posY")
.assignmentOptional()
.defaultValue(564)
.reconfigurable()
.commit(),
INT32_ELEMENT(ModuleColumn)
.key("orientationX")
.assignmentOptional()
.defaultValue(-1)
.reconfigurable()
.commit(),
INT32_ELEMENT(ModuleColumn)
.key("orientationY")
.assignmentOptional()
.defaultValue(-1)
.reconfigurable()
.commit(),
)
@KARABO_CLASSINFO("ManualGeometryBase", deviceVersion)
class ManualGeometryBase(PythonDevice):
@staticmethod
def expectedParameters(expected):
# "mandatory" for geometry serving device
(
OUTPUT_CHANNEL(expected)
.key("geometryOutput")
.dataSchema(geometry_schema)
.commit(),
SLOT_ELEMENT(expected).key("pleaseSendYourGeometry").commit(),
SLOT_ELEMENT(expected).key("reloadScenes").commit(),
OUTPUT_CHANNEL(expected)
.key("previewOutput")
.dataSchema(preview_schema)
.commit(),
IMAGEDATA_ELEMENT(expected).key("layoutPreview").commit(),
)
# scenes are fun
(
VECTOR_STRING_ELEMENT(expected)
.key("availableScenes")
.setSpecialDisplayType(DT_SCENES)
.readOnly()
.initialValue(["overview"])
.commit(),
)
def update_geom(self):
raise NotImplementedError()
def __init__(self, config):
super().__init__(config)
self.KARABO_SLOT(self.pleaseSendYourGeometry)
self.KARABO_SLOT(self.requestScene)
self.KARABO_SLOT(self.reloadScenes)
self.update_geom()
plt.switch_backend("agg")
self.updateState(State.ON)
def requestScene(self, params):
payload = Hash()
scene_name = params.get("name", default="")
payload["name"] = scene_name
payload["success"] = True
if scene_name == "overview":
payload["data"] = scenes.manual_geometry_overview(
device_id=self.getInstanceId()
)
else:
payload["success"] = False
response = Hash()
response["type"] = "deviceScene"
response["origin"] = self.getInstanceId()
response["payload"] = payload
self.reply(response)
def pleaseSendYourGeometry(self):
self.writeChannel("geometryOutput", Hash("pickledGeometry", self.pickled))
axis = self.geom.inspect()
axis.figure.tight_layout(pad=0)
axis.figure.set_facecolor("none")
# axis.figure.set_size_inches(6, 6)
# axis.figure.set_dpi(300)
canvas = FigureCanvasAgg(axis.figure)
canvas.draw()
image_buffer = np.frombuffer(canvas.buffer_rgba(), dtype=np.uint8).reshape(
canvas.get_width_height()[::-1] + (4,)
)
self.set(
"layoutPreview",
ImageData(image_buffer, encoding=Encoding.RGBA, bitsPerPixel=3 * 8),
)
def preReconfigure(self, config):
self._prereconfigure_update_hash = config
def postReconfigure(self):
if any(
path.startswith("quadrantCorners")
for path in self._prereconfigure_update_hash.getPaths()
):
self.update_geom()
del self._prereconfigure_update_hash
@KARABO_CLASSINFO("ManualQuadrantsGeometryBase", deviceVersion)
class ManualQuadrantsGeometryBase(ManualGeometryBase):
@staticmethod
def expectedParameters(expected):
# note: subclasses should set better defaults
(NODE_ELEMENT(expected).key("quadrantCorners").commit(),)
for q in range(1, 5):
(
NODE_ELEMENT(expected).key(f"quadrantCorners.Q{q}").commit(),
DOUBLE_ELEMENT(expected)
.key(f"quadrantCorners.Q{q}.x")
.assignmentOptional()
.defaultValue(0)
.reconfigurable()
.commit(),
DOUBLE_ELEMENT(expected)
.key(f"quadrantCorners.Q{q}.y")
.assignmentOptional()
.defaultValue(0)
.reconfigurable()
.commit(),
)
def update_geom(self):
self.quadrant_corners = tuple(
(self.get(f"quadrantCorners.Q{q}.x"), self.get(f"quadrantCorners.Q{q}.y"))
for q in range(1, 5)
)
self.geom = self.geometry_class.from_quad_positions(self.quadrant_corners)
self.pickled = pickle.dumps(self.geom)
# TODO: send to anyone who asks? make slot for that? send on connect?
self.writeChannel("geometryOutput", Hash("pickledGeometry", self.pickled))
@KARABO_CLASSINFO("ManualModulesGeometryBase", deviceVersion)
class ManualModulesGeometryBase(ManualGeometryBase):
@staticmethod
def expectedParameters(expected):
(
TABLE_ELEMENT(expected)
.key("modules")
.setColumns(ModuleColumn)
.assignmentOptional()
.defaultValue([])
.reconfigurable()
.commit(),
)
def update_geom(self):
modules = self.get("modules")
module_pos = [(module.get("posX"), module.get("posY")) for module in modules]
orientations = [
(module.get("orientationX"), module.get("orientationY"))
for module in modules
]
self.geom = self.geometry_class.from_module_positions(
module_pos, orientations=orientations
)
self.pickled = pickle.dumps(self.geom)
# TODO: send to anyone who asks? make slot for that?
self.writeChannel("geometryOutput", Hash("pickledGeometry", self.pickled))
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment