From 48f475127837ce8a814971f62a41b80cfb875471 Mon Sep 17 00:00:00 2001
From: David Hammer <david.hammer@xfel.eu>
Date: Thu, 2 Mar 2023 11:41:26 +0100
Subject: [PATCH] Make all previews GUI-friendly, replace standalone mode
---
docs/detectors.md | 2 +-
docs/devices.md | 33 +++--
docs/schemas/BaseCorrection.md | 34 ++++-
docs/schemas/CalibrationManager.md | 2 +-
docs/schemas/DetectorAssembler.md | 16 ---
docs/troubleshooting.md | 2 +-
src/calng/CalibrationManager.py | 5 +-
src/calng/DetectorAssembler.py | 82 +++++++----
src/calng/RoiTool.py | 51 +++----
src/calng/base_correction.py | 72 +++++-----
src/calng/corrections/AgipdCorrection.py | 37 ++---
src/calng/corrections/DsscCorrection.py | 20 +--
src/calng/corrections/Epix100Correction.py | 54 +++++--
src/calng/corrections/Gotthard2Correction.py | 89 +++++-------
src/calng/corrections/JungfrauCorrection.py | 63 +++-----
src/calng/corrections/LpdCorrection.py | 71 +++------
src/calng/preview_utils.py | 142 +++++++++---------
src/calng/scenes.py | 143 ++++++++++++-------
src/calng/schemas.py | 10 ++
19 files changed, 480 insertions(+), 448 deletions(-)
diff --git a/docs/detectors.md b/docs/detectors.md
index 863a039e..97d0191c 100644
--- a/docs/detectors.md
+++ b/docs/detectors.md
@@ -39,7 +39,7 @@ Configuring which subset to use is done under the node `corrections.badPixels.su
`calng` does not propagate the full bad pixel mask in the output data - it simply masks bad pixels with a configurable masking value.
-The default masking value `NaN` has the benefit that it excludes bad pixels from [preview statistics](devices.md#preview-configuration) and can be handled directly by the [preview assemblers](devices.md#preview-assemblers).
+The default masking value `NaN` has the benefit that it excludes bad pixels from [preview statistics](devices.md#assembled-preview-configuration) and can be handled directly by the [preview assemblers](devices.md#preview-assemblers).
Note that new deployments has an additional bad pixel field `NON_STANDARD_SIZE`.
This is used for AGIPD and JUNGFRAU to mask double size pixels.
diff --git a/docs/devices.md b/docs/devices.md
index 5abce532..bbdaa9b8 100644
--- a/docs/devices.md
+++ b/docs/devices.md
@@ -28,7 +28,7 @@ Note that a lot of the manager's schema (including constant retrieval operating
While the pipeline is running, the manager provides convenient reconfiguration of multiple devices simultaneously.
For instance, it allows simultaneously setting the [DAQ train stride](data-rates.md) on all DAQ devices setting change any of a number of settings on all correction devices.
-Thus, although a number of points below (such as [preview configuration](#preview-configuration) [retrieving constants](#retrieving-constants)) technically deal with settings on correction devices, these should almost always be set through the manager.
+Thus, although a number of points below (such as [preview configuration](#assembled-preview-configuration) [retrieving constants](#retrieving-constants)) technically deal with settings on correction devices, these should almost always be set through the manager.
@@ -42,13 +42,14 @@ Module grouping is configured via the [Modules](schemas/CalibrationManager.md#mo
This table must list all the detector modules with virtual names and each is assigned a group number (the remaining fields in each row typically left blank as they can be inferred).
Modules with the same group number are processed on the same device server - which device server is seen in the [Module groups](schemas/CalibrationManager.md#moduleGroups) table, which also has some options for automatically starting [group matchers](#group-matcher) and setting up their [karabo bridge output](trainmatcher.md#karabo-bridge-output).
-### Preview configuration
+### Assembled preview configuration
-As illustrated in the simplified overview, previews run on a separate specialized data flow (only a single frame is sent per train), starting at correction devices.
-Configuring correction device-specific preview settings must be done through the manager to keep settings consistent.
+As illustrated in the simplified overview, previews use a separate data flow.
+The main feature of this is to slice or summarize big detector data down to one frame already on the correction devices.
+Therefore, settings related to how this is done should typically be managed via the manager to keep settings consistent.
The parameters seen in the "Preview" box on the manager overview scene control:
-- How to select frame to preview (for non-negative indices)
+- How to select which frame to preview (for non-negative indices)
- See [Index selection mode](schemas/BaseCorrection.md#preview.selectionMode)
- Frame can be extracted directly from image ndarray (`frame` mode), by looking up corresponding frame in cell table (`cell` mode), or by looking up in pulse table (`pulse` mode)
- For XTDF detectors, the mapping tables are typically `image.cellId` and `image.pulseId`.
@@ -65,9 +66,12 @@ The parameters seen in the "Preview" box on the manager overview scene control:
- `-2` for mean
- `-3` for sum
- `-4` for standard deviation
-e
-Some additional parameters are set on the [preview assembler](#preview-assemblers) itself.
+
+These settings determine the data slicing / summarizing on the correction devices.
+Before the preview is sent to the GUI, some additional steps are taken.
+In case of assembled previews, these are set on the [preview assembler](#preview-assemblers) itself.
+In case of a [single module preview](#single-module-preview), the correction device exposes the same configuration settings as the assembler.
### Retrieving constants
@@ -114,14 +118,17 @@ Observations about the state of the correction device at the time this screensho
- All correction steps are available
- All correction steps except `forceMgIfBelow` and `forceHgIfBelow` are enabled (those two are off by default)
-### Standalone mode
+### Single module preview
+
+The section on [assembled preview configuration](#assembled-preview-configuration) deals with previewing a full multi-module detector, including using a [`DetectorAssembler`](#preview-assemblers) with detector geometry to assemble the full preview.
+One can, however, also directly access the preview data from a single correction device.
+This is found under the [`preview` node](schemas/BaseCorrection.md#preview) which contain both the settings described for assembled preview configuration - how to slice burst mode data for preview - and the GUI-specific settings necessary to display the preview in the KaraboGUI.
-Although the previous section states that correction devices are typically started by a manager and need not be in projects, there are single-module detector instances where it is simpler to run a single correction device in *standalone mode*.
-This mode is available for JUNGFRAU and LPD, and is deployed for single-module JUNGFRAU instances at FXE and HED along with a test setup for an LPD mini.
-Standalone mode means:
+The default correction device scene includes the corrected preview output and links to individual scenes with any additional preview channels.
-- Full data is sent on the `dataOutput` channel without using shared memory handles
-- The preview outputs of the device can be displayed directly by Karabo GUI with configuration options similar to those of [preview assemblers](#preview-assemblers)
+Single module previews are primarily intended for use in single-module detector installations such as JF500K.
+They can also be used to inspect single modules in multi-module detector setups, but one should avoid tweaking individual module preview settings in this case.
+Changing the GUI-related settings (flipping along SS / FS axes, downsampling) on the correction device level should be expected to break or render inconsistent the assembled preview.
### Warning lamps
diff --git a/docs/schemas/BaseCorrection.md b/docs/schemas/BaseCorrection.md
index 53989d40..9bde3db3 100644
--- a/docs/schemas/BaseCorrection.md
+++ b/docs/schemas/BaseCorrection.md
@@ -84,7 +84,7 @@ Description
Default value
: ``10``
-## <a name='runAsStandaloneModule'></a>Standalone mode (`runAsStandaloneModule`)
+## <a name='useShmemHandles'></a>Standalone mode (`useShmemHandles`)
Type
: BOOL
@@ -98,7 +98,7 @@ Description
: If enabled, full corrected data (not using shared memory handles) will be sent on main output and preview outputs will be configured to be suitable for use directly into Karabo GUI rather than through an assembler.
Default value
-: ``False``
+: ``True``
## <a name='useInfiniband'></a>Use infiniband (`useInfiniband`)
Type
@@ -267,6 +267,32 @@ Description
Default value
: ``[]``
+### <a name='dataFormat.cellId'></a>`dataFormat.cellId`
+Type
+: VECTOR_UINT32
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``[]``
+
+### <a name='dataFormat.pulseId'></a>`dataFormat.pulseId`
+Type
+: VECTOR_UINT32
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``[]``
+
## Workarounds (`workarounds`)
### <a name='workarounds.overrideInputAxisOrder'></a>Override input axis order (`workarounds.overrideInputAxisOrder`)
Type
@@ -350,10 +376,6 @@ Default value
: ``NORMAL``
## Preview (`preview`)
-### `preview.outputRaw`
-OutputChannel
-### `preview.outputCorrected`
-OutputChannel
### <a name='preview.index'></a>Index (or stat) for preview (`preview.index`)
Type
: INT32
diff --git a/docs/schemas/CalibrationManager.md b/docs/schemas/CalibrationManager.md
index 03bf0a8c..ecd3f021 100644
--- a/docs/schemas/CalibrationManager.md
+++ b/docs/schemas/CalibrationManager.md
@@ -394,7 +394,7 @@ Type
: Slot
Allowed in states
-: ERROR, ACTIVE
+: ACTIVE, ERROR
Access mode
: RECONFIGURABLE
diff --git a/docs/schemas/DetectorAssembler.md b/docs/schemas/DetectorAssembler.md
index b508390d..75d0826a 100644
--- a/docs/schemas/DetectorAssembler.md
+++ b/docs/schemas/DetectorAssembler.md
@@ -536,22 +536,6 @@ Description
Default value
: ``0.0``
-### <a name='preview.maxRate'></a>Max rate (`preview.maxRate`)
-Type
-: DOUBLE
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: Preview output is throttled to (at most) this speed. Data arriving too quickly after last send is silently dropped.
-
-Default value
-: ``2.0``
-
### `preview.output`
See description of parent node, 'preview'.
OutputChannel
\ No newline at end of file
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index 6a9b5073..8495c69a 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -8,7 +8,7 @@
- Are the [preview assemblers](devices.md#preview-assemblers) receiving data?
- Is the [geometry device](devices.md#geometry-devices) up?
- Are you simply seeing data from a different memory cell than you expect?
- Check [which frames is previewed](devices.md#preview-configuration) and maybe bad pixel masking.
+ Check [which frames is previewed](devices.md#assembled-preview-configuration) and maybe bad pixel masking.
3. Is preview matching breaking down?
- Note: for fast detectors, data transfer from DAQ to correction device can hit [network limits](data-rates.md).
This can cause **matching** on preview assemblers to break down.
diff --git a/src/calng/CalibrationManager.py b/src/calng/CalibrationManager.py
index 8af755ee..90947154 100644
--- a/src/calng/CalibrationManager.py
+++ b/src/calng/CalibrationManager.py
@@ -1303,9 +1303,8 @@ class CalibrationManager(DeviceClientBase, Device):
config['sources'] = [
Hash('select', True,
'source',
- f'{input_source_by_module[virtual_id]}'
- f'@{device_id}:{output_pipeline}')
- for (virtual_id, device_id)
+ f'{device_id}:{output_pipeline}')
+ for (_, device_id)
in correct_device_id_by_module.items()]
config['geometryDevice'] = self.geometryDevice.value
config['maxIdle'] = self.maxIdle.value
diff --git a/src/calng/DetectorAssembler.py b/src/calng/DetectorAssembler.py
index eee5cfeb..29c9b96c 100644
--- a/src/calng/DetectorAssembler.py
+++ b/src/calng/DetectorAssembler.py
@@ -11,6 +11,7 @@ from karabo.bound import (
STRING_ELEMENT,
ChannelMetaData,
Hash,
+ ImageData,
MetricPrefix,
Timestamp,
Unit,
@@ -21,6 +22,7 @@ from . import geom_utils, scenes, schemas, preview_utils
from ._version import version as deviceVersion
+corr_source_re = re.compile(r".*\/CORRECT(\d+)")
xtdf_source_re = re.compile(r".*\/DET\/(\d+)CH0:xtdf")
daq_source_re = re.compile(r".*\/DET\/.*?(\d+):daqOutput")
@@ -28,7 +30,6 @@ daq_source_re = re.compile(r".*\/DET\/.*?(\d+):daqOutput")
class BridgeOutputOptions(enum.Enum):
MATCHED = "matched"
ASSEMBLED = "assembled"
- PREVIEW = "preview"
@KARABO_CLASSINFO("DetectorAssembler", deviceVersion)
@@ -61,7 +62,7 @@ class DetectorAssembler(TrainMatcher.TrainMatcher):
OUTPUT_CHANNEL(expected)
.key("assembledOutput")
- .dataSchema(schemas.preview_schema())
+ .dataSchema(schemas.preview_schema(wrap_image_in_imagedata=False))
.commit(),
STRING_ELEMENT(expected)
@@ -90,7 +91,18 @@ class DetectorAssembler(TrainMatcher.TrainMatcher):
.assignmentMandatory()
.commit(),
)
- preview_utils.PreviewFriend.add_schema(expected, "preview")
+ preview_utils.PreviewFriend.add_schema(expected, "preview", create_node=True)
+ (
+ OVERWRITE_ELEMENT(expected)
+ .key("preview.flipSS")
+ .setNewDefaultValue(True)
+ .commit(),
+
+ OVERWRITE_ELEMENT(expected)
+ .key("preview.flipFS")
+ .setNewDefaultValue(True)
+ .commit(),
+ )
def __init__(self, conf):
super().__init__(conf)
@@ -160,8 +172,13 @@ class DetectorAssembler(TrainMatcher.TrainMatcher):
return
self._geometry = geom_utils.deserialize_geometry(serialized_geometry)
# TODO: allow multiple memory cells (extra geom notion of extra dimensions)
- self._stack_input_buffer = np.zeros(
- self._geometry.expected_data_shape, dtype=np.float32
+ self._stack_input_buffer = np.ma.masked_array(
+ data=np.zeros(self._geometry.expected_data_shape, dtype=np.float32),
+ mask=False,
+ )
+ self._assemble_buffer = np.ma.masked_array(
+ data=self._geometry.output_array_for_position_fast(),
+ mask=False,
)
def on_matched_data(self, train_id, sources):
@@ -177,6 +194,7 @@ class DetectorAssembler(TrainMatcher.TrainMatcher):
module_indices_unfilled = set(range(self._stack_input_buffer.shape[0]))
earliest_source_timestamp = float("inf")
+ self._stack_input_buffer.mask.fill(False)
for source, (data, source_timestamp) in sources.items():
# regular TrainMatcher output
self.output.write(
@@ -187,10 +205,23 @@ class DetectorAssembler(TrainMatcher.TrainMatcher):
# prepare for assembly
# TODO: handle failure to "parse" source, get data out
+ image_data = data["image.data"]
+ if isinstance(image_data, ImageData):
+ # TODO: maybe glance encoding here
+ image_data = image_data.getData()
+ image_data = image_data.astype(np.float32, copy=False) # TODO: set dtype based on input?
+ if data.has("image.mask"):
+ image_mask = data["image.mask"]
+ if isinstance(image_mask, ImageData):
+ image_mask = image_mask.getData()
+ else:
+ image_mask = False
module_index = self._source_to_index(source)
- self._stack_input_buffer[module_index] = data.get(
- self._path_to_stack
- ).astype(np.float32, copy=False) # TODO: set dtype based on input?
+ masked = np.ma.masked_array(
+ data=image_data,
+ mask=image_mask,
+ )
+ self._stack_input_buffer[module_index] = masked
module_indices_unfilled.discard(module_index)
earliest_source_timestamp = min(
earliest_source_timestamp, source_timestamp.toTimestamp()
@@ -201,16 +232,17 @@ class DetectorAssembler(TrainMatcher.TrainMatcher):
self.zmq_output.update()
for module_index in module_indices_unfilled:
- self._stack_input_buffer[module_index].fill(np.nan)
+ self._stack_input_buffer.mask[module_index].fill(True)
# consider configurable treatment of missing modules
- # TODO: reusable output buffer to save on allocation
- assembled, _ = self._geometry.position_modules_fast(self._stack_input_buffer)
-
- # TODO: optionally include control data
+ assembled, _ = self._geometry.position_modules_fast(
+ self._stack_input_buffer, out=self._assemble_buffer
+ )
output_hash = Hash(
"image.data",
assembled,
+ "image.mask",
+ assembled.mask,
)
self.assembled_output.write(
output_hash,
@@ -224,17 +256,10 @@ class DetectorAssembler(TrainMatcher.TrainMatcher):
)
self.zmq_output.update()
- preview_hash_sent = self._preview_friend.maybe_write([assembled])
- if (
- bridge_output_choice is BridgeOutputOptions.PREVIEW
- and preview_hash_sent is not None
- ):
- self.zmq_output.write(
- f"{my_device_id}:preview.output",
- preview_hash_sent,
- my_timestamp,
- )
- self.zmq_output.update()
+ self._preview_friend.write_outputs(
+ my_timestamp,
+ assembled,
+ )
self.info["timeOfFlight"] = (
Timestamp().toTimestamp() - earliest_source_timestamp
@@ -253,12 +278,13 @@ class DetectorAssembler(TrainMatcher.TrainMatcher):
# note: cache means warning only shows up once (also not performance-critical)
# TODO: allow user to inspect, modify the mapping
- match = xtdf_source_re.match(source)
- if match is not None:
+ if (match := corr_source_re.match(source)) is not None:
+ return int(match.group(1))
+
+ if (match := xtdf_source_re.match(source)) is not None:
return int(match.group(1))
- match = daq_source_re.match(source)
- if match is not None:
+ if (match := daq_source_re.match(source)) is not None:
return int(match.group(1)) - 1
self.log.WARN(f"Couldn't figure out index for source {source}")
diff --git a/src/calng/RoiTool.py b/src/calng/RoiTool.py
index af66fffc..8f8f4788 100644
--- a/src/calng/RoiTool.py
+++ b/src/calng/RoiTool.py
@@ -39,8 +39,10 @@ def image_data_node():
class DownsamplingNode(Configurable):
- method = String(options=["nanmax", "nanmean", "nanmin", "nanmedian"])
- factor = UInt32(options=[1, 2, 4, 8])
+ method = String(
+ options=["nanmax", "nanmean", "nanmin", "nanmedian"], defaultValue="nanmax"
+ )
+ factor = UInt32(options=[1, 2, 4, 8], defaultValue=1)
class PreviewSettingsNode(Configurable):
@@ -143,6 +145,7 @@ class RoiTool(Device):
imageDataPath = String(
defaultValue="image.data", accessMode=AccessMode.RECONFIGURABLE
)
+ maskPath = String(defaultValue="image.mask", accessMode=AccessMode.RECONFIGURABLE)
histogram = Node(HistogramSettingsNode)
preview = Node(PreviewSettingsNode)
rate = Double(defaultValue=0, accessMode=AccessMode.READONLY)
@@ -163,23 +166,28 @@ class RoiTool(Device):
@InputChannel()
async def imageInput(self, data, meta):
- image_data = rec_getattr(data, self.imageDataPath.value).astype(
- np.float32, copy=False
+ # TODO: handle streams without explicit mask?
+ image = np.ma.masked_array(
+ data=rec_getattr(data, self.imageDataPath.value).astype(
+ np.float32, copy=False
+ ),
+ mask=rec_getattr(data, self.maskPath.value).astype(np.uint8, copy=False),
)
+
+ # TODO: make handling of extra dimension(s) configurable
+ if image.ndim == 3:
+ image = image[0]
+
if self.preview.flipX.value:
- image_data = image_data[:, ::-1]
+ image = image[:, ::-1]
if self.preview.flipY.value:
- image_data = image_data[::-1]
+ image = image[::-1]
- # TODO: make handling of extra dimension(s) configurable
- if len(image_data.shape) == 3:
- image_data = image_data[0]
- image_data = np.ascontiguousarray(image_data)
x_min, x_max, y_min, y_max = self.output.schema.roiImage.roi.value
x_min = max(x_min, 0)
y_min = max(y_min, 0)
- x_max = min(x_max, image_data.shape[0])
- y_max = min(y_max, image_data.shape[1])
+ x_max = min(x_max, image.shape[0])
+ y_max = min(y_max, image.shape[1])
if (downsampling_factor := self.preview.downsampling.factor.value) > 1:
x_min *= downsampling_factor
y_min *= downsampling_factor
@@ -187,9 +195,9 @@ class RoiTool(Device):
y_max *= downsampling_factor
# data for analysis may contain NaNs, we should filter them out
- zoomed = image_data[y_min:y_max, x_min:x_max]
- zoomed = zoomed[np.isfinite(zoomed)]
- finite_pixel_count = zoomed.size
+ zoomed = image[y_min:y_max, x_min:x_max]
+ # TODO: still handle NaN in image?
+ finite_pixel_count = image.count()
update_histogram = finite_pixel_count > 0
if update_histogram:
@@ -207,7 +215,7 @@ class RoiTool(Device):
await self.histogram.resetBins()
if self._bins is None:
counts, bin_edges = np.histogram(
- zoomed,
+ zoomed.compressed(),
bins=self.histogram.numBins.value,
range=(
self.histogram.rangeMin.value,
@@ -224,9 +232,7 @@ class RoiTool(Device):
)
self._window_count_index = 0
else:
- counts, bin_edges = np.histogram(
- zoomed[np.isfinite(zoomed)], bins=self._bins
- )
+ counts, bin_edges = np.histogram(zoomed.compressed(), bins=self._bins)
counts = counts.astype(np.float32)
self._window_counts[self._window_count_index] = counts
self._window_count_index = (
@@ -246,12 +252,9 @@ class RoiTool(Device):
bin_edges,
)
- # data for preview should not contain NaNs
- np.nan_to_num(image_data, copy=False)
- zoomed = image_data[y_min:y_max, x_min:x_max]
self.output.schema.roiImage.data.image = ImageData(
utils.downsample_2d(
- image_data,
+ image.filled(fill_value=0),
downsampling_factor,
getattr(np, self.preview.downsampling.method.value),
),
@@ -259,7 +262,7 @@ class RoiTool(Device):
bitsPerPixel=32,
)
self.output.schema.zoomImage = ImageData(
- np.ascontiguousarray(zoomed),
+ zoomed.filled(fill_value=0),
encoding=EncodingType.GRAY,
bitsPerPixel=32,
)
diff --git a/src/calng/base_correction.py b/src/calng/base_correction.py
index 0ace7697..b50f7176 100644
--- a/src/calng/base_correction.py
+++ b/src/calng/base_correction.py
@@ -35,7 +35,7 @@ from karabo.bound import (
)
from karabo.common.api import KARABO_SCHEMA_DISPLAY_TYPE_SCENES as DT_SCENES
-from . import scenes, shmem_utils, schemas, utils
+from . import scenes, preview_utils, shmem_utils, schemas, utils
from ._version import version as deviceVersion
PROCESSING_STATE_TIMEOUT = 10
@@ -80,7 +80,7 @@ class BaseCorrection(PythonDevice):
"outputShmemBufferSize",
"preview.index",
"preview.selectionMode",
- "runAsStandaloneModule",
+ "useShmemHandles",
"useInfiniband",
} # subclass can extend this, /must/ put it in schema as managedKeys
_image_data_path = "image.data" # customize for *some* subclasses
@@ -230,16 +230,16 @@ class BaseCorrection(PythonDevice):
.commit(),
BOOL_ELEMENT(expected)
- .key("runAsStandaloneModule")
- .displayedName("Standalone mode")
+ .key("useShmemHandles")
+ .displayedName("Use shared memory")
.description(
- "If enabled, full corrected data (not using shared memory handles) "
- "will be sent on main output and preview outputs will be configured "
- "to be suitable for use directly into Karabo GUI rather than through "
- "an assembler."
+ "If enabled, shared memory handles will be used to avoid copying "
+ "the main image data on the dataOutput channel. Note that these "
+ "handles need to be dereferenced (typically by a group matcher) to "
+ "get the data off the correction node."
)
.assignmentOptional()
- .defaultValue(False)
+ .defaultValue(True)
.commit(),
BOOL_ELEMENT(expected)
@@ -449,16 +449,6 @@ class BaseCorrection(PythonDevice):
.displayedName("Preview")
.commit(),
- OUTPUT_CHANNEL(expected)
- .key("preview.outputRaw")
- .dataSchema(schemas.preview_schema())
- .commit(),
-
- OUTPUT_CHANNEL(expected)
- .key("preview.outputCorrected")
- .dataSchema(schemas.preview_schema())
- .commit(),
-
INT32_ELEMENT(expected)
.key("preview.index")
.displayedName("Index (or stat) for preview")
@@ -588,11 +578,6 @@ class BaseCorrection(PythonDevice):
ib_ip = find_infiniband_ip()
config["dataOutput.hostname"] = ib_ip
- if not config.get("runAsStandaloneModule", default=False):
- for key in config.get("preview").getKeys():
- path = f"preview.{key}"
- if key.startswith("output") and config.has(f"{path}.hostname"):
- config[f"{path}.hostname"] = ib_ip
super().__init__(config)
self.output_data_dtype = np.dtype(config["dataFormat.outputImageDtype"])
@@ -601,7 +586,8 @@ class BaseCorrection(PythonDevice):
self.kernel_runner = None # must call _update_buffers to initialize
self._shmem_buffer = None # ditto
- self._preview_friend = None # used in standalone mode (see JungfrauCorrection)
+ self._use_shmem_handles = config.get("useShmemHandles")
+ self._preview_friend = None
self._correction_flag_enabled = self._correction_flag_class.NONE
self._correction_flag_preview = self._correction_flag_class.NONE
@@ -702,6 +688,13 @@ class BaseCorrection(PythonDevice):
key: utils.ContextWarningLamp(self, key)
for key in self.getFullSchema().getDefaultValue("warningLamps")
}
+ self._preview_friend = preview_utils.PreviewFriend(
+ self,
+ output_channels=self._preview_outputs,
+ )
+ self["availableScenes"] = self["availableScenes"] + [
+ f"preview:{channel}" for channel in self._preview_outputs
+ ]
for constant in self._constant_enum_class:
key = f"foundConstants.{constant.name}.state"
self._warning_trackers[key] = utils.ContextWarningLamp(self, key)
@@ -816,6 +809,9 @@ class BaseCorrection(PythonDevice):
):
self._update_correction_flags()
+ if update.has("preview"):
+ self._preview_friend.reconfigure(update)
+
def _lock_and_update(self, method, background=True):
# TODO: securely handle errors (postReconfigure may succeed, device state not)
def runner():
@@ -899,7 +895,6 @@ class BaseCorrection(PythonDevice):
payload["data"] = scenes.correction_device_overview(
device_id=self.getInstanceId(),
schema=self.getFullSchema(),
- direct_preview=self.get("runAsStandaloneModule"),
)
elif name == "constant_overrides":
payload["data"] = scenes.correction_device_constant_overrides(
@@ -944,20 +939,6 @@ class BaseCorrection(PythonDevice):
channel.write(data, metadata, copyAllData=False)
channel.update()
- def _write_preview_outputs(self, channel_data_pairs, old_metadata):
- # consider allowing sending *all* frames for commissioning (request: Jola)
- timestamp = Timestamp.fromHashAttributes(
- old_metadata.getAttributes("timestamp")
- )
- preview_hash = Hash()
- preview_hash.set("image.trainId", timestamp.getTrainId())
- metadata = ChannelMetaData(old_metadata.get("source"), timestamp)
- for channel_name, data in channel_data_pairs:
- preview_hash.set("image.data", data)
- channel = self.signalSlotable.getOutputChannel(channel_name)
- channel.write(preview_hash, metadata, copyAllData=False)
- channel.update()
-
def _update_correction_flags(self):
"""Based on constants loaded and settings, update bit mask flags for kernel"""
enabled = self._correction_flag_class.NONE
@@ -1397,6 +1378,17 @@ def add_bad_pixel_config_node(schema, managed_keys, prefix="corrections.badPixel
managed_keys.add(f"corrections.badPixels.subsetToUse.{field.name}")
+def add_preview_outputs(schema, channels):
+ for channel in channels:
+ (
+ OUTPUT_CHANNEL(schema)
+ .key(f"preview.{channel}")
+ .dataSchema(schemas.preview_schema(wrap_image_in_imagedata=True))
+ .commit(),
+ )
+ preview_utils.PreviewFriend.add_schema(schema, output_channels=channels)
+
+
def get_bad_pixel_field_selection(self):
selection = 0
for field in utils.BadPixelValues:
diff --git a/src/calng/corrections/AgipdCorrection.py b/src/calng/corrections/AgipdCorrection.py
index 7d0803b5..f7e95344 100644
--- a/src/calng/corrections/AgipdCorrection.py
+++ b/src/calng/corrections/AgipdCorrection.py
@@ -514,6 +514,12 @@ class AgipdCorrection(base_correction.BaseCorrection):
_calcat_friend_class = AgipdCalcatFriend
_constant_enum_class = Constants
_managed_keys = base_correction.BaseCorrection._managed_keys.copy()
+ _preview_outputs = [
+ "outputRaw",
+ "outputCorrected",
+ "outputRawGain",
+ "outputGainMap",
+ ]
@staticmethod
def expectedParameters(expected):
@@ -534,18 +540,7 @@ class AgipdCorrection(base_correction.BaseCorrection):
.commit(),
)
- (
- OUTPUT_CHANNEL(expected)
- .key("preview.outputRawGain")
- .dataSchema(schemas.preview_schema())
- .commit(),
-
- OUTPUT_CHANNEL(expected)
- .key("preview.outputGainMap")
- .dataSchema(schemas.preview_schema())
- .commit(),
- )
-
+ base_correction.add_preview_outputs(expected, AgipdCorrection._preview_outputs)
AgipdCalcatFriend.add_schema(expected, AgipdCorrection._managed_keys)
# this is not automatically done by superclass for complicated class reasons
(
@@ -799,21 +794,19 @@ class AgipdCorrection(base_correction.BaseCorrection):
) = self.kernel_runner.compute_previews(preview_slice_index)
# reusing input data hash for sending
- data_hash.set("image.data", buffer_handle)
- data_hash.set("calngShmemPaths", ["image.data"])
+ if self._use_shmem_handles:
+ data_hash.set("image.data", buffer_handle)
+ data_hash.set("calngShmemPaths", ["image.data"])
+ else:
+ data_hash.set("image.data", buffer_array)
+ data_hash.set("calngShmemPaths", [])
data_hash.set("image.cellId", cell_table[:, np.newaxis])
data_hash.set("image.pulseId", pulse_table[:, np.newaxis])
self._write_output(data_hash, metadata)
- self._write_preview_outputs(
- (
- ("preview.outputRaw", preview_raw),
- ("preview.outputCorrected", preview_corrected),
- ("preview.outputRawGain", preview_raw_gain),
- ("preview.outputGainMap", preview_gain_map),
- ),
- metadata,
+ self._preview_friend.write_outputs(
+ metadata, preview_raw, preview_corrected, preview_raw_gain, preview_gain_map
)
def _load_constant_to_runner(self, constant, constant_data):
diff --git a/src/calng/corrections/DsscCorrection.py b/src/calng/corrections/DsscCorrection.py
index 3ae2ec2a..054bcd0d 100644
--- a/src/calng/corrections/DsscCorrection.py
+++ b/src/calng/corrections/DsscCorrection.py
@@ -194,6 +194,7 @@ class DsscCorrection(base_correction.BaseCorrection):
_calcat_friend_class = DsscCalcatFriend
_constant_enum_class = Constants
_managed_keys = base_correction.BaseCorrection._managed_keys.copy()
+ _preview_outputs = ["outputRaw", "outputCorrected"]
@staticmethod
def expectedParameters(expected):
@@ -235,6 +236,7 @@ class DsscCorrection(base_correction.BaseCorrection):
.commit(),
)
DsscCorrection._managed_keys.add("kernelType")
+ base_correction.add_preview_outputs(expected, DsscCorrection._preview_outputs)
base_correction.add_correction_step_schema(
expected,
DsscCorrection._managed_keys,
@@ -324,18 +326,18 @@ class DsscCorrection(base_correction.BaseCorrection):
preview_slice_index,
)
- data_hash.set(self._image_data_path, buffer_handle)
+ if self._use_shmem_handles:
+ data_hash.set(self._image_data_path, buffer_handle)
+ data_hash.set("calngShmemPaths", [self._image_data_path])
+ else:
+ data_hash.set(self._image_data_path, buffer_array)
+ data_hash.set("calngShmemPaths", [])
+
data_hash.set(self._cell_table_path, cell_table[:, np.newaxis])
data_hash.set("image.pulseId", pulse_table[:, np.newaxis])
- data_hash.set("calngShmemPaths", [self._image_data_path])
+
self._write_output(data_hash, metadata)
- self._write_preview_outputs(
- (
- ("preview.outputRaw", preview_raw),
- ("preview.outputCorrected", preview_corrected),
- ),
- metadata,
- )
+ self._preview_friend.write_outputs(metadata, preview_raw, preview_corrected)
def _load_constant_to_runner(self, constant, constant_data):
self.kernel_runner.load_offset_map(constant_data)
diff --git a/src/calng/corrections/Epix100Correction.py b/src/calng/corrections/Epix100Correction.py
index 2a052e58..3d72b18d 100644
--- a/src/calng/corrections/Epix100Correction.py
+++ b/src/calng/corrections/Epix100Correction.py
@@ -99,7 +99,7 @@ class Epix100CalcatFriend(base_calcat.BaseCalcatFriend):
.key("constantParameters.inVacuum")
.displayedName("In vacuum")
.assignmentOptional()
- .defaultValue(10)
+ .defaultValue(0)
.reconfigurable()
.commit(),
@@ -141,6 +141,24 @@ class Epix100CalcatFriend(base_calcat.BaseCalcatFriend):
class Epix100CpuRunner(base_kernel_runner.BaseKernelRunner):
_corrected_axis_order = "fxy"
+ _xp = np
+ _gpu_based = False
+
+ @property
+ def input_shape(self):
+ return (self.pixels_x, self.pixels_y)
+
+ @property
+ def preview_shape(self):
+ return (self.pixels_x, self.pixels_y)
+
+ @property
+ def processed_shape(self):
+ return self.input_shape
+
+ @property
+ def map_shape(self):
+ return (self.pixels_x, self.pixels_y)
def __init__(
self,
@@ -154,23 +172,18 @@ class Epix100CpuRunner(base_kernel_runner.BaseKernelRunner):
assert (
output_data_dtype == np.float32
), "Alternative output types not supported yet"
- self.input_shape = (pixels_x, pixels_y)
- self.preview_shape = (pixels_x, pixels_y)
- self.processed_shape = self.input_shape
super().__init__(
pixels_x,
pixels_y,
frames,
constant_memory_cells,
- input_data_dtype,
output_data_dtype,
)
self.input_data = None
self.processed_data = np.empty(self.processed_shape, dtype=np.float32)
- self.map_shape = (pixels_x, pixels_y)
self.offset_map = np.empty(self.map_shape, dtype=np.float32)
self.rel_gain_map = np.empty(self.map_shape, dtype=np.float32)
self.bad_pixel_map = np.empty(self.map_shape, dtype=np.uint32)
@@ -301,11 +314,12 @@ class Epix100Correction(base_correction.BaseCorrection):
("commonMode", CorrectionFlags.COMMONMODE, {Constants.NoiseEPix100}),
("badPixels", CorrectionFlags.BPMASK, {Constants.BadPixelsDarkEPix100}),
)
+ _image_data_path = "data.image.pixels"
_kernel_runner_class = Epix100CpuRunner
_calcat_friend_class = Epix100CalcatFriend
_constant_enum_class = Constants
_managed_keys = base_correction.BaseCorrection._managed_keys.copy()
- _image_data_path = "data.image.pixels"
+ _preview_outputs = ["outputRaw", "outputCorrected"]
_cell_table_path = None
_pulse_table_path = None
_warn_memory_cell_range = False
@@ -315,7 +329,7 @@ class Epix100Correction(base_correction.BaseCorrection):
(
OUTPUT_CHANNEL(expected)
.key("dataOutput")
- .dataSchema(schemas.jf_output_schema())
+ .dataSchema(schemas.jf_output_schema(use_shmem_handle=False))
.commit(),
OVERWRITE_ELEMENT(expected)
@@ -335,12 +349,20 @@ class Epix100Correction(base_correction.BaseCorrection):
# TODO: disable preview selection mode
+ OVERWRITE_ELEMENT(expected)
+ .key("useShmemHandles")
+ .setNewDefaultValue(False)
+ .commit(),
+
OVERWRITE_ELEMENT(expected)
.key("outputShmemBufferSize")
.setNewDefaultValue(2)
.commit(),
)
+ base_correction.add_preview_outputs(
+ expected, Epix100Correction._preview_outputs
+ )
base_correction.add_correction_step_schema(
expected,
Epix100Correction._managed_keys,
@@ -442,15 +464,19 @@ class Epix100Correction(base_correction.BaseCorrection):
out=buffer_array,
)
+ if self._use_shmem_handles:
+ # TODO: consider better name for key...
+ data_hash.set("data.adc", buffer_handle)
+ data_hash.set("calngShmemPaths", ["data.adc"])
+ else:
+ data_hash.set("data.adc", buffer_array)
+ data_hash.set("calngShmemPaths", [])
+
self._write_output(data_hash, metadata)
# note: base class preview machinery assumes burst mode, shortcut it
- self._write_preview_outputs(
- zip(
- ("preview.outputRaw", "preview.outputCorrected"),
- self.kernel_runner.preview_data_views,
- ),
- metadata,
+ self._preview_friend.write_outputs(
+ metadata, *self.kernel_runner.preview_data_views
)
def _load_constant_to_runner(self, constant, constant_data):
diff --git a/src/calng/corrections/Gotthard2Correction.py b/src/calng/corrections/Gotthard2Correction.py
index c07efd58..ec10c50d 100644
--- a/src/calng/corrections/Gotthard2Correction.py
+++ b/src/calng/corrections/Gotthard2Correction.py
@@ -3,14 +3,13 @@ import enum
import numpy as np
from karabo.bound import (
DOUBLE_ELEMENT,
- FLOAT_ELEMENT,
KARABO_CLASSINFO,
OUTPUT_CHANNEL,
OVERWRITE_ELEMENT,
VECTOR_STRING_ELEMENT,
- Dims,
- Encoding,
- ImageData,
+ ChannelMetaData,
+ Hash,
+ Timestamp,
)
from .. import base_calcat, base_correction, base_kernel_runner, schemas, utils
@@ -260,6 +259,7 @@ class Gotthard2Correction(base_correction.BaseCorrection):
_cell_table_path = "data.memoryCell"
_pulse_table_path = None
_warn_memory_cell_range = False # for now, receiver always writes 255
+ _preview_outputs = ["outputStreak"]
_cuda_pin_buffers = False
@staticmethod
@@ -297,27 +297,17 @@ class Gotthard2Correction(base_correction.BaseCorrection):
.commit(),
)
- (
- OUTPUT_CHANNEL(expected)
- .key("preview.outputStreak")
- .dataSchema(schemas.preview_schema(wrap_image_in_imagedata=True))
- .commit(),
-
- OUTPUT_CHANNEL(expected)
- .key("preview.outputFrameSums")
- .dataSchema(schemas.preview_schema())
- .commit(),
-
- FLOAT_ELEMENT(expected)
- .key("preview.replaceNanWith")
- .displayedName("Replace streak/sum NaNs by")
- .assignmentOptional()
- .defaultValue(0)
- .reconfigurable()
- .commit(),
+ base_correction.add_preview_outputs(
+ expected, Gotthard2Correction._preview_outputs
)
-
- Gotthard2CalcatFriend.add_schema(expected, Gotthard2Correction._managed_keys)
+ for channel in ("outputRaw", "outputCorrected", "outputFrameSums"):
+ # add these "manually" as the automated bits wrap ImageData
+ (
+ OUTPUT_CHANNEL(expected)
+ .key(f"preview.{channel}")
+ .dataSchema(schemas.preview_schema(wrap_image_in_imagedata=False))
+ .commit(),
+ )
base_correction.add_correction_step_schema(
expected,
Gotthard2Correction._managed_keys,
@@ -326,6 +316,7 @@ class Gotthard2Correction(base_correction.BaseCorrection):
base_correction.add_bad_pixel_config_node(
expected, Gotthard2Correction._managed_keys
)
+ Gotthard2CalcatFriend.add_schema(expected, Gotthard2Correction._managed_keys)
# mandatory: manager needs this in schema
(
@@ -447,36 +438,34 @@ class Gotthard2Correction(base_correction.BaseCorrection):
preview_corrected,
) = self.kernel_runner.compute_previews(preview_slice_index)
- # reusing input data hash for sending
- data_hash.set(self._image_data_path, buffer_handle)
- data_hash.set("calngShmemPaths", [self._image_data_path])
+ if self._use_shmem_handles:
+ data_hash.set(self._image_data_path, buffer_handle)
+ data_hash.set("calngShmemPaths", [self._image_data_path])
+ else:
+ data_hash.set(self._image_data_path, buffer_array)
+ data_hash.set("calngShmemPaths", [])
self._write_output(data_hash, metadata)
- streak_preview = buffer_array.copy()
- streak_preview = np.nan_to_num(
- streak_preview,
- copy=False,
- nan=self.unsafe_get("preview.replaceNanWith"),
- )
- frame_sums = np.sum(streak_preview, axis=1)
- self._write_preview_outputs(
- (
- ("preview.outputRaw", preview_raw),
- ("preview.outputCorrected", preview_corrected),
- (
- "preview.outputStreak",
- ImageData(
- streak_preview,
- Dims(*streak_preview.shape),
- Encoding.GRAY,
- bitsPerPixel=32,
- ),
+ frame_sums = np.nansum(buffer_array, axis=1)
+ timestamp = Timestamp.fromHashAttributes(metadata.getAttributes("timestamp"))
+ for channel, data in (
+ ("outputRaw", preview_raw),
+ ("outputCorrected", preview_corrected),
+ ("outputFrameSums", frame_sums),
+ ):
+ output = self.signalSlotable.getOutputChannel(f"preview.{channel}")
+ output.write(
+ Hash(
+ "image.data",
+ data,
+ "image.mask",
+ (~np.isfinite(data)).astype(np.uint8),
),
- ("preview.outputFrameSums", frame_sums),
- ),
- metadata,
- )
+ ChannelMetaData(f"{self.getInstanceId()}:preview.{channel}", timestamp),
+ )
+ output.update()
+ self._preview_friend.write_outputs(metadata, buffer_array)
def _load_constant_to_runner(self, constant, constant_data):
self.kernel_runner.load_constant(constant, constant_data)
diff --git a/src/calng/corrections/JungfrauCorrection.py b/src/calng/corrections/JungfrauCorrection.py
index e25add50..bb40a97f 100644
--- a/src/calng/corrections/JungfrauCorrection.py
+++ b/src/calng/corrections/JungfrauCorrection.py
@@ -450,6 +450,11 @@ class JungfrauCorrection(base_correction.BaseCorrection):
_calcat_friend_class = JungfrauCalcatFriend
_constant_enum_class = Constants
_managed_keys = base_correction.BaseCorrection._managed_keys.copy()
+ _preview_outputs = [
+ "outputRaw",
+ "outputCorrected",
+ "outputGainMap",
+ ]
_image_data_path = "data.adc"
_cell_table_path = "data.memoryCell"
_pulse_table_path = None
@@ -504,6 +509,10 @@ class JungfrauCorrection(base_correction.BaseCorrection):
.commit(),
)
JungfrauCorrection._managed_keys.add("kernelType")
+
+ base_correction.add_preview_outputs(
+ expected, JungfrauCorrection._preview_outputs
+ )
base_correction.add_correction_step_schema(
expected,
JungfrauCorrection._managed_keys,
@@ -520,16 +529,10 @@ class JungfrauCorrection(base_correction.BaseCorrection):
.setNewDefaultValue(False)
.commit(),
)
- JungfrauCalcatFriend.add_schema(expected, JungfrauCorrection._managed_keys)
base_correction.add_bad_pixel_config_node(
expected, JungfrauCorrection._managed_keys
)
- (
- OUTPUT_CHANNEL(expected)
- .key("preview.outputGainMap")
- .dataSchema(schemas.preview_schema())
- .commit(),
- )
+ JungfrauCalcatFriend.add_schema(expected, JungfrauCorrection._managed_keys)
# mandatory: manager needs this in schema
(
@@ -596,7 +599,7 @@ class JungfrauCorrection(base_correction.BaseCorrection):
except ValueError:
config["corrections.badPixels.maskingValue"] = "nan"
- if config.get("runAsStandaloneModule", default=False):
+ if config.get("useShmemHandles", default=True):
schema_override = Schema()
(
OUTPUT_CHANNEL(schema_override)
@@ -604,25 +607,8 @@ class JungfrauCorrection(base_correction.BaseCorrection):
.dataSchema(schemas.jf_output_schema(use_shmem_handle=False))
.commit(),
)
- preview_utils.PreviewFriend.add_schema(
- schema_override,
- output_channels=["outputRaw", "outputCorrected", "outputGainMap"],
- )
self.updateSchema(schema_override)
- def aux():
- self._preview_friend = preview_utils.PreviewFriend(
- self,
- output_channels=["outputRaw", "outputCorrected", "outputGainMap"],
- )
- self["availableScenes"] = self["availableScenes"] + [
- "preview:outputRaw",
- "preview:outputAssembled",
- "preview:outputGainMap",
- ]
-
- self.registerInitialFunction(aux)
-
def process_data(
self,
data_hash,
@@ -677,29 +663,19 @@ class JungfrauCorrection(base_correction.BaseCorrection):
) = self.kernel_runner.compute_previews(preview_slice_index)
# reusing input data hash for sending
- if self.unsafe_get("runAsStandaloneModule"):
+ if self._use_shmem_handles:
+ data_hash.set(self._image_data_path, buffer_handle)
+ data_hash.set("calngShmemPaths", [self._image_data_path])
+ else:
# TODO: use shmem for data.gain, too
data_hash.set(self._image_data_path, buffer_array)
data_hash.set("calngShmemPaths", [])
- else:
- data_hash.set(self._image_data_path, buffer_handle)
- data_hash.set("calngShmemPaths", [self._image_data_path])
self._write_output(data_hash, metadata)
- if self.unsafe_get("runAsStandaloneModule"):
- self._preview_friend.maybe_write(
- [preview_raw, preview_corrected, preview_gain_map]
- )
- else:
- self._write_preview_outputs(
- (
- ("preview.outputRaw", preview_raw),
- ("preview.outputCorrected", preview_corrected),
- ("preview.outputGainMap", preview_gain_map),
- ),
- metadata,
- )
+ self._preview_friend.write_outputs(
+ metadata, preview_raw, preview_corrected, preview_gain_map
+ )
def _load_constant_to_runner(self, constant, constant_data):
if constant in bad_pixel_constants:
@@ -727,6 +703,3 @@ class JungfrauCorrection(base_correction.BaseCorrection):
self._load_constant_to_runner(
constant, self.calcat_friend.cached_constants[constant]
)
-
- if self._preview_friend is not None:
- self._preview_friend.reconfigure(update)
diff --git a/src/calng/corrections/LpdCorrection.py b/src/calng/corrections/LpdCorrection.py
index 31c25ca0..913f8549 100644
--- a/src/calng/corrections/LpdCorrection.py
+++ b/src/calng/corrections/LpdCorrection.py
@@ -24,6 +24,7 @@ from ..base_correction import (
BaseCorrection,
WarningLampType,
add_correction_step_schema,
+ add_preview_outputs,
)
@@ -324,6 +325,7 @@ class LpdCorrection(BaseCorrection):
_calcat_friend_class = LpdCalcatFriend
_constant_enum_class = Constants
_managed_keys = BaseCorrection._managed_keys.copy()
+ _preview_outputs = ["outputRaw", "outputCorrected", "outputGainMap"]
@staticmethod
def expectedParameters(expected):
@@ -355,13 +357,7 @@ class LpdCorrection(BaseCorrection):
.commit(),
)
- (
- OUTPUT_CHANNEL(expected)
- .key("preview.outputGainMap")
- .dataSchema(schemas.preview_schema())
- .commit(),
- )
-
+ add_preview_outputs(expected, LpdCorrection._preview_outputs)
add_correction_step_schema(
expected,
LpdCorrection._managed_keys,
@@ -411,30 +407,15 @@ class LpdCorrection(BaseCorrection):
except ValueError:
config["corrections.badPixels.maskingValue"] = "nan"
- if config.get("runAsStandaloneModule", default=False):
- schema_override = Schema()
- (
- OUTPUT_CHANNEL(schema_override)
- .key("dataOutput")
- .dataSchema(schemas.xtdf_output_schema(use_shmem_handle=False))
- .commit(),
- )
- preview_utils.PreviewFriend.add_schema(
- schema_override,
- output_channels=["outputRaw", "outputCorrected", "outputGainMap"],
- )
- self.updateSchema(schema_override)
-
+ if config.get("useShmemHandles", default=False):
def aux():
- self._preview_friend = preview_utils.PreviewFriend(
- self,
- output_channels=["outputRaw", "outputCorrected", "outputGainMap"],
+ schema_override = Schema()
+ (
+ OUTPUT_CHANNEL(schema_override)
+ .key("dataOutput")
+ .dataSchema(schemas.xtdf_output_schema(use_shmem_handle=False))
+ .commit(),
)
- self["availableScenes"] = self["availableScenes"] + [
- "preview:outputRaw",
- "preview:outputAssembled",
- "preview:outputGainMap",
- ]
self.registerInitialFunction(aux)
@@ -511,32 +492,20 @@ class LpdCorrection(BaseCorrection):
preview_gain_map,
) = self.kernel_runner.compute_previews(preview_slice_index)
- if self.unsafe_get("runAsStandaloneModule"):
- data_hash.set(self._image_data_path, buffer_array)
- data_hash.set(self._cell_table_path, cell_table[:, np.newaxis])
- data_hash.set("image.pulseId", pulse_table[:, np.newaxis])
- data_hash.set("calngShmemPaths", [])
- else:
+ if self._use_shmem_handles:
data_hash.set(self._image_data_path, buffer_handle)
- data_hash.set(self._cell_table_path, cell_table[:, np.newaxis])
- data_hash.set("image.pulseId", pulse_table[:, np.newaxis])
data_hash.set("calngShmemPaths", [self._image_data_path])
+ else:
+ data_hash.set(self._image_data_path, buffer_array)
+ data_hash.set("calngShmemPaths", [])
- self._write_output(data_hash, metadata)
+ data_hash.set(self._cell_table_path, cell_table[:, np.newaxis])
+ data_hash.set("image.pulseId", pulse_table[:, np.newaxis])
- if self.unsafe_get("runAsStandaloneModule"):
- self._preview_friend.maybe_write(
- [preview_raw, preview_corrected, preview_gain_map]
- )
- else:
- self._write_preview_outputs(
- (
- ("preview.outputRaw", preview_raw),
- ("preview.outputCorrected", preview_corrected),
- ("preview.outputGainMap", preview_gain_map),
- ),
- metadata,
- )
+ self._write_output(data_hash, metadata)
+ self._preview_friend.write_outputs(
+ metadata, preview_raw, preview_corrected, preview_gain_map
+ )
def preReconfigure(self, config):
# TODO: DRY (taken from AGIPD device)
diff --git a/src/calng/preview_utils.py b/src/calng/preview_utils.py
index d7a4c013..98ab68c7 100644
--- a/src/calng/preview_utils.py
+++ b/src/calng/preview_utils.py
@@ -1,6 +1,5 @@
from karabo.bound import (
BOOL_ELEMENT,
- DOUBLE_ELEMENT,
FLOAT_ELEMENT,
NODE_ELEMENT,
OUTPUT_CHANNEL,
@@ -11,7 +10,7 @@ from karabo.bound import (
Encoding,
Hash,
ImageData,
- Unit,
+ Timestamp,
)
import numpy as np
@@ -21,25 +20,30 @@ from . import schemas, utils
class PreviewFriend:
@staticmethod
- def add_schema(schema, node_path="preview", output_channels=None):
+ def add_schema(
+ schema, node_path="preview", output_channels=None, create_node=False
+ ):
if output_channels is None:
output_channels = ["output"]
- (
- NODE_ELEMENT(schema)
- .key(node_path)
- .displayedName("Preview")
- .description(
- "Output specifically intended for preview in Karabo GUI. Includes "
- "some options for throttling and adjustments of the output data."
- )
- .commit(),
+ if create_node:
+ (
+ NODE_ELEMENT(schema)
+ .key(node_path)
+ .displayedName("Preview")
+ .description(
+ "Output specifically intended for preview in Karabo GUI. Includes "
+ "some options for throttling and adjustments of the output data."
+ )
+ .commit(),
+ )
+ (
BOOL_ELEMENT(schema)
.key(f"{node_path}.flipSS")
.displayedName("Flip SS")
.description("Flip image data along slow scan axis.")
.assignmentOptional()
- .defaultValue(True)
+ .defaultValue(False)
.reconfigurable()
.commit(),
@@ -48,7 +52,7 @@ class PreviewFriend:
.displayedName("Flip FS")
.description("Flip image data along fast scan axis.")
.assignmentOptional()
- .defaultValue(True)
+ .defaultValue(False)
.reconfigurable()
.commit(),
@@ -90,19 +94,6 @@ class PreviewFriend:
.defaultValue(0)
.reconfigurable()
.commit(),
-
- DOUBLE_ELEMENT(schema)
- .key(f"{node_path}.maxRate")
- .displayedName("Max rate")
- .description(
- "Preview output is throttled to (at most) this speed. Data arriving "
- "too quickly after last send is silently dropped."
- )
- .unit(Unit.HERTZ)
- .assignmentOptional()
- .defaultValue(2)
- .reconfigurable()
- .commit(),
)
for channel in output_channels:
(
@@ -118,6 +109,7 @@ class PreviewFriend:
output_channels = ["output"]
self.output_channels = output_channels
self.device = device
+ self.dev_id = self.device.getInstanceId()
self.node_name = node_name
self.outputs = [
self.device.signalSlotable.getOutputChannel(f"{self.node_name}.{channel}")
@@ -125,54 +117,62 @@ class PreviewFriend:
]
self.reconfigure(device._parameters)
- def maybe_write(self, datas, inplace=True):
- """If doing so would not exceed maxRate, apply preview settings to data and
- write preview hash to output channel. Returns written hash or None in case
- writing was skipped."""
- if self.throttler.test_and_set():
- timestamp = self.device.getActualTimestamp()
- dev_id = self.device.getInstanceId()
- for data, output, channel_name in zip(
- datas, self.outputs, self.output_channels
- ):
- if self.downsampling_factor > 1:
- data = utils.downsample_2d(
- data,
- self.downsampling_factor,
- reduction_fun=self.downsampling_function,
- )
- elif not inplace:
- data = data.copy()
- data = np.nan_to_num(
+ def write_outputs(self, timestamp, *datas, inplace=True, source=None):
+ """Applies GUI-friendly preview settings (replace NaN, downsample, wrap as
+ ImageData) and writes to output channels. Make sure datas length matches number
+ of channels!"""
+ if isinstance(timestamp, Hash):
+ timestamp = Timestamp.fromHashAttributes(
+ timestamp.getAttributes("timestamp")
+ )
+ for data, output, channel_name in zip(
+ datas, self.outputs, self.output_channels
+ ):
+ if self.downsampling_factor > 1:
+ data = utils.downsample_2d(
data,
- copy=False,
- nan=self.nan_replacement,
- )
- if self.flip_ss:
- data = np.flip(data, 0)
- if self.flip_fs:
- data = np.flip(data, 1)
- output_hash = Hash(
- "image.data",
- ImageData(
- data,
- Dims(*data.shape),
- Encoding.GRAY,
- bitsPerPixel=32,
- ),
+ self.downsampling_factor,
+ reduction_fun=self.downsampling_function,
)
- output.write(
- output_hash,
- ChannelMetaData(f"{dev_id}:{channel_name}", timestamp),
- copyAllData=False,
- )
- output.update()
+ elif not inplace:
+ data = data.copy()
+ if self.flip_ss:
+ data = np.flip(data, 0)
+ if self.flip_fs:
+ data = np.flip(data, 1)
+ if isinstance(data, np.ma.MaskedArray):
+ data, mask = data.data, data.mask | ~np.isfinite(data)
+ else:
+ mask = ~np.isfinite(data)
+ data[mask] = self.nan_replacement
+ mask = mask.astype(np.uint8)
+ output_hash = Hash(
+ "image.data",
+ ImageData(
+ data,
+ Dims(*data.shape),
+ Encoding.GRAY,
+ bitsPerPixel=32,
+ ),
+ "image.mask",
+ ImageData(
+ mask,
+ Dims(*mask.shape),
+ Encoding.GRAY,
+ bitsPerPixel=8,
+ ),
+ )
+ output.write(
+ output_hash,
+ ChannelMetaData(
+ f"{self.dev_id}:{self.node_name}.{channel_name}",
+ timestamp,
+ ),
+ copyAllData=False,
+ )
+ output.update()
def reconfigure(self, conf):
- if conf.has(f"{self.node_name}.maxRate"):
- self.throttler = utils.SkippingThrottler(
- 1 / conf[f"{self.node_name}.maxRate"]
- )
if conf.has(f"{self.node_name}.downsamplingFunction"):
self.downsampling_function = getattr(
np, conf[f"{self.node_name}.downsamplingFunction"]
diff --git a/src/calng/scenes.py b/src/calng/scenes.py
index 46a306f0..ba0f4a90 100644
--- a/src/calng/scenes.py
+++ b/src/calng/scenes.py
@@ -18,6 +18,7 @@ from karabo.common.scenemodel.api import (
LampModel,
LineEditModel,
LineModel,
+ NDArrayGraphModel,
RectangleModel,
SceneModel,
SceneTargetWindow,
@@ -1061,13 +1062,6 @@ class PreviewSettings(HorizontalLayout):
6,
4,
),
- EditableRow(
- device_id,
- schema_hash,
- f"{node_name}.maxRate",
- 6,
- 4,
- ),
HorizontalLayout(
EditableRow(
device_id,
@@ -1159,6 +1153,63 @@ class HistogramSettings(HorizontalLayout):
)
+class PreviewDisplayArea(VerticalLayout):
+ def __init__(
+ self,
+ device_id,
+ schema_hash,
+ channel_name,
+ data_width=None,
+ mask_width=None,
+ data_height=None,
+ mask_height=None,
+ ):
+ super().__init__()
+ try:
+ class_id = schema_hash.getAttribute(
+ f"{channel_name}.schema.image.data", "classId"
+ )
+ except RuntimeError:
+ warning = f"{channel_name} in schema of {device_id} missing classId"
+ print(warning)
+ self.children.append(
+ LabelModel(text=warning, width=10 * BASE_INC, height=BASE_INC)
+ )
+ return
+ if class_id == "ImageData":
+ self.children.extend(
+ [
+ DetectorGraphModel(
+ keys=[f"{device_id}.{channel_name}.schema.image.data"],
+ colormap="viridis",
+ width=60 * BASE_INC if data_width is None else data_width,
+ height=18 * BASE_INC if data_height is None else data_height,
+ ),
+ DetectorGraphModel(
+ keys=[f"{device_id}.{channel_name}.schema.image.mask"],
+ colormap="viridis",
+ width=60 * BASE_INC if mask_width is None else mask_width,
+ height=18 * BASE_INC if mask_height is None else mask_height,
+ ),
+ ]
+ )
+ else:
+ self.children.extend(
+ [
+ NDArrayGraphModel(
+ keys=[f"{device_id}.{channel_name}.schema.image.data"],
+ width=60 * BASE_INC if data_width is None else data_width,
+ height=18 * BASE_INC if data_height is None else data_height,
+ ),
+ NDArrayGraphModel(
+ keys=[f"{device_id}.{channel_name}.schema.image.mask"],
+ width=60 * BASE_INC if mask_width is None else mask_width,
+ height=12 * BASE_INC if mask_height is None else mask_height,
+ ),
+ ]
+ )
+
+
# section: generating actual scenes
@@ -1187,7 +1238,7 @@ def scene_generator(fun):
@scene_generator
-def correction_device_overview(device_id, schema, direct_preview=False):
+def correction_device_overview(device_id, schema):
schema_hash = schema_to_hash(schema)
main_overview = HorizontalLayout(
CorrectionDeviceStatus(device_id, schema_hash),
@@ -1211,37 +1262,29 @@ def correction_device_overview(device_id, schema, direct_preview=False):
max_depth=2,
),
)
-
- if direct_preview:
- return VerticalLayout(
- main_overview,
- LabelModel(
- text="Preview (corrected):",
- width=20 * BASE_INC,
+ return VerticalLayout(
+ main_overview,
+ LabelModel(
+ text="Preview (corrected):",
+ width=20 * BASE_INC,
+ height=BASE_INC,
+ ),
+ PreviewDisplayArea(device_id, schema_hash, "preview.outputCorrected"),
+ *(
+ DeviceSceneLinkModel(
+ text=f"Preview: {channel}",
+ keys=[f"{device_id}.availableScenes"],
+ target=f"preview:preview.{channel}",
+ target_window=SceneTargetWindow.Dialog,
+ width=16 * BASE_INC,
height=BASE_INC,
- ),
- DetectorGraphModel(
- keys=[f"{device_id}.preview.outputCorrected.schema.image.data"],
- height=20 * BASE_INC,
- width=60 * BASE_INC,
- ),
- *(
- DeviceSceneLinkModel(
- text=f"Preview: {channel}",
- keys=[f"{device_id}.availableScenes"],
- target=f"preview:{channel}",
- target_window=SceneTargetWindow.Dialog,
- width=16 * BASE_INC,
- height=BASE_INC,
- )
- for channel in schema_hash.get("preview").getKeys()
- if schema_hash.hasAttribute(f"preview.{channel}", "classId")
- and schema_hash.getAttribute(f"preview.{channel}", "classId")
- == "OutputChannel"
- ),
- )
- else:
- return main_overview
+ )
+ for channel in schema_hash.get("preview").getKeys()
+ if schema_hash.hasAttribute(f"preview.{channel}", "classId")
+ and schema_hash.getAttribute(f"preview.{channel}", "classId")
+ == "OutputChannel"
+ ),
+ )
@scene_generator
@@ -1263,14 +1306,7 @@ def correction_device_preview(device_id, schema, preview_channel):
)
],
),
- titled("Preview image")(boxed(dummy_wrap(DetectorGraphModel)))(
- keys=[f"{device_id}.preview.{preview_channel}.schema.image.data"],
- colormap="viridis",
- width=60 * BASE_INC,
- height=30 * BASE_INC,
- x=PADDING,
- y=PADDING,
- ),
+ PreviewDisplayArea(device_id, schema_hash, preview_channel),
)
@@ -1622,13 +1658,14 @@ def detector_assembler_overview(device_id, schema):
AssemblerDeviceStatus(device_id),
PreviewSettings(device_id, schema_hash),
),
- titled("Preview image")(boxed(dummy_wrap(DetectorGraphModel)))(
- keys=[f"{device_id}.preview.output.schema.image.data"],
- colormap="viridis",
- width=30 * BASE_INC,
- height=30 * BASE_INC,
- x=PADDING,
- y=PADDING,
+ PreviewDisplayArea(
+ device_id,
+ schema_hash,
+ "preview.output",
+ data_width=40 * BASE_INC,
+ data_height=40 * BASE_INC,
+ mask_width=40 * BASE_INC,
+ mask_height=15 * BASE_INC,
),
)
diff --git a/src/calng/schemas.py b/src/calng/schemas.py
index 340f1604..2443c9c3 100644
--- a/src/calng/schemas.py
+++ b/src/calng/schemas.py
@@ -22,6 +22,10 @@ def preview_schema(wrap_image_in_imagedata=False):
IMAGEDATA_ELEMENT(res)
.key("image.data")
.commit(),
+
+ IMAGEDATA_ELEMENT(res)
+ .key("image.mask")
+ .commit(),
)
else:
(
@@ -29,6 +33,12 @@ def preview_schema(wrap_image_in_imagedata=False):
.key("image.data")
.dtype("FLOAT")
.commit(),
+
+ NDARRAY_ELEMENT(res)
+ .key("image.mask")
+ .dtype("UINT8")
+ .commit(),
+
)
return res
--
GitLab