From f5ca09a35cc3f632af68c0678320983956e6ddad Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 29 Aug 2024 14:54:57 +0200
Subject: [PATCH] Split device docs, add concepts, overhaul schemas
---
Makefile | 2 +-
docs/concepts.md | 88 ++++
docs/detectors.md | 4 +-
docs/devices.md | 308 ------------
docs/devices/correction-devices.md | 108 +++++
docs/devices/matchers.md | 67 +++
docs/devices/middlelayer.md | 101 ++++
docs/devices/overview.md | 22 +
docs/{ => devices}/trainmatcher.md | 24 +-
docs/extensions.md | 6 +-
docs/index.md | 2 +-
docs/integrations.md | 15 +-
docs/output-formats.md | 12 +-
docs/schemas/Agipd1MGeometry.md | 726 +++++++++++++++++++++++++++-
docs/schemas/AgipdCorrection.md | 414 ++--------------
docs/schemas/BaseCorrection.md | 13 +
docs/schemas/CalibrationManager.md | 28 +-
docs/schemas/DetectorAssembler.md | 463 +-----------------
docs/schemas/Dssc1MGeometry.md | 33 +-
docs/schemas/DsscCorrection.md | 385 +--------------
docs/schemas/Epix100Geometry.md | 33 +-
docs/schemas/Gotthard2Correction.md | 493 +++++++++----------
docs/schemas/JungfrauCorrection.md | 501 +++++++++----------
docs/schemas/JungfrauGeometry.md | 33 +-
docs/schemas/Lpd1MGeometry.md | 33 +-
docs/schemas/LpdCorrection.md | 385 +--------------
docs/schemas/LpdminiGeometry.md | 33 +-
docs/schemas/PnccdGeometry.md | 33 +-
docs/schemas/ShmemTrainMatcher.md | 444 +----------------
docs/schemas/TrainMatcher.md | 403 +++++++++++++++
docs/troubleshooting.md | 29 +-
31 files changed, 2342 insertions(+), 2899 deletions(-)
create mode 100644 docs/concepts.md
delete mode 100644 docs/devices.md
create mode 100644 docs/devices/correction-devices.md
create mode 100644 docs/devices/matchers.md
create mode 100644 docs/devices/middlelayer.md
create mode 100644 docs/devices/overview.md
rename docs/{ => devices}/trainmatcher.md (74%)
create mode 100644 docs/schemas/TrainMatcher.md
diff --git a/Makefile b/Makefile
index 60413578..59ebb7a7 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
PYPI = pip install --index-url "https://devpi.exfldadev01.desy.de/root/pypi"
PROXIED = pip install --proxy "http://exflproxy01.desy.de:3128"
-.PHONY: all cupy jinja2 h5py xarray extra-geom posixshmem calng
+.PHONY: all cupy jinja2 h5py xarray extra-geom calng
all: calng
diff --git a/docs/concepts.md b/docs/concepts.md
new file mode 100644
index 00000000..f3172e54
--- /dev/null
+++ b/docs/concepts.md
@@ -0,0 +1,88 @@
+# General concepts
+
+## Preview
+Throughout calng, a distinction is made between preview data and "full" data.
+This distinction has two reasons behind it:
+
+1. the data we send should be formatted nicely for GUI preview
+ - some Karabo-specifics (`ImageData` element wrapping)
+ - there should be no `nan` or `inf` values
+ - optionally: downsampling for bandwidth reasons
+2. we don't want to send full detector data to the guiserver (this would simply make the GUI unusable)
+ - instead, we reduce data down to one frame per train
+ - additionally, the guiserver will throttle to 2 Hz by default
+
+
+This is all handled in two different steps and may need to be configured in two different places (until we figure out a nice way to package the configuration).
+Step 1 happens on the device which produces the preview output - can be [correction device](devices/correction-devices.md#single-module-preview) or [preview assembler](devices/matchers.md#preview-assemblers).
+Step 2 happens as early as possible - inside the correction devices.
+
+For step 1, the following configurations are available:
+
+- [NaN replacement](schemas/DetectorAssembler.md#preview.assembled.replaceNanWith): Karabo widgets don't play well with `nan` values.
+ Bad pixel masking by default sets bad pixels to `nan` and the space between modules when assembled is by default filled with `nan`.
+ Therefore, it is up to the operator to decide which value to show for these pixels.
+- Downsampling [factor](schemas/DetectorAssembler.md#preview.assembled.downsamplingFactor) and [function](schemas/DetectorAssembler.md#preview.assembled.downsamplingFunction): allows downsampling of preview image resolution; mostly relevant over slow connections, hopefully not in control room.
+ Downsampling (enabled if the factor is greater than one) is recursive halving, so factors can only be powers of two and image dimensions must be multiples of the factor.
+ The downsampling function defines the downscaling kernel.
+ This function is applied to each 2x2 group of pixels during recursive halving.
+
+For step 2, the options available (in the case of fast detectors) in terms of frame reduction are:
+
+- How to select which frame to preview (for non-negative indices)
+ - Frame can be extracted directly from image ndarray (`frame` mode), by looking up corresponding frame in cell table (`cell` mode), or by looking up in pulse table (`pulse` mode)
+ - For XTDF detectors, the mapping tables are typically `image.cellId` and `image.pulseId`.
+ Which selection mode makes sense depends on detector, veto pattern, and experimental setup.
+ - If the specified cell or pulse is not found in the respective table, a warning is issued and the first frame is sent instead.
+- Which frame or statistic to send
+ - If the index is non-negative, a single frame is sliced from the image data.
+ How this frame is found depends on the the index selection mode (previous point).
+ - If the index is negative, a statistic is computed across all frames for a summary preview.
+ Note that this is done individually per pixel and that `NaN` values are ignored.
+ Options are:
+ - `-1` for max
+ - `-2` for mean
+ - `-3` for sum
+ - `-4` for standard deviation
+
+See also:
+
+- [single module preview](devices/correction-devices.md#single-module-preview)
+- [assembled preview](devices/middlelayer.md#assembled-preview-configuration)
+- [preview assemblers](devices/matchers.md#preview-assemblers)
+
+## Shared memory handles
+To avoid expensive memory copies, correction devices (depending on [configuration](schemas/BaseCorrection.md#useShmemHandles)) put corrected big image data into shared memory.
+This means that they can send results to group matchers or other devices on the same node very cheaply.
+It also means that something like a group matcher is *necessary* for anything *not* on the same node to get access to the data; the group matcher will "dereference" the "shared memory handles" sent by the correction devices.
+
+The following is a description of the protocol used for this purpose.
+It is a bit technical and mostly relevant in case you want to develop a device to run on a correction node, exploiting the shared memory handles on there for cheap data access.
+
+`calng` uses shared memory to pass detector data around between devices running on the same node, in particular the actual correction device, train matchers, bridges and other analysis devices such as `MetroProcessor`. This avoids the serialization and memory transfer cost that even on the same node can be substantial at rates exceeding tens of Gb/s.
+
+The hash being sent from the `dataOutput` pipeline of a correction contains the `VECTOR_STRING` key `calngShmemPaths`. This may list a number of `NDARRAY` paths in same hash that do not actually contain data, but instead contain point to a shared memory region with the actual data. The format of this string is:
+
+```
+{name}${dtype}${shape}${index}
+```
+
+* `name` a unique identifier for the shared memory region, usually containing the correction device ID and pipeline name
+* `dtype` the string representation of the numpy data type, i.e. recoverable via `np.dtype(dtype)`
+* `shape` a comma-separated list of axis lengths
+* `index` an integer pointing to the position along the first array axis acting as a ring buffer
+
+e.g. `SQS_NQS_DSSC/CAL/CORRECT01_Q1M2:dataOutput$float16$50,512,128,60$5`
+
+The shared memory is allocated via the [POSIX shared memory API](https://man7.org/linux/man-pages/man7/shm_overview.7.html) - as of Python 3.8, `multiprocessing.shared_memory` provides classes wrapping the POSIX API.
+For convenience, [`shmem_utils.py`](https://git.xfel.eu/calibration/calngutils/-/blob/master/src/calngUtils/shmem_utils.py) in `calngUtils` provides some wrappers for sending and receiving using our protocol.
+The `ShmemCircularBufferReceiver` in there handles the following:
+
+* The allocated shared memory region (indicated by `name`) may change on any train, but can be assumed to be fairly constant and its mapping can be cached.
+ If and only if `name` stays the same, it can be assumed to be the same shared memory region of the same size.
+* Even with an unchanged memory region name, the `dtype` and `shape` may also change on any train and require wrapping a new `ndarray` around it to correctly find the data.
+* The array described by `dtype` and `shape` is used as a circular buffer with `0 ≤ index < shape[0]`.
+ It always starts at no offset, but may be smaller than the entire shared memory region.
+ There is no guarantee how long any given entry stays around, but is currently ≥ 10 s.
+* The lifetime of the shared memory region is bound to its creator (a correction device) and it should be assumed to be unlinked once it shuts down or `name` changed.
+ POSIX semantics guarantee any consumer the memory stays valid until closed on their side.
diff --git a/docs/detectors.md b/docs/detectors.md
index 1fc5d084..b2fa6d7c 100644
--- a/docs/detectors.md
+++ b/docs/detectors.md
@@ -35,11 +35,11 @@ The enumeration used in `calng` is the same as [in the offline calibration syste
These flags can (for now, for `AgipdCorrection` and `JungfrauCorrection`) be enabled or disabled individually.
By default, all fields are enabled, meaning that bad pixel masks loaded from constants are applied fully.
-Configuring which subset to use is done under the node `corrections.badPixels.subsetToUse`; this should typically be set via the [manager](devices.md#calibration-manager) and the device / manager overview scenes provide a link to a scene for convenient configuration:
+Configuring which subset to use is done under the node `corrections.badPixels.subsetToUse`; this should typically be set via the [manager](devices/middlelayer.md#calibration-manager) and the device / manager overview scenes provide a link to a scene for convenient configuration:
`calng` does not propagate the full bad pixel mask in the output data - it simply masks bad pixels with a configurable masking value.
-The default masking value `NaN` has the benefit that it excludes bad pixels from [preview statistics](devices.md#assembled-preview-configuration) and can be handled directly by the [preview assemblers](devices.md#preview-assemblers).
+The default masking value `nan` has the benefit that it excludes bad pixels from [preview statistics](concepts.md#preview) and can be handled directly by the [preview assemblers](devices/matchers.md#preview-assemblers).
Note that new deployments has an additional bad pixel field `NON_STANDARD_SIZE`.
This is used for AGIPD and JUNGFRAU to mask double size pixels.
diff --git a/docs/devices.md b/docs/devices.md
deleted file mode 100644
index 242caf3a..00000000
--- a/docs/devices.md
+++ /dev/null
@@ -1,308 +0,0 @@
-# Devices
-
-## Overview
-
-A typical calibration pipeline will contain the following device roles:
-
-- One calibration manager
- - Exception: single-module detectors (some JUNGFRAU, GOTTHARD-II)
-- One correction device per detector module, arranged in groups
- - AGIPD1M@SPB, AGIPD1M@MID, LPD1M@FXE, DSSC1M@1MSQS/SCS: 4 groups of 4 correction devices
- - JF4M@SPB (in GPU mode): 1 group with all 8 correction devices
-- Optionally one group matcher per group
-- Optionally one full matcher
-- One preview assembler per preview layer (at least raw and corrected)
-- One geometry device
-
-
-The following diagram illustrates the data flow between devices.
-For readability, the diagram only includes one module in one group, only shows one preview layer, and omits manager and geometry devices (these don't use fast data channels).
-
-
-
-## Calibration manager
-
-Responsible for instantiating, configuring, and restarting most other devices, the manager is the primary way to interact with the pipeline.
-Its basic configuration is saved in the pipeline Karabo project.
-Note that a lot of the manager's schema (including constant retrieval operating condition parameters) is dynamic and so will reset to default when the manager is restarted.
-
-While the pipeline is running, the manager provides convenient reconfiguration of multiple devices simultaneously.
-For instance, it allows simultaneously setting the [DAQ train stride](data-rates.md) on all DAQ devices setting change any of a number of settings on all correction devices.
-Thus, although a number of points below (such as [preview configuration](#assembled-preview-configuration) [retrieving constants](#retrieving-constants)) technically deal with settings on correction devices, these should almost always be set through the manager.
-
-
-
-### Group configuration
-
-For big multi-module detectors, correction devices are split across multiple compute nodes.
-We use the term *group* here to refer to a set of modules handled by a single node.
-For AGIPD / DSSC / LPD, a typical calng pipeline will have four groups of four modules - note that groups do not have to correspond to to quadrants.
-
-Module grouping is configured via the [Modules](schemas/CalibrationManager.md#modules) table.
-This table must list all the detector modules with virtual names and each is assigned a group number (the remaining fields in each row typically left blank as they can be inferred).
-Modules with the same group number are processed on the same device server - which device server is seen in the [Module groups](schemas/CalibrationManager.md#moduleGroups) table, which also has some options for automatically starting [group matchers](#group-matcher) and setting up their [karabo bridge output](trainmatcher.md#karabo-bridge-output).
-
-### Assembled preview configuration
-
-As illustrated in the simplified overview, previews use a separate data flow.
-The main feature of this is to slice or summarize big detector data down to one frame already on the correction devices.
-Therefore, settings related to how this is done should typically be managed via the manager to keep settings consistent.
-The parameters seen in the "Preview" box on the manager overview scene control:
-
-- How to select which frame to preview (for non-negative indices)
- - Frame can be extracted directly from image ndarray (`frame` mode), by looking up corresponding frame in cell table (`cell` mode), or by looking up in pulse table (`pulse` mode)
- - For XTDF detectors, the mapping tables are typically `image.cellId` and `image.pulseId`.
- Which selection mode makes sense depends on detector, veto pattern, and experimental setup.
- - If the specified cell or pulse is not found in the respective table, a warning is issued and the first frame is sent instead.
-- Which frame or statistic to send
- - If the index is non-negative, a single frame is sliced from the image data.
- How this frame is found depends on the the index selection mode (previous point).
- - If the index is negative, a statistic is computed across all frames for a summary preview.
- Note that this is done individually per pixel and that `NaN` values are ignored.
- Options are:
- - `-1` for max
- - `-2` for mean
- - `-3` for sum
- - `-4` for standard deviation
-
-
-These settings determine the data slicing / summarizing on the correction devices.
-Before the preview is sent to the GUI, some additional steps are taken.
-In case of assembled previews, these are set on the [preview assembler](#preview-assemblers) itself.
-In case of a [single module preview](#single-module-preview), the correction device exposes the same configuration settings as the assembler.
-
-### Retrieving constants
-
-For correction devices to apply their corrections, they need to load appropriate correction constants.
-To do so, they need a number of parameters about current operating conditions in order to query [CalCat](https://in.xfel.eu/calibration/).
-Through the manager - see the "Constant retrieval parameters" box in the screenshot - these parameters are set and propagated to the correction devices.
-
-Note that one needs to manually update these when changing detector configuration.
-Taking the AGIPD example in the screenshots, this typically means ensuring that memory cells, bias voltage, acquisition rate, and gain setting / mode corresponds to the current state of the detector.
-Efforts are being made to automate this process.
-
-Once parameters are set, click "Load most recent constants" to start the constant loading process.
-The lamps on the manager overview gives an indication of the progress.
-The individual correction provide additional information about what was found by the latest query, used to troubleshoot [constant loading issues](troubleshooting.md#cannot-load-new-constants).
-
-### Restoring device settings on restart
-
-In some cases, you may want to configure running manager-started devices (ex. group matchers) beyond settings maanged directly by the manager (ex. additional data sources).
-One way to persist such settings is to add such devices to the project.
-Then, however, these devices must be started manually for project-level custom settings to take effect - the manager does not instantiate via project settings.
-
-The ["Restored configurations"](schemas/CalibrationManager.md#restoredConfigurations) table offers an alternate solution.
-Each row in this table specifies a device and key regex.
-When instantiating devices, for each row where the device regex matches the device name, the manager will use the Karabo "get configuration from past" feature with the time set to "now" in order to get the last known values for all properties matching the key regex.
-This allows live reconfigured settings to be persisted across pipeline restarts.
-Make sure to save the manager configuration to the project, though.
-
-## Correction devices
-
-As they are started and configured by the manager, correction devices for big detectors need not be in the Karabo project.
-Similarly, most settings you will want to change on correction devices should be set via the manager in order to keep them consistent.
-
-The overview scene for a single correction device can, however, be useful for troubleshooting:
-
-
-
-Observations about the state of the correction device at the time this screenshot was taken:
-
-- The `deviceInternalsState` [warning lamp](#warning-lamps) is on
- - At the time the screenshot was taken, this was due to a benign issue connecting to timeserver
-- The last `status` was a warning about input hash having unknown source
- - Given that `inputDataState` is no longer red, this was not an issue any longer for the last train processed
-- All constants are loaded
- - All correction steps are available
- - All correction steps except `forceMgIfBelow` and `forceHgIfBelow` are enabled (those two are off by default)
-
-### Single module preview
-
-The section on [assembled preview configuration](#assembled-preview-configuration) deals with previewing a full multi-module detector, including using a [`DetectorAssembler`](#preview-assemblers) with detector geometry to assemble the full preview.
-One can, however, also directly access the preview data from a single correction device.
-This is found under the [`preview` node](schemas/BaseCorrection.md#preview) which contain both the settings described for assembled preview configuration - how to slice burst mode data for preview - and the GUI-specific settings necessary to display the preview in the KaraboGUI.
-
-The default correction device scene includes the corrected preview output and links to individual scenes with any additional preview channels.
-
-Single module previews are primarily intended for use in single-module detector installations such as JF500K.
-They can also be used to inspect single modules in multi-module detector setups, but one should avoid tweaking individual module preview settings in this case.
-Changing the GUI-related settings (flipping along SS / FS axes, downsampling) on the correction device level should be expected to break or render inconsistent the assembled preview.
-
-### Warning lamps
-
-The warning lamps discussed here are displayed with names on the [correction device overview scene](#correction-devices) and shown in the correction device overview of the [manager overview scene](#calibration-manager).
-They are intended to quickly notify the operator of issues with the pipeline.
-There are multiple lamps (rather than the regular device state going to `ERROR`) in order to give a hint as to where the errors may originate.
-The lamps and their error types are currently:
-
-`inputDataState` indicates a problem with the latest input received - can flicker if detector, DAQ, or network is unreliable
-
-: `EMPTY_HASH` is most commonly encountered when the DAQ is monitoring, but not getting data from the detector.
-In that setting, the DAQ will still send data to the pipeline, but it will be "empty".
-Specifically, this warning is triggered when opening image data causes a `RuntimeError` because the number of bytes in the array does not match the shape.
-
-: `TRAIN_ID` means that the train ID of the input seems fishy.
-Note that while the correction devices do not care about train IDs, train ID issues will affect matching later in the pipeline.
-Train IDs are fishy if they are much greater than "current" train ID (see [`trainFromFutureThreshold`](schemas/BaseCorrection.md#dataFormat.trainFromFutureThreshold)) or if they are not monotonic.
-In the former case, input is dropped, while in the latter, the warning is just issued (and the [received train percentage](schemas/BaseCorrection.md#performance.ratioOfRecentTrainsReceived) is reset).
-
-: `MISC_INPUT_DATA` covers other issues such as input source not being in [`fastSources`](schemas/BaseCorrection.md#fastSources) or input missing image data node.
-These issues seem to sometimes show for a single train when DAQ cycles through states.
-
-`processingState`
-
-: `MEMORY_CELL_RANGE` means that the memory cell range for the input (for XTDF detectors, the values in `image.cellId`) exceed the number of memory cells as used for constant queries (`constantParameters.memoryCells`).
-Most corrections are per pixel and per memory cell, so frames from cells outside the range of currently loaded constants may not get corrected.
-Make sure to set the constant parameters according to the current detector configuration.
-
-: `FRAME_FILTER` means an error occurred during application of the frame filter.
-
-: `PREVIEW_SETTINGS` means that there was a warning when picking the frame for preview, probably because the requested frame was out of bounds or the cell / pulse requested was not found.
-The warning text should give some details.
-
-: In future versions of `calng`, there will likely be a catch-all `MISC_PROCESSING` to gather uncaught exceptions in the detector-specific correction code.
-
-`deviceInternalsState` indicates problems within the correction device expected to prevent normal function.
-
-: `CALCAT_CONNECTION` means that the device failed to connect to [CalCat](https://in.xfel.eu/calibration).
-In this case, the device will be unable to retrieve constants.
-
-: `TIMESERVER_CONNECTION` means that the device (when doing `getActualTimestamp`) thought that the "current train ID" was zero, typically indicating that the device was not connected to the timeserver.
-In this case, the correction device may be unable to preemptively discard erroneous "future train IDs" (see [`trainFromFutureThreshold`](schemas/BaseCorrection.md#dataFormat.trainFromFutureThreshold) and the `TRAIN_ID` warning above).
-This does not impair processing, but if "future train IDs" are expected and this warning persists, one should try to fix the configuration.
-
-: `CORRECTION_RUNNER` means that data was received while the correction kernel(s) were not ready.
-Some reconfiguration (for instance, changing number of cells on the detector) may trigger recompilation of kernels, causing the lamp to blink briefly.
-While this warning is on, the device will be unable to correct data - if it is on and persists for a while after a restart, it is likely an issue for CAL OCD.
-
-: `FRAME_FILTER` - when applying frame filter fails for a given train, this sets a warning on `processingState`.
-As it is possible to specify "obviously" invalid frame filters (ex. with indices out of range for the number of frames on input), this warning can additionally be set here on `deviceInternalsState` after reconfiguration.
-
-If an issue is detected, a warning is issued once - this goes in the device `status` and in the log - and the appropriate lamp is turned on.
-The lamp is turned back off when the issue is noticed to be resolved - for example, the lamp warning about `inputDataState` might turn on if corrupt data is received for one train, but turned off again if the next train is valid.
-
-Pro tip: you can use Karabo's device property history feature to see recent history of device `status`.
-Select a correction device, double-click the "Current value on device" for "Status", select a time range, and "Request History".
-This is useful to see why a warning lamp is on in cases where it has been on for a while and other `status` messages might have come since.
-
-### Constant status
-
-For each calibration constant, the correction device exposes a state field to indicate whether the constant was successfully found and loaded.
-These are somewhat simpler than the warning lamps described in the previous section:
-
-- Initially, all constants are in the `OFF` state
-- Every time a constant is queried / reloaded, the state changes to:
- - `ON` if no exceptions happened
- - `ERROR` in case anything went wrong
-
-
-Consult the correction device status log for information about what specifically went wrong.
-A common source of issues is incorrectly set operating condition parameters, leading to "condition not found" when querying CalCat.
-
-## Geometry devices
-
-For multi-module detector setups, a geometry device is typically set up to inform the [preview assemblers](#preview-assemblers) about the current geometry of the detector.
-The geometry device should be saved in the Karabo project.
-It should not need to be restarted and is not managed as such by the manager.
-
-
-
-As the above screenshot suggests, the current geometry can be set manually via quadrant positions (or module list, in the case of JUNGFRAU) or loaded from a file.
-Geometry devices wrap geometry classes from [EXtra-geom](https://extra-geom.readthedocs.io/en/latest/).
-
-## Matchers
-
-A number of specialized versions of the [TrainMatcher](trainmatcher.md) are used in `calng` pipelines.
-The extensions provided in `ShmemTrainMatcher` used within `calng` are twofold:
-
-1. They handle [shared memory handles](https://git.xfel.eu/karaboDevices/calng/-/wikis/ShmemProtocol)
- - This is necessary for group matchers
-2. They allow stacking arrays from multiple sources
- - This is useful for connecting tools like OnDA to a full matcher
-
-
-### Group matcher
-
-Each detector module produces an independent data stream - to the control system, each module is essentially its own detector.
-The data from one module is typically passed through one DAQ device from which it is then sent to a `calng` correction device.
-As multiple data streams can be processed simultaneously, a **group** in `calng` denotes a set of modules which are *corrected* on the same compute node in the online cluster.
-Fast detectors like AGIPD, DSSC, or LPD are typically split into four groups of four modules.
-For slower detectors like JUNGFRAU, it may be feasible to process the entire detector wihin a sigle group.
-
-Each group can optionally run a group matcher, which provides the fastest possible access to the output stream from the group.
-If your online analysis relies only upon a subset of modules, configuring how the groups are divided (done via the manager) is the first step to set up an efficient output over a Karabo bridge.
-See [manager notes](#calibration-manager) for details on group matcher configuration.
-
-Here is a screenshot of the overview scene from a group matcher in the wild:
-
-
-
-### Full matcher
-
-Important to note:
-
-- Full matcher should connect to outputs of group matchers, not directly to correction devices
- - The group matchers will take care to send full data out (dereferencing shared memory handles from correction devices)
- - To configure sources, keep in mind that correction devices forward the source name they get from DAQs and that TrainMatcher needs to know source name and actual channel name in the format `[source]@[channel]`
- - Example source in `SPB_DET_AGIPD1M-1/CALNG/FULL_MATCHER`: `SPB_DET_AGIPD1M-1/DET/0CH0:xtdf@SPB_DET_AGIPD1M-1/CALNG/MATCH_G1:output`
-- Full matcher is likely to hit [network limits](data-rates.md)
- - Be aware of DAQ train stride and number of frames (or memory cells) coming from the detector
- - If possible, consider running the full matcher on the node where analysis is done
- - Configuring and testing full matcher with online analysis software should be done during commissioning (please contact calibration team ahead of time)
-
-## Preview assemblers
-
-Preview assemblers are also based on [TrainMatcher](trainmatcher.md) and as such, some configuration and troubleshooting notes overlap with those for [other matchers](#matchers).
-In particular, consider the value of the [`maxIdle`](schemas/ShmemTrainMatcher.md#maxIdle) parameter; if it is too low relative to DAQ train stride, it can cause [flickering](troubleshooting.md#preview-is-flickering).
-
-A preview assembler needs to contact a geometry device to get current detector geometry to use for assembly.
-The geometry classes and assembly methods directly use [EXtra-geom](https://extra-geom.readthedocs.io/en/latest/).
-
-Preview assemblers offer a few parameters to tweak the preview image sent to the GUI:
-
-- [NaN replacement](schemas/DetectorAssembler.md#preview.replaceNanWith): Karabo widgets don't play well with `NaN` values.
- Bad pixel masking by default sets bad pixels to `NaN` and the space between modules when assembled is by default filled with `NaN`.
- Therefore, it is up to the operator to decide which value to show for these pixels.
-- [Max rate](schemas/DetectorAssembler.md#preview.maxRate): how often to send preview updates.
- Karabo GUI is typically set to only accept updates at 2 Hz, so typically leave this parameter at this default value.
-- Downsampling [factor](schemas/DetectorAssembler.md#preview.downsamplingFactor) and [function](schemas/DetectorAssembler.md#preview.downsamplingFunction): allows downsampling of preview image resolution; mostly relevant over slow connections, hopefully not in control room.
- Downsampling (enabled if the factor is greater than one) is recursive halving, so factors can only be powers of two and image dimensions must be multiples of the factor.
- The downsampling function defines the downscaling kernel.
- This function is applied to each 2x2 group of pixels during recursive halving.
-
-
-Preview assemblers provide multiple outputs:
-
-- As a [TrainMatcher](trainmatcher.md), it has the typical "Pipeline Output" (`output`).
- This is unassembled, at full rate, and per source.
- Keep in mind that preview assemblers typically receive single-frame input - this is different to a [full matcher](#full-matcher).
-- `assembledOutput` provides the assembled images at full speed.
- The output only contains `image.data` (single frame, assembled) and `trainId`.
- As the preview settings have not been applied to this data, this output may be useful for inspecting with small analysis devices.
-- `preview.output` is the preview output, providing the image displayed on the default scene.
- The aforementioned throttling, `NaN` replacement, and downscaling is only applied to this output.
-
-
-
-As a TrainMatcher, a preview assembler can also provide [Karabo bridge output](trainmatcher.md#karabo-bridge-output).
-The `outputForBridgeOutput` parameter allows you to choose which of the three output channels the Karabo bridge output should mirror.
-
-
-
-## Condition watchers
-
-As described above, [retrieving constants](#retrieving-constants) requires querying with current operating conditions.
-In many cases, this means repeating values set by (a detector) control device(s).
-A *condition* device can partly automate the task of keeping the query values in sync.
-Still a work in progress, condition devices are available for AGIPD and JUNGFRAU.
-
-
-
-The above screenshot from the `calng` installation at MID reveals some details:
-
-- The condition checker distinguishes between the current value on the control device, the ideal value that the manager should use for constant querying based on that, and the current value actually set on the manager
- - For some values like gain mode, the condition device maps from an integer control value (`gainModeIndex`) to the descriptive enum used in `calng` (`AgipdGainMode` with members like `ADAPTIVE_GAIN`)
- - For other values like bias voltage, the condition device rounds the floating control value to avoid small fluctuations interfering with constant retrieval
-- Some parameters - like photon energy - are not found on detector control device which is indicated by `IGNORING`
- - In the screenshot, this includes `pixelsX` and `pixelsY` which are anyway fixed for the detector
- - It additionally shows `deviceMappingSnapshotAt` and `constantVersionEventAt` which are [CalCat](https://in.xfel.eu/calibration/)-specific parameters only relevant for certain testing scenarios and should generally be left blank for normal operation
diff --git a/docs/devices/correction-devices.md b/docs/devices/correction-devices.md
new file mode 100644
index 00000000..bce9f57a
--- /dev/null
+++ b/docs/devices/correction-devices.md
@@ -0,0 +1,108 @@
+## Correction devices
+
+As they are started and configured by the manager, correction devices for big detectors need not be in the Karabo project.
+Similarly, most settings you will want to change on correction devices should be set via the manager in order to keep them consistent.
+
+The overview scene for a single correction device can, however, be useful for troubleshooting:
+
+
+
+Observations about the state of the correction device at the time this screenshot was taken:
+
+- The `deviceInternalsState` [warning lamp](#warning-lamps) is on
+ - At the time the screenshot was taken, this was due to a benign issue connecting to timeserver
+- The last `status` was a warning about input hash having unknown source
+ - Given that `inputDataState` is no longer red, this was not an issue any longer for the last train processed
+- All constants are loaded
+ - All correction steps are available
+ - All correction steps except `forceMgIfBelow` and `forceHgIfBelow` are enabled (those two are off by default)
+
+### Single module preview
+
+For big detectors, one will usually be interested in the full assembled previews from the [preview assemblers](matchers.md#preview-assemblers).
+One can, however, also directly access the preview data from a single correction device.
+This is found under the `preview` node which contains one subnode for each preview output of the correction device.
+Preview outputs vary from device class to device class (so from detector to detector), but always include raw and corrected.
+
+Within the subnode for a preview output, there is the actual output channel, `preview.[preview type].output` and some configuration settings applicable to the preview; see [preview configuration](../concepts.md#preview).
+Note that which options (like image downsampling, reduction across frames, frame selection etc.) are available depends on the detector and preview type; not all settings apply to all types of preview.
+
+Important note: these settings are configured individually per preview channel.
+If, for example, you wish to compare two different preview outputs for an XTDF detector pipeline, make sure to set the same frame selection parameters (`index` in particular) on each.
+
+The default correction device scene includes the corrected preview output and links to individual scenes with any additional preview channels.
+
+Single module previews are primarily intended for use in single-module detector installations such as JF500K.
+They can also be used to inspect single modules in multi-module detector setups, but one should avoid tweaking individual module preview settings in this case.
+Changing the GUI-related settings (flipping along SS / FS axes, downsampling) on the correction device level should be expected to break or render inconsistent the assembled preview.
+
+### Warning lamps
+
+The warning lamps discussed here are displayed with names on the [correction device overview scene](#correction-devices) and shown in the correction device overview of the [manager overview scene](middlelayer.md#calibration-manager).
+They are intended to quickly notify the operator of issues with the pipeline.
+There are multiple lamps (rather than the regular device state going to `ERROR`) in order to give a hint as to where the errors may originate.
+The lamps and their error types are currently:
+
+`inputDataState` indicates a problem with the latest input received - can flicker if detector, DAQ, or network is unreliable
+
+: `EMPTY_HASH` is most commonly encountered when the DAQ is monitoring, but not getting data from the detector.
+In that setting, the DAQ will still send data to the pipeline, but it will be "empty".
+Specifically, this warning is triggered when opening image data causes a `RuntimeError` because the number of bytes in the array does not match the shape.
+
+: `TRAIN_ID` means that the train ID of the input seems fishy.
+Note that while the correction devices do not care about train IDs, train ID issues will affect matching later in the pipeline.
+Train IDs are fishy if they are much greater than "current" train ID (see [`trainFromFutureThreshold`](../schemas/BaseCorrection.md#workarounds.trainFromFutureThreshold)) or if they are not monotonic.
+In the former case, input is dropped, while in the latter, the warning is just issued (and the [received train percentage](../schemas/BaseCorrection.md#performance.ratioOfRecentTrainsReceived) is reset).
+
+: `MISC_INPUT_DATA` covers other issues such as input source not being in [`fastSources`](../schemas/BaseCorrection.md#fastSources) or input missing image data node.
+These issues seem to sometimes show for a single train when DAQ cycles through states.
+
+`processingState`
+
+: `MEMORY_CELL_RANGE` means that the memory cell range for the input (for XTDF detectors, the values in `image.cellId`) exceed the number of memory cells as used for constant queries (`constantParameters.memoryCells`).
+Most corrections are per pixel and per memory cell, so frames from cells outside the range of currently loaded constants may not get corrected.
+Make sure to set the constant parameters according to the current detector configuration.
+
+: `FRAME_FILTER` means an error occurred during application of the frame filter.
+
+: `PREVIEW_SETTINGS` means that there was a warning when picking the frame for preview, probably because the requested frame was out of bounds or the cell / pulse requested was not found.
+The warning text should give some details.
+
+: In future versions of `calng`, there will likely be a catch-all `MISC_PROCESSING` to gather uncaught exceptions in the detector-specific correction code.
+
+`deviceInternalsState` indicates problems within the correction device expected to prevent normal function.
+
+: `CALCAT_CONNECTION` means that the device failed to connect to [CalCat](https://in.xfel.eu/calibration).
+In this case, the device will be unable to retrieve constants.
+
+: `TIMESERVER_CONNECTION` means that the device (when doing `getActualTimestamp`) thought that the "current train ID" was zero, typically indicating that the device was not connected to the timeserver.
+In this case, the correction device may be unable to preemptively discard erroneous "future train IDs" (see [`trainFromFutureThreshold`](../schemas/BaseCorrection.md#workarounds.trainFromFutureThreshold) and the `TRAIN_ID` warning above).
+This does not impair processing, but if "future train IDs" are expected and this warning persists, one should try to fix the configuration.
+
+: `CORRECTION_RUNNER` means that data was received while the correction kernel(s) were not ready.
+Some reconfiguration (for instance, changing number of cells on the detector) may trigger recompilation of kernels, causing the lamp to blink briefly.
+While this warning is on, the device will be unable to correct data - if it is on and persists for a while after a restart, it is likely an issue for CAL OCD.
+
+: `FRAME_FILTER` - when applying frame filter fails for a given train, this sets a warning on `processingState`.
+As it is possible to specify "obviously" invalid frame filters (ex. with indices out of range for the number of frames on input), this warning can additionally be set here on `deviceInternalsState` after reconfiguration.
+
+If an issue is detected, a warning is issued once - this goes in the device `status` and in the log - and the appropriate lamp is turned on.
+The lamp is turned back off when the issue is noticed to be resolved - for example, the lamp warning about `inputDataState` might turn on if corrupt data is received for one train, but turned off again if the next train is valid.
+
+Pro tip: you can use Karabo's device property history feature to see recent history of device `status`.
+Select a correction device, double-click the "Current value on device" for "Status", select a time range, and "Request History".
+This is useful to see why a warning lamp is on in cases where it has been on for a while and other `status` messages might have come since.
+
+### Constant status
+
+For each calibration constant, the correction device exposes a state field to indicate whether the constant was successfully found and loaded.
+These are somewhat simpler than the warning lamps described in the previous section:
+
+- Initially, all constants are in the `OFF` state
+- Every time a constant is queried / reloaded, the state changes to:
+ - `ON` if no exceptions happened
+ - `ERROR` in case anything went wrong
+
+
+Consult the correction device status log for information about what specifically went wrong.
+A common source of issues is incorrectly set operating condition parameters, leading to "condition not found" when querying CalCat.
diff --git a/docs/devices/matchers.md b/docs/devices/matchers.md
new file mode 100644
index 00000000..68f8f9a2
--- /dev/null
+++ b/docs/devices/matchers.md
@@ -0,0 +1,67 @@
+A number of specialized versions of the [TrainMatcher](trainmatcher.md) are used in `calng` pipelines.
+The extensions provided in `ShmemTrainMatcher` (the device class for [group matchers](#group-matcher) and [full matchers](#full-matcher)) used within `calng` are:
+
+1. They handle [shared memory handles](https://git.xfel.eu/karaboDevices/calng/-/wikis/ShmemProtocol)
+ - This is necessary for group matchers
+2. They allow stacking arrays from multiple sources
+ - This is useful for connecting tools like OnDA to a full matcher
+3. They support frame selection for online data reduction
+ - Documentation for this feature is lacking
+
+
+### Group matcher
+
+For multi-module detectors, each detector module produces an independent data stream - to the control system, each module is essentially its own detector.
+The data from one module is typically passed through one DAQ device from which it is then sent to a `calng` correction device.
+As multiple data streams can be processed simultaneously, a **group** in `calng` denotes a set of modules which are *corrected* on the same compute node in the online cluster.
+Fast detectors like AGIPD, DSSC, or LPD are typically split into four groups of four modules.
+For slower detectors like JUNGFRAU, it may be feasible to process the entire detector wihin a sigle group.
+
+Each group can optionally run a group matcher, which provides the fastest possible access to the output stream from the group.
+If your online analysis relies only upon a subset of modules, configuring how the groups are divided (done via the manager) is the first step to set up an efficient output over a Karabo bridge.
+See [manager notes](middlelayer.md#calibration-manager) for details on group matcher configuration.
+
+Here is a screenshot of the overview scene from a group matcher in the wild:
+
+
+
+### Full matcher
+
+Important to note:
+
+- Full matcher should connect to outputs of group matchers, not directly to correction devices
+ - The group matchers will take care to send full data out (dereferencing shared memory handles from correction devices)
+ - To configure sources, keep in mind that correction devices forward the source name they get from DAQs and that TrainMatcher needs to know source name and actual channel name in the format `[source]@[channel]`
+ - Example source in `SPB_DET_AGIPD1M-1/CALNG/FULL_MATCHER`: `SPB_DET_AGIPD1M-1/DET/0CH0:xtdf@SPB_DET_AGIPD1M-1/CALNG/MATCH_G1:output`
+- Full matcher is likely to hit [network limits](../data-rates.md)
+ - Be aware of DAQ train stride and number of frames (or memory cells) coming from the detector
+ - If possible, consider running the full matcher on the node where analysis is done
+ - Configuring and testing full matcher with online analysis software should be done during commissioning (please contact calibration team ahead of time)
+
+## Preview assemblers
+As the name implies, a preview assembler (class name: [`DetectorAssembler`](../schemas/DetectorAssembler.md)) assembles preview images for full detectors by matching the data from the individual modules and applying a detector geometry to the data.
+A preview assembler needs to contact a geometry device to get current detector geometry to use for assembly.
+The [geometry classes](middlelayer.md#geometry-devices) and assembly methods directly use [EXtra-geom](https://extra-geom.readthedocs.io/en/latest/).
+
+
+Preview assemblers provide multiple outputs:
+
+- As a [TrainMatcher](trainmatcher.md), it has the typical "Pipeline Output" (`output`).
+ This is unassembled, at full rate, and per source.
+ Keep in mind that preview assemblers typically receive single-frame input - this is different to a [full matcher](#full-matcher).
+- `assembledOutput` provides the assembled images at full speed.
+ The output only contains `image.data` (single frame, assembled) and `trainId`.
+ As the preview settings have not been applied to this data, this output may be useful for inspecting with small analysis devices.
+- `preview.assembled.output` is the preview output, providing the image displayed on the default scene.
+ See [preview](../concepts.md#preview) for details on how to configure the preview output.
+ Keep in mind that frame reduction should be set [via the manager](middlelayer.md#assembled-preview-configuration) as it happens on the correction devices.
+
+As a TrainMatcher, a preview assembler can also provide [Karabo bridge output](trainmatcher.md#karabo-bridge-output).
+The `outputForBridgeOutput` parameter allows you to choose which of the three output channels the Karabo bridge output should mirror.
+
+
+
+Common issues:
+
+- Misconfigured [`maxIdle`](../schemas/TrainMatcher.md#maxIdle); if too low (relative to DAQ train stride), it can cause [flickering](../troubleshooting.md#preview-is-flickering)
+- Missing geometry can cause [preview to not update](../troubleshooting.md#preview-does-not-update)
diff --git a/docs/devices/middlelayer.md b/docs/devices/middlelayer.md
new file mode 100644
index 00000000..7a156950
--- /dev/null
+++ b/docs/devices/middlelayer.md
@@ -0,0 +1,101 @@
+These are the middlelayer devices typically accompanying calibration pipelines.
+They naturally run on a separate device server - a big detector pipeline typically has multiple correction servers (bound), a preview server (bound), and a single middlelayer server.
+The middlelayer devices are usually not restarted along with the pipeline, but rather stay around.
+
+## Calibration manager
+
+Responsible for instantiating, configuring, and restarting most other devices, the manager is the primary way to interact with the pipeline.
+Its basic configuration is saved in the pipeline Karabo project.
+Note that a lot of the manager's schema (including constant retrieval operating condition parameters) is dynamic and so will reset to default when the manager is restarted.
+
+While the pipeline is running, the manager provides convenient reconfiguration of multiple devices simultaneously.
+For instance, it allows simultaneously setting the [DAQ train stride](../data-rates.md) on all DAQ devices setting change any of a number of settings on all correction devices.
+Thus, although a number of points below (such as [preview configuration](#assembled-preview-configuration) [retrieving constants](#retrieving-constants)) technically deal with settings on correction devices, these should almost always be set through the manager.
+
+
+
+### Group configuration
+
+For big multi-module detectors, correction devices are split across multiple compute nodes.
+We use the term *group* here to refer to a set of modules handled by a single node.
+For AGIPD / DSSC / LPD, a typical calng pipeline will have four groups of four modules - note that groups do not have to correspond to to quadrants.
+
+Module grouping is configured via the [Modules](../schemas/CalibrationManager.md#modules) table.
+This table must list all the detector modules with virtual names and each is assigned a group number (the remaining fields in each row typically left blank as they can be inferred).
+Modules with the same group number are processed on the same device server - which device server is seen in the [Module groups](../schemas/CalibrationManager.md#moduleGroups) table, which also has some options for automatically starting [group matchers](matchers.md#group-matcher) and setting up their [karabo bridge output](trainmatcher.md#karabo-bridge-output).
+
+### Assembled preview configuration
+
+As illustrated in the simplified overview, previews use a separate data flow.
+The main feature of this is to slice or summarize big detector data down to one frame already on the correction devices.
+Therefore, settings related to how this is done should typically be managed via the manager to keep settings consistent.
+
+See [preview configuration](../concepts.md#preview) for details on the settings relevant to preview configuration.
+Note that the manager will configure the correction devices' frame reduction settings; the settings on the assemblers (such as downsampling / `nan` replacement) are not typically touched and the manager does not set these.
+Note also that there will be one [preview assembler](matchers.md#preview-assemblers) per [preview channel](correction-devices.md#single-module-preview) and each preview channel has its own, independent, configuration.
+
+### Retrieving constants
+
+For correction devices to apply their corrections, they need to load appropriate correction constants.
+To do so, they need a number of parameters about current operating conditions in order to query [CalCat](https://in.xfel.eu/calibration/).
+Through the manager - see the "Constant retrieval parameters" box in the screenshot - these parameters are set and propagated to the correction devices.
+
+Note that one needs to manually update these when changing detector configuration.
+Taking the AGIPD example in the screenshots, this typically means ensuring that memory cells, bias voltage, acquisition rate, and gain setting / mode corresponds to the current state of the detector.
+Efforts are being made to automate this process.
+
+Once parameters are set, click "Load most recent constants" to start the constant loading process.
+The lamps on the manager overview gives an indication of the progress.
+The individual correction provide additional information about what was found by the latest query, used to troubleshoot [constant loading issues](../troubleshooting.md#cannot-load-new-constants).
+
+### Restoring device settings on restart
+
+In some cases, you may want to configure running manager-started devices (ex. group matchers) beyond settings maanged directly by the manager (ex. additional data sources).
+One way to persist such settings is to add such devices to the project.
+Then, however, these devices must be started manually for project-level custom settings to take effect - the manager does not instantiate via project settings.
+
+The ["Restored configurations"](../schemas/CalibrationManager.md#restoredConfigurations) table offers an alternate solution.
+Each row in this table specifies a device and key regex.
+When instantiating devices, for each row where the device regex matches the device name, the manager will use the Karabo "get configuration from past" feature with the time set to "now" in order to get the last known values for all properties matching the key regex.
+This allows live reconfigured settings to be persisted across pipeline restarts.
+Make sure to save the manager configuration to the project, though.
+
+## Geometry devices
+
+For multi-module detector setups, a geometry device is typically set up to inform the [preview assemblers](matchers.md#preview-assemblers) about the current geometry of the detector.
+The geometry device should be saved in the Karabo project.
+It should not need to be restarted and is not managed as such by the manager.
+
+
+
+As the above screenshot suggests, the current geometry can be set manually via quadrant positions (or module list, in the case of JUNGFRAU) or loaded from a file.
+Geometry devices wrap geometry classes from [EXtra-geom](https://extra-geom.readthedocs.io/en/latest/).
+
+## Condition watchers
+
+As described above, [retrieving constants](#retrieving-constants) requires querying with current operating conditions.
+In many cases, this means repeating values set by (a detector) control device(s).
+A *condition* device can partly automate the task of keeping the query values in sync.
+Still a work in progress, condition devices are available for AGIPD and JUNGFRAU.
+
+
+
+The above screenshot from the `calng` installation at MID reveals some details:
+
+- The condition checker distinguishes between the current value on the control device, the ideal value that the manager should use for constant querying based on that, and the current value actually set on the manager
+ - For some values like gain mode, the condition device maps from an integer control value (`gainModeIndex`) to the descriptive enum used in `calng` (`AgipdGainMode` with members like `ADAPTIVE_GAIN`)
+ - For other values like bias voltage, the condition device rounds the floating control value to avoid small fluctuations interfering with constant retrieval
+- Some parameters - like photon energy - are not found on detector control device which is indicated by `IGNORING`
+ - In the screenshot, this includes `pixelsX` and `pixelsY` which are anyway fixed for the detector
+ - It additionally shows `deviceMappingSnapshotAt` and `constantVersionEventAt` which are [CalCat](https://in.xfel.eu/calibration/)-specific parameters only relevant for certain testing scenarios and should generally be left blank for normal operation
+
+## ROI tool
+
+The ROI tool is a small device developed to help keep track of pixel intensities.
+It is not very fast and is usually just connected to a preview output of corrected data.
+Note that using [preview data](../concepts.md#preview) for any kind of analysis comes with a lot of caveats and should generally be avoided.
+
+With these limitations in mind, the purpose of the ROI tool is to just monitor the intensity distribution in some region of interest.
+The device scene will give you a widget to select the ROI, show you a histogram of a running average distribution within this ROI, and some options to tweak the histogramming.
+
+Currently deployed for testing at SPB; see `SPB_DET_AGIPD1M-1/CALNG/ROI_HISTOGRAM_CORR` and `SPB_IRDA_JF4M/CALNG/ROI_TOOL_CORR`.
diff --git a/docs/devices/overview.md b/docs/devices/overview.md
new file mode 100644
index 00000000..17d03b0e
--- /dev/null
+++ b/docs/devices/overview.md
@@ -0,0 +1,22 @@
+# Device overview
+
+A typical calibration pipeline will contain the following device roles:
+
+- One [manager](middlelayer.md#calibration-manager) responsible for instantiating and configuring the pipeline
+ - Exception: some single-module detectors (some JUNGFRAU, GOTTHARD-II)
+- One [correction device](correction-devices.md) per detector module, processing the detector data
+ - Arranged in groups
+ - AGIPD1M@SPB, AGIPD1M@MID, LPD1M@FXE, DSSC1M@1MSQS/SCS: 4 groups of 4 correction devices
+ - JF4M@SPB (in GPU mode): 1 group with all 8 correction devices
+- Optionally one [group matcher](matchers.md#group-matcher) per group responsible for sending data out from the processing node
+ - Can send to online analysis via Karabo bridge
+ - Can send to full matcher via regular Karabo channel
+- Optionally one [full matcher](matchers.md#full-matcher) - same as group matcher, but for full detector
+- One preview assembler per preview layer (at least raw and corrected)
+- One geometry device
+
+
+The following diagram illustrates the data flow between devices.
+For readability, the diagram only includes one module in one group, only shows one preview layer, and omits manager and geometry devices (these don't use fast data channels).
+
+
diff --git a/docs/trainmatcher.md b/docs/devices/trainmatcher.md
similarity index 74%
rename from docs/trainmatcher.md
rename to docs/devices/trainmatcher.md
index 5c0581b5..062665e1 100644
--- a/docs/trainmatcher.md
+++ b/docs/devices/trainmatcher.md
@@ -3,7 +3,7 @@
As several variants of [TrainMatchers](https://git.xfel.eu/karaboDevices/TrainMatcher) are used in `calng` pipelines, it is useful to understand the basics of how a TrainMatcher works.
As the name implies, a TrainMatcher matches data based on train IDs.
It can monitor properties of other devices (also known as *slow sources* or *control sources*) and connect to a number of output channels (also known as *fast sources*).
-Generally, it will aim to forward data from all *selected* sources grouped by train IDs in one of two operation [modes](schemas/ShmemTrainMatcher.md#mode):
+Generally, it will aim to forward data from all *selected* sources grouped by train IDs in one of two operation [modes](../schemas/TrainMatcher.md#mode):
match
: This is the mode typically used in `calng` contexts.
@@ -11,7 +11,7 @@ In this mode, data for a train is sent once data from all selected sources has a
If data from only a subset of sources arrived for train `x`, but a full match for a later `y` has arrived, then the match is sent for `y` whereas `x` is discarded.
In other terms: a match is sent as soon as it is complete, but failure to match means nothing is sent.
-: The [**matching ratio**](schemas/ShmemTrainMatcher.md#matchingRatio) is the rate of matched (and sent) trains divided by the rate of incoming unique train IDs (across all sources).
+: The [**matching ratio**](../schemas/TrainMatcher.md#matchingRatio) is the rate of matched (and sent) trains divided by the rate of incoming unique train IDs (across all sources).
buffer
: In this mode, a configurable number, `n`, of trains is cached.
@@ -21,20 +21,20 @@ This means partially matched trains can be sent, and naturally implies a delay o
Data forwarded by a TrainMatcher is sent per source.
On the Karabo channel output, this means that data from each matched source is written before `update` is called on the channel.
On the Karabo bridge output, this means that the receiver will, get a dictionary with one entry per matched source.
-See also [output formats](output-formats.md), in particular if you need data [stacked](output-formats.md#stacking).
+See also [output formats](../output-formats.md), in particular if you need data [stacked](../output-formats.md#stacking).
## Configuring sources
-The TrainMatcher device comes with a useful description of the [`sources`](schemas/ShmemTrainMatcher.md#sources) parameter.
-This description is available within Karabo, on the overview scene (also in the screenshot below), or [here](schemas/ShmemTrainMatcher.md#sources).
-The following screenshot shows an example of the source configuration for a [group matcher](devices.md#group-matcher)
+The TrainMatcher device comes with a useful description of the [`sources`](../schemas/TrainMatcher.md#sources) parameter.
+This description is available within Karabo, on the overview scene (also in the screenshot below), or [here](../schemas/TrainMatcher.md#sources).
+The following screenshot shows an example of the source configuration for a [group matcher](matchers.md#group-matcher)
-
+
For each output channel source, the TrainMatcher needs to know the source name to watch and the device name plus channel name to connect to.
To explicitly specify both, the format is `[source name]@[device name]:[channel name]`.
In most cases, the device name plus channel name *is* the source name.
Correction devices - as the configuration seen in the screenshot above indicates - are an exception as they forward the source name of the source they are correcting.
-Another exception is other TrainMatchers as they also forward sources; relevant if setting up a [full matcher](devices.md#full-matcher).
+Another exception is other TrainMatchers as they also forward sources; relevant if setting up a [full matcher](matchers.md#full-matcher).
Note that device property sources (also known as *control sources*) are handled differently from channel sources.
The value associated with a property is updated when the property being watched is changed.
@@ -43,20 +43,20 @@ Channel sources which haven't sent data for a given train will simply not appear
## Handling missing sources
In `match` mode, missing data from any *selected* source causes a train to fail to match.
-This (for different causes of missing data from any source) is a common cause of [missing preview](troubleshooting.md#preview-does-not-update) or [missing data on bridge](troubleshooting.md#analysis-software-not-receiving-data).
+This (for different causes of missing data from any source) is a common cause of [missing preview](../troubleshooting.md#preview-does-not-update) or [missing data on bridge](../troubleshooting.md#analysis-software-not-receiving-data).
- If a source is known to not send data for a while (ex. defective detector module), consider unselecting it in the source configuration.
- If it's likely for some source(s) to suddenly go missing, the max idle parameter can be used to gracefully handle certain kinds of failures from individual sources.
- The [manager](devices.md#calibration-manager) by default sets a non-zero max idle value for the [preview assemblers](devices.md#preview-assemblers) it instantiates.
+ The [manager](middlelayer.md#calibration-manager) by default sets a non-zero max idle value for the [preview assemblers](matchers.md#preview-assemblers) it instantiates.
- If some latency is acceptable, consider using `buffer` mode instead of `match` mode.
### Max idle
-[This parameter](schemas/ShmemTrainMatcher.md#maxIdle), if set to a value greater than zero, allows the TrainMatcher to ignore any source from which it has not received data in a configurable number of seconds.
+[This parameter](../schemas/TrainMatcher.md#maxIdle), if set to a value greater than zero, allows the TrainMatcher to ignore any source from which it has not received data in a configurable number of seconds.
Any sources not heard from within the last `maxIdle` seconds will not be waited for in matching.
This is useful in case a source may suddenly disappear - for example, in case of an unreliable detector module.
Note that if data sporadically fails to arrive for a few trains at a time, max idle may not take effect quickly enough to help.
-In particular, hitting [network limits](data-rates.md) can cause this to happen for many detector module sources at once, rendering matching infeasible, regardless of the max idle function.
+In particular, hitting [network limits](../data-rates.md) can cause this to happen for many detector module sources at once, rendering matching infeasible, regardless of the max idle function.
## Karabo bridge output
Recent versions of TrainMatcher include a "built-in" [Karabo bridge output](https://rtd.xfel.eu/docs/data-analysis-user-documentation/en/latest/karabo_bridge/protocol.html).
diff --git a/docs/extensions.md b/docs/extensions.md
index 4227348a..a4e71883 100644
--- a/docs/extensions.md
+++ b/docs/extensions.md
@@ -1,5 +1,5 @@
# Correction addons and frame selection
-This page describes two different places in calng pipelines where user code can be executed to improve online analysis applications: correction addons, which run within each [correction device](devices.md#correction-devices) (and therefore per module), and frame selection kernels, which run in an arbiter, telling [group matchers](devices.md#group-matcher) which frames they should forward.
+This page describes two different places in calng pipelines where user code can be executed to improve online analysis applications: correction addons, which run within each [correction device](devices/correction-devices.md#correction-devices) (and therefore per module), and frame selection kernels, which run in an arbiter, telling [group matchers](devices/matchers.md#group-matcher) which frames they should forward.
The purpose of correction addons may be to perform additional computation on the corrected detector data to provide useful information either to downstream online analysis or for use with a frame selection arbiter.
The purpose of the arbiter is to select on a train-by-train basis which frames should be sent out of the pipeline, thus allowing flexible data reduction for online analysis.
@@ -14,7 +14,7 @@ The rest of the content of this page will be more technical than other pages in
From an operator point of view, these extensions mostly introduce additional configuration and thus complexity to running pipelines.
The correction addons each register their configuration subnodes in the correction device schema under `addons`.
-Configuring these will typically be done via the [manager](devices.md#calibration-manager).
+Configuring these will typically be done via the [manager](devices/middlelayer.md#calibration-manager).
Only addons which are marked as enabled are actually executed.
The frame selection feature involves an arbiter - essentially yet another train matcher - which runs user code and interation with each of the group matchers.
@@ -48,7 +48,7 @@ When writing a subclass, you will want to override a subset of these stubs:
- `processed_data` is the image data right after it has been corrected.
If running a GPU kernel, this data is still on GPU which you may want to exploit.
- `cell_table` and `pulse_table` are the mappings from frame to cells and pulses provided by the detector (if available).
- These are discussed in [preview configuration](devices.md#assembled-preview-configuration) and may be relevant for some kernel types.
+ These are discussed in [preview configuration](concepts.md#preview) and may be relevant for some kernel types.
- `output_hash` is the hash which will eventually be written to the main output channel.
The addon can do whatever it wants with this hash.
Typically, you will want to add some new computed keys to it (which you have hopefully mentioned in `extend_output_schema`).
diff --git a/docs/index.md b/docs/index.md
index c4216f9a..c9c95c4b 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -5,7 +5,7 @@ This is a work in progress.
If you are having issues with your pipeline right now, check out the [troubleshooting](troubleshooting.md) page.
To learn more about `calng`, peruse the other pages listed in the table of contents.
-The page on [devices](devices.md) will give an overview of devices involved in `calng` pipelines and [data rates](data-rates.md) and [output formats](output-formats.md) are useful to consider when planning an online analysis setup for an experiment.
+The page on [devices](devices/overview.md) will give an overview of devices involved in `calng` pipelines and [data rates](data-rates.md) and [output formats](output-formats.md) are useful to consider when planning an online analysis setup for an experiment.
Please do reach out to your data analysis contact and / or the calibration team well in advance of experiments which require novel data flows and processing.
The correction addons and frame selection features are described in the [extensions](extensions.md) page.
diff --git a/docs/integrations.md b/docs/integrations.md
index 1ce6d6fc..6644f3d7 100644
--- a/docs/integrations.md
+++ b/docs/integrations.md
@@ -1,18 +1,13 @@
+See also: [Karabo Bridge](https://rtd.xfel.eu/docs/data-analysis-user-documentation/en/latest/software/karabo-bridge/) documentation.
+
## EXtra-foam
[EXtra-foam documentation](https://extra-foam.readthedocs.io/en/latest/index.html)
-When connecting to bridge output of [group matchers](devices.md#group-matcher) or [full matcher](devices.md#full-matcher), the source type must be set to "run directory".
-This is due to the way a [TrainMatcher](trainmatcher.md#overview) forwards sources individually (similar to how data is stored in runs later).
+When connecting to bridge output of [group matchers](devices/matchers.md#group-matcher) or [full matcher](devices/matchers.md#full-matcher), the source type must be set to "run directory".
+This is due to the way a [TrainMatcher](devices/trainmatcher.md#overview) forwards sources individually (similar to how data is stored in runs later).
## OnDA
-It is possible to send full detector data (with appropriate [throttling](data-rates.md)) via a [full matcher](devices.md#full-matcher) to OnDA.
+It is possible to send full detector data (with appropriate [throttling](data-rates.md)) via a [full matcher](devices/matchers.md#full-matcher) to OnDA.
As OnDA expects stacked data, one needs to set up [stacking](output-formats.md#stacking) on the full matcher.
-
-
-## ROI tool
-
-The ROI tool is a small device developed to help keep track of pixel intensities.
-It is not currently installed as a part of `calng`, but is intended to be used with `calng` pipelines.
-Currently deployed for testing at SPB; see `SPB_DET_AGIPD1M-1/CALNG/ROI_HISTOGRAM_CORR` and `SPB_IRDA_JF4M/CALNG/ROI_TOOL_CORR`.
diff --git a/docs/output-formats.md b/docs/output-formats.md
index a4a5e090..d4d564de 100644
--- a/docs/output-formats.md
+++ b/docs/output-formats.md
@@ -9,7 +9,7 @@ Keep in mind that optional [stacking](#stacking) happens downstream on a (group
The module axis added by stacking is determined by the `stackingAxis` parameter in the stacking configuration.
## Stacking
-By default, a [TrainMatcher](trainmatcher.md) sends matched data per source.
+By default, a [TrainMatcher](devices/trainmatcher.md) sends matched data per source.
Some software may need data to be **stacked** (or interleaved): for instance, image data for multiple modules arranged within a single contiguous array.
To this end, the `ShmemTrainMatcher` extends the regular `TrainMatcher` by allowing for stacking of arbitrary keys from arbitrary sources.
Stacking can either work across sources - stacking arrays with the same name from multiple sources - or keys - stacking arrays multiple keys from one source.
@@ -104,25 +104,25 @@ With this setup, the new source `INSTRUMENT/DET/25UM-MODULE` will contain `data.
## One group, all frames
-This is the output one can get from [group matchers](devices.md#group-matcher).
+This is the output one can get from [group matchers](devices/matchers.md#group-matcher).
It is by default per source, though image data can be [stacked](#stacking).
If only a subset of modules are needed for online analysis, this is the recommended data stream to use.
-Refer to [group configuration](devices.md#group-configuration) for notes on how to rearrange modules between groups.
+Refer to [group configuration](devices/middlelayer.md#group-configuration) for notes on how to rearrange modules between groups.
## All modules, all frames
-If data from the full detector is needed for analysis, one can set up a [full matcher](devices.md#full-matcher).
+If data from the full detector is needed for analysis, one can set up a [full matcher](devices/matchers.md#full-matcher).
This pattern includes forwarding data from each of the group matchers - due to [network limits](data-rates.md), careful throttling will most likely be necessary.
Image data can optionally be [stacked](#stacking) (relevant when running [OnDA](integrations.md#onda), for instance).
## All modules, one frame, assembled
-This is the output from [preview assemblers](devices.md#preview-assemblers).
+This is the output from [preview assemblers](devices/matchers.md#preview-assemblers).
They provide fully assembled preview single-frame images for the full detector.
Although they only send up two 2 Hz of preview to the GUI, they typically received and process at 10 Hz and on a separate output provide the 10 Hz stream.
-Refer to [preview assembler](devices.md#preview-assemblers) for details on configuration and outputs.
+Refer to [preview assembler](devices/matchers.md#preview-assemblers) for details on configuration and outputs.
Keep in mind that all-trains, all-frames, all-modules data is generally not possible to gather on one node for fast detectors, assembled or not.
Hence the trade-off for the preview data flow is to reduce data to a single frame per train.
diff --git a/docs/schemas/Agipd1MGeometry.md b/docs/schemas/Agipd1MGeometry.md
index 72afd87c..2ed5f9bf 100644
--- a/docs/schemas/Agipd1MGeometry.md
+++ b/docs/schemas/Agipd1MGeometry.md
@@ -1,5 +1,18 @@
# Agipd1MGeometry
Base classes: ManualQuadrantsGeometryBase
+## <a name='availableScenes'></a>Available scenes (`availableScenes`)
+Type
+: VECTOR_STRING
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``['overview']``
+
## Geometry preview (`geometryPreview`)
ImageData
## Geometry file (`geometryFile`)
@@ -93,6 +106,22 @@ Description
Default value
: ``True``
+### <a name='geometryFile.updateMotorReferenceOnLoad'></a>Update motor reference (`geometryFile.updateMotorReferenceOnLoad`)
+Type
+: BOOL
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Description
+: If this flag is on, the motor reference positions in the motor tracking node will be updated according to the motor positions stored in geometry file.
+
+Default value
+: ``True``
+
### <a name='geometryFile.loadFromFile'></a>Load from file (`geometryFile.loadFromFile`)
Type
: Slot
@@ -223,7 +252,7 @@ Type
: Slot
Allowed in states
-: ACTIVE
+: ACTIVE, MONITORING
Access mode
: RECONFIGURABLE
@@ -239,7 +268,7 @@ Type
: Slot
Allowed in states
-: ACTIVE
+: ACTIVE, MONITORING
Access mode
: RECONFIGURABLE
@@ -441,3 +470,696 @@ Access mode
Assignment
: OPTIONAL
+
+## `motorTracking`
+### <a name='motorTracking.state'></a>`motorTracking.state`
+Type
+: STRING
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Options:
+: ``UNKNOWN``, ``KNOWN``, ``INIT``, ``ERROR``, ``INTERLOCKED``, ``NORMAL``, ``STATIC``, ``INTERLOCK_OK``, ``CHANGING``, ``DECREASING``, ``COOLING``, ``MOVING_LEFT``, ``MOVING_DOWN``, ``MOVING_BACK``, ``ROTATING_CNTCLK``, ``RAMPING_DOWN``, ``EXTRACTING``, ``STOPPING``, ``EMPTYING``, ``DISENGAGING``, ``SWITCHING_OFF``, ``HOMING``, ``ROTATING``, ``MOVING``, ``SWITCHING``, ``OPENING``, ``CLOSING``, ``SEARCHING``, ``INCREASING``, ``HEATING``, ``MOVING_RIGHT``, ``MOVING_UP``, ``MOVING_FORWARD``, ``ROTATING_CLK``, ``RAMPING_UP``, ``INSERTING``, ``STARTING``, ``FILLING``, ``ENGAGING``, ``SWITCHING_ON``, ``PAUSED``, ``RUNNING``, ``ACQUIRING``, ``PROCESSING``, ``PASSIVE``, ``WARM``, ``COLD``, ``PRESSURIZED``, ``CLOSED``, ``OFF``, ``INSERTED``, ``STOPPED``, ``UNLOCKED``, ``DISENGAGED``, ``IGNORING``, ``ACTIVE``, ``COOLED``, ``HEATED``, ``EVACUATED``, ``OPENED``, ``ON``, ``EXTRACTED``, ``STARTED``, ``LOCKED``, ``ENGAGED``, ``MONITORING``, ``DISABLED``, ``INTERLOCK_BROKEN``
+
+Default value
+: ``OFF``
+
+### <a name='motorTracking.runOnStartup'></a>`motorTracking.runOnStartup`
+Type
+: BOOL
+
+Access mode
+: INITONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``False``
+
+### <a name='motorTracking.motorPattern'></a>`motorTracking.motorPattern`
+Type
+: STRING
+
+Access mode
+: INITONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: empty string
+
+### `motorTracking.motorsConfig`
+#### `motorTracking.motorsConfig.Q1`
+##### `motorTracking.motorsConfig.Q1.M1`
+###### <a name='motorTracking.motorsConfig.Q1.M1.axisVector'></a>`motorTracking.motorsConfig.Q1.M1.axisVector`
+Type
+: VECTOR_DOUBLE
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``[0.0, -1.0]``
+
+###### <a name='motorTracking.motorsConfig.Q1.M1.deviceId'></a>`motorTracking.motorsConfig.Q1.M1.deviceId`
+Type
+: STRING
+
+Access mode
+: INITONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: empty string
+
+###### <a name='motorTracking.motorsConfig.Q1.M1.referencePos'></a>`motorTracking.motorsConfig.Q1.M1.referencePos`
+Type
+: DOUBLE
+
+Allowed in states
+: ACTIVE
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+###### <a name='motorTracking.motorsConfig.Q1.M1.currentPos'></a>`motorTracking.motorsConfig.Q1.M1.currentPos`
+Type
+: DOUBLE
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+##### <a name='motorTracking.motorsConfig.Q1.M1'></a>`motorTracking.motorsConfig.Q1.M1`
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+##### `motorTracking.motorsConfig.Q1.M2`
+###### <a name='motorTracking.motorsConfig.Q1.M2.axisVector'></a>`motorTracking.motorsConfig.Q1.M2.axisVector`
+Type
+: VECTOR_DOUBLE
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``[-1.0, 0.0]``
+
+###### <a name='motorTracking.motorsConfig.Q1.M2.deviceId'></a>`motorTracking.motorsConfig.Q1.M2.deviceId`
+Type
+: STRING
+
+Access mode
+: INITONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: empty string
+
+###### <a name='motorTracking.motorsConfig.Q1.M2.referencePos'></a>`motorTracking.motorsConfig.Q1.M2.referencePos`
+Type
+: DOUBLE
+
+Allowed in states
+: ACTIVE
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+###### <a name='motorTracking.motorsConfig.Q1.M2.currentPos'></a>`motorTracking.motorsConfig.Q1.M2.currentPos`
+Type
+: DOUBLE
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+##### <a name='motorTracking.motorsConfig.Q1.M2'></a>`motorTracking.motorsConfig.Q1.M2`
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+#### <a name='motorTracking.motorsConfig.Q1'></a>`motorTracking.motorsConfig.Q1`
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+#### `motorTracking.motorsConfig.Q2`
+##### `motorTracking.motorsConfig.Q2.M1`
+###### <a name='motorTracking.motorsConfig.Q2.M1.axisVector'></a>`motorTracking.motorsConfig.Q2.M1.axisVector`
+Type
+: VECTOR_DOUBLE
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``[0.0, -1.0]``
+
+###### <a name='motorTracking.motorsConfig.Q2.M1.deviceId'></a>`motorTracking.motorsConfig.Q2.M1.deviceId`
+Type
+: STRING
+
+Access mode
+: INITONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: empty string
+
+###### <a name='motorTracking.motorsConfig.Q2.M1.referencePos'></a>`motorTracking.motorsConfig.Q2.M1.referencePos`
+Type
+: DOUBLE
+
+Allowed in states
+: ACTIVE
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+###### <a name='motorTracking.motorsConfig.Q2.M1.currentPos'></a>`motorTracking.motorsConfig.Q2.M1.currentPos`
+Type
+: DOUBLE
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+##### <a name='motorTracking.motorsConfig.Q2.M1'></a>`motorTracking.motorsConfig.Q2.M1`
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+##### `motorTracking.motorsConfig.Q2.M2`
+###### <a name='motorTracking.motorsConfig.Q2.M2.axisVector'></a>`motorTracking.motorsConfig.Q2.M2.axisVector`
+Type
+: VECTOR_DOUBLE
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``[1.0, 0.0]``
+
+###### <a name='motorTracking.motorsConfig.Q2.M2.deviceId'></a>`motorTracking.motorsConfig.Q2.M2.deviceId`
+Type
+: STRING
+
+Access mode
+: INITONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: empty string
+
+###### <a name='motorTracking.motorsConfig.Q2.M2.referencePos'></a>`motorTracking.motorsConfig.Q2.M2.referencePos`
+Type
+: DOUBLE
+
+Allowed in states
+: ACTIVE
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+###### <a name='motorTracking.motorsConfig.Q2.M2.currentPos'></a>`motorTracking.motorsConfig.Q2.M2.currentPos`
+Type
+: DOUBLE
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+##### <a name='motorTracking.motorsConfig.Q2.M2'></a>`motorTracking.motorsConfig.Q2.M2`
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+#### <a name='motorTracking.motorsConfig.Q2'></a>`motorTracking.motorsConfig.Q2`
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+#### `motorTracking.motorsConfig.Q3`
+##### `motorTracking.motorsConfig.Q3.M1`
+###### <a name='motorTracking.motorsConfig.Q3.M1.axisVector'></a>`motorTracking.motorsConfig.Q3.M1.axisVector`
+Type
+: VECTOR_DOUBLE
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``[0.0, 1.0]``
+
+###### <a name='motorTracking.motorsConfig.Q3.M1.deviceId'></a>`motorTracking.motorsConfig.Q3.M1.deviceId`
+Type
+: STRING
+
+Access mode
+: INITONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: empty string
+
+###### <a name='motorTracking.motorsConfig.Q3.M1.referencePos'></a>`motorTracking.motorsConfig.Q3.M1.referencePos`
+Type
+: DOUBLE
+
+Allowed in states
+: ACTIVE
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+###### <a name='motorTracking.motorsConfig.Q3.M1.currentPos'></a>`motorTracking.motorsConfig.Q3.M1.currentPos`
+Type
+: DOUBLE
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+##### <a name='motorTracking.motorsConfig.Q3.M1'></a>`motorTracking.motorsConfig.Q3.M1`
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+##### `motorTracking.motorsConfig.Q3.M2`
+###### <a name='motorTracking.motorsConfig.Q3.M2.axisVector'></a>`motorTracking.motorsConfig.Q3.M2.axisVector`
+Type
+: VECTOR_DOUBLE
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``[1.0, 0.0]``
+
+###### <a name='motorTracking.motorsConfig.Q3.M2.deviceId'></a>`motorTracking.motorsConfig.Q3.M2.deviceId`
+Type
+: STRING
+
+Access mode
+: INITONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: empty string
+
+###### <a name='motorTracking.motorsConfig.Q3.M2.referencePos'></a>`motorTracking.motorsConfig.Q3.M2.referencePos`
+Type
+: DOUBLE
+
+Allowed in states
+: ACTIVE
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+###### <a name='motorTracking.motorsConfig.Q3.M2.currentPos'></a>`motorTracking.motorsConfig.Q3.M2.currentPos`
+Type
+: DOUBLE
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+##### <a name='motorTracking.motorsConfig.Q3.M2'></a>`motorTracking.motorsConfig.Q3.M2`
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+#### <a name='motorTracking.motorsConfig.Q3'></a>`motorTracking.motorsConfig.Q3`
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+#### `motorTracking.motorsConfig.Q4`
+##### `motorTracking.motorsConfig.Q4.M1`
+###### <a name='motorTracking.motorsConfig.Q4.M1.axisVector'></a>`motorTracking.motorsConfig.Q4.M1.axisVector`
+Type
+: VECTOR_DOUBLE
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``[0.0, 1.0]``
+
+###### <a name='motorTracking.motorsConfig.Q4.M1.deviceId'></a>`motorTracking.motorsConfig.Q4.M1.deviceId`
+Type
+: STRING
+
+Access mode
+: INITONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: empty string
+
+###### <a name='motorTracking.motorsConfig.Q4.M1.referencePos'></a>`motorTracking.motorsConfig.Q4.M1.referencePos`
+Type
+: DOUBLE
+
+Allowed in states
+: ACTIVE
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+###### <a name='motorTracking.motorsConfig.Q4.M1.currentPos'></a>`motorTracking.motorsConfig.Q4.M1.currentPos`
+Type
+: DOUBLE
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+##### <a name='motorTracking.motorsConfig.Q4.M1'></a>`motorTracking.motorsConfig.Q4.M1`
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+##### `motorTracking.motorsConfig.Q4.M2`
+###### <a name='motorTracking.motorsConfig.Q4.M2.axisVector'></a>`motorTracking.motorsConfig.Q4.M2.axisVector`
+Type
+: VECTOR_DOUBLE
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``[-1.0, 0.0]``
+
+###### <a name='motorTracking.motorsConfig.Q4.M2.deviceId'></a>`motorTracking.motorsConfig.Q4.M2.deviceId`
+Type
+: STRING
+
+Access mode
+: INITONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: empty string
+
+###### <a name='motorTracking.motorsConfig.Q4.M2.referencePos'></a>`motorTracking.motorsConfig.Q4.M2.referencePos`
+Type
+: DOUBLE
+
+Allowed in states
+: ACTIVE
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+###### <a name='motorTracking.motorsConfig.Q4.M2.currentPos'></a>`motorTracking.motorsConfig.Q4.M2.currentPos`
+Type
+: DOUBLE
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+##### <a name='motorTracking.motorsConfig.Q4.M2'></a>`motorTracking.motorsConfig.Q4.M2`
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+#### <a name='motorTracking.motorsConfig.Q4'></a>`motorTracking.motorsConfig.Q4`
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+### <a name='motorTracking.motorsConfig'></a>`motorTracking.motorsConfig`
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+### <a name='motorTracking.positionKey'></a>`motorTracking.positionKey`
+Type
+: STRING
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``actualPosition``
+
+### <a name='motorTracking.tolerance'></a>`motorTracking.tolerance`
+Type
+: DOUBLE
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.005``
+
+### `motorTracking.referencePositions`
+#### <a name='motorTracking.referencePositions.reset'></a>Reset (`motorTracking.referencePositions.reset`)
+Type
+: Slot
+
+Allowed in states
+: ACTIVE
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+#### <a name='motorTracking.referencePositions.fromGeometry'></a>From geometry (`motorTracking.referencePositions.fromGeometry`)
+Type
+: Slot
+
+Allowed in states
+: ACTIVE
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+#### <a name='motorTracking.referencePositions.useCurrentPos'></a>Use current positions (`motorTracking.referencePositions.useCurrentPos`)
+Type
+: Slot
+
+Allowed in states
+: ACTIVE
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+### <a name='motorTracking.referencePositions'></a>`motorTracking.referencePositions`
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+### <a name='motorTracking.update'></a>Update once (`motorTracking.update`)
+Type
+: Slot
+
+Allowed in states
+: ACTIVE
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+### <a name='motorTracking.toggle'></a>Toggle tracking (`motorTracking.toggle`)
+Type
+: Slot
+
+Allowed in states
+: ACTIVE, MONITORING
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+## <a name='motorTracking'></a>`motorTracking`
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
diff --git a/docs/schemas/AgipdCorrection.md b/docs/schemas/AgipdCorrection.md
index ea298da8..948335f6 100644
--- a/docs/schemas/AgipdCorrection.md
+++ b/docs/schemas/AgipdCorrection.md
@@ -1,357 +1,5 @@
# AgipdCorrection
-Base classes: BaseCorrection
-## `dataInput`
-InputChannel
-## <a name='fastSources'></a>Fast data sources (`fastSources`)
-Type
-: VECTOR_STRING
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Sources to get data from. Only incoming hashes from these sources will be processed. This will typically be a single entry of the form: '[instrument]_DET_[detector]/DET/[channel]:xtdf'.
-
-Default value
-: ``[]``
-
-## <a name='outputShmemBufferSize'></a>Output buffer size limit (`outputShmemBufferSize`)
-Type
-: UINT32
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Corrected trains are written to shared memory locations. These are pre-allocated and re-used (circular buffer). This parameter determines how much memory to set aside for that buffer.
-
-Default value
-: ``10``
-
-## <a name='useShmemHandles'></a>Use shared memory (`useShmemHandles`)
-Type
-: BOOL
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: If enabled, shared memory handles will be used to avoid copying the main image data on the dataOutput channel. Note that these handles need to be dereferenced (typically by a group matcher) to get the data off the correction node.
-
-Default value
-: ``True``
-
-## <a name='useInfiniband'></a>Use infiniband (`useInfiniband`)
-Type
-: BOOL
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: If enabled, device will during initialization try to bind its main data output channel (dataOutput) to its node's infiniband interface. Default interface is used if no 'ib0' interface is found.
-
-Default value
-: ``True``
-
-## <a name='geometryDevice'></a>Geometry device (`geometryDevice`)
-Type
-: STRING
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: empty string
-
-## Data format (in/out) (`dataFormat`)
-### <a name='dataFormat.outputImageDtype'></a>Output image data dtype (`dataFormat.outputImageDtype`)
-Type
-: STRING
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: The (numpy) dtype to use for outgoing image data. Input is cast to float32, corrections are applied, and only then will the result be cast to outputImageDtype. Be aware that casting to integer type causes truncation rather than rounding.
-
-Options:
-: ``float32``, ``float16``, ``uint16``
-
-Default value
-: ``float32``
-
-### <a name='dataFormat.inputFrames'></a>Frames (`dataFormat.inputFrames`)
-Type
-: UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Number of frames on input
-
-Default value
-: ``0``
-
-### <a name='dataFormat.outputAxisOrder'></a>Output axis order (`dataFormat.outputAxisOrder`)
-Type
-: STRING
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: Axes of main data output can be reordered after correction. Axis order is specified as string consisting of 'f' (frames), 'ss' (slow scan), and 'ff' (fast scan). Anything but the default f-ss-fs order implies reordering - this axis ordering is based on the data we get from the detector (/ receiver), so how ss and fs maps to what may be considered x and y is detector-dependent.
-
-Options:
-: ``f-ss-fs``, ``f-fs-ss``, ``ss-f-fs``, ``ss-fs-f``, ``fs-f-ss``, ``fs-ss-f``
-
-Default value
-: ``f-ss-fs``
-
-### <a name='dataFormat.inputDataShape'></a>Input data shape (`dataFormat.inputDataShape`)
-Type
-: VECTOR_UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Image data shape in incoming data (from reader / DAQ). Updated based on latest train processed. Note that axis order may look wonky due to DAQ quirk.
-
-Default value
-: ``[]``
-
-### <a name='dataFormat.outputDataShape'></a>Output data shape (`dataFormat.outputDataShape`)
-Type
-: VECTOR_UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Image data shape for data output from this device. Takes into account axis reordering, if applicable. Even without that, workarounds.overrideInputAxisOrder likely makes this differ from dataFormat.inputDataShape.
-
-Default value
-: ``[]``
-
-### <a name='dataFormat.cellId'></a>`dataFormat.cellId`
-Type
-: VECTOR_UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``[]``
-
-### <a name='dataFormat.pulseId'></a>`dataFormat.pulseId`
-Type
-: VECTOR_UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``[]``
-
-## Workarounds (`workarounds`)
-### <a name='workarounds.overrideInputAxisOrder'></a>Override input axis order (`workarounds.overrideInputAxisOrder`)
-Type
-: BOOL
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: The shape of the image data ndarray as received from the DataAggregator is sometimes wrong - the axes are actually in a different order than the ndarray shape suggests. If this flag is on, the shape of the ndarray will be overridden with the axis order which was expected.
-
-Default value
-: ``True``
-
-### <a name='workarounds.trainFromFutureThreshold'></a>Spurious future train ID threshold (`workarounds.trainFromFutureThreshold`)
-Type
-: UINT64
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: Some detectors occasionally send a train with incorrect and much too large train ID. To avoid these 'future trains' from interfering with train matching, use this threshold to discard them immediately. If a train arrives with an ID which exceeds the current train ID from the time server by more than this threshold, the train is ignored.
-
-Default value
-: ``10000``
-
-## <a name='loadMostRecentConstants'></a>Load most recent constants (`loadMostRecentConstants`)
-Type
-: Slot
-
-Access mode
-: RECONFIGURABLE
-
-Description
-: Calling this slot will flush all constant buffers and cause the device to start querying CalCat for the most recent constants (all constants applicable for this device) available with the currently set constant parameters. This should typically be called after instantiating pipeline, after changing parameters, or after generating new constants.
-
-## <a name='inputDataState'></a>`inputDataState`
-Type
-: STRING
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``NORMAL``
-
-## <a name='deviceInternalsState'></a>`deviceInternalsState`
-Type
-: STRING
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``NORMAL``
-
-## <a name='processingState'></a>`processingState`
-Type
-: STRING
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``NORMAL``
-
-## <a name='trainId'></a>Train ID (`trainId`)
-Type
-: UINT64
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: ID of latest train processed by this device.
-
-Default value
-: ``0``
-
-## Performance measures (`performance`)
-### <a name='performance.processingTime'></a>Processing time (`performance.processingTime`)
-Type
-: DOUBLE
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``0.0``
-
-### <a name='performance.rate'></a>Rate (`performance.rate`)
-Type
-: DOUBLE
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Actual rate with which this device gets, processes, and sends trains. This is a simple windowed moving average.
-
-Default value
-: ``0.0``
-
-### <a name='performance.inputDelay'></a>Input delay (`performance.inputDelay`)
-Type
-: DOUBLE
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Measured delay on the input channel. Measured by difference between 'now' and timestamp in input metadata.
-
-Default value
-: ``0.0``
-
-### <a name='performance.ratioOfRecentTrainsReceived'></a>`performance.ratioOfRecentTrainsReceived`
-Type
-: DOUBLE
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Of the latest trains (from last received train, going back [some buffer range]), how many did we receive? This estimate is updated when new trains come in, so is unreliable if nothing is coming at all.
-
-Default value
-: ``0.0``
-
+Base classes: [BaseCorrection](BaseCorrection.md)
## `corrections`
### `corrections.thresholding`
#### <a name='corrections.thresholding.available'></a>Available (`corrections.thresholding.available`)
@@ -676,7 +324,7 @@ Assignment
: OPTIONAL
Description
-: If set, the additional offset applied to medium gain pixels (assuming adjustMgBaseLine is set) will use the value from mdAdditionalOffset globally for. Otherwise, the additional offset is derived from the relevant constants.
+: If set, the additional offset applied to medium gain pixels (assuming adjustMgBaseline is set) will use the value from mdAdditionalOffset globally for. Otherwise, the additional offset is derived from the relevant constants.
Default value
: ``False``
@@ -1216,19 +864,6 @@ Assignment
Default value
: ``35.0``
-## <a name='warningLamps'></a>`warningLamps`
-Type
-: VECTOR_STRING
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``['deviceInternalsState', 'inputDataState', 'processingState']``
-
## Addons (`addons`)
### `addons.IntegratedIntensity`
#### <a name='addons.IntegratedIntensity.enable'></a>`addons.IntegratedIntensity.enable`
@@ -1270,6 +905,33 @@ Assignment
Default value
: ``20000.0``
+### `addons.LitPixelCounter`
+#### <a name='addons.LitPixelCounter.enable'></a>`addons.LitPixelCounter.enable`
+Type
+: BOOL
+
+Access mode
+: INITONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``False``
+
+#### <a name='addons.LitPixelCounter.threshold'></a>`addons.LitPixelCounter.threshold`
+Type
+: DOUBLE
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``6.0``
+
### `addons.Peakfinder9`
#### <a name='addons.Peakfinder9.enable'></a>`addons.Peakfinder9.enable`
Type
@@ -1535,7 +1197,25 @@ Description
Default value
: ``0``
+#### `addons.SaturationMonitor.output`
+OutputChannel
## Constant retrieval parameters (`constantParameters`)
+### <a name='constantParameters.secretsFile'></a>CalCat secrets file (`constantParameters.secretsFile`)
+Type
+: STRING
+
+Access mode
+: INITONLY
+
+Assignment
+: OPTIONAL
+
+Description
+: Path to JSON file specifying secret token and some parameters used to access CalCat.
+
+Default value
+: ``~/.calcat-secrets.json``
+
### <a name='constantParameters.deviceMappingSnapshotAt'></a>Snapshot timestamp (for device mapping) (`constantParameters.deviceMappingSnapshotAt`)
Type
: STRING
diff --git a/docs/schemas/BaseCorrection.md b/docs/schemas/BaseCorrection.md
index 80fadd38..a3fa0607 100644
--- a/docs/schemas/BaseCorrection.md
+++ b/docs/schemas/BaseCorrection.md
@@ -34,6 +34,19 @@ Description
Default value
: ``10``
+## <a name='availableScenes'></a>`availableScenes`
+Type
+: VECTOR_STRING
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``['overview', 'constant_overrides']``
+
## <a name='useShmemHandles'></a>Use shared memory (`useShmemHandles`)
Type
: BOOL
diff --git a/docs/schemas/CalibrationManager.md b/docs/schemas/CalibrationManager.md
index d31f0e3e..5a90064a 100644
--- a/docs/schemas/CalibrationManager.md
+++ b/docs/schemas/CalibrationManager.md
@@ -16,6 +16,19 @@ Description
Default value
: ``['DeviceInstantiator']``
+## <a name='availableScenes'></a>Available scenes (`availableScenes`)
+Type
+: VECTOR_STRING
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``['overview', 'managed_keys', 'correction_performance_overview', 'correction_constant_overview']``
+
## <a name='detectorType'></a>Detector type (`detectorType`)
Type
: STRING
@@ -389,12 +402,25 @@ Table schema and default value
| ``False`` | ``MATCH_G\d`` | ``^(sources|zmqConfig|sortSources|mode`` |
+## <a name='doNotCompressEvents'></a>`doNotCompressEvents`
+Type
+: BOOL
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``False``
+
## <a name='restartServers'></a>Restart servers (`restartServers`)
Type
: Slot
Allowed in states
-: ACTIVE, UNKNOWN, ERROR
+: ERROR, ACTIVE, UNKNOWN
Access mode
: RECONFIGURABLE
diff --git a/docs/schemas/DetectorAssembler.md b/docs/schemas/DetectorAssembler.md
index d1957f1d..c9775e5d 100644
--- a/docs/schemas/DetectorAssembler.md
+++ b/docs/schemas/DetectorAssembler.md
@@ -1,447 +1,5 @@
# DetectorAssembler
-Base classes: TrainMatcher
-## <a name='zmqConfig'></a>ZeroMQ configuration (`zmqConfig`)
-Type
-: VECTOR_HASH
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Configuration for ZeroMQ sockets
-
-Table schema and default value
-:
-| Pattern | HWM | RCVTIMEO | SNDTIMEO | Protocol | Use Infiniband | ZMQ publisher port |
-| - | - | - | - | - | - | - |
-
-
-## <a name='zmqOutputs'></a>ZeroMQ outputs (`zmqOutputs`)
-Type
-: VECTOR_HASH
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: List of ZeroMQ outputs
-
-Table schema and default value
-:
-| Sent | Pattern | Address |
-| - | - | - |
-
-
-## <a name='pollTimeout'></a>ZeroMQ Poller Timeout (`pollTimeout`)
-Type
-: UINT32
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: ZMQ poller timeout in milliseconds.
-
-Default value
-: ``0``
-
-## <a name='onSlowness'></a>On Slowness (`onSlowness`)
-Type
-: STRING
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Policy for what to do if this input is too slow for the fed data rate
-
-Options:
-: ``drop``, ``wait``, ``queue``, ``queueDrop``
-
-Default value
-: ``queueDrop``
-
-## <a name='maxQueueLength'></a>Max. Queue Length (`maxQueueLength`)
-Type
-: UINT32
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Maximum number of data items to be queued by connected Output Channels (only queue and queueDrop policies)
-
-Default value
-: ``5``
-
-## <a name='sources'></a>Data Sources (`sources`)
-Type
-: VECTOR_HASH
-
-Allowed in states
-: PASSIVE, ACTIVE
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: list data sources to monitor. To add a data source: Right click on the table below and select "add row below" Device properties: Write in the "source" column the device name and the property name joined with a dot, e.g. "SA2_XTD1_XGM/XGM/DOOCS.pulseEnergy.photonFlux" Device output channels: Write in the "Source" column the name of the channel you want to monitor, e.g. HED_XTD6_IMGPI/CAM/BEAMVIEW:output. If the device name and the data source name do not match, for example if you monitor data directly from the output of the DAQ or the calibration pipeline, use the following syntax: SOURCE_NAME@DEVICE:output. For example SQS_DIGITIZER_UTC2/ADC/1:network@SQS_DAQ_DATA/DIGI/2:output
-
-Table schema and default value
-:
-| Select | Source | Offset (#trains) | Status |
-| - | - | - | - |
-
-
-## <a name='stats'></a>Sources Stats (`stats`)
-Type
-: VECTOR_HASH
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Statistics on input data
-
-Table schema and default value
-:
-| Source | Update Rate (Hz) | Received | Train ID | Latency (#trains) |
-| - | - | - | - | - |
-
-
-## <a name='sortSources'></a>Sort Sources (`sortSources`)
-Type
-: BOOL
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: Sort sources by name after reconfiguration.
-
-Default value
-: ``True``
-
-## <a name='mode'></a>Mode (`mode`)
-Type
-: STRING
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: operation mode, "match": match all selected fast sources, "buffer": send (selected) sources that arrived when the buffer reached the limit
-
-Options:
-: ``match``, ``buffer``
-
-Default value
-: ``match``
-
-## <a name='start'></a>Start (`start`)
-Type
-: Slot
-
-Allowed in states
-: PASSIVE
-
-Access mode
-: RECONFIGURABLE
-
-## <a name='stop'></a>Stop (`stop`)
-Type
-: Slot
-
-Allowed in states
-: ACTIVE
-
-Access mode
-: RECONFIGURABLE
-
-## <a name='processing'></a>Pipeline Update (`processing`)
-Type
-: FLOAT
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``0.0``
-
-## <a name='uniqueProcessing'></a>Unique Trains Update (`uniqueProcessing`)
-Type
-: FLOAT
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``0.0``
-
-## <a name='ctrlProcessing'></a>Control Update (`ctrlProcessing`)
-Type
-: FLOAT
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``0.0``
-
-## <a name='outProcessing'></a>Output Update (`outProcessing`)
-Type
-: FLOAT
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``0.0``
-
-## <a name='received'></a>Received (`received`)
-Type
-: UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Count how many trains were received.
-
-Default value
-: ``0``
-
-## <a name='sent'></a>Sent (`sent`)
-Type
-: UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Count how many train data were sent.
-
-Default value
-: ``0``
-
-## <a name='matchingRatio'></a>Matching Ratio (`matchingRatio`)
-Type
-: FLOAT
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Exponential moving average of the relation between sent and received trains.
-
-Default value
-: ``0.0``
-
-## <a name='trainId'></a>Train ID (`trainId`)
-Type
-: UINT64
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: the last trainId data that matched.
-
-Default value
-: ``0``
-
-## <a name='buffer'></a>Buffer size (`buffer`)
-Type
-: UINT32
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: Maximum length of buffer to hold non-matched trains
-
-Default value
-: ``100``
-
-## <a name='buffered'></a>Buffered trains (`buffered`)
-Type
-: UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Number of trains buffered
-
-Default value
-: ``0``
-
-## <a name='delay'></a>Delay on output (`delay`)
-Type
-: UINT32
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: delay applied before sending data on output channel (in #trains)
-
-Default value
-: ``0``
-
-## <a name='offsetTid'></a>Overwrite Offset TrainIds (`offsetTid`)
-Type
-: BOOL
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: If this property is True, it will change the value of the trainId in the metadata for the data sources having an offset value different from 0. The trainId value assigned will be the "matched" trainId, i.e. trainId + offset for the source.
-
-Default value
-: ``False``
-
-## <a name='maxIdle'></a>Max Idle (`maxIdle`)
-Type
-: FLOAT
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: Max allowed IDLE time (in seconds). If last data received for a source is longer than this time, the source will be ignore when matching data.
-
-Default value
-: ``0.0``
-
-## <a name='idleChannels'></a>Idle Channels (`idleChannels`)
-Type
-: VECTOR_STRING
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: List of idle channels
-
-Default value
-: ``[]``
-
-## <a name='idleState'></a>Idle State (`idleState`)
-Type
-: STRING
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: State of idle channels (error if any monitored channel is idle)
-
-Default value
-: ``ON``
-
-## <a name='cpuPercent'></a>CPU usage (`cpuPercent`)
-Type
-: FLOAT
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``0.0``
-
-## <a name='memUsage'></a>MEM usage (`memUsage`)
-Type
-: FLOAT
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``0.0``
-
-## Pipeline Output (`output`)
-Output channel for combined train data.
-OutputChannel
+Base classes: [TrainMatcher](TrainMatcher.md)
## <a name='timeOfFlight'></a>Time of flight (`timeOfFlight`)
Type
: DOUBLE
@@ -496,6 +54,25 @@ Default value
## `assembledOutput`
OutputChannel
+## <a name='outputForBridgeOutput'></a>Bridge output (`outputForBridgeOutput`)
+Type
+: STRING
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Description
+: The bridge can send the same data as any one of the three Karabo output channels. If this setting is 'matched', it will follow 'output', if 'assembled', it will follow 'assembledOutput' and if 'preview', it will follow 'preview.output'.
+
+Options:
+: ``matched``, ``assembled``
+
+Default value
+: ``matched``
+
## <a name='geometryDevice'></a>Geometry device (`geometryDevice`)
Type
: STRING
diff --git a/docs/schemas/Dssc1MGeometry.md b/docs/schemas/Dssc1MGeometry.md
index 7a4b16d0..7821a3ef 100644
--- a/docs/schemas/Dssc1MGeometry.md
+++ b/docs/schemas/Dssc1MGeometry.md
@@ -1,5 +1,18 @@
# Dssc1MGeometry
Base classes: ManualQuadrantsGeometryBase
+## <a name='availableScenes'></a>Available scenes (`availableScenes`)
+Type
+: VECTOR_STRING
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``['overview']``
+
## Geometry preview (`geometryPreview`)
ImageData
## Geometry file (`geometryFile`)
@@ -93,6 +106,22 @@ Description
Default value
: ``True``
+### <a name='geometryFile.updateMotorReferenceOnLoad'></a>Update motor reference (`geometryFile.updateMotorReferenceOnLoad`)
+Type
+: BOOL
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Description
+: If this flag is on, the motor reference positions in the motor tracking node will be updated according to the motor positions stored in geometry file.
+
+Default value
+: ``True``
+
### <a name='geometryFile.loadFromFile'></a>Load from file (`geometryFile.loadFromFile`)
Type
: Slot
@@ -223,7 +252,7 @@ Type
: Slot
Allowed in states
-: ACTIVE
+: ACTIVE, MONITORING
Access mode
: RECONFIGURABLE
@@ -239,7 +268,7 @@ Type
: Slot
Allowed in states
-: ACTIVE
+: ACTIVE, MONITORING
Access mode
: RECONFIGURABLE
diff --git a/docs/schemas/DsscCorrection.md b/docs/schemas/DsscCorrection.md
index 9e870aa9..a1a59ea2 100644
--- a/docs/schemas/DsscCorrection.md
+++ b/docs/schemas/DsscCorrection.md
@@ -1,357 +1,5 @@
# DsscCorrection
-Base classes: BaseCorrection
-## `dataInput`
-InputChannel
-## <a name='fastSources'></a>Fast data sources (`fastSources`)
-Type
-: VECTOR_STRING
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Sources to get data from. Only incoming hashes from these sources will be processed. This will typically be a single entry of the form: '[instrument]_DET_[detector]/DET/[channel]:xtdf'.
-
-Default value
-: ``[]``
-
-## <a name='outputShmemBufferSize'></a>Output buffer size limit (`outputShmemBufferSize`)
-Type
-: UINT32
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Corrected trains are written to shared memory locations. These are pre-allocated and re-used (circular buffer). This parameter determines how much memory to set aside for that buffer.
-
-Default value
-: ``10``
-
-## <a name='useShmemHandles'></a>Use shared memory (`useShmemHandles`)
-Type
-: BOOL
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: If enabled, shared memory handles will be used to avoid copying the main image data on the dataOutput channel. Note that these handles need to be dereferenced (typically by a group matcher) to get the data off the correction node.
-
-Default value
-: ``True``
-
-## <a name='useInfiniband'></a>Use infiniband (`useInfiniband`)
-Type
-: BOOL
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: If enabled, device will during initialization try to bind its main data output channel (dataOutput) to its node's infiniband interface. Default interface is used if no 'ib0' interface is found.
-
-Default value
-: ``True``
-
-## <a name='geometryDevice'></a>Geometry device (`geometryDevice`)
-Type
-: STRING
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: empty string
-
-## Data format (in/out) (`dataFormat`)
-### <a name='dataFormat.outputImageDtype'></a>Output image data dtype (`dataFormat.outputImageDtype`)
-Type
-: STRING
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: The (numpy) dtype to use for outgoing image data. Input is cast to float32, corrections are applied, and only then will the result be cast to outputImageDtype. Be aware that casting to integer type causes truncation rather than rounding.
-
-Options:
-: ``float32``, ``float16``, ``uint16``
-
-Default value
-: ``float32``
-
-### <a name='dataFormat.inputFrames'></a>Frames (`dataFormat.inputFrames`)
-Type
-: UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Number of frames on input
-
-Default value
-: ``0``
-
-### <a name='dataFormat.outputAxisOrder'></a>Output axis order (`dataFormat.outputAxisOrder`)
-Type
-: STRING
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: Axes of main data output can be reordered after correction. Axis order is specified as string consisting of 'f' (frames), 'ss' (slow scan), and 'ff' (fast scan). Anything but the default f-ss-fs order implies reordering - this axis ordering is based on the data we get from the detector (/ receiver), so how ss and fs maps to what may be considered x and y is detector-dependent.
-
-Options:
-: ``f-ss-fs``, ``f-fs-ss``, ``ss-f-fs``, ``ss-fs-f``, ``fs-f-ss``, ``fs-ss-f``
-
-Default value
-: ``f-ss-fs``
-
-### <a name='dataFormat.inputDataShape'></a>Input data shape (`dataFormat.inputDataShape`)
-Type
-: VECTOR_UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Image data shape in incoming data (from reader / DAQ). Updated based on latest train processed. Note that axis order may look wonky due to DAQ quirk.
-
-Default value
-: ``[]``
-
-### <a name='dataFormat.outputDataShape'></a>Output data shape (`dataFormat.outputDataShape`)
-Type
-: VECTOR_UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Image data shape for data output from this device. Takes into account axis reordering, if applicable. Even without that, workarounds.overrideInputAxisOrder likely makes this differ from dataFormat.inputDataShape.
-
-Default value
-: ``[]``
-
-### <a name='dataFormat.cellId'></a>`dataFormat.cellId`
-Type
-: VECTOR_UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``[]``
-
-### <a name='dataFormat.pulseId'></a>`dataFormat.pulseId`
-Type
-: VECTOR_UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``[]``
-
-## Workarounds (`workarounds`)
-### <a name='workarounds.overrideInputAxisOrder'></a>Override input axis order (`workarounds.overrideInputAxisOrder`)
-Type
-: BOOL
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: The shape of the image data ndarray as received from the DataAggregator is sometimes wrong - the axes are actually in a different order than the ndarray shape suggests. If this flag is on, the shape of the ndarray will be overridden with the axis order which was expected.
-
-Default value
-: ``True``
-
-### <a name='workarounds.trainFromFutureThreshold'></a>Spurious future train ID threshold (`workarounds.trainFromFutureThreshold`)
-Type
-: UINT64
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: Some detectors occasionally send a train with incorrect and much too large train ID. To avoid these 'future trains' from interfering with train matching, use this threshold to discard them immediately. If a train arrives with an ID which exceeds the current train ID from the time server by more than this threshold, the train is ignored.
-
-Default value
-: ``10000``
-
-## <a name='loadMostRecentConstants'></a>Load most recent constants (`loadMostRecentConstants`)
-Type
-: Slot
-
-Access mode
-: RECONFIGURABLE
-
-Description
-: Calling this slot will flush all constant buffers and cause the device to start querying CalCat for the most recent constants (all constants applicable for this device) available with the currently set constant parameters. This should typically be called after instantiating pipeline, after changing parameters, or after generating new constants.
-
-## <a name='inputDataState'></a>`inputDataState`
-Type
-: STRING
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``NORMAL``
-
-## <a name='deviceInternalsState'></a>`deviceInternalsState`
-Type
-: STRING
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``NORMAL``
-
-## <a name='processingState'></a>`processingState`
-Type
-: STRING
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``NORMAL``
-
-## <a name='trainId'></a>Train ID (`trainId`)
-Type
-: UINT64
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: ID of latest train processed by this device.
-
-Default value
-: ``0``
-
-## Performance measures (`performance`)
-### <a name='performance.processingTime'></a>Processing time (`performance.processingTime`)
-Type
-: DOUBLE
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``0.0``
-
-### <a name='performance.rate'></a>Rate (`performance.rate`)
-Type
-: DOUBLE
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Actual rate with which this device gets, processes, and sends trains. This is a simple windowed moving average.
-
-Default value
-: ``0.0``
-
-### <a name='performance.inputDelay'></a>Input delay (`performance.inputDelay`)
-Type
-: DOUBLE
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Measured delay on the input channel. Measured by difference between 'now' and timestamp in input metadata.
-
-Default value
-: ``0.0``
-
-### <a name='performance.ratioOfRecentTrainsReceived'></a>`performance.ratioOfRecentTrainsReceived`
-Type
-: DOUBLE
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Of the latest trains (from last received train, going back [some buffer range]), how many did we receive? This estimate is updated when new trains come in, so is unreliable if nothing is coming at all.
-
-Default value
-: ``0.0``
-
+Base classes: [BaseCorrection](BaseCorrection.md)
## `corrections`
### `corrections.offset`
#### <a name='corrections.offset.available'></a>Available (`corrections.offset.available`)
@@ -402,19 +50,6 @@ Description
Default value
: ``True``
-## <a name='warningLamps'></a>`warningLamps`
-Type
-: VECTOR_STRING
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``['deviceInternalsState', 'inputDataState', 'processingState']``
-
## Addons (`addons`)
### `addons.IntegratedIntensity`
#### <a name='addons.IntegratedIntensity.enable'></a>`addons.IntegratedIntensity.enable`
@@ -721,7 +356,25 @@ Description
Default value
: ``0``
+#### `addons.SaturationMonitor.output`
+OutputChannel
## Constant retrieval parameters (`constantParameters`)
+### <a name='constantParameters.secretsFile'></a>CalCat secrets file (`constantParameters.secretsFile`)
+Type
+: STRING
+
+Access mode
+: INITONLY
+
+Assignment
+: OPTIONAL
+
+Description
+: Path to JSON file specifying secret token and some parameters used to access CalCat.
+
+Default value
+: ``~/.calcat-secrets.json``
+
### <a name='constantParameters.deviceMappingSnapshotAt'></a>Snapshot timestamp (for device mapping) (`constantParameters.deviceMappingSnapshotAt`)
Type
: STRING
diff --git a/docs/schemas/Epix100Geometry.md b/docs/schemas/Epix100Geometry.md
index d9fb3ba2..f913eab9 100644
--- a/docs/schemas/Epix100Geometry.md
+++ b/docs/schemas/Epix100Geometry.md
@@ -1,5 +1,18 @@
# Epix100Geometry
Base classes: ManualOriginGeometryBase
+## <a name='availableScenes'></a>Available scenes (`availableScenes`)
+Type
+: VECTOR_STRING
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``['overview']``
+
## Geometry preview (`geometryPreview`)
ImageData
## Geometry file (`geometryFile`)
@@ -93,6 +106,22 @@ Description
Default value
: ``True``
+### <a name='geometryFile.updateMotorReferenceOnLoad'></a>Update motor reference (`geometryFile.updateMotorReferenceOnLoad`)
+Type
+: BOOL
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Description
+: If this flag is on, the motor reference positions in the motor tracking node will be updated according to the motor positions stored in geometry file.
+
+Default value
+: ``True``
+
### <a name='geometryFile.loadFromFile'></a>Load from file (`geometryFile.loadFromFile`)
Type
: Slot
@@ -223,7 +252,7 @@ Type
: Slot
Allowed in states
-: ACTIVE
+: ACTIVE, MONITORING
Access mode
: RECONFIGURABLE
@@ -239,7 +268,7 @@ Type
: Slot
Allowed in states
-: ACTIVE
+: ACTIVE, MONITORING
Access mode
: RECONFIGURABLE
diff --git a/docs/schemas/Gotthard2Correction.md b/docs/schemas/Gotthard2Correction.md
index 600fee8a..874e299c 100644
--- a/docs/schemas/Gotthard2Correction.md
+++ b/docs/schemas/Gotthard2Correction.md
@@ -1,88 +1,91 @@
# Gotthard2Correction
-Base classes: BaseCorrection
-## `dataInput`
-InputChannel
-## <a name='fastSources'></a>Fast data sources (`fastSources`)
+Base classes: [BaseCorrection](BaseCorrection.md)
+## `corrections`
+### `corrections.lut`
+#### <a name='corrections.lut.available'></a>Available (`corrections.lut.available`)
Type
-: VECTOR_STRING
+: BOOL
Access mode
-: INITONLY
+: READONLY
Assignment
: OPTIONAL
Description
-: Sources to get data from. Only incoming hashes from these sources will be processed. This will typically be a single entry of the form: '[instrument]_DET_[detector]/DET/[channel]:xtdf'.
+: This boolean indicates whether the necessary constants have been loaded for this correction step to be applied. Enabling the correction will have no effect unless this is True.
Default value
-: ``[]``
+: ``False``
-## <a name='outputShmemBufferSize'></a>Output buffer size limit (`outputShmemBufferSize`)
+#### <a name='corrections.lut.enable'></a>Enable (`corrections.lut.enable`)
Type
-: UINT32
+: BOOL
Access mode
-: INITONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
Description
-: Corrected trains are written to shared memory locations. These are pre-allocated and re-used (circular buffer). This parameter determines how much memory to set aside for that buffer.
+: Controls whether to apply this correction step for main data output - subject to availability.
Default value
-: ``2``
+: ``True``
-## <a name='useShmemHandles'></a>Use shared memory (`useShmemHandles`)
+#### <a name='corrections.lut.preview'></a>Preview (`corrections.lut.preview`)
Type
: BOOL
Access mode
-: INITONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
Description
-: If enabled, shared memory handles will be used to avoid copying the main image data on the dataOutput channel. Note that these handles need to be dereferenced (typically by a group matcher) to get the data off the correction node.
+: Whether to apply this correction step for corrected preview output - subject to availability.
Default value
-: ``False``
+: ``True``
-## <a name='useInfiniband'></a>Use infiniband (`useInfiniband`)
+### `corrections.offset`
+#### <a name='corrections.offset.available'></a>Available (`corrections.offset.available`)
Type
: BOOL
Access mode
-: INITONLY
+: READONLY
Assignment
: OPTIONAL
Description
-: If enabled, device will during initialization try to bind its main data output channel (dataOutput) to its node's infiniband interface. Default interface is used if no 'ib0' interface is found.
+: This boolean indicates whether the necessary constants have been loaded for this correction step to be applied. Enabling the correction will have no effect unless this is True.
Default value
-: ``True``
+: ``False``
-## <a name='geometryDevice'></a>Geometry device (`geometryDevice`)
+#### <a name='corrections.offset.enable'></a>Enable (`corrections.offset.enable`)
Type
-: STRING
+: BOOL
Access mode
-: INITONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
+Description
+: Controls whether to apply this correction step for main data output - subject to availability.
+
Default value
-: empty string
+: ``True``
-## Data format (in/out) (`dataFormat`)
-### <a name='dataFormat.outputImageDtype'></a>Output image data dtype (`dataFormat.outputImageDtype`)
+#### <a name='corrections.offset.preview'></a>Preview (`corrections.offset.preview`)
Type
-: STRING
+: BOOL
Access mode
: RECONFIGURABLE
@@ -91,17 +94,15 @@ Assignment
: OPTIONAL
Description
-: The (numpy) dtype to use for outgoing image data. Input is cast to float32, corrections are applied, and only then will the result be cast to outputImageDtype. Be aware that casting to integer type causes truncation rather than rounding.
-
-Options:
-: ``float32``, ``float16``, ``uint16``
+: Whether to apply this correction step for corrected preview output - subject to availability.
Default value
-: ``float32``
+: ``True``
-### <a name='dataFormat.inputFrames'></a>Frames (`dataFormat.inputFrames`)
+### `corrections.gain`
+#### <a name='corrections.gain.available'></a>Available (`corrections.gain.available`)
Type
-: UINT32
+: BOOL
Access mode
: READONLY
@@ -110,14 +111,14 @@ Assignment
: OPTIONAL
Description
-: Number of frames on input
+: This boolean indicates whether the necessary constants have been loaded for this correction step to be applied. Enabling the correction will have no effect unless this is True.
Default value
-: ``0``
+: ``False``
-### <a name='dataFormat.outputAxisOrder'></a>Output axis order (`dataFormat.outputAxisOrder`)
+#### <a name='corrections.gain.enable'></a>Enable (`corrections.gain.enable`)
Type
-: STRING
+: BOOL
Access mode
: RECONFIGURABLE
@@ -126,33 +127,31 @@ Assignment
: OPTIONAL
Description
-: Axes of main data output can be reordered after correction. Axis order is specified as string consisting of 'f' (frames), 'ss' (slow scan), and 'ff' (fast scan). Anything but the default f-ss-fs order implies reordering - this axis ordering is based on the data we get from the detector (/ receiver), so how ss and fs maps to what may be considered x and y is detector-dependent.
-
-Options:
-: ``f-ss-fs``, ``f-fs-ss``, ``ss-f-fs``, ``ss-fs-f``, ``fs-f-ss``, ``fs-ss-f``
+: Controls whether to apply this correction step for main data output - subject to availability.
Default value
-: ``f-ss-fs``
+: ``True``
-### <a name='dataFormat.inputDataShape'></a>Input data shape (`dataFormat.inputDataShape`)
+#### <a name='corrections.gain.preview'></a>Preview (`corrections.gain.preview`)
Type
-: VECTOR_UINT32
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
Description
-: Image data shape in incoming data (from reader / DAQ). Updated based on latest train processed. Note that axis order may look wonky due to DAQ quirk.
+: Whether to apply this correction step for corrected preview output - subject to availability.
Default value
-: ``[]``
+: ``True``
-### <a name='dataFormat.outputDataShape'></a>Output data shape (`dataFormat.outputDataShape`)
+### `corrections.badPixels`
+#### <a name='corrections.badPixels.available'></a>Available (`corrections.badPixels.available`)
Type
-: VECTOR_UINT32
+: BOOL
Access mode
: READONLY
@@ -161,39 +160,28 @@ Assignment
: OPTIONAL
Description
-: Image data shape for data output from this device. Takes into account axis reordering, if applicable. Even without that, workarounds.overrideInputAxisOrder likely makes this differ from dataFormat.inputDataShape.
+: This boolean indicates whether the necessary constants have been loaded for this correction step to be applied. Enabling the correction will have no effect unless this is True.
Default value
-: ``[]``
+: ``False``
-### <a name='dataFormat.cellId'></a>`dataFormat.cellId`
+#### <a name='corrections.badPixels.enable'></a>Enable (`corrections.badPixels.enable`)
Type
-: VECTOR_UINT32
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
-Default value
-: ``[]``
-
-### <a name='dataFormat.pulseId'></a>`dataFormat.pulseId`
-Type
-: VECTOR_UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
+Description
+: Controls whether to apply this correction step for main data output - subject to availability.
Default value
-: ``[]``
+: ``True``
-## Workarounds (`workarounds`)
-### <a name='workarounds.overrideInputAxisOrder'></a>Override input axis order (`workarounds.overrideInputAxisOrder`)
+#### <a name='corrections.badPixels.preview'></a>Preview (`corrections.badPixels.preview`)
Type
: BOOL
@@ -204,14 +192,14 @@ Assignment
: OPTIONAL
Description
-: The shape of the image data ndarray as received from the DataAggregator is sometimes wrong - the axes are actually in a different order than the ndarray shape suggests. If this flag is on, the shape of the ndarray will be overridden with the axis order which was expected.
+: Whether to apply this correction step for corrected preview output - subject to availability.
Default value
: ``True``
-### <a name='workarounds.trainFromFutureThreshold'></a>Spurious future train ID threshold (`workarounds.trainFromFutureThreshold`)
+#### <a name='corrections.badPixels.maskingValue'></a>Bad pixel masking value (`corrections.badPixels.maskingValue`)
Type
-: UINT64
+: STRING
Access mode
: RECONFIGURABLE
@@ -220,157 +208,144 @@ Assignment
: OPTIONAL
Description
-: Some detectors occasionally send a train with incorrect and much too large train ID. To avoid these 'future trains' from interfering with train matching, use this threshold to discard them immediately. If a train arrives with an ID which exceeds the current train ID from the time server by more than this threshold, the train is ignored.
+: Any pixels masked by the bad pixel mask will have their value replaced with this. Note that this parameter is to be interpreted as a numpy.float32; use 'nan' to get NaN value.
Default value
-: ``10000``
+: ``nan``
-## <a name='loadMostRecentConstants'></a>Load most recent constants (`loadMostRecentConstants`)
+#### Bad pixel flags to use (`corrections.badPixels.subsetToUse`)
+The booleans under this node allow for selecting a subset of bad pixel types to take into account when doing bad pixel masking. Upon updating these flags, the map used for bad pixel masking will be ANDed with this selection. Turning disabled flags back on causes reloading of cached constants.
+##### <a name='corrections.badPixels.subsetToUse.OFFSET_OUT_OF_THRESHOLD'></a>`corrections.badPixels.subsetToUse.OFFSET_OUT_OF_THRESHOLD`
Type
-: Slot
+: BOOL
Access mode
: RECONFIGURABLE
-Description
-: Calling this slot will flush all constant buffers and cause the device to start querying CalCat for the most recent constants (all constants applicable for this device) available with the currently set constant parameters. This should typically be called after instantiating pipeline, after changing parameters, or after generating new constants.
+Assignment
+: OPTIONAL
+
+Default value
+: ``True``
-## <a name='inputDataState'></a>`inputDataState`
+##### <a name='corrections.badPixels.subsetToUse.NOISE_OUT_OF_THRESHOLD'></a>`corrections.badPixels.subsetToUse.NOISE_OUT_OF_THRESHOLD`
Type
-: STRING
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
Default value
-: ``NORMAL``
+: ``True``
-## <a name='deviceInternalsState'></a>`deviceInternalsState`
+##### <a name='corrections.badPixels.subsetToUse.OFFSET_NOISE_EVAL_ERROR'></a>`corrections.badPixels.subsetToUse.OFFSET_NOISE_EVAL_ERROR`
Type
-: STRING
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
Default value
-: ``NORMAL``
+: ``True``
-## <a name='processingState'></a>`processingState`
+##### <a name='corrections.badPixels.subsetToUse.NO_DARK_DATA'></a>`corrections.badPixels.subsetToUse.NO_DARK_DATA`
Type
-: STRING
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
Default value
-: ``NORMAL``
+: ``True``
-## <a name='trainId'></a>Train ID (`trainId`)
+##### <a name='corrections.badPixels.subsetToUse.CI_GAIN_OUT_OF_THRESHOLD'></a>`corrections.badPixels.subsetToUse.CI_GAIN_OUT_OF_THRESHOLD`
Type
-: UINT64
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
-Description
-: ID of latest train processed by this device.
-
Default value
-: ``0``
+: ``True``
-## Performance measures (`performance`)
-### <a name='performance.processingTime'></a>Processing time (`performance.processingTime`)
+##### <a name='corrections.badPixels.subsetToUse.CI_LINEAR_DEVIATION'></a>`corrections.badPixels.subsetToUse.CI_LINEAR_DEVIATION`
Type
-: DOUBLE
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
Default value
-: ``0.0``
+: ``True``
-### <a name='performance.rate'></a>Rate (`performance.rate`)
+##### <a name='corrections.badPixels.subsetToUse.CI_EVAL_ERROR'></a>`corrections.badPixels.subsetToUse.CI_EVAL_ERROR`
Type
-: DOUBLE
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
-Description
-: Actual rate with which this device gets, processes, and sends trains. This is a simple windowed moving average.
-
Default value
-: ``0.0``
+: ``True``
-### <a name='performance.inputDelay'></a>Input delay (`performance.inputDelay`)
+##### <a name='corrections.badPixels.subsetToUse.FF_GAIN_EVAL_ERROR'></a>`corrections.badPixels.subsetToUse.FF_GAIN_EVAL_ERROR`
Type
-: DOUBLE
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
-Description
-: Measured delay on the input channel. Measured by difference between 'now' and timestamp in input metadata.
-
Default value
-: ``0.0``
+: ``True``
-### <a name='performance.ratioOfRecentTrainsReceived'></a>`performance.ratioOfRecentTrainsReceived`
+##### <a name='corrections.badPixels.subsetToUse.FF_GAIN_DEVIATION'></a>`corrections.badPixels.subsetToUse.FF_GAIN_DEVIATION`
Type
-: DOUBLE
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
-Description
-: Of the latest trains (from last received train, going back [some buffer range]), how many did we receive? This estimate is updated when new trains come in, so is unreliable if nothing is coming at all.
-
Default value
-: ``0.0``
+: ``True``
-## `corrections`
-### `corrections.lut`
-#### <a name='corrections.lut.available'></a>Available (`corrections.lut.available`)
+##### <a name='corrections.badPixels.subsetToUse.FF_NO_ENTRIES'></a>`corrections.badPixels.subsetToUse.FF_NO_ENTRIES`
Type
: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
-Description
-: This boolean indicates whether the necessary constants have been loaded for this correction step to be applied. Enabling the correction will have no effect unless this is True.
-
Default value
-: ``False``
+: ``True``
-#### <a name='corrections.lut.enable'></a>Enable (`corrections.lut.enable`)
+##### <a name='corrections.badPixels.subsetToUse.CI2_EVAL_ERROR'></a>`corrections.badPixels.subsetToUse.CI2_EVAL_ERROR`
Type
: BOOL
@@ -380,13 +355,10 @@ Access mode
Assignment
: OPTIONAL
-Description
-: Controls whether to apply this correction step for main data output - subject to availability.
-
Default value
: ``True``
-#### <a name='corrections.lut.preview'></a>Preview (`corrections.lut.preview`)
+##### <a name='corrections.badPixels.subsetToUse.VALUE_IS_NAN'></a>`corrections.badPixels.subsetToUse.VALUE_IS_NAN`
Type
: BOOL
@@ -396,30 +368,23 @@ Access mode
Assignment
: OPTIONAL
-Description
-: Whether to apply this correction step for corrected preview output - subject to availability.
-
Default value
: ``True``
-### `corrections.offset`
-#### <a name='corrections.offset.available'></a>Available (`corrections.offset.available`)
+##### <a name='corrections.badPixels.subsetToUse.VALUE_OUT_OF_RANGE'></a>`corrections.badPixels.subsetToUse.VALUE_OUT_OF_RANGE`
Type
: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
-Description
-: This boolean indicates whether the necessary constants have been loaded for this correction step to be applied. Enabling the correction will have no effect unless this is True.
-
Default value
-: ``False``
+: ``True``
-#### <a name='corrections.offset.enable'></a>Enable (`corrections.offset.enable`)
+##### <a name='corrections.badPixels.subsetToUse.GAIN_THRESHOLDING_ERROR'></a>`corrections.badPixels.subsetToUse.GAIN_THRESHOLDING_ERROR`
Type
: BOOL
@@ -429,13 +394,10 @@ Access mode
Assignment
: OPTIONAL
-Description
-: Controls whether to apply this correction step for main data output - subject to availability.
-
Default value
: ``True``
-#### <a name='corrections.offset.preview'></a>Preview (`corrections.offset.preview`)
+##### <a name='corrections.badPixels.subsetToUse.DATA_STD_IS_ZERO'></a>`corrections.badPixels.subsetToUse.DATA_STD_IS_ZERO`
Type
: BOOL
@@ -445,30 +407,23 @@ Access mode
Assignment
: OPTIONAL
-Description
-: Whether to apply this correction step for corrected preview output - subject to availability.
-
Default value
: ``True``
-### `corrections.gain`
-#### <a name='corrections.gain.available'></a>Available (`corrections.gain.available`)
+##### <a name='corrections.badPixels.subsetToUse.ASIC_STD_BELOW_NOISE'></a>`corrections.badPixels.subsetToUse.ASIC_STD_BELOW_NOISE`
Type
: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
-Description
-: This boolean indicates whether the necessary constants have been loaded for this correction step to be applied. Enabling the correction will have no effect unless this is True.
-
Default value
-: ``False``
+: ``True``
-#### <a name='corrections.gain.enable'></a>Enable (`corrections.gain.enable`)
+##### <a name='corrections.badPixels.subsetToUse.INTERPOLATED'></a>`corrections.badPixels.subsetToUse.INTERPOLATED`
Type
: BOOL
@@ -478,13 +433,10 @@ Access mode
Assignment
: OPTIONAL
-Description
-: Controls whether to apply this correction step for main data output - subject to availability.
-
Default value
: ``True``
-#### <a name='corrections.gain.preview'></a>Preview (`corrections.gain.preview`)
+##### <a name='corrections.badPixels.subsetToUse.NOISY_ADC'></a>`corrections.badPixels.subsetToUse.NOISY_ADC`
Type
: BOOL
@@ -494,30 +446,23 @@ Access mode
Assignment
: OPTIONAL
-Description
-: Whether to apply this correction step for corrected preview output - subject to availability.
-
Default value
: ``True``
-### `corrections.badPixels`
-#### <a name='corrections.badPixels.available'></a>Available (`corrections.badPixels.available`)
+##### <a name='corrections.badPixels.subsetToUse.OVERSCAN'></a>`corrections.badPixels.subsetToUse.OVERSCAN`
Type
: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
-Description
-: This boolean indicates whether the necessary constants have been loaded for this correction step to be applied. Enabling the correction will have no effect unless this is True.
-
Default value
-: ``False``
+: ``True``
-#### <a name='corrections.badPixels.enable'></a>Enable (`corrections.badPixels.enable`)
+##### <a name='corrections.badPixels.subsetToUse.NON_SENSITIVE'></a>`corrections.badPixels.subsetToUse.NON_SENSITIVE`
Type
: BOOL
@@ -527,13 +472,10 @@ Access mode
Assignment
: OPTIONAL
-Description
-: Controls whether to apply this correction step for main data output - subject to availability.
-
Default value
: ``True``
-#### <a name='corrections.badPixels.preview'></a>Preview (`corrections.badPixels.preview`)
+##### <a name='corrections.badPixels.subsetToUse.NON_LIN_RESPONSE_REGION'></a>`corrections.badPixels.subsetToUse.NON_LIN_RESPONSE_REGION`
Type
: BOOL
@@ -543,15 +485,12 @@ Access mode
Assignment
: OPTIONAL
-Description
-: Whether to apply this correction step for corrected preview output - subject to availability.
-
Default value
: ``True``
-#### <a name='corrections.badPixels.maskingValue'></a>Bad pixel masking value (`corrections.badPixels.maskingValue`)
+##### <a name='corrections.badPixels.subsetToUse.WRONG_GAIN_VALUE'></a>`corrections.badPixels.subsetToUse.WRONG_GAIN_VALUE`
Type
-: STRING
+: BOOL
Access mode
: RECONFIGURABLE
@@ -559,15 +498,10 @@ Access mode
Assignment
: OPTIONAL
-Description
-: Any pixels masked by the bad pixel mask will have their value replaced with this. Note that this parameter is to be interpreted as a numpy.float32; use 'nan' to get NaN value.
-
Default value
-: ``nan``
+: ``True``
-#### Bad pixel flags to use (`corrections.badPixels.subsetToUse`)
-The booleans under this node allow for selecting a subset of bad pixel types to take into account when doing bad pixel masking. Upon updating these flags, the map used for bad pixel masking will be ANDed with this selection. Turning disabled flags back on causes reloading of cached constants.
-##### <a name='corrections.badPixels.subsetToUse.OFFSET_OUT_OF_THRESHOLD'></a>`corrections.badPixels.subsetToUse.OFFSET_OUT_OF_THRESHOLD`
+##### <a name='corrections.badPixels.subsetToUse.NON_STANDARD_SIZE'></a>`corrections.badPixels.subsetToUse.NON_STANDARD_SIZE`
Type
: BOOL
@@ -580,22 +514,24 @@ Assignment
Default value
: ``True``
-##### <a name='corrections.badPixels.subsetToUse.NOISE_OUT_OF_THRESHOLD'></a>`corrections.badPixels.subsetToUse.NOISE_OUT_OF_THRESHOLD`
+## Addons (`addons`)
+### `addons.IntegratedIntensity`
+#### <a name='addons.IntegratedIntensity.enable'></a>`addons.IntegratedIntensity.enable`
Type
: BOOL
Access mode
-: RECONFIGURABLE
+: INITONLY
Assignment
: OPTIONAL
Default value
-: ``True``
+: ``False``
-##### <a name='corrections.badPixels.subsetToUse.OFFSET_NOISE_EVAL_ERROR'></a>`corrections.badPixels.subsetToUse.OFFSET_NOISE_EVAL_ERROR`
+#### <a name='addons.IntegratedIntensity.valueMin'></a>`addons.IntegratedIntensity.valueMin`
Type
-: BOOL
+: DOUBLE
Access mode
: RECONFIGURABLE
@@ -604,11 +540,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``-10.0``
-##### <a name='corrections.badPixels.subsetToUse.NO_DARK_DATA'></a>`corrections.badPixels.subsetToUse.NO_DARK_DATA`
+#### <a name='addons.IntegratedIntensity.valueMax'></a>`addons.IntegratedIntensity.valueMax`
Type
-: BOOL
+: DOUBLE
Access mode
: RECONFIGURABLE
@@ -617,24 +553,25 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``20000.0``
-##### <a name='corrections.badPixels.subsetToUse.CI_GAIN_OUT_OF_THRESHOLD'></a>`corrections.badPixels.subsetToUse.CI_GAIN_OUT_OF_THRESHOLD`
+### `addons.Peakfinder9`
+#### <a name='addons.Peakfinder9.enable'></a>`addons.Peakfinder9.enable`
Type
: BOOL
Access mode
-: RECONFIGURABLE
+: INITONLY
Assignment
: OPTIONAL
Default value
-: ``True``
+: ``False``
-##### <a name='corrections.badPixels.subsetToUse.CI_LINEAR_DEVIATION'></a>`corrections.badPixels.subsetToUse.CI_LINEAR_DEVIATION`
+#### <a name='addons.Peakfinder9.windowRadius'></a>`addons.Peakfinder9.windowRadius`
Type
-: BOOL
+: UINT32
Access mode
: RECONFIGURABLE
@@ -643,11 +580,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``2``
-##### <a name='corrections.badPixels.subsetToUse.CI_EVAL_ERROR'></a>`corrections.badPixels.subsetToUse.CI_EVAL_ERROR`
+#### <a name='addons.Peakfinder9.maxPeaks'></a>`addons.Peakfinder9.maxPeaks`
Type
-: BOOL
+: UINT32
Access mode
: RECONFIGURABLE
@@ -656,11 +593,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``500``
-##### <a name='corrections.badPixels.subsetToUse.FF_GAIN_EVAL_ERROR'></a>`corrections.badPixels.subsetToUse.FF_GAIN_EVAL_ERROR`
+#### <a name='addons.Peakfinder9.minPeakValueOverNeighbors'></a>`addons.Peakfinder9.minPeakValueOverNeighbors`
Type
-: BOOL
+: FLOAT
Access mode
: RECONFIGURABLE
@@ -669,11 +606,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``10.0``
-##### <a name='corrections.badPixels.subsetToUse.FF_GAIN_DEVIATION'></a>`corrections.badPixels.subsetToUse.FF_GAIN_DEVIATION`
+#### <a name='addons.Peakfinder9.minSnrMaxPixel'></a>`addons.Peakfinder9.minSnrMaxPixel`
Type
-: BOOL
+: FLOAT
Access mode
: RECONFIGURABLE
@@ -682,11 +619,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``5.0``
-##### <a name='corrections.badPixels.subsetToUse.FF_NO_ENTRIES'></a>`corrections.badPixels.subsetToUse.FF_NO_ENTRIES`
+#### <a name='addons.Peakfinder9.minSnrPeakPixels'></a>`addons.Peakfinder9.minSnrPeakPixels`
Type
-: BOOL
+: FLOAT
Access mode
: RECONFIGURABLE
@@ -695,11 +632,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``4.0``
-##### <a name='corrections.badPixels.subsetToUse.CI2_EVAL_ERROR'></a>`corrections.badPixels.subsetToUse.CI2_EVAL_ERROR`
+#### <a name='addons.Peakfinder9.minSnrWholePeak'></a>`addons.Peakfinder9.minSnrWholePeak`
Type
-: BOOL
+: FLOAT
Access mode
: RECONFIGURABLE
@@ -708,11 +645,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``6.0``
-##### <a name='corrections.badPixels.subsetToUse.VALUE_IS_NAN'></a>`corrections.badPixels.subsetToUse.VALUE_IS_NAN`
+#### <a name='addons.Peakfinder9.minSigma'></a>`addons.Peakfinder9.minSigma`
Type
-: BOOL
+: FLOAT
Access mode
: RECONFIGURABLE
@@ -721,11 +658,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``5.0``
-##### <a name='corrections.badPixels.subsetToUse.VALUE_OUT_OF_RANGE'></a>`corrections.badPixels.subsetToUse.VALUE_OUT_OF_RANGE`
+#### <a name='addons.Peakfinder9.blockX'></a>`addons.Peakfinder9.blockX`
Type
-: BOOL
+: UINT32
Access mode
: RECONFIGURABLE
@@ -734,11 +671,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``1``
-##### <a name='corrections.badPixels.subsetToUse.GAIN_THRESHOLDING_ERROR'></a>`corrections.badPixels.subsetToUse.GAIN_THRESHOLDING_ERROR`
+#### <a name='addons.Peakfinder9.blockY'></a>`addons.Peakfinder9.blockY`
Type
-: BOOL
+: UINT32
Access mode
: RECONFIGURABLE
@@ -747,11 +684,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``1``
-##### <a name='corrections.badPixels.subsetToUse.DATA_STD_IS_ZERO'></a>`corrections.badPixels.subsetToUse.DATA_STD_IS_ZERO`
+#### <a name='addons.Peakfinder9.blockZ'></a>`addons.Peakfinder9.blockZ`
Type
-: BOOL
+: UINT32
Access mode
: RECONFIGURABLE
@@ -760,24 +697,25 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``64``
-##### <a name='corrections.badPixels.subsetToUse.ASIC_STD_BELOW_NOISE'></a>`corrections.badPixels.subsetToUse.ASIC_STD_BELOW_NOISE`
+### `addons.RandomFrames`
+#### <a name='addons.RandomFrames.enable'></a>`addons.RandomFrames.enable`
Type
: BOOL
Access mode
-: RECONFIGURABLE
+: INITONLY
Assignment
: OPTIONAL
Default value
-: ``True``
+: ``False``
-##### <a name='corrections.badPixels.subsetToUse.INTERPOLATED'></a>`corrections.badPixels.subsetToUse.INTERPOLATED`
+#### <a name='addons.RandomFrames.probability'></a>`addons.RandomFrames.probability`
Type
-: BOOL
+: DOUBLE
Access mode
: RECONFIGURABLE
@@ -786,24 +724,25 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``50.0``
-##### <a name='corrections.badPixels.subsetToUse.NOISY_ADC'></a>`corrections.badPixels.subsetToUse.NOISY_ADC`
+### `addons.SaturationMonitor`
+#### <a name='addons.SaturationMonitor.enable'></a>`addons.SaturationMonitor.enable`
Type
: BOOL
Access mode
-: RECONFIGURABLE
+: INITONLY
Assignment
: OPTIONAL
Default value
-: ``True``
+: ``False``
-##### <a name='corrections.badPixels.subsetToUse.OVERSCAN'></a>`corrections.badPixels.subsetToUse.OVERSCAN`
+#### <a name='addons.SaturationMonitor.alarmThreshold'></a>`addons.SaturationMonitor.alarmThreshold`
Type
-: BOOL
+: FLOAT
Access mode
: RECONFIGURABLE
@@ -811,12 +750,15 @@ Access mode
Assignment
: OPTIONAL
+Description
+: Alarm threshold per pixel.
+
Default value
-: ``True``
+: ``0.0``
-##### <a name='corrections.badPixels.subsetToUse.NON_SENSITIVE'></a>`corrections.badPixels.subsetToUse.NON_SENSITIVE`
+#### <a name='addons.SaturationMonitor.warnThreshold'></a>`addons.SaturationMonitor.warnThreshold`
Type
-: BOOL
+: FLOAT
Access mode
: RECONFIGURABLE
@@ -824,12 +766,15 @@ Access mode
Assignment
: OPTIONAL
+Description
+: Warning threshold per pixel.
+
Default value
-: ``True``
+: ``0.0``
-##### <a name='corrections.badPixels.subsetToUse.NON_LIN_RESPONSE_REGION'></a>`corrections.badPixels.subsetToUse.NON_LIN_RESPONSE_REGION`
+#### <a name='addons.SaturationMonitor.alarmMaxCount'></a>`addons.SaturationMonitor.alarmMaxCount`
Type
-: BOOL
+: UINT64
Access mode
: RECONFIGURABLE
@@ -837,12 +782,15 @@ Access mode
Assignment
: OPTIONAL
+Description
+: Maximum number of pixel above alarm threshold.
+
Default value
-: ``True``
+: ``0``
-##### <a name='corrections.badPixels.subsetToUse.WRONG_GAIN_VALUE'></a>`corrections.badPixels.subsetToUse.WRONG_GAIN_VALUE`
+#### <a name='addons.SaturationMonitor.warnMaxCount'></a>`addons.SaturationMonitor.warnMaxCount`
Type
-: BOOL
+: UINT64
Access mode
: RECONFIGURABLE
@@ -850,12 +798,15 @@ Access mode
Assignment
: OPTIONAL
+Description
+: Maximum number of pixel above warn threshold.
+
Default value
-: ``True``
+: ``0``
-##### <a name='corrections.badPixels.subsetToUse.NON_STANDARD_SIZE'></a>`corrections.badPixels.subsetToUse.NON_STANDARD_SIZE`
+#### <a name='addons.SaturationMonitor.frameAxis'></a>Multi-frame axis (`addons.SaturationMonitor.frameAxis`)
Type
-: BOOL
+: UINT64
Access mode
: RECONFIGURABLE
@@ -863,12 +814,18 @@ Access mode
Assignment
: OPTIONAL
+Description
+: Axis for frames. Used to take the max over this axis.
+
Default value
-: ``True``
+: ``0``
-## <a name='warningLamps'></a>`warningLamps`
+#### `addons.SaturationMonitor.output`
+OutputChannel
+## Constant retrieval parameters (`constantParameters`)
+### <a name='constantParameters.secretsFile'></a>CalCat secrets file (`constantParameters.secretsFile`)
Type
-: VECTOR_STRING
+: STRING
Access mode
: INITONLY
@@ -876,14 +833,12 @@ Access mode
Assignment
: OPTIONAL
-Default value
-: ``['deviceInternalsState', 'inputDataState', 'processingState']``
+Description
+: Path to JSON file specifying secret token and some parameters used to access CalCat.
-## <a name='addons'></a>Addons (`addons`)
-Access mode
-: RECONFIGURABLE
+Default value
+: ``~/.calcat-secrets.json``
-## Constant retrieval parameters (`constantParameters`)
### <a name='constantParameters.deviceMappingSnapshotAt'></a>Snapshot timestamp (for device mapping) (`constantParameters.deviceMappingSnapshotAt`)
Type
: STRING
diff --git a/docs/schemas/JungfrauCorrection.md b/docs/schemas/JungfrauCorrection.md
index 0060f6b6..d8521ef1 100644
--- a/docs/schemas/JungfrauCorrection.md
+++ b/docs/schemas/JungfrauCorrection.md
@@ -1,88 +1,91 @@
# JungfrauCorrection
-Base classes: BaseCorrection
-## `dataInput`
-InputChannel
-## <a name='fastSources'></a>Fast data sources (`fastSources`)
+Base classes: [BaseCorrection](BaseCorrection.md)
+## `corrections`
+### `corrections.offset`
+#### <a name='corrections.offset.available'></a>Available (`corrections.offset.available`)
Type
-: VECTOR_STRING
+: BOOL
Access mode
-: INITONLY
+: READONLY
Assignment
: OPTIONAL
Description
-: Sources to get data from. Only incoming hashes from these sources will be processed. This will typically be a single entry of the form: '[instrument]_DET_[detector]/DET/[channel]:xtdf'.
+: This boolean indicates whether the necessary constants have been loaded for this correction step to be applied. Enabling the correction will have no effect unless this is True.
Default value
-: ``[]``
+: ``False``
-## <a name='outputShmemBufferSize'></a>Output buffer size limit (`outputShmemBufferSize`)
+#### <a name='corrections.offset.enable'></a>Enable (`corrections.offset.enable`)
Type
-: UINT32
+: BOOL
Access mode
-: INITONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
Description
-: Corrected trains are written to shared memory locations. These are pre-allocated and re-used (circular buffer). This parameter determines how much memory to set aside for that buffer.
+: Controls whether to apply this correction step for main data output - subject to availability.
Default value
-: ``2``
+: ``True``
-## <a name='useShmemHandles'></a>Use shared memory (`useShmemHandles`)
+#### <a name='corrections.offset.preview'></a>Preview (`corrections.offset.preview`)
Type
: BOOL
Access mode
-: INITONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
Description
-: If enabled, shared memory handles will be used to avoid copying the main image data on the dataOutput channel. Note that these handles need to be dereferenced (typically by a group matcher) to get the data off the correction node.
+: Whether to apply this correction step for corrected preview output - subject to availability.
Default value
-: ``False``
+: ``True``
-## <a name='useInfiniband'></a>Use infiniband (`useInfiniband`)
+### `corrections.relGain`
+#### <a name='corrections.relGain.available'></a>Available (`corrections.relGain.available`)
Type
: BOOL
Access mode
-: INITONLY
+: READONLY
Assignment
: OPTIONAL
Description
-: If enabled, device will during initialization try to bind its main data output channel (dataOutput) to its node's infiniband interface. Default interface is used if no 'ib0' interface is found.
+: This boolean indicates whether the necessary constants have been loaded for this correction step to be applied. Enabling the correction will have no effect unless this is True.
Default value
-: ``True``
+: ``False``
-## <a name='geometryDevice'></a>Geometry device (`geometryDevice`)
+#### <a name='corrections.relGain.enable'></a>Enable (`corrections.relGain.enable`)
Type
-: STRING
+: BOOL
Access mode
-: INITONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
+Description
+: Controls whether to apply this correction step for main data output - subject to availability.
+
Default value
-: empty string
+: ``True``
-## Data format (in/out) (`dataFormat`)
-### <a name='dataFormat.outputImageDtype'></a>Output image data dtype (`dataFormat.outputImageDtype`)
+#### <a name='corrections.relGain.preview'></a>Preview (`corrections.relGain.preview`)
Type
-: STRING
+: BOOL
Access mode
: RECONFIGURABLE
@@ -91,17 +94,15 @@ Assignment
: OPTIONAL
Description
-: The (numpy) dtype to use for outgoing image data. Input is cast to float32, corrections are applied, and only then will the result be cast to outputImageDtype. Be aware that casting to integer type causes truncation rather than rounding.
-
-Options:
-: ``float32``, ``float16``, ``uint16``
+: Whether to apply this correction step for corrected preview output - subject to availability.
Default value
-: ``float32``
+: ``True``
-### <a name='dataFormat.inputFrames'></a>Frames (`dataFormat.inputFrames`)
+### `corrections.badPixels`
+#### <a name='corrections.badPixels.available'></a>Available (`corrections.badPixels.available`)
Type
-: UINT32
+: BOOL
Access mode
: READONLY
@@ -110,14 +111,14 @@ Assignment
: OPTIONAL
Description
-: Number of frames on input
+: This boolean indicates whether the necessary constants have been loaded for this correction step to be applied. Enabling the correction will have no effect unless this is True.
Default value
-: ``0``
+: ``True``
-### <a name='dataFormat.outputAxisOrder'></a>Output axis order (`dataFormat.outputAxisOrder`)
+#### <a name='corrections.badPixels.enable'></a>Enable (`corrections.badPixels.enable`)
Type
-: STRING
+: BOOL
Access mode
: RECONFIGURABLE
@@ -126,74 +127,72 @@ Assignment
: OPTIONAL
Description
-: Axes of main data output can be reordered after correction. Axis order is specified as string consisting of 'f' (frames), 'ss' (slow scan), and 'ff' (fast scan). Anything but the default f-ss-fs order implies reordering - this axis ordering is based on the data we get from the detector (/ receiver), so how ss and fs maps to what may be considered x and y is detector-dependent.
-
-Options:
-: ``f-ss-fs``, ``f-fs-ss``, ``ss-f-fs``, ``ss-fs-f``, ``fs-f-ss``, ``fs-ss-f``
+: Controls whether to apply this correction step for main data output - subject to availability.
Default value
-: ``f-ss-fs``
+: ``True``
-### <a name='dataFormat.inputDataShape'></a>Input data shape (`dataFormat.inputDataShape`)
+#### <a name='corrections.badPixels.preview'></a>Preview (`corrections.badPixels.preview`)
Type
-: VECTOR_UINT32
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
Description
-: Image data shape in incoming data (from reader / DAQ). Updated based on latest train processed. Note that axis order may look wonky due to DAQ quirk.
+: Whether to apply this correction step for corrected preview output - subject to availability.
Default value
-: ``[]``
+: ``True``
-### <a name='dataFormat.outputDataShape'></a>Output data shape (`dataFormat.outputDataShape`)
+#### <a name='corrections.badPixels.maskingValue'></a>Bad pixel masking value (`corrections.badPixels.maskingValue`)
Type
-: VECTOR_UINT32
+: STRING
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
Description
-: Image data shape for data output from this device. Takes into account axis reordering, if applicable. Even without that, workarounds.overrideInputAxisOrder likely makes this differ from dataFormat.inputDataShape.
+: Any pixels masked by the bad pixel mask will have their value replaced with this. Note that this parameter is to be interpreted as a numpy.float32; use 'nan' to get NaN value.
Default value
-: ``[]``
+: ``nan``
-### <a name='dataFormat.cellId'></a>`dataFormat.cellId`
+#### Bad pixel flags to use (`corrections.badPixels.subsetToUse`)
+The booleans under this node allow for selecting a subset of bad pixel types to take into account when doing bad pixel masking. Upon updating these flags, the map used for bad pixel masking will be ANDed with this selection. Turning disabled flags back on causes reloading of cached constants.
+##### <a name='corrections.badPixels.subsetToUse.OFFSET_OUT_OF_THRESHOLD'></a>`corrections.badPixels.subsetToUse.OFFSET_OUT_OF_THRESHOLD`
Type
-: VECTOR_UINT32
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
Default value
-: ``[]``
+: ``True``
-### <a name='dataFormat.pulseId'></a>`dataFormat.pulseId`
+##### <a name='corrections.badPixels.subsetToUse.NOISE_OUT_OF_THRESHOLD'></a>`corrections.badPixels.subsetToUse.NOISE_OUT_OF_THRESHOLD`
Type
-: VECTOR_UINT32
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
Default value
-: ``[]``
+: ``True``
-## Workarounds (`workarounds`)
-### <a name='workarounds.overrideInputAxisOrder'></a>Override input axis order (`workarounds.overrideInputAxisOrder`)
+##### <a name='corrections.badPixels.subsetToUse.OFFSET_NOISE_EVAL_ERROR'></a>`corrections.badPixels.subsetToUse.OFFSET_NOISE_EVAL_ERROR`
Type
: BOOL
@@ -203,15 +202,12 @@ Access mode
Assignment
: OPTIONAL
-Description
-: The shape of the image data ndarray as received from the DataAggregator is sometimes wrong - the axes are actually in a different order than the ndarray shape suggests. If this flag is on, the shape of the ndarray will be overridden with the axis order which was expected.
-
Default value
: ``True``
-### <a name='workarounds.trainFromFutureThreshold'></a>Spurious future train ID threshold (`workarounds.trainFromFutureThreshold`)
+##### <a name='corrections.badPixels.subsetToUse.NO_DARK_DATA'></a>`corrections.badPixels.subsetToUse.NO_DARK_DATA`
Type
-: UINT64
+: BOOL
Access mode
: RECONFIGURABLE
@@ -219,158 +215,127 @@ Access mode
Assignment
: OPTIONAL
-Description
-: Some detectors occasionally send a train with incorrect and much too large train ID. To avoid these 'future trains' from interfering with train matching, use this threshold to discard them immediately. If a train arrives with an ID which exceeds the current train ID from the time server by more than this threshold, the train is ignored.
-
Default value
-: ``10000``
+: ``True``
-## <a name='loadMostRecentConstants'></a>Load most recent constants (`loadMostRecentConstants`)
+##### <a name='corrections.badPixels.subsetToUse.CI_GAIN_OUT_OF_THRESHOLD'></a>`corrections.badPixels.subsetToUse.CI_GAIN_OUT_OF_THRESHOLD`
Type
-: Slot
+: BOOL
Access mode
: RECONFIGURABLE
-Description
-: Calling this slot will flush all constant buffers and cause the device to start querying CalCat for the most recent constants (all constants applicable for this device) available with the currently set constant parameters. This should typically be called after instantiating pipeline, after changing parameters, or after generating new constants.
-
-## <a name='inputDataState'></a>`inputDataState`
-Type
-: STRING
-
-Access mode
-: READONLY
-
Assignment
: OPTIONAL
Default value
-: ``NORMAL``
+: ``True``
-## <a name='deviceInternalsState'></a>`deviceInternalsState`
+##### <a name='corrections.badPixels.subsetToUse.CI_LINEAR_DEVIATION'></a>`corrections.badPixels.subsetToUse.CI_LINEAR_DEVIATION`
Type
-: STRING
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
Default value
-: ``NORMAL``
+: ``True``
-## <a name='processingState'></a>`processingState`
+##### <a name='corrections.badPixels.subsetToUse.CI_EVAL_ERROR'></a>`corrections.badPixels.subsetToUse.CI_EVAL_ERROR`
Type
-: STRING
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
Default value
-: ``NORMAL``
+: ``True``
-## <a name='trainId'></a>Train ID (`trainId`)
+##### <a name='corrections.badPixels.subsetToUse.FF_GAIN_EVAL_ERROR'></a>`corrections.badPixels.subsetToUse.FF_GAIN_EVAL_ERROR`
Type
-: UINT64
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
-Description
-: ID of latest train processed by this device.
-
Default value
-: ``0``
+: ``True``
-## Performance measures (`performance`)
-### <a name='performance.processingTime'></a>Processing time (`performance.processingTime`)
+##### <a name='corrections.badPixels.subsetToUse.FF_GAIN_DEVIATION'></a>`corrections.badPixels.subsetToUse.FF_GAIN_DEVIATION`
Type
-: DOUBLE
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
Default value
-: ``0.0``
+: ``True``
-### <a name='performance.rate'></a>Rate (`performance.rate`)
+##### <a name='corrections.badPixels.subsetToUse.FF_NO_ENTRIES'></a>`corrections.badPixels.subsetToUse.FF_NO_ENTRIES`
Type
-: DOUBLE
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
-Description
-: Actual rate with which this device gets, processes, and sends trains. This is a simple windowed moving average.
-
Default value
-: ``0.0``
+: ``True``
-### <a name='performance.inputDelay'></a>Input delay (`performance.inputDelay`)
+##### <a name='corrections.badPixels.subsetToUse.CI2_EVAL_ERROR'></a>`corrections.badPixels.subsetToUse.CI2_EVAL_ERROR`
Type
-: DOUBLE
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
-Description
-: Measured delay on the input channel. Measured by difference between 'now' and timestamp in input metadata.
-
Default value
-: ``0.0``
+: ``True``
-### <a name='performance.ratioOfRecentTrainsReceived'></a>`performance.ratioOfRecentTrainsReceived`
+##### <a name='corrections.badPixels.subsetToUse.VALUE_IS_NAN'></a>`corrections.badPixels.subsetToUse.VALUE_IS_NAN`
Type
-: DOUBLE
+: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
-Description
-: Of the latest trains (from last received train, going back [some buffer range]), how many did we receive? This estimate is updated when new trains come in, so is unreliable if nothing is coming at all.
-
Default value
-: ``0.0``
+: ``True``
-## `corrections`
-### `corrections.offset`
-#### <a name='corrections.offset.available'></a>Available (`corrections.offset.available`)
+##### <a name='corrections.badPixels.subsetToUse.VALUE_OUT_OF_RANGE'></a>`corrections.badPixels.subsetToUse.VALUE_OUT_OF_RANGE`
Type
: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
-Description
-: This boolean indicates whether the necessary constants have been loaded for this correction step to be applied. Enabling the correction will have no effect unless this is True.
-
Default value
-: ``False``
+: ``True``
-#### <a name='corrections.offset.enable'></a>Enable (`corrections.offset.enable`)
+##### <a name='corrections.badPixels.subsetToUse.GAIN_THRESHOLDING_ERROR'></a>`corrections.badPixels.subsetToUse.GAIN_THRESHOLDING_ERROR`
Type
: BOOL
@@ -380,13 +345,10 @@ Access mode
Assignment
: OPTIONAL
-Description
-: Controls whether to apply this correction step for main data output - subject to availability.
-
Default value
: ``True``
-#### <a name='corrections.offset.preview'></a>Preview (`corrections.offset.preview`)
+##### <a name='corrections.badPixels.subsetToUse.DATA_STD_IS_ZERO'></a>`corrections.badPixels.subsetToUse.DATA_STD_IS_ZERO`
Type
: BOOL
@@ -396,30 +358,23 @@ Access mode
Assignment
: OPTIONAL
-Description
-: Whether to apply this correction step for corrected preview output - subject to availability.
-
Default value
: ``True``
-### `corrections.relGain`
-#### <a name='corrections.relGain.available'></a>Available (`corrections.relGain.available`)
+##### <a name='corrections.badPixels.subsetToUse.ASIC_STD_BELOW_NOISE'></a>`corrections.badPixels.subsetToUse.ASIC_STD_BELOW_NOISE`
Type
: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
-Description
-: This boolean indicates whether the necessary constants have been loaded for this correction step to be applied. Enabling the correction will have no effect unless this is True.
-
Default value
-: ``False``
+: ``True``
-#### <a name='corrections.relGain.enable'></a>Enable (`corrections.relGain.enable`)
+##### <a name='corrections.badPixels.subsetToUse.INTERPOLATED'></a>`corrections.badPixels.subsetToUse.INTERPOLATED`
Type
: BOOL
@@ -429,13 +384,10 @@ Access mode
Assignment
: OPTIONAL
-Description
-: Controls whether to apply this correction step for main data output - subject to availability.
-
Default value
: ``True``
-#### <a name='corrections.relGain.preview'></a>Preview (`corrections.relGain.preview`)
+##### <a name='corrections.badPixels.subsetToUse.NOISY_ADC'></a>`corrections.badPixels.subsetToUse.NOISY_ADC`
Type
: BOOL
@@ -445,30 +397,23 @@ Access mode
Assignment
: OPTIONAL
-Description
-: Whether to apply this correction step for corrected preview output - subject to availability.
-
Default value
: ``True``
-### `corrections.badPixels`
-#### <a name='corrections.badPixels.available'></a>Available (`corrections.badPixels.available`)
+##### <a name='corrections.badPixels.subsetToUse.OVERSCAN'></a>`corrections.badPixels.subsetToUse.OVERSCAN`
Type
: BOOL
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
-Description
-: This boolean indicates whether the necessary constants have been loaded for this correction step to be applied. Enabling the correction will have no effect unless this is True.
-
Default value
: ``True``
-#### <a name='corrections.badPixels.enable'></a>Enable (`corrections.badPixels.enable`)
+##### <a name='corrections.badPixels.subsetToUse.NON_SENSITIVE'></a>`corrections.badPixels.subsetToUse.NON_SENSITIVE`
Type
: BOOL
@@ -478,13 +423,10 @@ Access mode
Assignment
: OPTIONAL
-Description
-: Controls whether to apply this correction step for main data output - subject to availability.
-
Default value
: ``True``
-#### <a name='corrections.badPixels.preview'></a>Preview (`corrections.badPixels.preview`)
+##### <a name='corrections.badPixels.subsetToUse.NON_LIN_RESPONSE_REGION'></a>`corrections.badPixels.subsetToUse.NON_LIN_RESPONSE_REGION`
Type
: BOOL
@@ -494,15 +436,12 @@ Access mode
Assignment
: OPTIONAL
-Description
-: Whether to apply this correction step for corrected preview output - subject to availability.
-
Default value
: ``True``
-#### <a name='corrections.badPixels.maskingValue'></a>Bad pixel masking value (`corrections.badPixels.maskingValue`)
+##### <a name='corrections.badPixels.subsetToUse.WRONG_GAIN_VALUE'></a>`corrections.badPixels.subsetToUse.WRONG_GAIN_VALUE`
Type
-: STRING
+: BOOL
Access mode
: RECONFIGURABLE
@@ -510,15 +449,10 @@ Access mode
Assignment
: OPTIONAL
-Description
-: Any pixels masked by the bad pixel mask will have their value replaced with this. Note that this parameter is to be interpreted as a numpy.float32; use 'nan' to get NaN value.
-
Default value
-: ``nan``
+: ``True``
-#### Bad pixel flags to use (`corrections.badPixels.subsetToUse`)
-The booleans under this node allow for selecting a subset of bad pixel types to take into account when doing bad pixel masking. Upon updating these flags, the map used for bad pixel masking will be ANDed with this selection. Turning disabled flags back on causes reloading of cached constants.
-##### <a name='corrections.badPixels.subsetToUse.OFFSET_OUT_OF_THRESHOLD'></a>`corrections.badPixels.subsetToUse.OFFSET_OUT_OF_THRESHOLD`
+##### <a name='corrections.badPixels.subsetToUse.NON_STANDARD_SIZE'></a>`corrections.badPixels.subsetToUse.NON_STANDARD_SIZE`
Type
: BOOL
@@ -531,20 +465,24 @@ Assignment
Default value
: ``True``
-##### <a name='corrections.badPixels.subsetToUse.NOISE_OUT_OF_THRESHOLD'></a>`corrections.badPixels.subsetToUse.NOISE_OUT_OF_THRESHOLD`
+### `corrections.strixel`
+#### <a name='corrections.strixel.available'></a>Available (`corrections.strixel.available`)
Type
: BOOL
Access mode
-: RECONFIGURABLE
+: READONLY
Assignment
: OPTIONAL
+Description
+: This boolean indicates whether the necessary constants have been loaded for this correction step to be applied. Enabling the correction will have no effect unless this is True.
+
Default value
: ``True``
-##### <a name='corrections.badPixels.subsetToUse.OFFSET_NOISE_EVAL_ERROR'></a>`corrections.badPixels.subsetToUse.OFFSET_NOISE_EVAL_ERROR`
+#### <a name='corrections.strixel.enable'></a>Enable (`corrections.strixel.enable`)
Type
: BOOL
@@ -554,10 +492,13 @@ Access mode
Assignment
: OPTIONAL
+Description
+: Controls whether to apply this correction step for main data output - subject to availability.
+
Default value
-: ``True``
+: ``False``
-##### <a name='corrections.badPixels.subsetToUse.NO_DARK_DATA'></a>`corrections.badPixels.subsetToUse.NO_DARK_DATA`
+#### <a name='corrections.strixel.preview'></a>Preview (`corrections.strixel.preview`)
Type
: BOOL
@@ -567,12 +508,15 @@ Access mode
Assignment
: OPTIONAL
+Description
+: Whether to apply this correction step for corrected preview output - subject to availability.
+
Default value
-: ``True``
+: ``False``
-##### <a name='corrections.badPixels.subsetToUse.CI_GAIN_OUT_OF_THRESHOLD'></a>`corrections.badPixels.subsetToUse.CI_GAIN_OUT_OF_THRESHOLD`
+#### <a name='corrections.strixel.type'></a>`corrections.strixel.type`
Type
-: BOOL
+: STRING
Access mode
: RECONFIGURABLE
@@ -580,25 +524,33 @@ Access mode
Assignment
: OPTIONAL
+Description
+: Which kind of strixel layout is used for this module? cols_A0123 is the first strixel layout deployed at HED and rows_A1256 is the one later deployed at SCS.
+
+Options:
+: ``cols_A0123(HED-type)``, ``rows_A1256(SCS-type)``
+
Default value
-: ``True``
+: ``cols_A0123(HED-type)``
-##### <a name='corrections.badPixels.subsetToUse.CI_LINEAR_DEVIATION'></a>`corrections.badPixels.subsetToUse.CI_LINEAR_DEVIATION`
+## Addons (`addons`)
+### `addons.IntegratedIntensity`
+#### <a name='addons.IntegratedIntensity.enable'></a>`addons.IntegratedIntensity.enable`
Type
: BOOL
Access mode
-: RECONFIGURABLE
+: INITONLY
Assignment
: OPTIONAL
Default value
-: ``True``
+: ``False``
-##### <a name='corrections.badPixels.subsetToUse.CI_EVAL_ERROR'></a>`corrections.badPixels.subsetToUse.CI_EVAL_ERROR`
+#### <a name='addons.IntegratedIntensity.valueMin'></a>`addons.IntegratedIntensity.valueMin`
Type
-: BOOL
+: DOUBLE
Access mode
: RECONFIGURABLE
@@ -607,11 +559,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``-10.0``
-##### <a name='corrections.badPixels.subsetToUse.FF_GAIN_EVAL_ERROR'></a>`corrections.badPixels.subsetToUse.FF_GAIN_EVAL_ERROR`
+#### <a name='addons.IntegratedIntensity.valueMax'></a>`addons.IntegratedIntensity.valueMax`
Type
-: BOOL
+: DOUBLE
Access mode
: RECONFIGURABLE
@@ -620,24 +572,25 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``20000.0``
-##### <a name='corrections.badPixels.subsetToUse.FF_GAIN_DEVIATION'></a>`corrections.badPixels.subsetToUse.FF_GAIN_DEVIATION`
+### `addons.Peakfinder9`
+#### <a name='addons.Peakfinder9.enable'></a>`addons.Peakfinder9.enable`
Type
: BOOL
Access mode
-: RECONFIGURABLE
+: INITONLY
Assignment
: OPTIONAL
Default value
-: ``True``
+: ``False``
-##### <a name='corrections.badPixels.subsetToUse.FF_NO_ENTRIES'></a>`corrections.badPixels.subsetToUse.FF_NO_ENTRIES`
+#### <a name='addons.Peakfinder9.windowRadius'></a>`addons.Peakfinder9.windowRadius`
Type
-: BOOL
+: UINT32
Access mode
: RECONFIGURABLE
@@ -646,11 +599,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``2``
-##### <a name='corrections.badPixels.subsetToUse.CI2_EVAL_ERROR'></a>`corrections.badPixels.subsetToUse.CI2_EVAL_ERROR`
+#### <a name='addons.Peakfinder9.maxPeaks'></a>`addons.Peakfinder9.maxPeaks`
Type
-: BOOL
+: UINT32
Access mode
: RECONFIGURABLE
@@ -659,11 +612,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``500``
-##### <a name='corrections.badPixels.subsetToUse.VALUE_IS_NAN'></a>`corrections.badPixels.subsetToUse.VALUE_IS_NAN`
+#### <a name='addons.Peakfinder9.minPeakValueOverNeighbors'></a>`addons.Peakfinder9.minPeakValueOverNeighbors`
Type
-: BOOL
+: FLOAT
Access mode
: RECONFIGURABLE
@@ -672,11 +625,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``10.0``
-##### <a name='corrections.badPixels.subsetToUse.VALUE_OUT_OF_RANGE'></a>`corrections.badPixels.subsetToUse.VALUE_OUT_OF_RANGE`
+#### <a name='addons.Peakfinder9.minSnrMaxPixel'></a>`addons.Peakfinder9.minSnrMaxPixel`
Type
-: BOOL
+: FLOAT
Access mode
: RECONFIGURABLE
@@ -685,11 +638,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``5.0``
-##### <a name='corrections.badPixels.subsetToUse.GAIN_THRESHOLDING_ERROR'></a>`corrections.badPixels.subsetToUse.GAIN_THRESHOLDING_ERROR`
+#### <a name='addons.Peakfinder9.minSnrPeakPixels'></a>`addons.Peakfinder9.minSnrPeakPixels`
Type
-: BOOL
+: FLOAT
Access mode
: RECONFIGURABLE
@@ -698,11 +651,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``4.0``
-##### <a name='corrections.badPixels.subsetToUse.DATA_STD_IS_ZERO'></a>`corrections.badPixels.subsetToUse.DATA_STD_IS_ZERO`
+#### <a name='addons.Peakfinder9.minSnrWholePeak'></a>`addons.Peakfinder9.minSnrWholePeak`
Type
-: BOOL
+: FLOAT
Access mode
: RECONFIGURABLE
@@ -711,11 +664,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``6.0``
-##### <a name='corrections.badPixels.subsetToUse.ASIC_STD_BELOW_NOISE'></a>`corrections.badPixels.subsetToUse.ASIC_STD_BELOW_NOISE`
+#### <a name='addons.Peakfinder9.minSigma'></a>`addons.Peakfinder9.minSigma`
Type
-: BOOL
+: FLOAT
Access mode
: RECONFIGURABLE
@@ -724,11 +677,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``5.0``
-##### <a name='corrections.badPixels.subsetToUse.INTERPOLATED'></a>`corrections.badPixels.subsetToUse.INTERPOLATED`
+#### <a name='addons.Peakfinder9.blockX'></a>`addons.Peakfinder9.blockX`
Type
-: BOOL
+: UINT32
Access mode
: RECONFIGURABLE
@@ -737,11 +690,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``1``
-##### <a name='corrections.badPixels.subsetToUse.NOISY_ADC'></a>`corrections.badPixels.subsetToUse.NOISY_ADC`
+#### <a name='addons.Peakfinder9.blockY'></a>`addons.Peakfinder9.blockY`
Type
-: BOOL
+: UINT32
Access mode
: RECONFIGURABLE
@@ -750,11 +703,11 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``1``
-##### <a name='corrections.badPixels.subsetToUse.OVERSCAN'></a>`corrections.badPixels.subsetToUse.OVERSCAN`
+#### <a name='addons.Peakfinder9.blockZ'></a>`addons.Peakfinder9.blockZ`
Type
-: BOOL
+: UINT32
Access mode
: RECONFIGURABLE
@@ -763,24 +716,25 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``64``
-##### <a name='corrections.badPixels.subsetToUse.NON_SENSITIVE'></a>`corrections.badPixels.subsetToUse.NON_SENSITIVE`
+### `addons.RandomFrames`
+#### <a name='addons.RandomFrames.enable'></a>`addons.RandomFrames.enable`
Type
: BOOL
Access mode
-: RECONFIGURABLE
+: INITONLY
Assignment
: OPTIONAL
Default value
-: ``True``
+: ``False``
-##### <a name='corrections.badPixels.subsetToUse.NON_LIN_RESPONSE_REGION'></a>`corrections.badPixels.subsetToUse.NON_LIN_RESPONSE_REGION`
+#### <a name='addons.RandomFrames.probability'></a>`addons.RandomFrames.probability`
Type
-: BOOL
+: DOUBLE
Access mode
: RECONFIGURABLE
@@ -789,24 +743,25 @@ Assignment
: OPTIONAL
Default value
-: ``True``
+: ``50.0``
-##### <a name='corrections.badPixels.subsetToUse.WRONG_GAIN_VALUE'></a>`corrections.badPixels.subsetToUse.WRONG_GAIN_VALUE`
+### `addons.SaturationMonitor`
+#### <a name='addons.SaturationMonitor.enable'></a>`addons.SaturationMonitor.enable`
Type
: BOOL
Access mode
-: RECONFIGURABLE
+: INITONLY
Assignment
: OPTIONAL
Default value
-: ``True``
+: ``False``
-##### <a name='corrections.badPixels.subsetToUse.NON_STANDARD_SIZE'></a>`corrections.badPixels.subsetToUse.NON_STANDARD_SIZE`
+#### <a name='addons.SaturationMonitor.alarmThreshold'></a>`addons.SaturationMonitor.alarmThreshold`
Type
-: BOOL
+: FLOAT
Access mode
: RECONFIGURABLE
@@ -814,29 +769,31 @@ Access mode
Assignment
: OPTIONAL
+Description
+: Alarm threshold per pixel.
+
Default value
-: ``True``
+: ``0.0``
-### `corrections.strixel`
-#### <a name='corrections.strixel.available'></a>Available (`corrections.strixel.available`)
+#### <a name='addons.SaturationMonitor.warnThreshold'></a>`addons.SaturationMonitor.warnThreshold`
Type
-: BOOL
+: FLOAT
Access mode
-: READONLY
+: RECONFIGURABLE
Assignment
: OPTIONAL
Description
-: This boolean indicates whether the necessary constants have been loaded for this correction step to be applied. Enabling the correction will have no effect unless this is True.
+: Warning threshold per pixel.
Default value
-: ``True``
+: ``0.0``
-#### <a name='corrections.strixel.enable'></a>Enable (`corrections.strixel.enable`)
+#### <a name='addons.SaturationMonitor.alarmMaxCount'></a>`addons.SaturationMonitor.alarmMaxCount`
Type
-: BOOL
+: UINT64
Access mode
: RECONFIGURABLE
@@ -845,14 +802,14 @@ Assignment
: OPTIONAL
Description
-: Controls whether to apply this correction step for main data output - subject to availability.
+: Maximum number of pixel above alarm threshold.
Default value
-: ``False``
+: ``0``
-#### <a name='corrections.strixel.preview'></a>Preview (`corrections.strixel.preview`)
+#### <a name='addons.SaturationMonitor.warnMaxCount'></a>`addons.SaturationMonitor.warnMaxCount`
Type
-: BOOL
+: UINT64
Access mode
: RECONFIGURABLE
@@ -861,14 +818,14 @@ Assignment
: OPTIONAL
Description
-: Whether to apply this correction step for corrected preview output - subject to availability.
+: Maximum number of pixel above warn threshold.
Default value
-: ``False``
+: ``0``
-#### <a name='corrections.strixel.type'></a>`corrections.strixel.type`
+#### <a name='addons.SaturationMonitor.frameAxis'></a>Multi-frame axis (`addons.SaturationMonitor.frameAxis`)
Type
-: STRING
+: UINT64
Access mode
: RECONFIGURABLE
@@ -877,17 +834,17 @@ Assignment
: OPTIONAL
Description
-: Which kind of strixel layout is used for this module? cols_A0123 is the first strixel layout deployed at HED and rows_A1256 is the one later deployed at SCS.
-
-Options:
-: ``cols_A0123(HED-type)``, ``rows_A1256(SCS-type)``
+: Axis for frames. Used to take the max over this axis.
Default value
-: ``cols_A0123(HED-type)``
+: ``0``
-## <a name='warningLamps'></a>`warningLamps`
+#### `addons.SaturationMonitor.output`
+OutputChannel
+## Constant retrieval parameters (`constantParameters`)
+### <a name='constantParameters.secretsFile'></a>CalCat secrets file (`constantParameters.secretsFile`)
Type
-: VECTOR_STRING
+: STRING
Access mode
: INITONLY
@@ -895,14 +852,12 @@ Access mode
Assignment
: OPTIONAL
-Default value
-: ``['deviceInternalsState', 'inputDataState', 'processingState']``
+Description
+: Path to JSON file specifying secret token and some parameters used to access CalCat.
-## <a name='addons'></a>Addons (`addons`)
-Access mode
-: RECONFIGURABLE
+Default value
+: ``~/.calcat-secrets.json``
-## Constant retrieval parameters (`constantParameters`)
### <a name='constantParameters.deviceMappingSnapshotAt'></a>Snapshot timestamp (for device mapping) (`constantParameters.deviceMappingSnapshotAt`)
Type
: STRING
diff --git a/docs/schemas/JungfrauGeometry.md b/docs/schemas/JungfrauGeometry.md
index df5d9733..535ca790 100644
--- a/docs/schemas/JungfrauGeometry.md
+++ b/docs/schemas/JungfrauGeometry.md
@@ -1,5 +1,18 @@
# JungfrauGeometry
Base classes: ManualOrientableModuleListGeometryBase
+## <a name='availableScenes'></a>Available scenes (`availableScenes`)
+Type
+: VECTOR_STRING
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``['overview']``
+
## Geometry preview (`geometryPreview`)
ImageData
## Geometry file (`geometryFile`)
@@ -93,6 +106,22 @@ Description
Default value
: ``True``
+### <a name='geometryFile.updateMotorReferenceOnLoad'></a>Update motor reference (`geometryFile.updateMotorReferenceOnLoad`)
+Type
+: BOOL
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Description
+: If this flag is on, the motor reference positions in the motor tracking node will be updated according to the motor positions stored in geometry file.
+
+Default value
+: ``True``
+
### <a name='geometryFile.loadFromFile'></a>Load from file (`geometryFile.loadFromFile`)
Type
: Slot
@@ -223,7 +252,7 @@ Type
: Slot
Allowed in states
-: ACTIVE
+: ACTIVE, MONITORING
Access mode
: RECONFIGURABLE
@@ -239,7 +268,7 @@ Type
: Slot
Allowed in states
-: ACTIVE
+: ACTIVE, MONITORING
Access mode
: RECONFIGURABLE
diff --git a/docs/schemas/Lpd1MGeometry.md b/docs/schemas/Lpd1MGeometry.md
index 906b7a1c..b9ce4039 100644
--- a/docs/schemas/Lpd1MGeometry.md
+++ b/docs/schemas/Lpd1MGeometry.md
@@ -1,5 +1,18 @@
# Lpd1MGeometry
Base classes: ManualQuadrantsGeometryBase
+## <a name='availableScenes'></a>Available scenes (`availableScenes`)
+Type
+: VECTOR_STRING
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``['overview']``
+
## Geometry preview (`geometryPreview`)
ImageData
## Geometry file (`geometryFile`)
@@ -93,6 +106,22 @@ Description
Default value
: ``True``
+### <a name='geometryFile.updateMotorReferenceOnLoad'></a>Update motor reference (`geometryFile.updateMotorReferenceOnLoad`)
+Type
+: BOOL
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Description
+: If this flag is on, the motor reference positions in the motor tracking node will be updated according to the motor positions stored in geometry file.
+
+Default value
+: ``True``
+
### <a name='geometryFile.loadFromFile'></a>Load from file (`geometryFile.loadFromFile`)
Type
: Slot
@@ -223,7 +252,7 @@ Type
: Slot
Allowed in states
-: ACTIVE
+: ACTIVE, MONITORING
Access mode
: RECONFIGURABLE
@@ -239,7 +268,7 @@ Type
: Slot
Allowed in states
-: ACTIVE
+: ACTIVE, MONITORING
Access mode
: RECONFIGURABLE
diff --git a/docs/schemas/LpdCorrection.md b/docs/schemas/LpdCorrection.md
index 0886d4f5..525b52d5 100644
--- a/docs/schemas/LpdCorrection.md
+++ b/docs/schemas/LpdCorrection.md
@@ -1,357 +1,5 @@
# LpdCorrection
-Base classes: BaseCorrection
-## `dataInput`
-InputChannel
-## <a name='fastSources'></a>Fast data sources (`fastSources`)
-Type
-: VECTOR_STRING
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Sources to get data from. Only incoming hashes from these sources will be processed. This will typically be a single entry of the form: '[instrument]_DET_[detector]/DET/[channel]:xtdf'.
-
-Default value
-: ``[]``
-
-## <a name='outputShmemBufferSize'></a>Output buffer size limit (`outputShmemBufferSize`)
-Type
-: UINT32
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Corrected trains are written to shared memory locations. These are pre-allocated and re-used (circular buffer). This parameter determines how much memory to set aside for that buffer.
-
-Default value
-: ``10``
-
-## <a name='useShmemHandles'></a>Use shared memory (`useShmemHandles`)
-Type
-: BOOL
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: If enabled, shared memory handles will be used to avoid copying the main image data on the dataOutput channel. Note that these handles need to be dereferenced (typically by a group matcher) to get the data off the correction node.
-
-Default value
-: ``True``
-
-## <a name='useInfiniband'></a>Use infiniband (`useInfiniband`)
-Type
-: BOOL
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: If enabled, device will during initialization try to bind its main data output channel (dataOutput) to its node's infiniband interface. Default interface is used if no 'ib0' interface is found.
-
-Default value
-: ``True``
-
-## <a name='geometryDevice'></a>Geometry device (`geometryDevice`)
-Type
-: STRING
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: empty string
-
-## Data format (in/out) (`dataFormat`)
-### <a name='dataFormat.outputImageDtype'></a>Output image data dtype (`dataFormat.outputImageDtype`)
-Type
-: STRING
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: The (numpy) dtype to use for outgoing image data. Input is cast to float32, corrections are applied, and only then will the result be cast to outputImageDtype. Be aware that casting to integer type causes truncation rather than rounding.
-
-Options:
-: ``float32``, ``float16``, ``uint16``
-
-Default value
-: ``float32``
-
-### <a name='dataFormat.inputFrames'></a>Frames (`dataFormat.inputFrames`)
-Type
-: UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Number of frames on input
-
-Default value
-: ``0``
-
-### <a name='dataFormat.outputAxisOrder'></a>Output axis order (`dataFormat.outputAxisOrder`)
-Type
-: STRING
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: Axes of main data output can be reordered after correction. Axis order is specified as string consisting of 'f' (frames), 'ss' (slow scan), and 'ff' (fast scan). Anything but the default f-ss-fs order implies reordering - this axis ordering is based on the data we get from the detector (/ receiver), so how ss and fs maps to what may be considered x and y is detector-dependent.
-
-Options:
-: ``f-ss-fs``, ``f-fs-ss``, ``ss-f-fs``, ``ss-fs-f``, ``fs-f-ss``, ``fs-ss-f``
-
-Default value
-: ``f-ss-fs``
-
-### <a name='dataFormat.inputDataShape'></a>Input data shape (`dataFormat.inputDataShape`)
-Type
-: VECTOR_UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Image data shape in incoming data (from reader / DAQ). Updated based on latest train processed. Note that axis order may look wonky due to DAQ quirk.
-
-Default value
-: ``[]``
-
-### <a name='dataFormat.outputDataShape'></a>Output data shape (`dataFormat.outputDataShape`)
-Type
-: VECTOR_UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Image data shape for data output from this device. Takes into account axis reordering, if applicable. Even without that, workarounds.overrideInputAxisOrder likely makes this differ from dataFormat.inputDataShape.
-
-Default value
-: ``[]``
-
-### <a name='dataFormat.cellId'></a>`dataFormat.cellId`
-Type
-: VECTOR_UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``[]``
-
-### <a name='dataFormat.pulseId'></a>`dataFormat.pulseId`
-Type
-: VECTOR_UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``[]``
-
-## Workarounds (`workarounds`)
-### <a name='workarounds.overrideInputAxisOrder'></a>Override input axis order (`workarounds.overrideInputAxisOrder`)
-Type
-: BOOL
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: The shape of the image data ndarray as received from the DataAggregator is sometimes wrong - the axes are actually in a different order than the ndarray shape suggests. If this flag is on, the shape of the ndarray will be overridden with the axis order which was expected.
-
-Default value
-: ``True``
-
-### <a name='workarounds.trainFromFutureThreshold'></a>Spurious future train ID threshold (`workarounds.trainFromFutureThreshold`)
-Type
-: UINT64
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: Some detectors occasionally send a train with incorrect and much too large train ID. To avoid these 'future trains' from interfering with train matching, use this threshold to discard them immediately. If a train arrives with an ID which exceeds the current train ID from the time server by more than this threshold, the train is ignored.
-
-Default value
-: ``10000``
-
-## <a name='loadMostRecentConstants'></a>Load most recent constants (`loadMostRecentConstants`)
-Type
-: Slot
-
-Access mode
-: RECONFIGURABLE
-
-Description
-: Calling this slot will flush all constant buffers and cause the device to start querying CalCat for the most recent constants (all constants applicable for this device) available with the currently set constant parameters. This should typically be called after instantiating pipeline, after changing parameters, or after generating new constants.
-
-## <a name='inputDataState'></a>`inputDataState`
-Type
-: STRING
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``NORMAL``
-
-## <a name='deviceInternalsState'></a>`deviceInternalsState`
-Type
-: STRING
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``NORMAL``
-
-## <a name='processingState'></a>`processingState`
-Type
-: STRING
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``NORMAL``
-
-## <a name='trainId'></a>Train ID (`trainId`)
-Type
-: UINT64
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: ID of latest train processed by this device.
-
-Default value
-: ``0``
-
-## Performance measures (`performance`)
-### <a name='performance.processingTime'></a>Processing time (`performance.processingTime`)
-Type
-: DOUBLE
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``0.0``
-
-### <a name='performance.rate'></a>Rate (`performance.rate`)
-Type
-: DOUBLE
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Actual rate with which this device gets, processes, and sends trains. This is a simple windowed moving average.
-
-Default value
-: ``0.0``
-
-### <a name='performance.inputDelay'></a>Input delay (`performance.inputDelay`)
-Type
-: DOUBLE
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Measured delay on the input channel. Measured by difference between 'now' and timestamp in input metadata.
-
-Default value
-: ``0.0``
-
-### <a name='performance.ratioOfRecentTrainsReceived'></a>`performance.ratioOfRecentTrainsReceived`
-Type
-: DOUBLE
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Of the latest trains (from last received train, going back [some buffer range]), how many did we receive? This estimate is updated when new trains come in, so is unreliable if nothing is coming at all.
-
-Default value
-: ``0.0``
-
+Base classes: [BaseCorrection](BaseCorrection.md)
## `corrections`
### `corrections.offset`
#### <a name='corrections.offset.available'></a>Available (`corrections.offset.available`)
@@ -915,19 +563,6 @@ Assignment
Default value
: ``True``
-## <a name='warningLamps'></a>`warningLamps`
-Type
-: VECTOR_STRING
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``['deviceInternalsState', 'inputDataState', 'processingState']``
-
## Addons (`addons`)
### `addons.IntegratedIntensity`
#### <a name='addons.IntegratedIntensity.enable'></a>`addons.IntegratedIntensity.enable`
@@ -1234,7 +869,25 @@ Description
Default value
: ``0``
+#### `addons.SaturationMonitor.output`
+OutputChannel
## Constant retrieval parameters (`constantParameters`)
+### <a name='constantParameters.secretsFile'></a>CalCat secrets file (`constantParameters.secretsFile`)
+Type
+: STRING
+
+Access mode
+: INITONLY
+
+Assignment
+: OPTIONAL
+
+Description
+: Path to JSON file specifying secret token and some parameters used to access CalCat.
+
+Default value
+: ``~/.calcat-secrets.json``
+
### <a name='constantParameters.deviceMappingSnapshotAt'></a>Snapshot timestamp (for device mapping) (`constantParameters.deviceMappingSnapshotAt`)
Type
: STRING
diff --git a/docs/schemas/LpdminiGeometry.md b/docs/schemas/LpdminiGeometry.md
index 071829fb..31b0329b 100644
--- a/docs/schemas/LpdminiGeometry.md
+++ b/docs/schemas/LpdminiGeometry.md
@@ -1,5 +1,18 @@
# LpdminiGeometry
Base classes: ManualRotatableModuleListGeometryBase
+## <a name='availableScenes'></a>Available scenes (`availableScenes`)
+Type
+: VECTOR_STRING
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``['overview']``
+
## Geometry preview (`geometryPreview`)
ImageData
## Geometry file (`geometryFile`)
@@ -93,6 +106,22 @@ Description
Default value
: ``True``
+### <a name='geometryFile.updateMotorReferenceOnLoad'></a>Update motor reference (`geometryFile.updateMotorReferenceOnLoad`)
+Type
+: BOOL
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Description
+: If this flag is on, the motor reference positions in the motor tracking node will be updated according to the motor positions stored in geometry file.
+
+Default value
+: ``True``
+
### <a name='geometryFile.loadFromFile'></a>Load from file (`geometryFile.loadFromFile`)
Type
: Slot
@@ -223,7 +252,7 @@ Type
: Slot
Allowed in states
-: ACTIVE
+: ACTIVE, MONITORING
Access mode
: RECONFIGURABLE
@@ -239,7 +268,7 @@ Type
: Slot
Allowed in states
-: ACTIVE
+: ACTIVE, MONITORING
Access mode
: RECONFIGURABLE
diff --git a/docs/schemas/PnccdGeometry.md b/docs/schemas/PnccdGeometry.md
index f29c1a20..ddd7d739 100644
--- a/docs/schemas/PnccdGeometry.md
+++ b/docs/schemas/PnccdGeometry.md
@@ -1,5 +1,18 @@
# PnccdGeometry
Base classes: ManualRelativePositionsGeometryBase
+## <a name='availableScenes'></a>Available scenes (`availableScenes`)
+Type
+: VECTOR_STRING
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``['overview']``
+
## Geometry preview (`geometryPreview`)
ImageData
## Geometry file (`geometryFile`)
@@ -93,6 +106,22 @@ Description
Default value
: ``True``
+### <a name='geometryFile.updateMotorReferenceOnLoad'></a>Update motor reference (`geometryFile.updateMotorReferenceOnLoad`)
+Type
+: BOOL
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Description
+: If this flag is on, the motor reference positions in the motor tracking node will be updated according to the motor positions stored in geometry file.
+
+Default value
+: ``True``
+
### <a name='geometryFile.loadFromFile'></a>Load from file (`geometryFile.loadFromFile`)
Type
: Slot
@@ -223,7 +252,7 @@ Type
: Slot
Allowed in states
-: ACTIVE
+: ACTIVE, MONITORING
Access mode
: RECONFIGURABLE
@@ -239,7 +268,7 @@ Type
: Slot
Allowed in states
-: ACTIVE
+: ACTIVE, MONITORING
Access mode
: RECONFIGURABLE
diff --git a/docs/schemas/ShmemTrainMatcher.md b/docs/schemas/ShmemTrainMatcher.md
index 1e8be054..6ecd9bd4 100644
--- a/docs/schemas/ShmemTrainMatcher.md
+++ b/docs/schemas/ShmemTrainMatcher.md
@@ -1,447 +1,5 @@
# ShmemTrainMatcher
-Base classes: TrainMatcher
-## <a name='zmqConfig'></a>ZeroMQ configuration (`zmqConfig`)
-Type
-: VECTOR_HASH
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Configuration for ZeroMQ sockets
-
-Table schema and default value
-:
-| Pattern | HWM | RCVTIMEO | SNDTIMEO | Protocol | Use Infiniband | ZMQ publisher port |
-| - | - | - | - | - | - | - |
-
-
-## <a name='zmqOutputs'></a>ZeroMQ outputs (`zmqOutputs`)
-Type
-: VECTOR_HASH
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: List of ZeroMQ outputs
-
-Table schema and default value
-:
-| Sent | Pattern | Address |
-| - | - | - |
-
-
-## <a name='pollTimeout'></a>ZeroMQ Poller Timeout (`pollTimeout`)
-Type
-: UINT32
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: ZMQ poller timeout in milliseconds.
-
-Default value
-: ``0``
-
-## <a name='onSlowness'></a>On Slowness (`onSlowness`)
-Type
-: STRING
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Policy for what to do if this input is too slow for the fed data rate
-
-Options:
-: ``drop``, ``wait``, ``queue``, ``queueDrop``
-
-Default value
-: ``queueDrop``
-
-## <a name='maxQueueLength'></a>Max. Queue Length (`maxQueueLength`)
-Type
-: UINT32
-
-Access mode
-: INITONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Maximum number of data items to be queued by connected Output Channels (only queue and queueDrop policies)
-
-Default value
-: ``5``
-
-## <a name='sources'></a>Data Sources (`sources`)
-Type
-: VECTOR_HASH
-
-Allowed in states
-: PASSIVE, ACTIVE
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: list data sources to monitor. To add a data source: Right click on the table below and select "add row below" Device properties: Write in the "source" column the device name and the property name joined with a dot, e.g. "SA2_XTD1_XGM/XGM/DOOCS.pulseEnergy.photonFlux" Device output channels: Write in the "Source" column the name of the channel you want to monitor, e.g. HED_XTD6_IMGPI/CAM/BEAMVIEW:output. If the device name and the data source name do not match, for example if you monitor data directly from the output of the DAQ or the calibration pipeline, use the following syntax: SOURCE_NAME@DEVICE:output. For example SQS_DIGITIZER_UTC2/ADC/1:network@SQS_DAQ_DATA/DIGI/2:output
-
-Table schema and default value
-:
-| Select | Source | Offset (#trains) | Status |
-| - | - | - | - |
-
-
-## <a name='stats'></a>Sources Stats (`stats`)
-Type
-: VECTOR_HASH
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Statistics on input data
-
-Table schema and default value
-:
-| Source | Update Rate (Hz) | Received | Train ID | Latency (#trains) |
-| - | - | - | - | - |
-
-
-## <a name='sortSources'></a>Sort Sources (`sortSources`)
-Type
-: BOOL
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Sort sources by name after reconfiguration.
-
-Default value
-: ``False``
-
-## <a name='mode'></a>Mode (`mode`)
-Type
-: STRING
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: operation mode, "match": match all selected fast sources, "buffer": send (selected) sources that arrived when the buffer reached the limit
-
-Options:
-: ``match``, ``buffer``
-
-Default value
-: ``match``
-
-## <a name='start'></a>Start (`start`)
-Type
-: Slot
-
-Allowed in states
-: PASSIVE
-
-Access mode
-: RECONFIGURABLE
-
-## <a name='stop'></a>Stop (`stop`)
-Type
-: Slot
-
-Allowed in states
-: ACTIVE
-
-Access mode
-: RECONFIGURABLE
-
-## <a name='processing'></a>Pipeline Update (`processing`)
-Type
-: FLOAT
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``0.0``
-
-## <a name='uniqueProcessing'></a>Unique Trains Update (`uniqueProcessing`)
-Type
-: FLOAT
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``0.0``
-
-## <a name='ctrlProcessing'></a>Control Update (`ctrlProcessing`)
-Type
-: FLOAT
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``0.0``
-
-## <a name='outProcessing'></a>Output Update (`outProcessing`)
-Type
-: FLOAT
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``0.0``
-
-## <a name='received'></a>Received (`received`)
-Type
-: UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Count how many trains were received.
-
-Default value
-: ``0``
-
-## <a name='sent'></a>Sent (`sent`)
-Type
-: UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Count how many train data were sent.
-
-Default value
-: ``0``
-
-## <a name='matchingRatio'></a>Matching Ratio (`matchingRatio`)
-Type
-: FLOAT
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Exponential moving average of the relation between sent and received trains.
-
-Default value
-: ``0.0``
-
-## <a name='trainId'></a>Train ID (`trainId`)
-Type
-: UINT64
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: the last trainId data that matched.
-
-Default value
-: ``0``
-
-## <a name='buffer'></a>Buffer size (`buffer`)
-Type
-: UINT32
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: Maximum length of buffer to hold non-matched trains
-
-Default value
-: ``100``
-
-## <a name='buffered'></a>Buffered trains (`buffered`)
-Type
-: UINT32
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: Number of trains buffered
-
-Default value
-: ``0``
-
-## <a name='delay'></a>Delay on output (`delay`)
-Type
-: UINT32
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: delay applied before sending data on output channel (in #trains)
-
-Default value
-: ``0``
-
-## <a name='offsetTid'></a>Overwrite Offset TrainIds (`offsetTid`)
-Type
-: BOOL
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: If this property is True, it will change the value of the trainId in the metadata for the data sources having an offset value different from 0. The trainId value assigned will be the "matched" trainId, i.e. trainId + offset for the source.
-
-Default value
-: ``False``
-
-## <a name='maxIdle'></a>Max Idle (`maxIdle`)
-Type
-: FLOAT
-
-Access mode
-: RECONFIGURABLE
-
-Assignment
-: OPTIONAL
-
-Description
-: Max allowed IDLE time (in seconds). If last data received for a source is longer than this time, the source will be ignore when matching data.
-
-Default value
-: ``0.0``
-
-## <a name='idleChannels'></a>Idle Channels (`idleChannels`)
-Type
-: VECTOR_STRING
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: List of idle channels
-
-Default value
-: ``[]``
-
-## <a name='idleState'></a>Idle State (`idleState`)
-Type
-: STRING
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Description
-: State of idle channels (error if any monitored channel is idle)
-
-Default value
-: ``ON``
-
-## <a name='cpuPercent'></a>CPU usage (`cpuPercent`)
-Type
-: FLOAT
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``0.0``
-
-## <a name='memUsage'></a>MEM usage (`memUsage`)
-Type
-: FLOAT
-
-Access mode
-: READONLY
-
-Assignment
-: OPTIONAL
-
-Default value
-: ``0.0``
-
-## Pipeline Output (`output`)
-Output channel for combined train data.
-OutputChannel
+Base classes: [TrainMatcher](TrainMatcher.md)
## <a name='enableKaraboOutput'></a>Enable Karabo channel (`enableKaraboOutput`)
Type
: BOOL
diff --git a/docs/schemas/TrainMatcher.md b/docs/schemas/TrainMatcher.md
new file mode 100644
index 00000000..2330ae84
--- /dev/null
+++ b/docs/schemas/TrainMatcher.md
@@ -0,0 +1,403 @@
+# TrainMatcher
+Base classes: ZeroMQOutput
+## <a name='availableScenes'></a>`availableScenes`
+Type
+: VECTOR_STRING
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``['scene']``
+
+## <a name='onSlowness'></a>On Slowness (`onSlowness`)
+Type
+: STRING
+
+Access mode
+: INITONLY
+
+Assignment
+: OPTIONAL
+
+Description
+: Policy for what to do if this input is too slow for the fed data rate
+
+Options:
+: ``drop``, ``wait``, ``queue``, ``queueDrop``
+
+Default value
+: ``queueDrop``
+
+## <a name='maxQueueLength'></a>Max. Queue Length (`maxQueueLength`)
+Type
+: UINT32
+
+Access mode
+: INITONLY
+
+Assignment
+: OPTIONAL
+
+Description
+: Maximum number of data items to be queued by connected Output Channels (only queue and queueDrop policies)
+
+Default value
+: ``5``
+
+## <a name='sources'></a>Data Sources (`sources`)
+Type
+: VECTOR_HASH
+
+Allowed in states
+: PASSIVE, ACTIVE
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Description
+: list data sources to monitor. To add a data source: Right click on the table below and select "add row below" Device properties: Write in the "source" column the device name and the property name joined with a dot, e.g. "SA2_XTD1_XGM/XGM/DOOCS.pulseEnergy.photonFlux" Device output channels: Write in the "Source" column the name of the channel you want to monitor, e.g. HED_XTD6_IMGPI/CAM/BEAMVIEW:output. If the device name and the data source name do not match, for example if you monitor data directly from the output of the DAQ or the calibration pipeline, use the following syntax: SOURCE_NAME@DEVICE:output. For example SQS_DIGITIZER_UTC2/ADC/1:network@SQS_DAQ_DATA/DIGI/2:output
+
+Table schema and default value
+:
+| Select | Source | Offset (#trains) | Status |
+| - | - | - | - |
+
+
+## <a name='stats'></a>Sources Stats (`stats`)
+Type
+: VECTOR_HASH
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Description
+: Statistics on input data
+
+Table schema and default value
+:
+| Source | Update Rate (Hz) | Received | Train ID | Latency (#trains) |
+| - | - | - | - | - |
+
+
+## <a name='sortSources'></a>Sort Sources (`sortSources`)
+Type
+: BOOL
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Description
+: Sort sources by name after reconfiguration.
+
+Default value
+: ``True``
+
+## <a name='mode'></a>Mode (`mode`)
+Type
+: STRING
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Description
+: operation mode, "match": match all selected fast sources, "buffer": send (selected) sources that arrived when the buffer reached the limit
+
+Options:
+: ``match``, ``buffer``
+
+Default value
+: ``match``
+
+## <a name='start'></a>Start (`start`)
+Type
+: Slot
+
+Allowed in states
+: PASSIVE
+
+Access mode
+: RECONFIGURABLE
+
+## <a name='stop'></a>Stop (`stop`)
+Type
+: Slot
+
+Allowed in states
+: ACTIVE
+
+Access mode
+: RECONFIGURABLE
+
+## <a name='processing'></a>Pipeline Update (`processing`)
+Type
+: FLOAT
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+## <a name='uniqueProcessing'></a>Unique Trains Update (`uniqueProcessing`)
+Type
+: FLOAT
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+## <a name='ctrlProcessing'></a>Control Update (`ctrlProcessing`)
+Type
+: FLOAT
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+## <a name='outProcessing'></a>Output Update (`outProcessing`)
+Type
+: FLOAT
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+## <a name='received'></a>Received (`received`)
+Type
+: UINT32
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Description
+: Count how many trains were received.
+
+Default value
+: ``0``
+
+## <a name='sent'></a>Sent (`sent`)
+Type
+: UINT32
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Description
+: Count how many train data were sent.
+
+Default value
+: ``0``
+
+## <a name='matchingRatio'></a>Matching Ratio (`matchingRatio`)
+Type
+: FLOAT
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Description
+: Exponential moving average of the relation between sent and received trains.
+
+Default value
+: ``0.0``
+
+## <a name='trainId'></a>Train ID (`trainId`)
+Type
+: UINT64
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Description
+: the last trainId data that matched.
+
+Default value
+: ``0``
+
+## <a name='buffer'></a>Buffer size (`buffer`)
+Type
+: UINT32
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Description
+: Maximum length of buffer to hold non-matched trains
+
+Default value
+: ``100``
+
+## <a name='buffered'></a>Buffered trains (`buffered`)
+Type
+: UINT32
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Description
+: Number of trains buffered
+
+Default value
+: ``0``
+
+## <a name='delay'></a>Delay on output (`delay`)
+Type
+: UINT32
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Description
+: delay applied before sending data on output channel (in #trains)
+
+Default value
+: ``0``
+
+## <a name='offsetTid'></a>Overwrite Offset TrainIds (`offsetTid`)
+Type
+: BOOL
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Description
+: If this property is True, it will change the value of the trainId in the metadata for the data sources having an offset value different from 0. The trainId value assigned will be the "matched" trainId, i.e. trainId + offset for the source.
+
+Default value
+: ``False``
+
+## <a name='maxIdle'></a>Max Idle (`maxIdle`)
+Type
+: FLOAT
+
+Access mode
+: RECONFIGURABLE
+
+Assignment
+: OPTIONAL
+
+Description
+: Max allowed IDLE time (in seconds). If last data received for a source is longer than this time, the source will be ignore when matching data.
+
+Default value
+: ``0.0``
+
+## <a name='idleChannels'></a>Idle Channels (`idleChannels`)
+Type
+: VECTOR_STRING
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Description
+: List of idle channels
+
+Default value
+: ``[]``
+
+## <a name='idleState'></a>Idle State (`idleState`)
+Type
+: STRING
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Description
+: State of idle channels (error if any monitored channel is idle)
+
+Default value
+: ``ON``
+
+## <a name='cpuPercent'></a>CPU usage (`cpuPercent`)
+Type
+: FLOAT
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+## <a name='memUsage'></a>MEM usage (`memUsage`)
+Type
+: FLOAT
+
+Access mode
+: READONLY
+
+Assignment
+: OPTIONAL
+
+Default value
+: ``0.0``
+
+## Pipeline Output (`output`)
+Output channel for combined train data.
+OutputChannel
\ No newline at end of file
diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md
index 8495c69a..f00f55f2 100644
--- a/docs/troubleshooting.md
+++ b/docs/troubleshooting.md
@@ -2,27 +2,30 @@
## Preview does not update
1. Is detector not sending data?
- - Open the device scene for individual [correction devices](devices.md#correction-devices) and look at the device status log.
+ - Open the device scene for individual [correction devices](devices/correction-devices.md) and look at the device status log.
If the DAQ is monitoring, but without detector data coming in, log should show warnings about this.
2. Is pipeline down or misconfigured?
- - Are the [preview assemblers](devices.md#preview-assemblers) receiving data?
- - Is the [geometry device](devices.md#geometry-devices) up?
+ - Are the [preview assemblers](devices/matchers.md#preview-assemblers) receiving data?
+ - Is the [geometry device](devices/middlelayer.md#geometry-devices) up?
- Are you simply seeing data from a different memory cell than you expect?
- Check [which frames is previewed](devices.md#assembled-preview-configuration) and maybe bad pixel masking.
+ Check [which frame is selected for preview](concepts.md#preview) and maybe bad pixel masking.
3. Is preview matching breaking down?
- Note: for fast detectors, data transfer from DAQ to correction device can hit [network limits](data-rates.md).
This can cause **matching** on preview assemblers to break down.
If rates on correction devices fluctuate or are lower than expected, try increasing the **DAQ train stride** to slow down incoming data.
-
+4. Does the preview assembler have a geometry?
+ - The [preview assemblers](devices/matchers.md#preview-assemblers) need a detector geometry in order to assemble previews
+ - There can sometimes be timing issues about sending the geometry from the geometry device - or the geometry device may be down entirely
+ - Try looking in the project / topology for `[detector prefix]/CAL/GEOMETRY`, select it and hit "Send geometry"
+ - We're working to make it more obvious when a geometry is missing (and eliminate this issue in most cases)
## Preview is flickering
-A common reason for flickering is the [max idle](trainmatcher.md#max-idle) parameter on the [preview assemblers](devices.md#preview-assemblers) being set too close to `10/DAQ train stride`.
+A common reason for flickering is the [max idle](devices/trainmatcher.md#max-idle) parameter on the [preview assemblers](devices/matchers.md#preview-assemblers) being set too close to `10/DAQ train stride`.
If for instance max idle is 1 second and DAQ train stride is 10, then every detector source is expected to provide data to the assembler exactly once per second - and even slight jitter will cause the last sources for a given train to show up too late, thus ignoring them for matching.
-
## Analysis software not receiving data
-Assumption: you are running some online analysis suite connected to the [Karabo Bridge](https://rtd.xfel.eu/docs/data-analysis-user-documentation/en/latest/karabo_bridge/protocol.html) output of a [group matcher](devices.md#group-matcher) or a [full matcher](devices.md#full-matcher).
-These matchers (along with the preview assemblers) are built around [TrainMatchers](trainmatcher.md).
+Assumption: you are running some online analysis suite connected to the [Karabo Bridge](https://rtd.xfel.eu/docs/data-analysis-user-documentation/en/latest/karabo_bridge/protocol.html) output of a [group matcher](devices/matchers.md#group-matcher) or a [full matcher](devices/matchers.md#full-matcher).
+These matchers (along with the preview assemblers) are built around [TrainMatchers](devices/trainmatcher.md).
1. Is the matcher not receiving data?
- Check that detector is sending data and calibration pipeline generally works.
@@ -37,10 +40,9 @@ These matchers (along with the preview assemblers) are built around [TrainMatche
- To gracefully handle situations where individual modules disappear / stop working for a while, consider using the **Max Idle** TrainMatcher feature on the matcher.
In case of a full detector matcher, do check what this is set to for the individual group matchers, too.
-
## Cannot load new constants
Assumption: you have just generated new constants (for example, dark constants via typical procedure) and the pipeline doesn't seem to load them.
-While investigating, it is recommended to open the "correction constant overview" found on the [calibration manager overview](devices.md#calibration-manager) to see the timestamps for currently found constants.
+While investigating, it is recommended to open the "correction constant overview" found on the [calibration manager overview](devices/middlelayer.md#calibration-manager) to see the timestamps for currently found constants.
If constants are found - individual device logs do not complain about "condition not found" / "calibration not found", but get stuck actually loading - see [loading new constants is very slow](#loading-new-constants-is-very-slow).
1. Are the operating conditions used for constant querying correct?
@@ -50,12 +52,11 @@ If constants are found - individual device logs do not complain about "condition
- Can you find the expected [calibration constant versions](https://in.xfel.eu/calibration/calibration_constant_versions) in CalCat?
- Does the calibration report from constant generation mention any issues?
3. If constants are there, do the device logs indicate some issue when loading them?
- - Inspect the device status log on the scene for one of the [correction devices](devices.md#correction-devices); it may give some information about the issue at hand.
+ - Inspect the device status log on the scene for one of the [correction devices](devices/correction-devices.md); it may give some information about the issue at hand.
- (For experts) Check the values on a specific correction device under "Constants retrieved" - "(correction name)".
During querying, the fields tagged with `[CalCat]` should fill out; if you have sufficient [CalCat](https://in.xfel.eu/calibration/) rights, you can cross-reference the found condition / constant (version) IDs.
4. Try restarting the pipeline to clear out all caches.
-
## Loading new constants is very slow
Due to file system and caching circumstances, constants which have just been generated typically take longer to load for the first time than older constants.
This was for a time exacerbated by an issue with the filesystem configuration.
@@ -63,6 +64,6 @@ Said issue has been resolved and newer `calng` devices additionally include miti
- The loading process may hang for a while and can fail
- Mitigation: file loading done by correction device in separate process to protect device process from stalling
- - If a device is taking a long time loading new constants, the [status log](devices.md#correction-devices) should give some information about current progress
+ - If a device is taking a long time loading new constants, the [status log](devices/correction-devices.md) should give some information about current progress
- Loading many files at once may exacerbate issues
- Instead of using the "load most recent constants" slot on the manager, consider using macro to stagger loading for different devices
--
GitLab