From c3a72c6d329a33c9273cbfe04ec554ef95bb4c83 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 23 Sep 2021 14:40:59 +0200
Subject: [PATCH] Working configurability of bad pixel flags,
md_additional_offset
---
src/calng/AgipdCorrection.py | 91 +++++++++++++++++++++++------------
src/calng/agipd_gpu.py | 93 ++++++++++++++++++++++--------------
src/calng/base_correction.py | 1 -
3 files changed, 119 insertions(+), 66 deletions(-)
diff --git a/src/calng/AgipdCorrection.py b/src/calng/AgipdCorrection.py
index dec34dd3..451cd011 100644
--- a/src/calng/AgipdCorrection.py
+++ b/src/calng/AgipdCorrection.py
@@ -3,10 +3,10 @@ import timeit
import numpy as np
from karabo.bound import (
BOOL_ELEMENT,
+ NODE_ELEMENT,
FLOAT_ELEMENT,
KARABO_CLASSINFO,
STRING_ELEMENT,
- TABLE_ELEMENT,
UINT32_ELEMENT,
Hash,
Schema,
@@ -28,7 +28,7 @@ class AgipdCorrection(BaseCorrection):
("offset", CorrectionFlags.OFFSET),
("relGainPc", CorrectionFlags.REL_GAIN_PC),
("relGainXray", CorrectionFlags.REL_GAIN_XRAY),
- ("badpixels", CorrectionFlags.BPMASK),
+ ("badPixels", CorrectionFlags.BPMASK),
)
_gpu_runner_class = AgipdGpuRunner
@@ -121,29 +121,30 @@ class AgipdCorrection(BaseCorrection):
.commit()
)
# TODO: hook this up to actual correction done
- bad_pixel_selection_schema = Schema()
+ # NOTE: wanted to configure this as table, could not make reaonly table with reconfigurable bools in rows
(
- STRING_ELEMENT(bad_pixel_selection_schema).key("name").readOnly().commit(),
- UINT32_ELEMENT(bad_pixel_selection_schema).key("value").readOnly().commit(),
- BOOL_ELEMENT(bad_pixel_selection_schema)
- .key("apply")
- .assignmentOptional()
- .defaultValue(True)
- .reconfigurable()
- .commit(),
- TABLE_ELEMENT(expected)
- .key("corrections.badPixelFlagsApplied")
- .setNodeSchema(bad_pixel_selection_schema)
- .assignmentOptional()
- .defaultValue(
- [
- Hash("name", field.name, "value", field.value, "apply", True)
- for field in BadPixelValues
- ]
+ NODE_ELEMENT(expected)
+ .key("corrections.badPixelFlagsToUse")
+ .displayedName("Bad pixel flags to use")
+ .description(
+ "The booleans under this node allow for selecting a subset of bad "
+ "pixel values to take into account when doing bad pixel masking. "
+ "Upon updating these flags, the map used for bad pixel masking will "
+ "be ANDed with this selection. TEMPORARY NOTE: if you want to toggle "
+ "a disabled flag back on, please reload constants for this to take "
+ "effect (will be triggered automatically in future version)."
)
- .reconfigurable()
.commit(),
)
+ for field in BadPixelValues:
+ (
+ BOOL_ELEMENT(expected)
+ .key(f"corrections.badPixelFlagsToUse.{field.name}")
+ .assignmentOptional()
+ .defaultValue(True)
+ .reconfigurable()
+ .commit()
+ )
@property
def input_data_shape(self):
@@ -177,12 +178,19 @@ class AgipdCorrection(BaseCorrection):
"no-reshape": None,
}[config.get("dataFormat.outputAxisOrder")]
self._update_shapes()
+
+ # configurability: overriding md_additional_offset
if config.get("corrections.overrideMdAdditionalOffset"):
self._override_md_additional_offset = config.get(
"corrections.mdAdditionalOffset"
)
else:
self._override_md_additional_offset = None
+
+ # configurability: disabling subset of bad pixel masking bits
+ self._has_updated_bad_pixel_selection = False
+ self._update_bad_pixel_selection()
+
self.updateState(State.ON)
def process_input(self, data, metadata):
@@ -307,12 +315,12 @@ class AgipdCorrection(BaseCorrection):
)
def _load_constant_to_gpu(self, constant_name, constant_data):
+ # TODO: encode correction / constant dependencies in a clever way
if constant_name == "ThresholdsDark":
if self.gain_mode is not AgipdGainMode.ADAPTIVE_GAIN:
self.log.INFO("Loaded ThresholdsDark ignored due to fixed gain mode")
return
self.gpu_runner.load_thresholds(constant_data)
- # TODO: encode correction / constant dependencies in a clever way
if not self.get("corrections.available.thresholding"):
self.set("corrections.available.thresholding", True)
self.set("corrections.enabled.thresholding", True)
@@ -340,12 +348,13 @@ class AgipdCorrection(BaseCorrection):
self.set("corrections.enabled.relGainXray", True)
self.set("corrections.preview.relGainXray", True)
elif "BadPixels" in constant_name:
- # TODO: implement loading bad pixels
- self.gpu_runner.load_badpixels_map(constant_data)
- if not self.get("corrections.available.badpixels"):
- self.set("corrections.available.badpixels", True)
- self.set("corrections.enabled.badpixels", True)
- self.set("corrections.preview.badpixels", True)
+ self.gpu_runner.load_bad_pixels_map(
+ constant_data, override_flags_to_use=self._override_bad_pixel_flags
+ )
+ if not self.get("corrections.available.badPixels"):
+ self.set("corrections.available.badPixels", True)
+ self.set("corrections.enabled.badPixels", True)
+ self.set("corrections.preview.badPixels", True)
def _update_pulse_filter(self, filter_string):
"""Called whenever the pulse filter changes, typically followed by
@@ -358,12 +367,34 @@ class AgipdCorrection(BaseCorrection):
assert np.max(new_filter) < self.get("dataFormat.memoryCells")
self.pulse_filter = new_filter
+ def _update_bad_pixel_selection(self):
+ selection = 0
+ for field in BadPixelValues:
+ if self.get(f"corrections.badPixelFlagsToUse.{field.name}"):
+ selection |= field
+ self._override_bad_pixel_flags = selection
+
+ def preReconfigure(self, config):
+ super().preReconfigure(config)
+ if any(
+ path.startswith("corrections.badPixelFlagsToUse")
+ for path in config.getPaths()
+ ):
+ self._has_updated_bad_pixel_selection = False
+
def postReconfigure(self):
super().postReconfigure()
+
if self.get("corrections.overrideMdAdditionalOffset"):
- self._override_md_additional_offset = self.get("corrections.mdAdditionalOffset")
- self.gpu_runner.md_additional_offset_gpu.fill(
+ self._override_md_additional_offset = self.get(
+ "corrections.mdAdditionalOffset"
+ )
+ self.gpu_runner.md_additional_offset_gpu.override_md_additional_offset(
self._override_md_additional_offset
)
else:
self._override_md_additional_offset = None
+
+ if not self._has_updated_bad_pixel_selection:
+ self._update_bad_pixel_selection()
+ self.gpu_runner.override_bad_pixel_flags_to_use(self._override_bad_pixel_flags)
diff --git a/src/calng/agipd_gpu.py b/src/calng/agipd_gpu.py
index b03ad918..808a9de3 100644
--- a/src/calng/agipd_gpu.py
+++ b/src/calng/agipd_gpu.py
@@ -34,16 +34,16 @@ class AgipdGpuRunner(base_gpu.BaseGpuRunner):
memory_cells,
constant_memory_cells,
output_transpose=(1, 2, 0), # default: memorycells-fast
- input_data_dtype=np.uint16,
- output_data_dtype=np.float32,
- badpixel_mask_value=np.float32(np.nan),
+ input_data_dtype=cupy.uint16,
+ output_data_dtype=cupy.float32,
+ bad_pixel_mask_value=cupy.float32(cupy.nan),
gain_mode=AgipdGainMode.ADAPTIVE_GAIN,
):
self.gain_mode = gain_mode
if self.gain_mode is AgipdGainMode.ADAPTIVE_GAIN:
- self.default_gain = np.uint8(gain_mode)
+ self.default_gain = cupy.uint8(gain_mode)
else:
- self.default_gain = np.uint8(gain_mode - 1)
+ self.default_gain = cupy.uint8(gain_mode - 1)
self.input_shape = (memory_cells, 2, pixels_x, pixels_y)
self.processed_shape = (memory_cells, pixels_x, pixels_y)
super().__init__(
@@ -55,22 +55,22 @@ class AgipdGpuRunner(base_gpu.BaseGpuRunner):
input_data_dtype,
output_data_dtype,
)
- self.gain_map_gpu = cupy.empty(self.processed_shape, dtype=np.uint8)
+ self.gain_map_gpu = cupy.empty(self.processed_shape, dtype=cupy.uint8)
self.map_shape = (self.constant_memory_cells, self.pixels_x, self.pixels_y)
self.gm_map_shape = self.map_shape + (3,) # for gain-mapped constants
self.threshold_map_shape = self.map_shape + (2,)
# constants
self.gain_thresholds_gpu = cupy.empty(
- self.threshold_map_shape, dtype=np.float32
+ self.threshold_map_shape, dtype=cupy.float32
)
- self.offset_map_gpu = cupy.zeros(self.gm_map_shape, dtype=np.float32)
- self.rel_gain_pc_map_gpu = cupy.ones(self.gm_map_shape, dtype=np.float32)
+ self.offset_map_gpu = cupy.zeros(self.gm_map_shape, dtype=cupy.float32)
+ self.rel_gain_pc_map_gpu = cupy.ones(self.gm_map_shape, dtype=cupy.float32)
# not gm_map_shape because it only applies to medium gain pixels
- self.md_additional_offset_gpu = cupy.zeros(self.map_shape, dtype=np.float32)
- self.rel_gain_xray_map_gpu = cupy.ones(self.map_shape, dtype=np.float32)
- self.badpixel_map_gpu = cupy.zeros(self.gm_map_shape, dtype=np.uint32)
- self.badpixel_mask_value = badpixel_mask_value # TODO: make this configurable
+ self.md_additional_offset_gpu = cupy.zeros(self.map_shape, dtype=cupy.float32)
+ self.rel_gain_xray_map_gpu = cupy.ones(self.map_shape, dtype=cupy.float32)
+ self.bad_pixel_map_gpu = cupy.zeros(self.gm_map_shape, dtype=cupy.uint32)
+ self.bad_pixel_mask_value = bad_pixel_mask_value # TODO: make this configurable
self.update_block_size((1, 1, 64))
@@ -93,7 +93,7 @@ class AgipdGpuRunner(base_gpu.BaseGpuRunner):
np.transpose(offset_map, (2, 1, 0, 3)).astype(np.float32)
)
- def load_rel_gain_pc_map(self, slopes_pc_map):
+ def load_rel_gain_pc_map(self, slopes_pc_map, override_md_additional_offset=None):
# pc has funny shape (11, 352, 128, 512) from file
# this is (fi, memory cell, y, x)
slopes_pc_map = slopes_pc_map.astype(np.float32)
@@ -120,18 +120,28 @@ class AgipdGpuRunner(base_gpu.BaseGpuRunner):
naughty_array[too_high_bool] = medians[too_high_cell]
frac_hg_mg = hg_slope / mg_slope
- md_additional_offset = (hg_intercept - mg_intercept * frac_hg_mg).astype(
- np.float32
- )
rel_gain_map = np.ones(
(3, self.constant_memory_cells, self.pixels_y, self.pixels_x),
dtype=np.float32,
)
rel_gain_map[1] = rel_gain_map[0] * frac_hg_mg
rel_gain_map[2] = rel_gain_map[1] * 4.48
- # TODO: enable overriding md_additional_offset_gpu based on user input
self.rel_gain_pc_map_gpu.set(np.transpose(rel_gain_map, (1, 3, 2, 0)))
- self.md_additional_offset_gpu.set(np.transpose(md_additional_offset, (0, 2, 1)))
+ if override_md_additional_offset is None:
+ md_additional_offset = (hg_intercept - mg_intercept * frac_hg_mg).astype(
+ np.float32
+ )
+ self.md_additional_offset_gpu.set(
+ np.transpose(md_additional_offset, (0, 2, 1))
+ )
+ else:
+ self.override_md_additional_offset(override_md_additional_offset)
+ print(
+ f"Rel gain pc min max: {self.rel_gain_pc_map_gpu.min()}, {self.rel_gain_pc_map_gpu.max()}"
+ )
+
+ def override_md_additional_offset(self, override_value):
+ self.md_additional_offset_gpu.fill(override_value)
def load_rel_gain_ff_map(self, slopes_ff_map):
# constant shape: y, x, memory cell
@@ -148,54 +158,66 @@ class AgipdGpuRunner(base_gpu.BaseGpuRunner):
# TODO: maybe clamp and replace NaNs like slopes_pc
self.rel_gain_xray_map_gpu.set(np.transpose(slopes_ff_map).astype(np.float32))
- def load_badpixels_map(self, badpixels_map):
- print(f"Loading bad pixels with shape: {badpixels_map.shape}")
+ def load_bad_pixels_map(self, bad_pixels_map, override_flags_to_use=None):
+ print(f"Loading bad pixels with shape: {bad_pixels_map.shape}")
# will simply OR with already loaded, does not take into account which ones
# TODO: allow configuring subset of bad pixels to care about
# TODO: allow configuring value for masked pixels
# TODO: inquire what "mask for double size pixels" means
- if len(badpixels_map.shape) == 3:
- if badpixels_map.shape == (
+ if len(bad_pixels_map.shape) == 3:
+ if bad_pixels_map.shape == (
self.pixels_y,
self.pixels_x,
self.constant_memory_cells,
):
# BadPixelsFF is not per gain stage - broadcasting along gain dimension
- self.badpixel_map_gpu |= cupy.asarray(
+ self.bad_pixel_map_gpu |= cupy.asarray(
np.broadcast_to(
- np.transpose(badpixels_map)[..., np.newaxis],
+ np.transpose(bad_pixels_map)[..., np.newaxis],
self.gm_map_shape,
),
dtype=np.uint32,
)
- elif badpixels_map.shape == (
+ elif bad_pixels_map.shape == (
self.constant_memory_cells,
self.pixels_y,
self.pixels_x,
):
# oh, can also be old bad pixels pc?
- self.badpixel_map_gpu |= cupy.asarray(
+ self.bad_pixel_map_gpu |= cupy.asarray(
np.broadcast_to(
- np.transpose(badpixels_map, (0, 2, 1))[..., np.newaxis],
+ np.transpose(bad_pixels_map, (0, 2, 1))[..., np.newaxis],
self.gm_map_shape,
),
dtype=np.uint32,
)
else:
raise ValueError(
- f"What in the world is this shape? {badpixels_map.shape}"
+ f"What in the world is this shape? {bad_pixels_map.shape}"
)
else:
- self.badpixel_map_gpu |= cupy.asarray(
- np.transpose(badpixels_map, (2, 1, 0, 3)), dtype=np.uint32
+ self.bad_pixel_map_gpu |= cupy.asarray(
+ np.transpose(bad_pixels_map, (2, 1, 0, 3)), dtype=np.uint32
)
+ if override_flags_to_use is not None:
+ self.override_bad_pixel_flags_to_use(override_flags_to_use)
+ print(
+ f"Percentage of bad pixels now: {cupy.count_nonzero(self.bad_pixel_map_gpu) / self.bad_pixel_map_gpu.size * 100:.02f}"
+ )
+
+ def override_bad_pixel_flags_to_use(self, override_value):
+ self.bad_pixel_map_gpu &= cupy.uint32(override_value)
+ print(
+ f"Percentage of bad pixels now: {cupy.count_nonzero(self.bad_pixel_map_gpu) / self.bad_pixel_map_gpu.size * 100:.02f}"
+ )
+
def flush_buffers(self):
self.offset_map_gpu.fill(0)
self.rel_gain_pc_map_gpu.fill(1)
self.md_additional_offset_gpu.fill(0)
self.rel_gain_xray_map_gpu.fill(1)
- self.badpixel_map_gpu.fill(0)
+ self.bad_pixel_map_gpu.fill(0)
# TODO: baseline shift
@@ -206,21 +228,22 @@ class AgipdGpuRunner(base_gpu.BaseGpuRunner):
flags & CorrectionFlags.THRESHOLD
):
raise ValueError("Cannot do gain thresholding in fixed gain mode")
+
self.correction_kernel(
self.full_grid,
self.full_block,
(
self.input_data_gpu,
self.cell_table_gpu,
- np.uint8(flags),
+ cupy.uint8(flags),
self.default_gain,
self.gain_thresholds_gpu,
self.offset_map_gpu,
self.rel_gain_pc_map_gpu,
self.md_additional_offset_gpu,
self.rel_gain_xray_map_gpu,
- self.badpixel_map_gpu,
- self.badpixel_mask_value,
+ self.bad_pixel_map_gpu,
+ self.bad_pixel_mask_value,
self.gain_map_gpu,
self.processed_data_gpu,
),
diff --git a/src/calng/base_correction.py b/src/calng/base_correction.py
index 15ed974d..117ebb86 100644
--- a/src/calng/base_correction.py
+++ b/src/calng/base_correction.py
@@ -570,7 +570,6 @@ class BaseCorrection(calibrationBase.CalibrationReceiverBaseDevice):
def _update_shapes(self):
"""(Re)initialize buffers according to expected data shapes"""
self.log.INFO("Updating shapes")
-
# reflect the axis reordering in the expected output shape
self.set("dataFormat.inputDataShape", list(self.input_data_shape))
self.set("dataFormat.outputDataShape", list(self.output_data_shape))
--
GitLab