diff --git a/setup.py b/setup.py index 04c4f70ab985ee570d35aa939434aa4fe9b0397d..b1d961fc93c80fac998017ebdf816f524ed19d1a 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ setup(name='calng', 'karabo.bound_device': [ 'AgipdCorrection = calng.corrections.AgipdCorrection:AgipdCorrection', 'DsscCorrection = calng.corrections.DsscCorrection:DsscCorrection', + 'Epix100Correction = calng.corrections.Epix100Correction:Epix100Correction', 'Gotthard2Correction = calng.corrections.Gotthard2Correction:Gotthard2Correction', 'JungfrauCorrection = calng.corrections.JungfrauCorrection:JungfrauCorrection', 'LpdCorrection = calng.corrections.LpdCorrection:LpdCorrection', @@ -41,6 +42,7 @@ setup(name='calng', 'JungfrauCondition = calng.conditions.JungfrauCondition:JungfrauCondition', 'Agipd1MGeometry = calng.geometries.Agipd1MGeometry:Agipd1MGeometry', 'Dssc1MGeometry = calng.geometries:Dssc1MGeometry.Dssc1MGeometry', + 'Epix100Geometry = calng.geometries:Epix100Geometry.Epix100Geometry', 'Lpd1MGeometry = calng.geometries:Lpd1MGeometry.Lpd1MGeometry', 'JungfrauGeometry = calng.geometries:JungfrauGeometry.JungfrauGeometry', 'RoiTool = calng.RoiTool:RoiTool', diff --git a/src/calng/base_correction.py b/src/calng/base_correction.py index 03f31b0f9ce6f6b33a377cae74c4d2e477e01566..0fd0edbcd2c5e41f486ea16c89689243084f7904 100644 --- a/src/calng/base_correction.py +++ b/src/calng/base_correction.py @@ -1100,9 +1100,12 @@ class BaseCorrection(PythonDevice): ) as warn: try: image_data = np.asarray(data_hash.get(self._image_data_path)) - cell_table = np.asarray( - data_hash.get(self._cell_table_path) - ).ravel() + if self._cell_table_path is None: + cell_table = None + else: + cell_table = np.asarray( + data_hash.get(self._cell_table_path) + ).ravel() except RuntimeError as err: warn( "Failed to load image data; " @@ -1171,17 +1174,20 @@ class BaseCorrection(PythonDevice): self._train_ratio_tracker.reset() self._train_ratio_tracker.update(train_id) - with self.warning_context( - "processingState", WarningLampType.MEMORY_CELL_RANGE - ) as warn: - if ( - self._warn_memory_cell_range - and self.unsafe_get("constantParameters.memoryCells") - <= cell_table.max() - ): - warn("Input cell IDs out of range of constants") - - if cell_table.size != self.unsafe_get("dataFormat.frames"): + if self._warn_memory_cell_range: + with self.warning_context( + "processingState", WarningLampType.MEMORY_CELL_RANGE + ) as warn: + if ( + self.unsafe_get("constantParameters.memoryCells") + <= cell_table.max() + ): + warn("Input cell IDs out of range of constants") + + # TODO: avoid branching multiple times on cell_table + if cell_table is not None and cell_table.size != self.unsafe_get( + "dataFormat.frames" + ): self.log_status_info( f"Updating new input shape to account for {cell_table.size} cells" ) diff --git a/src/calng/base_geometry.py b/src/calng/base_geometry.py index c4964c787dcb1f24f2db251ff8ed305884422d97..6581581b5760e7bca854fc2a67a6a9c2f7610a69 100644 --- a/src/calng/base_geometry.py +++ b/src/calng/base_geometry.py @@ -339,6 +339,49 @@ class ManualGeometryBase(Device): self.logger.log(level, text) +def make_origin_node(default_x, default_y): + class OriginNode(BaseManualGeometryConfigNode): + x = Double( + defaultValue=default_x, + accessMode=AccessMode.RECONFIGURABLE, + assignment=Assignment.OPTIONAL, + ) + y = Double( + defaultValue=default_y, + accessMode=AccessMode.RECONFIGURABLE, + assignment=Assignment.OPTIONAL, + ) + + return Node(OriginNode) + + +class ManualOriginGeometryBase(ManualGeometryBase): + origin = None # subclass must define + + async def _set_from_manual_config(self): + self._set_status("Updating geometry from manual configuration") + geometry = self.geometry_class.from_origin( + (self.origin.x.value, self.origin.y.value) + ) + await self._set_geometry(geometry) + # TODO: allow offset + + @slot + def requestScene(self, params): + name = params.get("name", default="overview") + if name == "overview": + scene_data = scenes.origin_geometry_overview( + self.deviceId, + self.getDeviceSchema(), + ) + payload = Hash("success", True, "name", name, "data", scene_data) + + return Hash("type", "deviceScene", "origin", self.deviceId, "payload", payload) + + async def _update_manual_from_current(self): + raise NotImplementedError() + + def make_quadrant_corners_node(default_values): assert len(default_values) == 4 assert all(len(x) == 2 for x in default_values) @@ -443,10 +486,6 @@ class ManualModuleListGeometryBase(ManualGeometryBase): return Hash("type", "deviceScene", "origin", self.deviceId, "payload", payload) - @Slot( - displayedName="Set from device", - allowedStates=[State.ACTIVE], - ) async def _set_from_manual_config(self): self._set_status("Updating geometry from manual configuration") with self.push_state(State.CHANGING): diff --git a/src/calng/corrections/Epix100Correction.py b/src/calng/corrections/Epix100Correction.py new file mode 100644 index 0000000000000000000000000000000000000000..2235ffeea56921b4ffc8606fa333c60176d762b5 --- /dev/null +++ b/src/calng/corrections/Epix100Correction.py @@ -0,0 +1,456 @@ +import concurrent.futures +import enum +import functools + +import bottleneck as bn +import numpy as np +from karabo.bound import ( + BOOL_ELEMENT, + DOUBLE_ELEMENT, + KARABO_CLASSINFO, + OUTPUT_CHANNEL, + OVERWRITE_ELEMENT, + VECTOR_STRING_ELEMENT, +) + +from .. import ( + base_calcat, + base_correction, + base_kernel_runner, + schemas, + preview_utils, + utils, +) +from .._version import version as deviceVersion + + +class Constants(enum.Enum): + BadPixelsDarkEPix100 = enum.auto() + RelativeGainEPix100 = enum.auto() + OffsetEPix100 = enum.auto() + NoiseEPix100 = enum.auto() + + +class CorrectionFlags(enum.IntFlag): + NONE = 0 + OFFSET = 2**1 + COMMONMODE = 2**2 + RELGAIN = 2**3 + BPMASK = 2**4 + + +class Epix100CalcatFriend(base_calcat.BaseCalcatFriend): + _constant_enum_class = Constants + + @property + def _constants_need_conditions(self): + return { + Constants.OffsetEPix100: self.dark_condition, + Constants.BadPixelsDarkEPix100: self.dark_condition, + Constants.NoiseEPix100: self.dark_condition, + Constants.RelativeGainEPix100: self.illuminated_condition, + } + + @staticmethod + def add_schema(schema, managed_keys): + super(Epix100CalcatFriend, Epix100CalcatFriend).add_schema( + schema, managed_keys, "ePix100-Type" + ) + + # set some defaults for common parameters + ( + OVERWRITE_ELEMENT(schema) + .key("constantParameters.memoryCells") + .setNewDefaultValue(1) + .commit(), + + OVERWRITE_ELEMENT(schema) + .key("constantParameters.biasVoltage") + .setNewDefaultValue(200) + .commit(), + + OVERWRITE_ELEMENT(schema) + .key("constantParameters.pixelsX") + .setNewDefaultValue(708) + .commit(), + + OVERWRITE_ELEMENT(schema) + .key("constantParameters.pixelsY") + .setNewDefaultValue(768) + .commit(), + ) + + ( + DOUBLE_ELEMENT(schema) + .key("constantParameters.integrationTime") + .displayedName("Integration Time") + .assignmentOptional() + .defaultValue(10) + .reconfigurable() + .commit(), + + DOUBLE_ELEMENT(schema) + .key("constantParameters.sensorTemperature") + .assignmentOptional() + .defaultValue(264.32933824373004) + .reconfigurable() + .commit(), + + DOUBLE_ELEMENT(schema) + .key("constantParameters.inVacuum") + .displayedName("In vacuum") + .assignmentOptional() + .defaultValue(10) + .reconfigurable() + .commit(), + + DOUBLE_ELEMENT(schema) + .key("constantParameters.sourceEnergy") + .displayedName("Source Energy") + .assignmentOptional() + .defaultValue(8.04778) + .reconfigurable() + .commit(), + ) + + managed_keys.add("constantParameters.integrationTime") + managed_keys.add("constantParameters.inVacuum") + managed_keys.add("constantParameters.sourceEnergy") + managed_keys.add("constantParameters.sensorTemperature") + + base_calcat.add_status_schema_from_enum(schema, Constants) + + def dark_condition(self): + res = base_calcat.OperatingConditions() + res["Memory cells"] = self._get_param("memoryCells") + res["Sensor Bias Voltage"] = self._get_param("biasVoltage") + res["Pixels X"] = self._get_param("pixelsX") + res["Pixels Y"] = self._get_param("pixelsY") + res["Integration Time"] = self._get_param("integrationTime") + res["Sensor Temperature"] = self._get_param("sensorTemperature") + res["In vacuum"] = self._get_param("inVacuum") + + return res + + def illuminated_condition(self): + res = self.dark_condition() + + res["Source Energy"] = self._get_param("sourceEnergy") + + return res + + +class Epix100CpuRunner(base_kernel_runner.BaseKernelRunner): + _corrected_axis_order = "fxy" + + def __init__( + self, + pixels_x, + pixels_y, + frames, # will be 1, will be ignored + constant_memory_cells, + input_data_dtype=np.uint16, + output_data_dtype=np.float32, + ): + assert ( + output_data_dtype == np.float32 + ), "Alternative output types not supported yet" + self.input_shape = (pixels_x, pixels_y) + self.preview_shape = (pixels_x, pixels_y) + self.processed_shape = self.input_shape + + super().__init__( + pixels_x, + pixels_y, + frames, + constant_memory_cells, + input_data_dtype, + output_data_dtype, + ) + + self.input_data = None + self.processed_data = np.empty(self.processed_shape, dtype=np.float32) + + self.map_shape = (pixels_x, pixels_y) + self.offset_map = np.empty(self.map_shape, dtype=np.float32) + self.rel_gain_map = np.empty(self.map_shape, dtype=np.float32) + self.bad_pixel_map = np.empty(self.map_shape, dtype=np.uint32) + self.noise_map = np.empty(self.map_shape, dtype=np.float32) + # will do everything by quadrant + self.thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=4) + self._q_input_data = None + self._q_processed_data = utils.quadrant_views(self.processed_data) + self._q_offset_map = utils.quadrant_views(self.offset_map) + self._q_rel_gain_map = utils.quadrant_views(self.rel_gain_map) + self._q_bad_pixel_map = utils.quadrant_views(self.bad_pixel_map) + self._q_noise_map = utils.quadrant_views(self.noise_map) + + def __del__(self): + self.thread_pool.shutdown() + + @property + def preview_data_views(self): + # NOTE: apparently there are "calibration rows" + # geometry assumes these are cut out already + return (self.input_data[2:-2], self.processed_data[2:-2]) + + def load_data(self, image_data): + # should almost never squeeze, but ePix doesn't do burst mode, right? + self.input_data = image_data.astype(np.uint16, copy=False).squeeze() + self._q_input_data = utils.quadrant_views(self.input_data) + + def load_constant(self, constant_type, data): + if constant_type is Constants.OffsetEPix100: + self.offset_map[:] = data.squeeze().astype(np.float32) + elif constant_type is Constants.RelativeGainEPix100: + self.rel_gain_map[:] = data.squeeze().astype(np.float32) + elif constant_type is Constants.BadPixelsDarkEPix100: + self.bad_pixel_map[:] = data.squeeze() + elif constant_type is Constants.NoiseEPix100: + self.noise_map[:] = data.squeeze() + else: + raise Exception(f"Unknown constant type {constant_type}") + + def flush_buffers(self, constants): + if Constants.OffsetEPix100 in constants: + self.offset_map.fill(0) + if Constants.RelativeGainEPix100 in constants: + self.rel_gain_map.fill(1) + if constants & {Constants.BadPixelsDarkEPix100}: + self.bad_pixel_map.fill(0) + if Constants.NoiseEPix100 in constants: + self.noise_map.fill(np.inf) + + def _correct_quadrant( + self, + q, + flags, + bad_pixel_mask_value, + cm_noise_sigma, + cm_min_frac, + cm_row, + cm_col, + cm_block, + ): + output = self._q_processed_data[q] + output[:] = self._q_input_data[q].astype(np.float32) + if flags & CorrectionFlags.OFFSET: + output -= self._q_offset_map[q] + + if flags & CorrectionFlags.COMMONMODE: + # per rectangular block that looks like something is going on + masked = np.ma.masked_array( + data=output, + mask=(self._q_bad_pixel_map[q] != 0) + | (output > self._q_noise_map[q] * cm_noise_sigma), + ) + if cm_block: + for block in np.hsplit(masked, 4): + if block.count() < block.size * cm_min_frac: + continue + block.data[:] -= np.ma.median(block) + + if cm_row: + subset_rows = masked.count(axis=1) >= masked.shape[1] * cm_min_frac + output[subset_rows] -= np.ma.median( + masked[subset_rows], axis=1, keepdims=True + ) + + if cm_col: + subset_cols = masked.count(axis=0) >= masked.shape[0] * cm_min_frac + output[:, subset_cols] -= np.ma.median(masked[:, subset_cols], axis=0) + + if flags & CorrectionFlags.RELGAIN: + output *= self._q_rel_gain_map[q] + + if flags & CorrectionFlags.BPMASK: + output[self._q_bad_pixel_map[q] != 0] = bad_pixel_mask_value + + def correct( + self, + flags, + bad_pixel_mask_value=np.nan, + cm_noise_sigma=5, + cm_min_frac=0.25, + cm_row=True, + cm_col=True, + cm_block=True, + ): + # NOTE: how to best clean up all these duplicated parameters? + for result in self.thread_pool.map( + functools.partial( + self._correct_quadrant, + flags=flags, + bad_pixel_mask_value=bad_pixel_mask_value, + cm_noise_sigma=cm_noise_sigma, + cm_min_frac=cm_min_frac, + cm_row=cm_row, + cm_col=cm_col, + cm_block=cm_block, + ), + range(4), + ): + ... + + +@KARABO_CLASSINFO("Epix100Correction", deviceVersion) +class Epix100Correction(base_correction.BaseCorrection): + _correction_flag_class = CorrectionFlags + _correction_steps = ( + ("offset", CorrectionFlags.OFFSET, {Constants.OffsetEPix100}), + ("relGain", CorrectionFlags.RELGAIN, {Constants.RelativeGainEPix100}), + ("commonMode", CorrectionFlags.COMMONMODE, {Constants.NoiseEPix100}), + ("badPixels", CorrectionFlags.BPMASK, {Constants.BadPixelsDarkEPix100}), + ) + _kernel_runner_class = Epix100CpuRunner + _calcat_friend_class = Epix100CalcatFriend + _constant_enum_class = Constants + _managed_keys = base_correction.BaseCorrection._managed_keys.copy() + _image_data_path = "data.image.pixels" + _cell_table_path = None + _warn_memory_cell_range = False + + @staticmethod + def expectedParameters(expected): + ( + OUTPUT_CHANNEL(expected) + .key("dataOutput") + .dataSchema(schemas.jf_output_schema()) + .commit(), + + OVERWRITE_ELEMENT(expected) + .key("dataFormat.pixelsX") + .setNewDefaultValue(708) + .commit(), + + OVERWRITE_ELEMENT(expected) + .key("dataFormat.pixelsY") + .setNewDefaultValue(768) + .commit(), + + OVERWRITE_ELEMENT(expected) + .key("dataFormat.frames") + .setNewDefaultValue(1) + .commit(), + + # TODO: disable preview selection mode + + OVERWRITE_ELEMENT(expected) + .key("outputShmemBufferSize") + .setNewDefaultValue(2) + .commit(), + ) + + base_correction.add_correction_step_schema( + expected, + Epix100Correction._managed_keys, + Epix100Correction._correction_steps, + ) + ( + DOUBLE_ELEMENT(expected) + .key("corrections.commonMode.noiseSigma") + .assignmentOptional() + .defaultValue(5) + .reconfigurable() + .commit(), + + DOUBLE_ELEMENT(expected) + .key("corrections.commonMode.minFrac") + .assignmentOptional() + .defaultValue(0.25) + .reconfigurable() + .commit(), + + BOOL_ELEMENT(expected) + .key("corrections.commonMode.enableRow") + .assignmentOptional() + .defaultValue(True) + .reconfigurable() + .commit(), + + BOOL_ELEMENT(expected) + .key("corrections.commonMode.enableCol") + .assignmentOptional() + .defaultValue(True) + .reconfigurable() + .commit(), + + BOOL_ELEMENT(expected) + .key("corrections.commonMode.enableBlock") + .assignmentOptional() + .defaultValue(True) + .reconfigurable() + .commit(), + ) + Epix100Correction._managed_keys |= { + "corrections.commonMode.noiseSigma", + "corrections.commonMode.minFrac", + "corrections.commonMode.enableRow", + "corrections.commonMode.enableCol", + "corrections.commonMode.enableBlock", + } + Epix100CalcatFriend.add_schema(expected, Epix100Correction._managed_keys) + # TODO: bad pixel node? + + # mandatory: manager needs this in schema + ( + VECTOR_STRING_ELEMENT(expected) + .key("managedKeys") + .assignmentOptional() + .defaultValue(list(Epix100Correction._managed_keys)) + .commit() + ) + + @property + def input_data_shape(self): + # TODO: check + return ( + self.unsafe_get("dataFormat.pixelsX"), + self.unsafe_get("dataFormat.pixelsY"), + ) + + def process_data( + self, + data_hash, + metadata, + source, + train_id, + image_data, + cell_table, # will be None + ): + self.kernel_runner.load_data(image_data) + + buffer_handle, buffer_array = self._shmem_buffer.next_slot() + args_which_should_be_cached = dict( + cm_noise_sigma=self.unsafe_get("corrections.commonMode.noiseSigma"), + cm_min_frac=self.unsafe_get("corrections.commonMode.minFrac"), + cm_row=self.unsafe_get("corrections.commonMode.enableRow"), + cm_col=self.unsafe_get("corrections.commonMode.enableCol"), + cm_block=self.unsafe_get("corrections.commonMode.enableBlock"), + ) + self.kernel_runner.correct( + flags=self._correction_flag_enabled, **args_which_should_be_cached + ) + if self._correction_flag_enabled != self._correction_flag_preview: + self.kernel_runner.correct( + flags=self._correction_flag_preview, + **args_which_should_be_cached, + ) + self.kernel_runner.reshape( + output_order=self.unsafe_get("dataFormat.outputAxisOrder"), + out=buffer_array, + ) + + self._write_output(data_hash, metadata) + + # note: base class preview machinery assumes burst mode, shortcut it + self._write_preview_outputs( + zip( + ("preview.outputRaw", "preview.outputCorrected"), + self.kernel_runner.preview_data_views, + ), + metadata, + ) + + def _load_constant_to_runner(self, constant, constant_data): + self.kernel_runner.load_constant(constant, constant_data) diff --git a/src/calng/geometries/Epix100Geometry.py b/src/calng/geometries/Epix100Geometry.py new file mode 100644 index 0000000000000000000000000000000000000000..20729ea3cd60a1009041b0cf831b36997126668f --- /dev/null +++ b/src/calng/geometries/Epix100Geometry.py @@ -0,0 +1,8 @@ +import extra_geom + +from ..base_geometry import ManualOriginGeometryBase, make_origin_node + + +class Epix100Geometry(ManualOriginGeometryBase): + geometry_class = extra_geom.Epix100Geometry + origin = make_origin_node(0, 0) diff --git a/src/calng/geometries/__init__.py b/src/calng/geometries/__init__.py index 091660d02ce4110a0cf979269dabe1c171b5805d..1526d0b903f59ff4810264dfa0f470258e1633bf 100644 --- a/src/calng/geometries/__init__.py +++ b/src/calng/geometries/__init__.py @@ -1,2 +1,8 @@ # flake8: noqa: F401 -from . import Agipd1MGeometry, Dssc1MGeometry, Lpd1MGeometry, JungfrauGeometry +from . import ( + Agipd1MGeometry, + Dssc1MGeometry, + Epix100Geometry, + Lpd1MGeometry, + JungfrauGeometry, +) diff --git a/src/calng/scenes.py b/src/calng/scenes.py index 08dd04a6c6dd5b0e8d30fbd499df1fb0869ba399..46a306f022793fde00d71c5aaf7334cfcd6d852e 100644 --- a/src/calng/scenes.py +++ b/src/calng/scenes.py @@ -823,6 +823,34 @@ class ManualQuadrantGeometrySettings(VerticalLayout): ) +@titled("Manual geometry settings") +@boxed +class ManualOriginGeometrySettings(VerticalLayout): + def __init__(self, device_id): + super().__init__(padding=0) + self.children.append( + HorizontalLayout( + DoubleLineEditModel( + keys=[f"{device_id}.origin.x"], + width=4 * BASE_INC, + height=BASE_INC, + ), + DoubleLineEditModel( + keys=[f"{device_id}.origin.y"], + width=4 * BASE_INC, + height=BASE_INC, + ), + ) + ) + self.children.append( + DisplayCommandModel( + keys=[f"{device_id}.origin.setManual"], + width=6 * BASE_INC, + height=BASE_INC, + ), + ) + + @titled("Manual geometry settings") @boxed class ManualModulesGeometrySettings(VerticalLayout): @@ -1618,6 +1646,19 @@ def quadrant_geometry_overview(device_id, schema): ) +@scene_generator +def origin_geometry_overview(device_id, schema): + schema_hash = schema_to_hash(schema) + return VerticalLayout( + HorizontalLayout( + ManualOriginGeometrySettings(device_id), + # GeometryFromFileSettings(device_id, schema_hash), + # TweakCurrentGeometry(device_id), + ), + GeometryPreview(device_id), + ) + + @scene_generator def modules_geometry_overview(device_id, schema): schema_hash = schema_to_hash(schema) diff --git a/src/calng/utils.py b/src/calng/utils.py index 17be2a9cc8510727704477c0ed1d758d5eaa1036..464a1958cf6695d0a227214fbf763bea0efe1828 100644 --- a/src/calng/utils.py +++ b/src/calng/utils.py @@ -533,3 +533,24 @@ def downsample_2d(arr, factor, reduction_fun=np.nanmax): axis=0, ) return arr + + +def blocked_view_2d(A, blocks_ss, blocks_fs): + assert A.shape[0] % blocks_ss == 0 + assert A.shape[1] % blocks_fs == 0 + return A.reshape( + blocks_ss, + A.shape[0] // blocks_ss, + blocks_fs, + A.shape[1] // blocks_fs, + ).swapaxes( + 1, + 2, + ) + + +def quadrant_views(A): + res = [] + for row in np.vsplit(A, 2): + res.extend(np.hsplit(row, 2)) + return res