From fa018d56012ffdb685e93200c70f81c6fd52fd19 Mon Sep 17 00:00:00 2001
From: midonc <midonc@exflong10.desy.de>
Date: Wed, 17 Apr 2024 15:32:22 +0200
Subject: [PATCH] added photonizer and xpcs code from online cluster
---
photonizer/setup.py | 29 +++++
photonizer/src/_version.py | 4 +
photonizer/src/photonizer/photonizer.py | 39 +++++++
xpcs/setup.py | 31 ++++++
xpcs/src/_version.py | 4 +
xpcs/src/xpcs/xpcs.py | 139 ++++++++++++++++++++++++
6 files changed, 246 insertions(+)
create mode 100644 photonizer/setup.py
create mode 100644 photonizer/src/_version.py
create mode 100644 photonizer/src/photonizer/photonizer.py
create mode 100644 xpcs/setup.py
create mode 100644 xpcs/src/_version.py
create mode 100644 xpcs/src/xpcs/xpcs.py
diff --git a/photonizer/setup.py b/photonizer/setup.py
new file mode 100644
index 0000000..f543877
--- /dev/null
+++ b/photonizer/setup.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+
+from os.path import dirname, join, realpath
+from setuptools import setup, find_packages, Extension
+
+from karabo.packaging.versioning import device_scm_version
+
+
+ROOT_FOLDER = dirname(realpath(__file__))
+scm_version = device_scm_version(ROOT_FOLDER, join(ROOT_FOLDER, "src", "_version.py"))
+
+
+setup(
+ name="photonizer",
+ use_scm_version=scm_version,
+ author="James Wrigley",
+ author_email="da-support@xfel.eu",
+ description="",
+ long_description="",
+ url="",
+ package_dir={"": "src"},
+ packages=find_packages("src"),
+ entry_points={
+ "calng.correction_addon": [
+ "Photonizer = photonizer.photonizer:Photonizer",
+ ],
+ },
+ requires=[],
+)
diff --git a/photonizer/src/_version.py b/photonizer/src/_version.py
new file mode 100644
index 0000000..0731bb7
--- /dev/null
+++ b/photonizer/src/_version.py
@@ -0,0 +1,4 @@
+# coding: utf-8
+# file generated by setuptools_scm
+# don't change, don't track in version control
+version = '0.0.1-2.17.0-0-g250da4f-dirty'
diff --git a/photonizer/src/photonizer/photonizer.py b/photonizer/src/photonizer/photonizer.py
new file mode 100644
index 0000000..465656b
--- /dev/null
+++ b/photonizer/src/photonizer/photonizer.py
@@ -0,0 +1,39 @@
+import numpy as np
+
+from karabo.bound import (
+ NODE_ELEMENT,
+ DOUBLE_ELEMENT
+)
+
+from calng.correction_addons.base_addon import BaseCorrectionAddon
+
+
+class Photonizer(BaseCorrectionAddon):
+ def __init__(self, config):
+ global cupy
+ import cupy
+
+ self._photon_energy = config.get("photonEnergy")
+
+ @staticmethod
+ def extend_device_schema(schema, prefix):
+ (
+ DOUBLE_ELEMENT(schema)
+ .key(f"{prefix}.photonEnergy")
+ .tags("managed")
+ .assignmentOptional()
+ .defaultValue(8.0)
+ .reconfigurable()
+ .commit(),
+ )
+
+ def reconfigure(self, changed_config):
+ if changed_config.has("photonEnergy"):
+ self._photon_energy = changed_config["photonEnergy"]
+
+ def post_correction(self, data, cell_table, pulse_table, output_hash):
+ data /= self._photon_energy
+ np.around(data, out=data)
+
+ def post_reshape(self, data, cell_table, pulse_table, output_hash):
+ print(output_hash["header"].keys())
diff --git a/xpcs/setup.py b/xpcs/setup.py
new file mode 100644
index 0000000..c661daa
--- /dev/null
+++ b/xpcs/setup.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+
+from os.path import dirname, join, realpath
+from setuptools import setup, find_packages, Extension
+
+from karabo.packaging.versioning import device_scm_version
+
+
+ROOT_FOLDER = dirname(realpath(__file__))
+scm_version = device_scm_version(ROOT_FOLDER, join(ROOT_FOLDER, "src", "_version.py"))
+
+
+setup(
+ name="xpcs",
+ use_scm_version=scm_version,
+ author="James Wrigley",
+ author_email="da-support@xfel.eu",
+ description="",
+ long_description="",
+ url="",
+ package_dir={"": "src"},
+ packages=find_packages("src"),
+ entry_points={
+ "calng.correction_addon": [
+ "Xpcs = xpcs.xpcs:Xpcs",
+ ],
+ },
+ requires=[
+ "fastXPCS"
+ ],
+)
diff --git a/xpcs/src/_version.py b/xpcs/src/_version.py
new file mode 100644
index 0000000..0731bb7
--- /dev/null
+++ b/xpcs/src/_version.py
@@ -0,0 +1,4 @@
+# coding: utf-8
+# file generated by setuptools_scm
+# don't change, don't track in version control
+version = '0.0.1-2.17.0-0-g250da4f-dirty'
diff --git a/xpcs/src/xpcs/xpcs.py b/xpcs/src/xpcs/xpcs.py
new file mode 100644
index 0000000..da7e463
--- /dev/null
+++ b/xpcs/src/xpcs/xpcs.py
@@ -0,0 +1,139 @@
+import time
+from pathlib import Path
+
+import numpy as np
+from karabo.bound import NODE_ELEMENT, NDARRAY_ELEMENT, STRING_ELEMENT
+
+from fastXPCS.algos import TTCdata, do_sparse_train
+from fastXPCS.fxpcs_sparse import sparsify, sparsify2
+from calng.correction_addons.base_addon import BaseCorrectionAddon
+
+
+class Xpcs(BaseCorrectionAddon):
+ def __init__(self, config):
+ self._ttcdata = TTCdata()
+ self._mask_path = config["mask_path"]
+ self._module_no = None
+
+ self._current_image_mean = 0
+ self._current_image = None
+ self._sparse_array = None
+ self._cal_mask = None
+
+ global cupy
+ import cupy
+
+ @staticmethod
+ def extend_device_schema(schema, prefix):
+ (
+ STRING_ELEMENT(schema)
+ .key(f"{prefix}.mask_path")
+ .tags("managed")
+ .assignmentOptional()
+ .defaultValue("")
+ .reconfigurable()
+ .commit(),
+ )
+
+ @staticmethod
+ def extend_output_schema(schema):
+ (
+ NODE_ELEMENT(schema)
+ .key("xpcs")
+ .commit(),
+
+ NDARRAY_ELEMENT(schema)
+ .key("xpcs.ttcf_on")
+ .dtype('FLOAT')
+ .commit(),
+
+ NDARRAY_ELEMENT(schema)
+ .key("xpcs.ttcf_off")
+ .dtype('FLOAT')
+ .commit(),
+ )
+
+ def set_mask(self):
+ if Path(self._mask_path).is_file() and self._module_no is not None:
+ mask = np.load(self._mask_path)
+ self._ttcdata.user_q_mask = mask[:, self._module_no].copy()
+ print(f"Loaded mask for module {self._module_no} with shape {self._ttcdata.user_q_mask.shape}, mean {self._ttcdata.user_q_mask.mean()}")
+
+ def reconfigure(self, changed_config):
+ if changed_config.has("mask_path"):
+ self._mask_path = changed_config["mask_path"]
+ self.set_mask()
+
+ def post_correction(self, data, cell_table, pulse_table, output_hash):
+ global cupy
+
+ # Insert dummy values
+ data[0] = 0
+
+ # Photonize
+ # print(f"Unphotonized mean ({data.shape}): {np.nanmean(data)}")
+ start = time.time()
+ data /= 10
+ data[data < 0] = 0
+ np.around(data, out=data)
+ # print(f"Photonized mean: {np.nanmean(data)}")
+
+ # Sparsify
+ start = time.time()
+ flat_image_shape = data.reshape(data.shape[0], -1).shape
+ reshaped_data = data.reshape(flat_image_shape).T
+ indices = cupy.indices(reshaped_data.shape)
+
+ lit_mask = reshaped_data > 0
+ lit_pixels = reshaped_data[lit_mask].astype(cupy.uint32)
+ lit_indices = indices[:, lit_mask].astype(cupy.uint32)
+
+ lit_pixels_cpu = lit_pixels.get()
+ lit_indices_cpu = lit_indices.get()
+
+ sparse_array = np.empty_like(lit_pixels_cpu, dtype=np.uint32)
+ sparsify2(lit_pixels_cpu, lit_indices_cpu, sparse_array)
+
+ self._current_image = data.astype(cupy.int32).get()
+ self._current_image_mean = np.nanmean(data)
+ self._cal_mask = np.isnan(data).get()
+ self._sparse_array = sparse_array
+ # print(f"Sparsified in {time.time() - start:.3f} down to: {self._sparse_array.shape}")
+
+ if self._module_no == 3:
+ print(f"Found lit pixels: {sparse_array.shape}")
+
+ def post_reshape(self, data, cell_table, pulse_table, output_hash):
+ if self._module_no is None:
+ source = list(self._device.sources)[0]
+ self._module_no = int(source.split("/")[-1].split(":")[0][:-3])
+ self.set_mask()
+
+ if self._module_no != 3 or self._current_image_mean > 20:
+ return
+
+ # print(f"Processing with mean {self._current_image_mean:.3e} and lit pixels: {self._sparse_array.shape}")
+
+ sparse_ref = sparsify(self._current_image.reshape(self._current_image.shape[0], -1))
+ # print(sparse_ref.shape, self._sparse_array.shape)
+ # if sparse_ref.shape == self._sparse_array.shape:
+ # print(np.all(np.equal(sparse_ref, self._sparse_array)))
+
+ # print(f"Using lit pixels: {self._sparse_array.shape}")
+ start = time.time()
+ self._ttcdata.current_image = self._current_image
+ self._ttcdata.current_image_sparse = sparse_ref # self._sparse_array
+ self._ttcdata.current_cal_mask = self._cal_mask
+
+ elapsed = time.time() - start
+ # print(f"Init in: {elapsed:.3f}")
+
+ start = time.time()
+ do_sparse_train(self._ttcdata)
+ elapsed = time.time() - start
+ # print(f"TTCF in: {elapsed:.3f}")
+
+ print(np.nanmean(self._ttcdata.ttc_on), self._ttcdata.s_cnt)
+
+ output_hash["xpcs.ttcf_on"] = self._ttcdata.ttc_on.astype(np.int32)
+ # output_hash["xpcs.ttcf_off"] = self._ttcdata.ttc_off.astype(np.int32)
--
GitLab