From ac9da3b8946d17f54a9e299eef65b84caa7bfb04 Mon Sep 17 00:00:00 2001
From: Egor Sobolev <egor.sobolev@xfel.eu>
Date: Thu, 12 May 2022 10:43:59 +0200
Subject: [PATCH] Add image selection with offline LitFrameFinder
---
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 70 +++++++++++++------
setup.py | 1 +
src/cal_tools/agipdlib.py | 52 +++++++-------
3 files changed, 72 insertions(+), 51 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index ce2c71bf3..03f5dc521 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -17,22 +17,22 @@
"metadata": {},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/SPB/202131/p900230/raw\" # the folder to read data from, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/agipd_resolve_conf\" # the folder to output to, required\n",
+ "in_folder = \"/gpfs/exfel/exp/MID/202201/p002834/raw\" # the folder to read data from, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/esobolev/pycal_litfrm/p002834/r0225\" # the folder to output to, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
"train_ids = [-1] # train IDs to correct, set to -1 for all, range allowed\n",
- "run = 275 # runs to process, required\n",
+ "run = 225 # runs to process, required\n",
"\n",
- "karabo_id = \"SPB_DET_AGIPD1M-1\" # karabo karabo_id\n",
+ "karabo_id = \"MID_DET_AGIPD1M-1\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
"receiver_template = \"{}CH0\" # inset for receiver devices\n",
"path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # the template to use to access data\n",
"instrument_source_template = '{}/DET/{}:xtdf' # path in the HDF5 file to images\n",
"index_source_template = 'INDEX/{}/DET/{}:xtdf/' # path in the HDF5 file to images\n",
"ctrl_source_template = '{}/MDL/FPGA_COMP' # path to control information\n",
- "karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for control device\n",
+ "karabo_id_control = \"MID_EXP_AGIPD1M1\" # karabo-id for control device\n",
"\n",
"slopes_ff_from_files = \"\" # Path to locally stored SlopesFF and BadPixelsFF constants, loaded in precorrection notebook\n",
"\n",
@@ -88,7 +88,8 @@
"use_ppu_device = '' # Device ID for a pulse picker device to only process picked trains, empty string to disable\n",
"ppu_train_offset = 0 # When using the pulse picker, offset between the PPU's sequence start and actually picked train\n",
"\n",
- "use_litframe_device = '' # Device ID for a lit frame finder device to only process illuminated frames, empty string to disable\n",
+ "use_litframe_finder = 'auto' # Process only illuminated frames: 'off' - disable, 'device' - use online device data, 'offline' - use offline algorithm, 'auto' - choose online/offline source automatically (default)\n",
+ "litframe_device_id = '' # Device ID for a lit frame finder device, empty string to auto detection\n",
"energy_threshold = -1000 # The low limit for the energy (uJ) exposed by frames subject to processing. If -1000, selection by pulse energy is disabled\n",
"\n",
"use_xgm_device = '' # DoocsXGM device ID to obtain actual photon energy, operating condition else.\n",
@@ -153,6 +154,9 @@
"sns.set_context(\"paper\", font_scale=1.4)\n",
"sns.set_style(\"ticks\")\n",
"\n",
+ "from extra_redu import make_litframe_finder\n",
+ "from extra_redu.litfrm.utils import litfrm_run_report\n",
+ "\n",
"import cal_tools\n",
"import seaborn as sns\n",
"from cal_tools import agipdalgs as calgs\n",
@@ -450,16 +454,35 @@
"metadata": {},
"outputs": [],
"source": [
- "if use_litframe_device:\n",
- " # check run for the AgipdLitFrameFinder device\n",
+ "if use_litframe_finder != 'off':\n",
+ " if use_litframe_finder not in ['auto', 'offline', 'online']:\n",
+ " raise ValueError(\"Unexpected value in 'use_litframe_finder'.\")\n",
"\n",
- " if use_litframe_device + ':output' in dc.instrument_sources:\n",
- " # Use selection provided by the AgipdLitFrameFinder (if the device is recorded)\n",
- " cell_sel = LitFrameSelection(use_litframe_device, dc, train_ids, max_pulses, energy_threshold)\n",
- " train_ids = cell_sel.train_ids\n",
- " else:\n",
- " # Use range selection (if the device is not recorded)\n",
- " print(f\"WARNING: LitFrameFinder {use_litframe_device} device is not found.\")\n",
+ " inst = karabo_id_control[:3]\n",
+ " litfrm = make_litframe_finder(inst, dc, litframe_device_id)\n",
+ " try:\n",
+ " if use_litframe_finder != 'auto':\n",
+ " r = litfrm.read_or_process()\n",
+ " elif use_litframe_finder != 'offline':\n",
+ " r = litfrm.process()\n",
+ " elif use_litframe_finder != 'online':\n",
+ " r = litfrm.read()\n",
+ "\n",
+ " report = litfrm_run_report(r)\n",
+ " print(\"Lit-frame patterns:\")\n",
+ " for rec in report:\n",
+ " frmintf = ', '.join(\n",
+ " [':'.join([str(n) for n in slc]) for slc in rec[6]]\n",
+ " )\n",
+ " trsintf = ':'.join([str(n) for n in rec[2]])\n",
+ " print(\n",
+ " f\"{rec[1]:2d} {trsintf:25s} {rec[3]:4d} {rec[4]:3d} \"\n",
+ " f\"{rec[5]:3d} [{frmintf}]\"\n",
+ " )\n",
+ " cell_sel = LitFrameSelection(r, train_ids, max_pulses, energy_threshold)\n",
+ " except Exception as err:\n",
+ " print(\"Cannot use AgipdLitFrameFinder due to:\")\n",
+ " print(err)\n",
" cell_sel = CellRange(max_pulses, max_cells=mem_cells)\n",
"else:\n",
" # Use range selection\n",
@@ -861,10 +884,11 @@
" tid, data = run_data.select(f'{detector_id}/DET/*', source).train_from_id(tid)\n",
" else:\n",
" tid, data = next(iter(run_data.select(f'{detector_id}/DET/*', source).trains(require_all=True)))\n",
- "\n",
+ " \n",
" # TODO: remove and use the keep_dims version after updating Extra-data.\n",
" # Avoid using default axis with sources of an expected scalar value per train.\n",
- " if len(range(*cell_sel.crange)) == 1 and source in ['image.blShift', 'image.cellId', 'image.pulseId']:\n",
+ " nfrm = cell_sel.get_cells_on_trains([tid]).sum()\n",
+ " if nfrm == 1 and source in ['image.blShift', 'image.cellId', 'image.pulseId']:\n",
" axis = 0\n",
" else:\n",
" axis = -3\n",
@@ -872,10 +896,8 @@
" stacked_data = stack_detector_data(\n",
" train=data, data=source, fillvalue=fillvalue, modules=modules, axis=axis)\n",
" # Add cellId dimension when correcting one cellId only.\n",
- " if (\n",
- " len(range(*cell_sel.crange)) == 1 and\n",
- " data_folder != run_folder # avoid adding pulse dims for raw data.\n",
- " ):\n",
+ " # avoid adding pulse dims for raw data.\n",
+ " if (nfrm == 1 and data_folder != run_folder):\n",
" stacked_data = stacked_data[np.newaxis, ...]\n",
"\n",
" return tid, stacked_data"
@@ -990,7 +1012,9 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "scrolled": false
+ },
"outputs": [],
"source": [
"pulse_range = [np.min(pulseId[pulseId>=0]), np.max(pulseId[pulseId>=0])]\n",
@@ -1301,7 +1325,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.12"
+ "version": "3.8.11"
}
},
"nbformat": 4,
diff --git a/setup.py b/setup.py
index 0b6b5cd16..ef4fbe0a2 100644
--- a/setup.py
+++ b/setup.py
@@ -103,6 +103,7 @@ install_requires = [
"tabulate==0.8.6",
"traitlets==4.3.3",
"xarray==2022.3.0",
+ "EXtra-redu @ git+ssh://git@git.xfel.eu:10022/dataAnalysis/data-reduction.git@41571d9cca7ec3811d56caa6a1f0177b275fdb02", # noqa
]
if "readthedocs.org" not in sys.executable:
diff --git a/src/cal_tools/agipdlib.py b/src/cal_tools/agipdlib.py
index 6a5ff22e5..2d6e8ab9f 100644
--- a/src/cal_tools/agipdlib.py
+++ b/src/cal_tools/agipdlib.py
@@ -253,11 +253,11 @@ class AgipdCtrl:
# TODO: Validate if removing this and
# and using NB value for old RAW data.
error = ("ERROR: Unable to read bias_voltage from"
- f" {voltage_src[0]}/{voltage_src[1].replace('.','/')}.")
+ f" {voltage_src[0]}/{voltage_src[1].replace('.','/')}.")
if default_voltage:
print(f"{error} Returning {default_voltage} "
- "as default bias voltage value.")
+ "as default bias voltage value.")
else:
raise ValueError(error)
return default_voltage
@@ -485,8 +485,8 @@ class AgipdCorrections:
data_dict["valid_trains"][:n_valid_trains] = valid_train_ids
# get cell selection for the images in this file
- cm = ( self.cell_sel.CM_NONE if apply_sel_pulses
- else self.cell_sel.CM_PRESEL )
+ cm = (self.cell_sel.CM_NONE if apply_sel_pulses
+ else self.cell_sel.CM_PRESEL)
img_selected = self.cell_sel.get_cells_on_trains(
valid_train_ids, cm=cm)
@@ -502,22 +502,21 @@ class AgipdCorrections:
kw = {
"unstack_pulses": False,
- "pulses": np.nonzero(img_selected),
}
-
# [n_modules, n_imgs, 2, x, y]
raw_data = agipd_comp.get_array("image.data", **kw)[0]
- n_img = raw_data.shape[0]
+ frm_ix = np.flatnonzero(img_selected)
+ n_img = frm_ix.size
data_dict['nImg'][0] = n_img
- data_dict['data'][:n_img] = raw_data[:, 0]
- data_dict['rawgain'][:n_img] = raw_data[:, 1]
+ data_dict['data'][:n_img] = raw_data[frm_ix, 0]
+ data_dict['rawgain'][:n_img] = raw_data[frm_ix, 1]
data_dict['cellId'][:n_img] = agipd_comp.get_array(
- "image.cellId", **kw)[0]
+ "image.cellId", **kw)[0, frm_ix]
data_dict['pulseId'][:n_img] = agipd_comp.get_array(
- "image.pulseId", **kw)[0]
+ "image.pulseId", **kw)[0, frm_ix]
data_dict['trainId'][:n_img] = agipd_comp.get_array(
- "image.trainId", **kw)[0]
+ "image.trainId", **kw)[0, frm_ix]
return n_img
def write_file(self, i_proc, file_name, ofile_name):
@@ -601,8 +600,8 @@ class AgipdCorrections:
in a single thread.
"""
def _compress_frame(i):
- # Equivalent to the HDF5 'shuffle' filter: transpose bytes for better
- # compression.
+ # Equivalent to the HDF5 'shuffle' filter: transpose bytes for
+ # better compression.
shuffled = np.ascontiguousarray(
arr[i].view(np.uint8).reshape((-1, arr.itemsize)).transpose()
)
@@ -973,7 +972,7 @@ class AgipdCorrections:
# exclude non valid trains
valid_trains = valid * dc_trains
- return valid_trains[valid_trains!=0]
+ return valid_trains[valid_trains != 0]
def apply_selected_pulses(self, i_proc: int) -> int:
"""Select sharedmem data indices to correct based on selected
@@ -1587,34 +1586,31 @@ class CellRange(CellSelection):
return np.tile(self._sel_for_cm(self.flag, self.flag_cm, cm),
len(train_sel))
+
class LitFrameSelection(CellSelection):
"""Selection of detector memery cells indicated as lit frames
by the AgipdLitFrameFinder
"""
- def __init__(self, dev: str, dc: DataCollection, train_ids: List[int],
+ def __init__(self, litfrmdata, train_ids: List[int],
crange: Optional[List[int]] = None,
energy_threshold: Optional[float] = None):
"""Initialize lit frame selection
- :param dev: AgipdLitFrameFinder device name
- :param dc: EXtra-data DataCollection of a run
+ :param litfrmdata: AgipdLitFrameFinder output data
:param train_ids: the list of selected trains
:param crange: range parameters of selected cells,
list up to 3 elements
"""
# read AgipdLitFrameFinder data
- self.dev = dev
+ self.dev = litfrmdata.meta.litFrmDev
self.crange = validate_selected_pulses(crange, self.ncell_max)
self.ethr = energy_threshold
- intr_src = dev + ':output'
- nfrm = dc[intr_src, 'data.nFrame'].ndarray()
- litfrm_train_ids = dc[intr_src, 'data.trainId'].ndarray()
- litfrm = dc[intr_src, 'data.nPulsePerFrame'].ndarray() > 0
- if (energy_threshold != -1000 and
- 'data.energyPerFrame' in dc.keys_for_source(intr_src)):
-
- litfrm &= (dc[intr_src, 'data.energyPerFrame'].ndarray()
- > energy_threshold)
+
+ nfrm = litfrmdata.output.nFrame
+ litfrm_train_ids = litfrmdata.meta.trainId
+ litfrm = litfrmdata.output.nPulsePerFrame > 0
+ if energy_threshold != -1000:
+ litfrm &= litfrmdata.output.energyPerFrame > energy_threshold
# apply range selection
if crange is None:
--
GitLab