Skip to content
Snippets Groups Projects
Commit 7daba04f authored by Egor Sobolev's avatar Egor Sobolev
Browse files

Support multiple module detector in Correct_DynamicFF_NBC.ipynb

parent 91e05cb3
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
notebooks/DynamicFF/Correct_DynamicFF_NBC.ipynb
+ 153
− 170
View file @ 7daba04f
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Shimadzu HPVX2 Offline Correction # Dynamic Flat-field Offline Correction
Author: Egor Sobolev Author: Egor Sobolev
Offline dynamic flat-field correction for Shimadzu HPVX2 cameras Offline dynamic flat-field correction
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
in_folder = "/gpfs/exfel/exp/SPB/202121/p002919/raw/" # input folder, required in_folder = "/gpfs/exfel/exp/SPB/202430/p900425/raw" # input folder, required
out_folder = '/gpfs/exfel/data/scratch/esobolev/test/shimadzu' # output folder, required out_folder = '/gpfs/exfel/data/scratch/esobolev/test/shimadzu' # output folder, required
metadata_folder = "" # Directory containing calibration_metadata.yml when run by xfel-calibrate metadata_folder = "" # Directory containing calibration_metadata.yml when run by xfel-calibrate
run = 30 # which run to read data from, required run = 3 # which run to read data from, required
# Data files parameters. # Data files parameters.
karabo_da = ['HPVX01'] # data aggregators karabo_da = ['HPVX01/1', 'HPVX01/2'] # data aggregators
karabo_id = "SPB_EHD_HPVX2_2" # karabo prefix of Shimadzu HPV-X2 devices karabo_id = "SPB_EHD_MIC" # karabo prefix of Shimadzu HPV-X2 devices
#receiver_id = "PNCCD_FMT-0" # inset for receiver devices #receiver_id = "PNCCD_FMT-0" # inset for receiver devices
#path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data #path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data
instrument_source_template = '{}/CAM/CAMERA:daqOutput' # data source path in h5file. Template filled with karabo_id instrument_source_template = 'SPB_EHD_MIC/CAM/HPVX2_{module}:daqOutput' # data source path in h5file.
image_key = "data.image.pixels" # image data key in Karabo or exdf notation image_key = "data.image.pixels" # image data key in Karabo or exdf notation
# Database access parameters. # Database access parameters.
use_dir_creation_date = True # use dir creation date as data production reference date use_dir_creation_date = True # use dir creation date as data production reference date
cal_db_interface = "tcp://max-exfl-cal001:8021" # calibration DB interface to use cal_db_interface = "tcp://max-exfl-cal001:8021" # calibration DB interface to use
cal_db_timeout = 300000 # timeout on caldb requests cal_db_timeout = 300000 # timeout on caldb requests
db_output = False # if True, the notebook sends dark constants to the calibration database db_output = False # if True, the notebook sends dark constants to the calibration database
local_output = True # if True, the notebook saves dark constants locally local_output = True # if True, the notebook saves dark constants locally
creation_time = "" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. 2019-07-04 11:02:41.00 creation_time = "" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. 2019-07-04 11:02:41.00
n_components = 20 # number of principal components of flat-field to use in correction n_components = 20 # number of principal components of flat-field to use in correction
downsample_factors = [1, 1] # list of downsample factors for each image dimention (y, x) downsample_factors = [1, 1] # list of downsample factors for each image dimention (y, x)
constants_folder = "/gpfs/exfel/data/scratch/esobolev/test/shimadzu" constants_folder = "/gpfs/exfel/data/scratch/esobolev/test/shimadzu"
db_module = "SHIMADZU_HPVX2_M001" db_module_template = "Shimadzu_HPVX2_{}"
num_proc = 32 # number of processes running correction in parallel num_proc = 32 # number of processes running correction in parallel
corrected_source_template = '{}/CORR/CAMERA:daqOutput' # data source path in h5file. Template filled with karabo_id corrected_source_template = 'SPB_EHD_MIC/CORR/HPVX2_{module}:output' # data source path in h5file.
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import os import os
import h5py import h5py
import numpy as np import numpy as np
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
from IPython.display import display, Markdown
from extra_data import RunDirectory from extra_data import RunDirectory, by_id
%matplotlib inline %matplotlib inline
from cal_tools.step_timing import StepTimer from cal_tools.step_timing import StepTimer
from cal_tools.files import sequence_trains, DataFile from cal_tools.files import sequence_trains, DataFile
from dffc.correction import DynamicFlatFieldCorrectionCython as DynamicFlatFieldCorrection from dffc.correction import DynamicFlatFieldCorrectionCython as DynamicFlatFieldCorrection
from dffc.offline import FlatFieldCorrectionFileProcessor from dffc.offline import FlatFieldCorrectionFileProcessor
from dffc.draw import plot_images, plot_camera_image from dffc.draw import plot_images, plot_camera_image
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
instrument = karabo_id.split("_")[0] index_group = image_key.partition('.')[0]
source = instrument_source_template.format(karabo_id) instrument, part, component = karabo_id.split('_')
aggregators = {}
sources = {}
source_to_db = {}
print("Sources:")
for da in karabo_da:
aggr, _, module = da.partition('/')
instrument_source_name = instrument_source_template.format(
instrument=instrument, part=part, component=component,
module=module
)
corrected_source_name = corrected_source_template.format(
instrument=instrument, part=part, component=component,
module=module
)
aggregators.setdefault(aggr, []).append(
(instrument_source_name, corrected_source_name))
sources[instrument_source_name] = aggr
source_to_db[instrument_source_name] = db_module_template.format(module)
print('-', instrument_source_name)
print()
print(f"Detector in use is {karabo_id}") print(f"Detector in use is {karabo_id}")
print(f"Instrument {instrument}") print(f"Instrument {instrument}")
step_timer = StepTimer() step_timer = StepTimer()
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Calibration constants # Calibration constants
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
requested_conditions = {
"frame_size": 1.0,
}
step_timer.start() step_timer.start()
# Offsets corrections = {}
constant_name = "Offset" constant_types = ["Offset", "DynamicFF"]
const_file = f"{constants_folder}/const_{constant_name}_{db_module}.h5" for source, db_module in source_to_db.items():
if not os.path.isfile(const_file): constants = {}
raise FileNotFoundError(f"{constant_name} constants are not found for {karabo_id}.") for constant_name in constant_types:
const_file = f"{constants_folder}/const_{constant_name}_{db_module}.h5"
with h5py.File(const_file, 'r') as f: if not os.path.isfile(const_file):
dark_conditions = dict( raise FileNotFoundError(f"{constant_name} constants are not found for {karabo_id}.")
num_frames=int(f["condition/Memory cells/value"][()]),
nx=int(f["condition/Pixels X/value"][()]), with h5py.File(const_file, 'r') as f:
ny=int(f["condition/Pixels Y/value"][()]), conditions = dict(
n_components=int(f["condition/FF components/value"][()]), frame_size=int(f["condition/Frame Size/value"][()])
) )
dark = f["data"][:] data = f["data"][:]
dark_creation_time = f["creation_time"][()].decode() data_creation_time = f["creation_time"][()].decode()
if not all(conditions[key] == value for key, value in requested_conditions.items()):
raise ValueError("Conditions for {constant_name} are not match")
print(f"{source} {db_module} {constant_name}: {data_creation_time}")
constants[constant_name] = data
dark = constants["Offset"]
flat = constants["DynamicFF"][0]
components = constants["DynamicFF"][1:][:n_components]
print(f"{constant_name}: {dark_creation_time}") dffc = DynamicFlatFieldCorrection.from_constants(
dark, flat, components, downsample_factors)
# Flat-field components corrections[source] = dffc
constant_name = "ComponentsFF"
const_file = f"{constants_folder}/const_{constant_name}_{db_module}.h5"
if not os.path.isfile(const_file):
raise FileNotFoundError(f"{constant_name} constants are not found for {karabo_id}.")
with h5py.File(const_file, 'r') as f:
flat_conditions = dict(
num_frames=int(f["condition/Memory cells/value"][()]),
nx=int(f["condition/Pixels X/value"][()]),
ny=int(f["condition/Pixels Y/value"][()]),
n_components=int(f["condition/FF components/value"][()]),
)
flat = f["data"][:]
components = flat[1:]
flat = flat[0]
flat_creation_time = f["creation_time"][()].decode()
print(f"{constant_name}: {dark_creation_time}")
if not all(flat_conditions[key] == value for key, value in dark_conditions.items()):
raise ValueError("Conditions for offsets and flat-field components are different")
conditions = type("Conditions", (), flat_conditions)
print(f"Image size: {conditions.nx} x {conditions.ny} px")
print(f"Number of flat-field components: {conditions.n_components}")
if conditions.n_components < n_components:
warnings.warn(
f"The correction set to use {n_components} flat-field components, "
f"but constants contains only {conditions.n_components}."
"The settings adjusted to the number of available components."
)
else:
components = components[:n_components]
step_timer.done_step("Load calibration constants") step_timer.done_step("Load calibration constants")
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Correction # Correction
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
step_timer.start() report = []
dc = RunDirectory(f"{in_folder}/r{run:04d}") for aggr, sources in aggregators.items():
dc = RunDirectory(f"{in_folder}/r{run:04d}", f"RAW-R{run:04d}-{aggr}-S*.h5")
num_trains, num_cells = dc[source][image_key].shape[:2]
num_images = num_trains * num_cells train_ids = set()
print("Number of trains:", num_trains) keydata_cache = {}
print("Number of images:", num_images) for instrument_source, corrected_source in sources:
keydata = dc[instrument_source][image_key].drop_empty_trains()
dffc = DynamicFlatFieldCorrection.from_constants( train_ids.update(keydata.train_ids)
dark, flat, components, downsample_factors) keydata_cache[instrument_source] = keydata
train_ids = np.array(sorted(train_ids))
proc = FlatFieldCorrectionFileProcessor(dffc, num_proc, source, image_key) ts = dc.select_trains(by_id[train_ids]).train_timestamps().astype(np.uint64)
for seq_id, train_mask in sequence_trains(train_ids, 200):
step_timer.start()
print('* sequience', seq_id)
seq_train_ids = train_ids[train_mask]
seq_timestamps = ts[train_mask]
dc_seq = dc.select_trains(by_id[seq_train_ids])
ntrains = len(seq_train_ids)
# create output file
channels = [f"{s[1]}/{index_group}" for s in sources]
f = DataFile.from_details(out_folder, aggr, run, seq_id)
f.create_metadata(like=dc, instrument_channels=channels)
f.create_index(seq_train_ids, timestamps=seq_timestamps)
seq_report = {}
image_datasets = {}
for instrument_source, corrected_source in sources:
keydata = dc_seq[instrument_source][image_key].drop_empty_trains()
count = keydata.data_counts()
i = np.flatnonzero(count.values)
raw_images = keydata.select_trains(np.s_[i]).ndarray()
# not pulse resolved
shape = keydata.shape
count = np.in1d(seq_train_ids, keydata.train_ids).astype(int)
src = f.create_instrument_source(corrected_source)
src.create_index(index_group=count)
ds_data = src.create_key(image_key, shape=shape, dtype=np.float32)
image_datasets[corrected_source] = ds_data
step_timer.done_step("Create output file")
for instrument_source, corrected_source in sources:
step_timer.start()
dc_seq = dc.select_trains(by_id[seq_train_ids])
dffc = corrections[instrument_source]
proc = FlatFieldCorrectionFileProcessor(dffc, num_proc, instrument_source, image_key)
proc.start_workers()
proc.run(dc_seq)
proc.join_workers()
# not pulse resolved
corrected_images = np.stack(proc.rdr.results, 0)
image_datasets[corrected_source][:] = corrected_images
proc.start_workers() seq_report[instrument_source] = (raw_images[0, 0], corrected_images[:20, 0])
proc.run(dc) step_timer.done_step("Correct flat-field")
proc.join_workers()
train_ids = proc.rdr.trains f.close()
corrected_images = np.stack(proc.rdr.results, 0) report.append(seq_report)
step_timer.done_step("Correct images")
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
step_timer.start() step_timer.start()
corr_source = corrected_source_template.format(karabo_id) for source, (raw_image, corrected_images) in report[0].items():
channel = image_key.partition('.')[0] display(Markdown(f"# {source}"))
data_source_id = corr_source + '/' + channel
ts = dc.train_timestamps().astype(np.uint64) display(Markdown("## The first raw image"))
ts = ts[np.in1d(dc.train_ids, train_ids)] plot_camera_image(raw_images[0, 0])
plt.show()
display(Markdown("## The first corrected image"))
plot_camera_image(corrected_images[0])
plt.show()
display(Markdown("## The first corrected images in the trains (up to 20)"))
plot_images(corrected_images, figsize=(13, 8))
plt.show()
for seq_id, train_mask in sequence_trains(train_ids): step_timer.done_step("Draw images")
seq_train_ids = train_ids[train_mask]
seq_timestamps = ts[train_mask]
ntrains = len(seq_train_ids)
f = DataFile.from_details(out_folder, karabo_da[0], run, seq_id)
src = f.create_instrument_source(corr_source)
f.create_metadata(like=dc, instrument_channels=(data_source_id,))
f.create_index(seq_train_ids, timestamps=seq_timestamps)
channels = {
image_key.partition('.')[0]: np.ones(ntrains, int)
}
src.create_index(**channels)
src.create_key(image_key, corrected_images[train_mask])
f.close()
step_timer.done_step("Save corrected images")
```
%% Cell type:markdown id: tags:
## The first raw image
%% Cell type:code id: tags:
``` python
step_timer.start()
counts = dc[source][image_key].data_counts()
i = np.flatnonzero(counts.values)
raw_images = dc[source][image_key].select_trains(np.s_[i]).ndarray()
plot_camera_image(raw_images[0, 0])
plt.show()
```
%% Cell type:markdown id: tags:
## The first corrected image
%% Cell type:code id: tags:
``` python
plot_camera_image(corrected_images[0, 0])
plt.show()
```
%% Cell type:markdown id: tags:
## The first corrected images in the trains (up to 20)
%% Cell type:code id: tags:
``` python
plot_images(corrected_images[:20, 0], figsize=(13, 8))
plt.show()
step_timer.done_step("Draw examples of corrected images")
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print(f"Total processing time {step_timer.timespan():.01f} s") print(f"Total processing time {step_timer.timespan():.01f} s")
step_timer.print_summary() step_timer.print_summary()
``` ```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts