Skip to content
Snippets Groups Projects
Commit 7daba04f authored by Egor Sobolev's avatar Egor Sobolev
Browse files

Support multiple module detector in Correct_DynamicFF_NBC.ipynb

parent 91e05cb3
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
notebooks/DynamicFF/Correct_DynamicFF_NBC.ipynb
+ 153
− 170
View file @ 7daba04f
%% Cell type:markdown id: tags:
# Shimadzu HPVX2 Offline Correction
# Dynamic Flat-field Offline Correction
Author: Egor Sobolev
Offline dynamic flat-field correction for Shimadzu HPVX2 cameras
Offline dynamic flat-field correction
%% Cell type:code id: tags:
``` python
in_folder = "/gpfs/exfel/exp/SPB/202121/p002919/raw/" # input folder, required
in_folder = "/gpfs/exfel/exp/SPB/202430/p900425/raw" # input folder, required
out_folder = '/gpfs/exfel/data/scratch/esobolev/test/shimadzu' # output folder, required
metadata_folder = "" # Directory containing calibration_metadata.yml when run by xfel-calibrate
run = 30 # which run to read data from, required
run = 3 # which run to read data from, required
# Data files parameters.
karabo_da = ['HPVX01'] # data aggregators
karabo_id = "SPB_EHD_HPVX2_2" # karabo prefix of Shimadzu HPV-X2 devices
karabo_da = ['HPVX01/1', 'HPVX01/2'] # data aggregators
karabo_id = "SPB_EHD_MIC" # karabo prefix of Shimadzu HPV-X2 devices
#receiver_id = "PNCCD_FMT-0" # inset for receiver devices
#path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data
instrument_source_template = '{}/CAM/CAMERA:daqOutput' # data source path in h5file. Template filled with karabo_id
instrument_source_template = 'SPB_EHD_MIC/CAM/HPVX2_{module}:daqOutput' # data source path in h5file.
image_key = "data.image.pixels" # image data key in Karabo or exdf notation
# Database access parameters.
use_dir_creation_date = True # use dir creation date as data production reference date
cal_db_interface = "tcp://max-exfl-cal001:8021" # calibration DB interface to use
cal_db_timeout = 300000 # timeout on caldb requests
db_output = False # if True, the notebook sends dark constants to the calibration database
local_output = True # if True, the notebook saves dark constants locally
creation_time = "" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. 2019-07-04 11:02:41.00
n_components = 20 # number of principal components of flat-field to use in correction
downsample_factors = [1, 1] # list of downsample factors for each image dimention (y, x)
constants_folder = "/gpfs/exfel/data/scratch/esobolev/test/shimadzu"
db_module = "SHIMADZU_HPVX2_M001"
db_module_template = "Shimadzu_HPVX2_{}"
num_proc = 32 # number of processes running correction in parallel
corrected_source_template = '{}/CORR/CAMERA:daqOutput' # data source path in h5file. Template filled with karabo_id
corrected_source_template = 'SPB_EHD_MIC/CORR/HPVX2_{module}:output' # data source path in h5file.
```
%% Cell type:code id: tags:
``` python
import os
import h5py
import numpy as np
import matplotlib.pyplot as plt
from IPython.display import display, Markdown
from extra_data import RunDirectory
from extra_data import RunDirectory, by_id
%matplotlib inline
from cal_tools.step_timing import StepTimer
from cal_tools.files import sequence_trains, DataFile
from dffc.correction import DynamicFlatFieldCorrectionCython as DynamicFlatFieldCorrection
from dffc.offline import FlatFieldCorrectionFileProcessor
from dffc.draw import plot_images, plot_camera_image
```
%% Cell type:code id: tags:
``` python
instrument = karabo_id.split("_")[0]
source = instrument_source_template.format(karabo_id)
index_group = image_key.partition('.')[0]
instrument, part, component = karabo_id.split('_')
aggregators = {}
sources = {}
source_to_db = {}
print("Sources:")
for da in karabo_da:
aggr, _, module = da.partition('/')
instrument_source_name = instrument_source_template.format(
instrument=instrument, part=part, component=component,
module=module
)
corrected_source_name = corrected_source_template.format(
instrument=instrument, part=part, component=component,
module=module
)
aggregators.setdefault(aggr, []).append(
(instrument_source_name, corrected_source_name))
sources[instrument_source_name] = aggr
source_to_db[instrument_source_name] = db_module_template.format(module)
print('-', instrument_source_name)
print()
print(f"Detector in use is {karabo_id}")
print(f"Instrument {instrument}")
step_timer = StepTimer()
```
%% Cell type:markdown id: tags:
# Calibration constants
%% Cell type:code id: tags:
``` python
requested_conditions = {
"frame_size": 1.0,
}
step_timer.start()
# Offsets
constant_name = "Offset"
const_file = f"{constants_folder}/const_{constant_name}_{db_module}.h5"
if not os.path.isfile(const_file):
raise FileNotFoundError(f"{constant_name} constants are not found for {karabo_id}.")
with h5py.File(const_file, 'r') as f:
dark_conditions = dict(
num_frames=int(f["condition/Memory cells/value"][()]),
nx=int(f["condition/Pixels X/value"][()]),
ny=int(f["condition/Pixels Y/value"][()]),
n_components=int(f["condition/FF components/value"][()]),
)
dark = f["data"][:]
dark_creation_time = f["creation_time"][()].decode()
corrections = {}
constant_types = ["Offset", "DynamicFF"]
for source, db_module in source_to_db.items():
constants = {}
for constant_name in constant_types:
const_file = f"{constants_folder}/const_{constant_name}_{db_module}.h5"
if not os.path.isfile(const_file):
raise FileNotFoundError(f"{constant_name} constants are not found for {karabo_id}.")
with h5py.File(const_file, 'r') as f:
conditions = dict(
frame_size=int(f["condition/Frame Size/value"][()])
)
data = f["data"][:]
data_creation_time = f["creation_time"][()].decode()
if not all(conditions[key] == value for key, value in requested_conditions.items()):
raise ValueError("Conditions for {constant_name} are not match")
print(f"{source} {db_module} {constant_name}: {data_creation_time}")
constants[constant_name] = data
dark = constants["Offset"]
flat = constants["DynamicFF"][0]
components = constants["DynamicFF"][1:][:n_components]
print(f"{constant_name}: {dark_creation_time}")
dffc = DynamicFlatFieldCorrection.from_constants(
dark, flat, components, downsample_factors)
# Flat-field components
constant_name = "ComponentsFF"
const_file = f"{constants_folder}/const_{constant_name}_{db_module}.h5"
if not os.path.isfile(const_file):
raise FileNotFoundError(f"{constant_name} constants are not found for {karabo_id}.")
with h5py.File(const_file, 'r') as f:
flat_conditions = dict(
num_frames=int(f["condition/Memory cells/value"][()]),
nx=int(f["condition/Pixels X/value"][()]),
ny=int(f["condition/Pixels Y/value"][()]),
n_components=int(f["condition/FF components/value"][()]),
)
flat = f["data"][:]
components = flat[1:]
flat = flat[0]
flat_creation_time = f["creation_time"][()].decode()
print(f"{constant_name}: {dark_creation_time}")
if not all(flat_conditions[key] == value for key, value in dark_conditions.items()):
raise ValueError("Conditions for offsets and flat-field components are different")
conditions = type("Conditions", (), flat_conditions)
print(f"Image size: {conditions.nx} x {conditions.ny} px")
print(f"Number of flat-field components: {conditions.n_components}")
if conditions.n_components < n_components:
warnings.warn(
f"The correction set to use {n_components} flat-field components, "
f"but constants contains only {conditions.n_components}."
"The settings adjusted to the number of available components."
)
else:
components = components[:n_components]
corrections[source] = dffc
step_timer.done_step("Load calibration constants")
```
%% Cell type:markdown id: tags:
# Correction
%% Cell type:code id: tags:
``` python
step_timer.start()
dc = RunDirectory(f"{in_folder}/r{run:04d}")
num_trains, num_cells = dc[source][image_key].shape[:2]
num_images = num_trains * num_cells
print("Number of trains:", num_trains)
print("Number of images:", num_images)
dffc = DynamicFlatFieldCorrection.from_constants(
dark, flat, components, downsample_factors)
proc = FlatFieldCorrectionFileProcessor(dffc, num_proc, source, image_key)
report = []
for aggr, sources in aggregators.items():
dc = RunDirectory(f"{in_folder}/r{run:04d}", f"RAW-R{run:04d}-{aggr}-S*.h5")
train_ids = set()
keydata_cache = {}
for instrument_source, corrected_source in sources:
keydata = dc[instrument_source][image_key].drop_empty_trains()
train_ids.update(keydata.train_ids)
keydata_cache[instrument_source] = keydata
train_ids = np.array(sorted(train_ids))
ts = dc.select_trains(by_id[train_ids]).train_timestamps().astype(np.uint64)
for seq_id, train_mask in sequence_trains(train_ids, 200):
step_timer.start()
print('* sequience', seq_id)
seq_train_ids = train_ids[train_mask]
seq_timestamps = ts[train_mask]
dc_seq = dc.select_trains(by_id[seq_train_ids])
ntrains = len(seq_train_ids)
# create output file
channels = [f"{s[1]}/{index_group}" for s in sources]
f = DataFile.from_details(out_folder, aggr, run, seq_id)
f.create_metadata(like=dc, instrument_channels=channels)
f.create_index(seq_train_ids, timestamps=seq_timestamps)
seq_report = {}
image_datasets = {}
for instrument_source, corrected_source in sources:
keydata = dc_seq[instrument_source][image_key].drop_empty_trains()
count = keydata.data_counts()
i = np.flatnonzero(count.values)
raw_images = keydata.select_trains(np.s_[i]).ndarray()
# not pulse resolved
shape = keydata.shape
count = np.in1d(seq_train_ids, keydata.train_ids).astype(int)
src = f.create_instrument_source(corrected_source)
src.create_index(index_group=count)
ds_data = src.create_key(image_key, shape=shape, dtype=np.float32)
image_datasets[corrected_source] = ds_data
step_timer.done_step("Create output file")
for instrument_source, corrected_source in sources:
step_timer.start()
dc_seq = dc.select_trains(by_id[seq_train_ids])
dffc = corrections[instrument_source]
proc = FlatFieldCorrectionFileProcessor(dffc, num_proc, instrument_source, image_key)
proc.start_workers()
proc.run(dc_seq)
proc.join_workers()
# not pulse resolved
corrected_images = np.stack(proc.rdr.results, 0)
image_datasets[corrected_source][:] = corrected_images
proc.start_workers()
proc.run(dc)
proc.join_workers()
seq_report[instrument_source] = (raw_images[0, 0], corrected_images[:20, 0])
step_timer.done_step("Correct flat-field")
train_ids = proc.rdr.trains
corrected_images = np.stack(proc.rdr.results, 0)
step_timer.done_step("Correct images")
f.close()
report.append(seq_report)
```
%% Cell type:code id: tags:
``` python
step_timer.start()
corr_source = corrected_source_template.format(karabo_id)
channel = image_key.partition('.')[0]
data_source_id = corr_source + '/' + channel
for source, (raw_image, corrected_images) in report[0].items():
display(Markdown(f"# {source}"))
ts = dc.train_timestamps().astype(np.uint64)
ts = ts[np.in1d(dc.train_ids, train_ids)]
display(Markdown("## The first raw image"))
plot_camera_image(raw_images[0, 0])
plt.show()
display(Markdown("## The first corrected image"))
plot_camera_image(corrected_images[0])
plt.show()
display(Markdown("## The first corrected images in the trains (up to 20)"))
plot_images(corrected_images, figsize=(13, 8))
plt.show()
for seq_id, train_mask in sequence_trains(train_ids):
seq_train_ids = train_ids[train_mask]
seq_timestamps = ts[train_mask]
ntrains = len(seq_train_ids)
f = DataFile.from_details(out_folder, karabo_da[0], run, seq_id)
src = f.create_instrument_source(corr_source)
f.create_metadata(like=dc, instrument_channels=(data_source_id,))
f.create_index(seq_train_ids, timestamps=seq_timestamps)
channels = {
image_key.partition('.')[0]: np.ones(ntrains, int)
}
src.create_index(**channels)
src.create_key(image_key, corrected_images[train_mask])
f.close()
step_timer.done_step("Save corrected images")
```
%% Cell type:markdown id: tags:
## The first raw image
%% Cell type:code id: tags:
``` python
step_timer.start()
counts = dc[source][image_key].data_counts()
i = np.flatnonzero(counts.values)
raw_images = dc[source][image_key].select_trains(np.s_[i]).ndarray()
plot_camera_image(raw_images[0, 0])
plt.show()
```
%% Cell type:markdown id: tags:
## The first corrected image
%% Cell type:code id: tags:
``` python
plot_camera_image(corrected_images[0, 0])
plt.show()
```
%% Cell type:markdown id: tags:
## The first corrected images in the trains (up to 20)
%% Cell type:code id: tags:
``` python
plot_images(corrected_images[:20, 0], figsize=(13, 8))
plt.show()
step_timer.done_step("Draw examples of corrected images")
step_timer.done_step("Draw images")
```
%% Cell type:code id: tags:
``` python
print(f"Total processing time {step_timer.timespan():.01f} s")
step_timer.print_summary()
```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts