Skip to content
Snippets Groups Projects
Commit 1b96322f authored by Egor Sobolev's avatar Egor Sobolev Committed by Philipp Schmidt
Browse files

Add correction notebook for Shimadzu HPVX2 camera

parent 8d3fa197
No related branches found
No related tags found
1 merge request!939[Generic][Shimadzu] Dynamic flat-field characterization and correction for MHz microscopy
Hide whitespace changes
Inline Side-by-side
notebooks/ShimadzuHPVX2/Correct_ShimadzuHPVX2_NBC.ipynb 0 → 100644
+ 324
− 0
View file @ 1b96322f
%% Cell type:markdown id: tags:
# Shimadzu HPVX2 Offline Correction
Author: Egor Sobolev
Offline dynamic flat-field correction for Shimadzu HPVX2 cameras
%% Cell type:code id: tags:
``` python
in_folder = "/gpfs/exfel/exp/SPB/202121/p002919/raw/" # input folder, required
out_folder = '/gpfs/exfel/data/scratch/esobolev/test/shimadzu' # output folder, required
metadata_folder = "" # Directory containing calibration_metadata.yml when run by xfel-calibrate
run = 30 # which run to read data from, required
# Data files parameters.
karabo_da = ['HPVX01'] # data aggregators
karabo_id = "SPB_EHD_HPVX2_2" # karabo prefix of Shimadzu HPV-X2 devices
#receiver_id = "PNCCD_FMT-0" # inset for receiver devices
#path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data
instrument_source_template = '{}/CAM/CAMERA:daqOutput' # data source path in h5file. Template filled with karabo_id
image_key = "data.image.pixels" # image data key in Karabo or exdf notation
# Database access parameters.
use_dir_creation_date = True # use dir creation date as data production reference date
cal_db_interface = "tcp://max-exfl-cal001:8021" # calibration DB interface to use
cal_db_timeout = 300000 # timeout on caldb requests
db_output = False # if True, the notebook sends dark constants to the calibration database
local_output = True # if True, the notebook saves dark constants locally
creation_time = "" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. 2019-07-04 11:02:41.00
n_components = 20 # number of principal components of flat-field to use in correction
downsample_factors = [1, 1] # list of downsample factors for each image dimention (y, x)
constants_folder = "/gpfs/exfel/data/scratch/esobolev/test/shimadzu"
db_module = "SHIMADZU_HPVX2_M001"
num_proc = 32 # number of processes running correction in parallel
corrected_source_template = '{}/CORR/CAMERA:daqOutput' # data source path in h5file. Template filled with karabo_id
```
%% Cell type:code id: tags:
``` python
import os
import h5py
import numpy as np
import matplotlib.pyplot as plt
from extra_data import RunDirectory
%matplotlib inline
from cal_tools.step_timing import StepTimer
from cal_tools.files import sequence_trains, DataFile
from dffc.correction import DynamicFlatFieldCorrectionCython as DynamicFlatFieldCorrection
from dffc.offline import FlatFieldCorrectionFileProcessor
from dffc.draw import plot_images, plot_camera_image
```
%% Cell type:code id: tags:
``` python
instrument = karabo_id.split("_")[0]
source = instrument_source_template.format(karabo_id)
print(f"Detector in use is {karabo_id}")
print(f"Instrument {instrument}")
step_timer = StepTimer()
```
%% Cell type:markdown id: tags:
# Calibration constants
%% Cell type:code id: tags:
``` python
step_timer.start()
# Offsets
constant_name = "Offset"
const_file = f"{constants_folder}/const_{constant_name}_{db_module}.h5"
if not os.path.isfile(const_file):
raise FileNotFoundError(f"{constant_name} constants are not found for {karabo_id}.")
with h5py.File(const_file, 'r') as f:
dark_conditions = dict(
num_frames=int(f["condition/Memory cells/value"][()]),
nx=int(f["condition/Pixels X/value"][()]),
ny=int(f["condition/Pixels Y/value"][()]),
n_components=int(f["condition/FF components/value"][()]),
)
dark = f["data"][:]
dark_creation_time = f["creation_time"][()].decode()
print(f"{constant_name}: {dark_creation_time}")
# Flat-field components
constant_name = "ComponentsFF"
const_file = f"{constants_folder}/const_{constant_name}_{db_module}.h5"
if not os.path.isfile(const_file):
raise FileNotFoundError(f"{constant_name} constants are not found for {karabo_id}.")
with h5py.File(const_file, 'r') as f:
flat_conditions = dict(
num_frames=int(f["condition/Memory cells/value"][()]),
nx=int(f["condition/Pixels X/value"][()]),
ny=int(f["condition/Pixels Y/value"][()]),
n_components=int(f["condition/FF components/value"][()]),
)
flat = f["data"][:]
components = flat[1:]
flat = flat[0]
flat_creation_time = f["creation_time"][()].decode()
print(f"{constant_name}: {dark_creation_time}")
if not all(flat_conditions[key] == value for key, value in dark_conditions.items()):
raise ValueError("Conditions for offsets and flat-field components are different")
conditions = type("Conditions", (), flat_conditions)
print(f"Image size: {conditions.nx} x {conditions.ny} px")
print(f"Number of flat-field components: {conditions.n_components}")
if conditions.n_components < n_components:
warnings.warn(
f"The correction set to use {n_components} flat-field components, "
f"but constants contains only {conditions.n_components}."
"The settings adjusted to the number of available components."
)
else:
components = components[:n_components]
step_timer.done_step("Load calibration constants")
```
%% Cell type:markdown id: tags:
# Correction
%% Cell type:code id: tags:
``` python
step_timer.start()
dc = RunDirectory(f"{in_folder}/r{run:04d}")
num_trains, num_cells = dc[source][image_key].shape[:2]
num_images = num_trains * num_cells
print("Number of trains:", num_trains)
print("Number of images:", num_images)
dffc = DynamicFlatFieldCorrection.from_constants(
dark, flat, components, downsample_factors)
proc = FlatFieldCorrectionFileProcessor(dffc, num_proc, source, image_key)
proc.start_workers()
proc.run(dc)
proc.join_workers()
train_ids = proc.rdr.trains
corrected_images = np.stack(proc.rdr.results, 0)
step_timer.done_step("Correct images")
```
%% Cell type:code id: tags:
``` python
step_timer.start()
corr_source = corrected_source_template.format(karabo_id)
channel = image_key.partition('.')[0]
data_source_id = corr_source + '/' + channel
ts = dc.train_timestamps().astype(np.uint64)
ts = ts[np.in1d(dc.train_ids, train_ids)]
for seq_id, train_mask in sequence_trains(train_ids):
seq_train_ids = train_ids[train_mask]
seq_timestamps = ts[train_mask]
ntrains = len(seq_train_ids)
f = DataFile.from_details(out_folder, karabo_da[0], run, seq_id)
src = f.create_instrument_source(corr_source)
f.create_metadata(like=dc, instrument_channels=(data_source_id,))
f.create_index(seq_train_ids, timestamps=seq_timestamps)
channels = {
image_key.partition('.')[0]: np.ones(ntrains, int)
}
src.create_index(**channels)
src.create_key(image_key, corrected_images[train_mask])
f.close()
step_timer.done_step("Save corrected images")
```
%% Cell type:markdown id: tags:
## The first raw image
%% Cell type:code id: tags:
``` python
step_timer.start()
counts = dc[source][image_key].data_counts()
i = np.flatnonzero(counts.values)
raw_images = dc[source][image_key].select_trains(np.s_[i]).ndarray()
plot_camera_image(raw_images[0, 0])
plt.show()
```
%% Cell type:markdown id: tags:
## The first corrected image
%% Cell type:code id: tags:
``` python
plot_camera_image(corrected_images[0, 0])
plt.show()
```
%% Cell type:markdown id: tags:
## The first corrected images in the trains (up to 20)
%% Cell type:code id: tags:
``` python
plot_images(corrected_images[:20, 0], figsize=(13, 8))
plt.show()
step_timer.done_step("Draw examples of corrected images")
```
%% Cell type:code id: tags:
``` python
print(f"Total processing time {step_timer.timespan():.01f} s")
step_timer.print_summary()
```
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts