Skip to content
Snippets Groups Projects
Commit 8d3fa197 authored by Egor Sobolev's avatar Egor Sobolev Committed by Philipp Schmidt
Browse files

Add dark and flat field characterization notebook for Shimadzu HPVX2

parent 132079a3
1 merge request!939[Generic][Shimadzu] Dynamic flat-field characterization and correction for MHz microscopy
Hide whitespace changes
Inline Side-by-side
notebooks/ShimadzuHPVX2/Characterize_Darks_ShimadzuHPVX2_NBC.ipynb 0 → 100644
+ 354
− 0
View file @ 8d3fa197
%% Cell type:markdown id: tags:
# Shimadzu HPVX2 Characterization of dark and flat field
Author: Egor Sobolev
Computation of dark offsets and flat-field principal components
%% Cell type:code id: tags:
``` python
in_folder = "/gpfs/exfel/exp/SPB/202121/p002919/raw/" # input folder, required
out_folder = '/gpfs/exfel/data/scratch/esobolev/test/shimadzu' # output folder, required
metadata_folder = "" # Directory containing calibration_metadata.yml when run by xfel-calibrate
dark_run = 59 # which run to read data from, required
flat_run = 40 # which run to read
# Data files parameters.
karabo_da = ['HPVX01'] # data aggregators
karabo_id = "SPB_EHD_HPVX2_2" # karabo prefix of Shimadzu HPV-X2 devices
#receiver_id = "PNCCD_FMT-0" # inset for receiver devices
#path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data
instrument_source_template = '{}/CAM/CAMERA:daqOutput' # data source path in h5file. Template filled with karabo_id and receiver_id
image_key = "data.image.pixels" # image data key in Karabo or exdf notation
# Database access parameters.
use_dir_creation_date = True # use dir creation date as data production reference date
cal_db_interface = "tcp://max-exfl-cal001:8021" # calibration DB interface to use
cal_db_timeout = 300000 # timeout on caldb requests
db_output = False # if True, the notebook sends dark constants to the calibration database
local_output = True # if True, the notebook saves dark constants locally
creation_time = "" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. 2019-07-04 11:02:41.00
n_components = 50
```
%% Cell type:code id: tags:
``` python
import datetime
import os
import warnings
warnings.filterwarnings('ignore')
import time
import numpy as np
import matplotlib.pyplot as plt
from extra_data import RunDirectory
%matplotlib inline
from cal_tools.step_timing import StepTimer
from cal_tools.tools import (
get_dir_creation_date,
get_pdu_from_db,
get_random_db_interface,
get_report,
save_const_to_h5,
save_dict_to_hdf5,
send_to_db,
run_prop_seq_from_path,
)
import dffc
from dffc.draw import plot_images, plot_camera_image
```
%% Cell type:code id: tags:
``` python
creation_time=None
if use_dir_creation_date:
creation_time = get_dir_creation_date(in_folder, max(dark_run, flat_run))
print(f"Using {creation_time} as creation time of constant.")
run, prop, seq = run_prop_seq_from_path(in_folder)
file_loc = f'proposal: {prop}, runs: {dark_run} {flat_run}'
# Read report path and create file location tuple to add with the injection
file_loc = f"proposal:{prop} runs:{dark_run} {flat_run}"
report = get_report(metadata_folder)
cal_db_interface = get_random_db_interface(cal_db_interface)
print(f'Calibration database interface: {cal_db_interface}')
instrument = karabo_id.split("_")[0]
source = instrument_source_template.format(karabo_id)
print(f"Detector in use is {karabo_id}")
print(f"Instrument {instrument}")
step_timer = StepTimer()
```
%% Cell type:markdown id: tags:
# Offset map
%% Cell type:code id: tags:
``` python
step_timer.start()
dark_dc = RunDirectory(f"{in_folder}/r{dark_run:04d}")
dark_dc = dark_dc.select([(source, image_key)])
key_data = dark_dc[source][image_key]
images_dark = key_data.ndarray()
ntrain, npulse, ny, nx = images_dark.shape
print(f"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})")
print(f"Image size: {ny} x {nx} px")
step_timer.done_step("Read dark images")
```
%% Cell type:code id: tags:
``` python
step_timer.start()
dark = dffc.process_dark(images_dark)
step_timer.done_step("Process dark images")
```
%% Cell type:code id: tags:
``` python
step_timer.start()
plot_camera_image(dark)
plt.show()
step_timer.done_step("Draw offset map")
```
%% Cell type:markdown id: tags:
# Flat-field PCA decomposition
%% Cell type:code id: tags:
``` python
step_timer.start()
flat_dc = RunDirectory(f"{in_folder}/r{flat_run:04d}")
flat_dc = flat_dc.select([(source, image_key)])
key_data = flat_dc[source][image_key]
images_flat = key_data.ndarray()
ntrain, npulse, ny, nx = images_flat.shape
print(f"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})")
print(f"Image size: {ny} x {nx} px")
step_timer.done_step("Read flat-field images")
tm0 = time.monotonic()
tm_cm = time.monotonic() - tm0
```
%% Cell type:code id: tags:
``` python
step_timer.start()
flat, components, explained_variance_ratio = dffc.process_flat(
images_flat, dark, n_components)
step_timer.done_step("Process flat-field images")
```
%% Cell type:markdown id: tags:
## Average flat-field
%% Cell type:code id: tags:
``` python
step_timer.start()
plot_camera_image(flat)
plt.show()
```
%% Cell type:markdown id: tags:
## Explained variance ratio
%% Cell type:code id: tags:
``` python
fig, ax = plt.subplots(1, 1, figsize=(10,4), tight_layout=True)
ax.semilogy(explained_variance_ratio, 'o')
ax.set_xticks(np.arange(len(explained_variance_ratio)))
ax.set_xlabel("Component no.")
ax.set_ylabel("Variance fraction")
plt.show()
```
%% Cell type:markdown id: tags:
# The first principal components (up to 20)
%% Cell type:code id: tags:
``` python
plot_images(components[:20], figsize=(13, 8))
plt.show()
step_timer.done_step("Draw flat-field map and components")
```
%% Cell type:markdown id: tags:
## Calibration constants
%% Cell type:code id: tags:
``` python
step_timer.start()
# Output Folder Creation:
os.makedirs(out_folder, exist_ok=True)
db_module = "SHIMADZU_HPVX2_M001"
constant_name = "Offset"
conditions = {
'Memory cells': {'value': 128},
'Pixels X': {'value': flat.shape[1]},
'Pixels Y': {'value': flat.shape[0]},
'FF components': {'value': components.shape[0]}
}
data_to_store = {
'condition': conditions,
'db_module': db_module,
'karabo_id': karabo_id,
'constant': constant_name,
'data': dark,
'creation_time': creation_time.replace(microsecond=0),
'file_loc': file_loc,
'report': report,
}
ofile = f"{out_folder}/const_{constant_name}_{db_module}.h5"
if os.path.isfile(ofile):
print(f'File {ofile} already exists and will be overwritten')
save_dict_to_hdf5(data_to_store, ofile)
constant_name = "ComponentsFF"
data_to_store.update({
'constant': constant_name,
'data': np.concatenate([flat[None, ...], components]),
})
ofile = f"{out_folder}/const_{constant_name}_{db_module}.h5"
if os.path.isfile(ofile):
print(f'File {ofile} already exists and will be overwritten')
save_dict_to_hdf5(data_to_store, ofile)
step_timer.done_step("Storing calibration constants")
```
%% Cell type:code id: tags:
``` python
print(f"Total processing time {step_timer.timespan():.01f} s")
step_timer.print_summary()
```
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts