Skip to content
Snippets Groups Projects
Commit 2d3f03bf authored by Egor Sobolev's avatar Egor Sobolev Committed by Philipp Schmidt
Browse files

Support multiple module detector in Characterize_DynamicFF_NBC.ipynb

parent 5ca049e8
No related branches found
No related tags found
1 merge request!939[Generic][Shimadzu] Dynamic flat-field characterization and correction for MHz microscopy
Hide whitespace changes
Inline Side-by-side
notebooks/DynamicFF/Characterize_DynamicFF_NBC.ipynb
+ 132
− 171
View file @ 2d3f03bf
%% Cell type:markdown id: tags:
# Shimadzu HPVX2 Characterization of dark and flat field
# Characterization of dark and flat field for Dynamic Flat Field correction
Author: Egor Sobolev
Computation of dark offsets and flat-field principal components
%% Cell type:code id: tags:
``` python
in_folder = "/gpfs/exfel/exp/SPB/202121/p002919/raw/" # input folder, required
in_folder = "/gpfs/exfel/exp/SPB/202430/p900425/raw" # input folder, required
out_folder = '/gpfs/exfel/data/scratch/esobolev/test/shimadzu' # output folder, required
metadata_folder = "" # Directory containing calibration_metadata.yml when run by xfel-calibrate
dark_run = 59 # which run to read data from, required
flat_run = 40 # which run to read
dark_run = 1 # which run to read data from, required
flat_run = 2 # which run to read
# Data files parameters.
karabo_da = ['HPVX01'] # data aggregators
karabo_id = "SPB_EHD_HPVX2_2" # karabo prefix of Shimadzu HPV-X2 devices
karabo_da = ['HPVX01/1', 'HPVX01/2'] # data aggregators
karabo_id = "SPB_EHD_MIC" # karabo prefix of Shimadzu HPV-X2 devices
#receiver_id = "PNCCD_FMT-0" # inset for receiver devices
#path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data
instrument_source_template = '{}/CAM/CAMERA:daqOutput' # data source path in h5file. Template filled with karabo_id and receiver_id
instrument_source_template = 'SPB_EHD_MIC/CAM/HPVX2_{module}:daqOutput' # data source path in h5file.
#instrument_source_template = 'SPB_EHD_HPVX2_{module}/CAM/CAMERA:daqOutput'
image_key = "data.image.pixels" # image data key in Karabo or exdf notation
db_module_template = "Shimadzu_HPVX2_{}"
# Database access parameters.
use_dir_creation_date = True # use dir creation date as data production reference date
cal_db_interface = "tcp://max-exfl-cal001:8021" # calibration DB interface to use
cal_db_timeout = 300000 # timeout on caldb requests
db_output = False # if True, the notebook sends dark constants to the calibration database
local_output = True # if True, the notebook saves dark constants locally
creation_time = "" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. 2019-07-04 11:02:41.00
n_components = 50
n_components = 50 # Number of principal components to compute
```
%% Cell type:code id: tags:
``` python
import datetime
import os
import warnings
warnings.filterwarnings('ignore')
import time
import numpy as np
import matplotlib.pyplot as plt
from IPython.display import display, Markdown
from extra_data import RunDirectory
%matplotlib inline
from cal_tools.step_timing import StepTimer
from cal_tools.tools import (
get_dir_creation_date,
get_pdu_from_db,
get_random_db_interface,
get_report,
save_const_to_h5,
save_dict_to_hdf5,
send_to_db,
run_prop_seq_from_path,
)
import dffc
from dffc.draw import plot_images, plot_camera_image
```
%% Cell type:code id: tags:
``` python
creation_time=None
if use_dir_creation_date:
creation_time = get_dir_creation_date(in_folder, max(dark_run, flat_run))
print(f"Using {creation_time} as creation time of constant.")
run, prop, seq = run_prop_seq_from_path(in_folder)
file_loc = f'proposal: {prop}, runs: {dark_run} {flat_run}'
# Read report path and create file location tuple to add with the injection
file_loc = f"proposal:{prop} runs:{dark_run} {flat_run}"
report = get_report(metadata_folder)
cal_db_interface = get_random_db_interface(cal_db_interface)
print(f'Calibration database interface: {cal_db_interface}')
print()
instrument, part, component = karabo_id.split('_')
instrument = karabo_id.split("_")[0]
source = instrument_source_template.format(karabo_id)
sources = {}
source_to_db = {}
print("Sources:")
for da in karabo_da:
aggr, _, module = da.partition('/')
source_name = instrument_source_template.format(
instrument=instrument, part=part, component=component,
module=module
)
sources[source_name] = aggr
source_to_db[source_name] = db_module_template.format(module)
print('-', source_name)
print()
print(f"Detector in use is {karabo_id}")
print(f"Instrument {instrument}")
step_timer = StepTimer()
constants = {}
```
%% Cell type:markdown id: tags:
# Offset map
%% Cell type:code id: tags:
``` python
step_timer.start()
for source, aggr in sources.items():
display(Markdown(f"## {source}"))
dark_dc = RunDirectory(f"{in_folder}/r{dark_run:04d}")
dark_dc = dark_dc.select([(source, image_key)])
key_data = dark_dc[source][image_key]
images_dark = key_data.ndarray()
ntrain, npulse, ny, nx = images_dark.shape
print(f"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})")
print(f"Image size: {ny} x {nx} px")
step_timer.done_step("Read dark images")
```
%% Cell type:code id: tags:
``` python
step_timer.start()
dark = dffc.process_dark(images_dark)
step_timer.done_step("Process dark images")
```
%% Cell type:code id: tags:
``` python
step_timer.start()
plot_camera_image(dark)
plt.show()
step_timer.done_step("Draw offset map")
# read
step_timer.start()
dark_dc = RunDirectory(f"{in_folder}/r{dark_run:04d}",
include=f"RAW-R{dark_run:04d}-{aggr}-S*.h5")
dark_dc = dark_dc.select([(source, image_key)])
key_data = dark_dc[source][image_key]
images_dark = key_data.ndarray()
ntrain, npulse, ny, nx = images_dark.shape
print(f"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})")
print(f"Image size: {ny} x {nx} px")
step_timer.done_step("Read dark images")
# process
step_timer.start()
dark = dffc.process_dark(images_dark)
module_constants = constants.setdefault(source, {})
module_constants["Offset"] = dark
step_timer.done_step("Process dark images")
display()
# draw plots
step_timer.start()
plot_camera_image(dark)
plt.show()
step_timer.done_step("Draw offsets")
```
%% Cell type:markdown id: tags:
# Flat-field PCA decomposition
%% Cell type:code id: tags:
``` python
step_timer.start()
flat_dc = RunDirectory(f"{in_folder}/r{flat_run:04d}")
flat_dc = flat_dc.select([(source, image_key)])
key_data = flat_dc[source][image_key]
images_flat = key_data.ndarray()
ntrain, npulse, ny, nx = images_flat.shape
print(f"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})")
print(f"Image size: {ny} x {nx} px")
step_timer.done_step("Read flat-field images")
tm0 = time.monotonic()
tm_cm = time.monotonic() - tm0
```
for source, aggr in sources.items():
display(Markdown(f"## {source}"))
%% Cell type:code id: tags:
# read
step_timer.start()
flat_dc = RunDirectory(f"{in_folder}/r{flat_run:04d}",
include=f"RAW-R{flat_run:04d}-{aggr}-S*.h5")
flat_dc = flat_dc.select([(source, image_key)])
key_data = flat_dc[source][image_key]
images_flat = key_data.ndarray()
ntrain, npulse, ny, nx = images_flat.shape
print(f"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})")
print(f"Image size: {ny} x {nx} px")
step_timer.done_step("Read flat-field images")
# process
step_timer.start()
flat, components, explained_variance_ratio = dffc.process_flat(
images_flat, dark, n_components)
module_constants = constants.setdefault(source, {})
module_constants["DynamicFF"] = np.concatenate([flat[None, ...], components])
step_timer.done_step("Process flat-field images")
# draw plots
step_timer.start()
display(Markdown("### Average flat-field"))
plot_camera_image(flat)
plt.show()
display(Markdown("### Explained variance ratio"))
fig, ax = plt.subplots(1, 1, figsize=(10,4), tight_layout=True)
ax.semilogy(explained_variance_ratio, 'o')
ax.set_xticks(np.arange(len(explained_variance_ratio)))
ax.set_xlabel("Component no.")
ax.set_ylabel("Variance fraction")
plt.show()
display(Markdown("### The first principal components (up to 20)"))
plot_images(components[:20], figsize=(13, 8))
plt.show()
``` python
step_timer.start()
flat, components, explained_variance_ratio = dffc.process_flat(
images_flat, dark, n_components)
step_timer.done_step("Process flat-field images")
```
%% Cell type:markdown id: tags:
## Average flat-field
%% Cell type:code id: tags:
``` python
step_timer.start()
plot_camera_image(flat)
plt.show()
```
%% Cell type:markdown id: tags:
## Explained variance ratio
%% Cell type:code id: tags:
``` python
fig, ax = plt.subplots(1, 1, figsize=(10,4), tight_layout=True)
ax.semilogy(explained_variance_ratio, 'o')
ax.set_xticks(np.arange(len(explained_variance_ratio)))
ax.set_xlabel("Component no.")
ax.set_ylabel("Variance fraction")
plt.show()
```
%% Cell type:markdown id: tags:
# The first principal components (up to 20)
%% Cell type:code id: tags:
``` python
plot_images(components[:20], figsize=(13, 8))
plt.show()
step_timer.done_step("Draw flat-field map and components")
step_timer.done_step("Draw flat-field")
```
%% Cell type:markdown id: tags:
## Calibration constants
%% Cell type:code id: tags:
``` python
step_timer.start()
# Output Folder Creation:
os.makedirs(out_folder, exist_ok=True)
db_module = "SHIMADZU_HPVX2_M001"
constant_name = "Offset"
conditions = {
'Memory cells': {'value': 128},
'Pixels X': {'value': flat.shape[1]},
'Pixels Y': {'value': flat.shape[0]},
'FF components': {'value': components.shape[0]}
}
data_to_store = {
'condition': conditions,
'db_module': db_module,
'karabo_id': karabo_id,
'constant': constant_name,
'data': dark,
'creation_time': creation_time.replace(microsecond=0),
'file_loc': file_loc,
'report': report,
}
ofile = f"{out_folder}/const_{constant_name}_{db_module}.h5"
if os.path.isfile(ofile):
print(f'File {ofile} already exists and will be overwritten')
save_dict_to_hdf5(data_to_store, ofile)
constant_name = "ComponentsFF"
data_to_store.update({
'constant': constant_name,
'data': np.concatenate([flat[None, ...], components]),
})
ofile = f"{out_folder}/const_{constant_name}_{db_module}.h5"
if os.path.isfile(ofile):
print(f'File {ofile} already exists and will be overwritten')
save_dict_to_hdf5(data_to_store, ofile)
for source, module_constants in constants.items():
for constant_name, data in module_constants.items():
db_module = source_to_db[source]
conditions = {
'Frame Size': {'value': 1.0},
}
data_to_store = {
'condition': conditions,
'db_module': db_module,
'karabo_id': karabo_id,
'constant': constant_name,
'data': data,
'creation_time': creation_time.replace(microsecond=0),
'file_loc': file_loc,
'report': report,
}
ofile = f"{out_folder}/const_{constant_name}_{db_module}.h5"
if os.path.isfile(ofile):
print(f'File {ofile} already exists and will be overwritten')
save_dict_to_hdf5(data_to_store, ofile)
step_timer.done_step("Storing calibration constants")
```
%% Cell type:code id: tags:
``` python
print(f"Total processing time {step_timer.timespan():.01f} s")
step_timer.print_summary()
```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts