Skip to content
Snippets Groups Projects
Commit 5100047f authored by Philipp Schmidt's avatar Philipp Schmidt
Browse files

(fixup) clean-up and typo fixes in characterization notebook

parent e7564ff5
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
notebooks/DynamicFF/Characterize_DynamicFF_NBC.ipynb
+ 4
− 9
View file @ 5100047f
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Characterization of dark and flat field for Dynamic Flat Field correction # Characterization of dark and flat field for Dynamic Flat Field correction
Author: Egor Sobolev Author: Egor Sobolev
Computation of dark offsets and flat-field principal components Computation of dark offsets and flat-field principal components
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
in_folder = "/gpfs/exfel/exp/SPB/202430/p900425/raw" # input folder, required in_folder = "/gpfs/exfel/exp/SPB/202430/p900425/raw" # input folder, required
out_folder = '/gpfs/exfel/data/scratch/esobolev/test/shimadzu' # output folder, required out_folder = '/gpfs/exfel/data/scratch/esobolev/test/shimadzu' # output folder, required
metadata_folder = "" # Directory containing calibration_metadata.yml when run by xfel-calibrate metadata_folder = "" # Directory containing calibration_metadata.yml when run by xfel-calibrate
run_high = 1 # run number in which dark data was recorded, required run_high = 1 # run number in which dark data was recorded, required
run_low = 2 # run number in which flat-field data was recorded, required run_low = 2 # run number in which flat-field data was recorded, required
operation_mode = "TI_DynamicFF" # Detector operation mode, optional (defaults to "TI_DynamicFF") operation_mode = "DynamicFF" # Detector operation mode, optional (defaults to "TI_DynamicFF")
# Data files parameters. # Data files parameters.
karabo_da = ['-1'] # data aggregators karabo_da = ['-1'] # data aggregators
karabo_id = "SPB_MIC_HPVX2" # karabo prefix of Shimadzu HPV-X2 devices karabo_id = "SPB_MIC_HPVX2" # karabo prefix of Shimadzu HPV-X2 devices
# Database access parameters. # Database access parameters.
cal_db_interface = "tcp://max-exfl-cal001:8021" # calibration DB interface to use cal_db_interface = "tcp://max-exfl-cal001:8021" # Unused, calibration DB interface to use
db_output = True # if True, the notebook sends dark constants to the calibration database db_output = True # if True, the notebook sends dark constants to the calibration database
local_output = True # if True, the notebook saves dark constants locally local_output = True # if True, the notebook saves dark constants locally
# Calibration constants parameters # Calibration constants parameters
n_components = 50 # Number of principal components of flat-field to compute (default: 50) n_components = 50 # Number of principal components of flat-field to compute (default: 50)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import datetime import datetime
import os import os
import warnings import warnings
from logging import warning from logging import warning
warnings.filterwarnings('ignore') warnings.filterwarnings('ignore')
import time import time
import numpy as np import numpy as np
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
from IPython.display import display, Markdown from IPython.display import display, Markdown
from extra_data import RunDirectory from extra_data import RunDirectory
%matplotlib inline %matplotlib inline
from cal_tools.step_timing import StepTimer from cal_tools.step_timing import StepTimer
from cal_tools.tools import ( from cal_tools.tools import (
get_dir_creation_date, get_dir_creation_date,
get_random_db_interface,
get_report, get_report,
save_dict_to_hdf5, save_dict_to_hdf5,
run_prop_seq_from_path, run_prop_seq_from_path,
) )
from cal_tools.restful_config import calibration_client from cal_tools.restful_config import calibration_client
from cal_tools.shimadzu import ShimadzuHPVX2 from cal_tools.shimadzu import ShimadzuHPVX2
import dynflatfield as dffc import dynflatfield as dffc
from dynflatfield.draw import plot_images, plot_camera_image from dynflatfield.draw import plot_images, plot_camera_image
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
cal_db_interface = get_random_db_interface(cal_db_interface)
print(f'Calibration database interface: {cal_db_interface}')
print()
cc = calibration_client() cc = calibration_client()
pdus = cc.get_all_phy_det_units_from_detector( pdus = cc.get_all_phy_det_units_from_detector(
{"detector_identifier": karabo_id}) {"detector_identifier": karabo_id})
if not pdus["success"]: if not pdus["success"]:
raise ValueException("Failed to retrieve PDUs") raise ValueError("Failed to retrieve PDUs")
detector_info = pdus['data'][0]['detector'] detector_info = pdus['data'][0]['detector']
detector = ShimadzuHPVX2(detector_info["source_name_pattern"]) detector = ShimadzuHPVX2(detector_info["source_name_pattern"])
print(f"Instrument {detector.instrument}") print(f"Instrument {detector.instrument}")
print(f"Detector in use is {karabo_id}") print(f"Detector in use is {karabo_id}")
modules = {} modules = {}
for pdu_no, pdu in enumerate(pdus["data"]): for pdu_no, pdu in enumerate(pdus["data"]):
db_module = pdu["physical_name"] db_module = pdu["physical_name"]
module = pdu["module_number"] module = pdu["module_number"]
da = pdu["karabo_da"] da = pdu["karabo_da"]
if karabo_da[0] != "-1" and da not in karabo_da: if karabo_da[0] != "-1" and da not in karabo_da:
continue continue
instrument_source_name = detector.instrument_source(module) instrument_source_name = detector.instrument_source(module)
print('-', da, db_module, module, instrument_source_name) print('-', da, db_module, module, instrument_source_name)
modules[da] = dict( modules[da] = dict(
db_module=db_module, db_module=db_module,
module=module, module=module,
raw_source_name=instrument_source_name, raw_source_name=instrument_source_name,
pdu_no=pdu_no, pdu_no=pdu_no,
) )
constants = {} constants = {}
step_timer = StepTimer() step_timer = StepTimer()
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Offset map # Offset map
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
dark_run = run_high dark_run = run_high
dark_creation_time = get_dir_creation_date(in_folder, dark_run) dark_creation_time = get_dir_creation_date(in_folder, dark_run)
print(f"Using {dark_creation_time} as creation time of Offset constant.") print(f"Using {dark_creation_time} as creation time of Offset constant.")
for da, meta in modules.items(): for da, meta in modules.items():
source_name = detector.instrument_source(meta["module"]) source_name = detector.instrument_source(meta["module"])
image_key = detector.image_key image_key = detector.image_key
display(Markdown(f"## {source_name}")) display(Markdown(f"## {source_name}"))
# read # read
step_timer.start() step_timer.start()
file_da, _, _ = da.partition('/') file_da, _, _ = da.partition('/')
dark_dc = RunDirectory(f"{in_folder}/r{dark_run:04d}", dark_dc = RunDirectory(f"{in_folder}/r{dark_run:04d}",
include=f"RAW-R{dark_run:04d}-{file_da}-S*.h5") include=f"RAW-R{dark_run:04d}-{file_da}-S*.h5")
if source_name not in dark_dc.all_sources: if source_name not in dark_dc.all_sources:
print(f"Source {source_name} for module {da} is missed") print(f"Source {source_name} for module {da} is missed")
continue continue
dark_dc = dark_dc.select([(source_name, image_key)]) dark_dc = dark_dc.select([(source_name, image_key)])
conditions = detector.conditions(dark_dc, meta["module"]) conditions = detector.conditions(dark_dc, meta["module"])
key_data = dark_dc[source_name, image_key] key_data = dark_dc[source_name, image_key]
images_dark = key_data.ndarray() images_dark = key_data.ndarray()
ntrain, npulse, ny, nx = images_dark.shape ntrain, npulse, ny, nx = images_dark.shape
print(f"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})") print(f"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})")
print(f"Image size: {ny} x {nx} px") print(f"Image size: {ny} x {nx} px")
step_timer.done_step("Read dark images") step_timer.done_step("Read dark images")
# process # process
step_timer.start() step_timer.start()
dark = dffc.process_dark(images_dark) dark = dffc.process_dark(images_dark) # Amounts to a per-pixel mean right now.
# put results in the dict # put results in the dict
module_constants = constants.setdefault(meta["db_module"], {}) module_constants = constants.setdefault(meta["db_module"], {})
module_constants["Offset"] = dict( module_constants["Offset"] = dict(
conditions=conditions, data=dark, pdu_no=meta["pdu_no"], conditions=conditions, data=dark, pdu_no=meta["pdu_no"],
creation_time=dark_creation_time creation_time=dark_creation_time
) )
step_timer.done_step("Process dark images") step_timer.done_step("Process dark images")
display() display()
# draw plots # draw plots
step_timer.start() step_timer.start()
plot_camera_image(dark) plot_camera_image(dark)
plt.show() plt.show()
step_timer.done_step("Draw offsets") step_timer.done_step("Draw offsets")
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Flat-field PCA decomposition # Flat-field PCA decomposition
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
flat_run = run_low flat_run = run_low
flat_creation_time = get_dir_creation_date(in_folder, flat_run) flat_creation_time = get_dir_creation_date(in_folder, flat_run)
print(f"Using {flat_creation_time} as creation time of DynamicFF constant.") print(f"Using {flat_creation_time} as creation time of DynamicFF constant.")
for da, meta in modules.items(): for da, meta in modules.items():
source_name = detector.instrument_source(meta["module"]) source_name = detector.instrument_source(meta["module"])
image_key = detector.image_key image_key = detector.image_key
display(Markdown(f"## {source_name}")) display(Markdown(f"## {source_name}"))
# read # read
step_timer.start() step_timer.start()
file_da, _, _ = da.partition('/') file_da, _, _ = da.partition('/')
flat_dc = RunDirectory(f"{in_folder}/r{flat_run:04d}", flat_dc = RunDirectory(f"{in_folder}/r{flat_run:04d}",
include=f"RAW-R{flat_run:04d}-{file_da}-S*.h5") include=f"RAW-R{flat_run:04d}-{file_da}-S*.h5")
if source_name not in flat_dc.all_sources: if source_name not in flat_dc.all_sources:
print(f"Source {source_name} for module {da} is missed") print(f"Source {source_name} for module {da} is missed")
continue continue
flat_dc = flat_dc.select([(source_name, image_key)]) flat_dc = flat_dc.select([(source_name, image_key)])
conditions = detector.conditions(flat_dc, meta["module"]) conditions = detector.conditions(flat_dc, meta["module"])
dark = constants[meta["db_module"]]["Offset"]["data"] dark = constants[meta["db_module"]]["Offset"]["data"]
dark_conditions = constants[meta["db_module"]]["Offset"]["conditions"] dark_conditions = constants[meta["db_module"]]["Offset"]["conditions"]
if conditions != dark_conditions: if conditions != dark_conditions:
ValueError("The conditions for flat-field run does not match " ValueError("The conditions for flat-field run does not match "
"the dark run conditions. Skip flat-field characterization.") "the dark run conditions. Skip flat-field characterization.")
key_data = flat_dc[source_name][image_key] key_data = flat_dc[source_name][image_key]
images_flat = key_data.ndarray() images_flat = key_data.ndarray()
ntrain, npulse, ny, nx = images_flat.shape ntrain, npulse, ny, nx = images_flat.shape
print(f"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})") print(f"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})")
print(f"Image size: {ny} x {nx} px") print(f"Image size: {ny} x {nx} px")
step_timer.done_step("Read flat-field images") step_timer.done_step("Read flat-field images")
# process # process
step_timer.start() step_timer.start()
flat, components, explained_variance_ratio = dffc.process_flat( flat, components, explained_variance_ratio = dffc.process_flat(
images_flat, dark, n_components) images_flat, dark, n_components)
flat_data = np.concatenate([flat[None, ...], components]) flat_data = np.concatenate([flat[None, ...], components])
# put results in the dict # put results in the dict
conditions = detector.conditions(flat_dc, meta["module"]) conditions = detector.conditions(flat_dc, meta["module"])
module_constants = constants.setdefault(meta["db_module"], {}) module_constants = constants.setdefault(meta["db_module"], {})
module_constants["DynamicFF"] = dict( module_constants["DynamicFF"] = dict(
conditions=conditions, data=flat_data, pdu_no=meta["pdu_no"], conditions=conditions, data=flat_data, pdu_no=meta["pdu_no"],
creation_time=flat_creation_time creation_time=flat_creation_time
) )
step_timer.done_step("Process flat-field images") step_timer.done_step("Process flat-field images")
# draw plots # draw plots
step_timer.start() step_timer.start()
display(Markdown("### Average flat-field")) display(Markdown("### Average flat-field"))
plot_camera_image(flat) plot_camera_image(flat)
plt.show() plt.show()
display(Markdown("### Explained variance ratio")) display(Markdown("### Explained variance ratio"))
fig, ax = plt.subplots(1, 1, figsize=(10,4), tight_layout=True) fig, ax = plt.subplots(1, 1, figsize=(10,4), tight_layout=True)
ax.semilogy(explained_variance_ratio, 'o') ax.semilogy(explained_variance_ratio, 'o')
ax.set_xticks(np.arange(len(explained_variance_ratio))) ax.set_xticks(np.arange(len(explained_variance_ratio)))
ax.set_xlabel("Component no.") ax.set_xlabel("Component no.")
ax.set_ylabel("Variance fraction") ax.set_ylabel("Variance fraction")
plt.show() plt.show()
display(Markdown("### The first principal components (up to 20)")) display(Markdown("### The first principal components (up to 20)"))
plot_images(components[:20], figsize=(13, 8)) plot_images(components[:20], figsize=(13, 8))
plt.show() plt.show()
step_timer.done_step("Draw flat-field") step_timer.done_step("Draw flat-field")
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## Calibration constants ## Calibration constants
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
step_timer.start() step_timer.start()
# Output Folder Creation: # Output Folder Creation:
os.makedirs(out_folder, exist_ok=True) os.makedirs(out_folder, exist_ok=True)
def inject_ccv(in_folder, metadata_folder, runs, calibration, cond, pdu, const_input, begin_at): def inject_ccv(in_folder, metadata_folder, runs, calibration, cond, pdu, const_input, begin_at):
print("* Send to db:", const_input) print("* Send to db:", const_input)
print(" - in folder:", in_folder) print(" - in folder:", in_folder)
print(" - metadata folder:", metadata_folder) print(" - metadata folder:", metadata_folder)
print(" - runs:", runs) print(" - runs:", runs)
print(" -", calibration) print(" -", calibration)
print(" -", cond) print(" -", cond)
print(" -", begin_at) print(" -", begin_at)
for db_module, module_constants in constants.items(): for db_module, module_constants in constants.items():
for constant_name, constant in module_constants.items(): for constant_name, constant in module_constants.items():
conditions = constant["conditions"] conditions = constant["conditions"]
conditions_dict = conditions.make_dict( conditions_dict = conditions.make_dict(
conditions.calibration_types[constant_name]) conditions.calibration_types[constant_name])
data_to_store = {db_module: {constant_name: {'0': { data_to_store = {db_module: {constant_name: {'0': {
'conditions': conditions_dict, 'conditions': conditions_dict,
'data': constant["data"], 'data': constant["data"],
}}}} }}}}
ofile = f"{out_folder}/const_{constant_name}_{db_module}.h5" ofile = f"{out_folder}/const_{constant_name}_{db_module}.h5"
if os.path.isfile(ofile): if os.path.isfile(ofile):
print(f'File {ofile} already exists and will be overwritten') print(f'File {ofile} already exists and will be overwritten')
save_dict_to_hdf5(data_to_store, ofile) save_dict_to_hdf5(data_to_store, ofile)
if db_output: if db_output:
inject_ccv( inject_ccv(
in_folder, metadata_folder, [dark_run, flat_run], in_folder, metadata_folder, [dark_run, flat_run],
constant_name, conditions, pdus["data"][constant["pdu_no"]], constant_name, conditions, pdus["data"][constant["pdu_no"]],
ofile, constant["creation_time"] ofile, constant["creation_time"]
) )
if not local_output: if not local_output:
os.unlink(ofile) os.unlink(ofile)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print(f"Total processing time {step_timer.timespan():.01f} s") print(f"Total processing time {step_timer.timespan():.01f} s")
step_timer.print_summary() step_timer.print_summary()
``` ```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts