Skip to content
Snippets Groups Projects
Commit 8b07d324 authored by Egor Sobolev's avatar Egor Sobolev Committed by Philipp Schmidt
Browse files

Fix parameters and inject_cvv signature according to the webserver call

parent a13bd103
No related branches found
No related tags found
1 merge request!939[Generic][Shimadzu] Dynamic flat-field characterization and correction for MHz microscopy
Hide whitespace changes
Inline Side-by-side
notebooks/DynamicFF/Characterize_DynamicFF_NBC.ipynb
+ 26
− 41
View file @ 8b07d324
%% Cell type:markdown id: tags:
# Characterization of dark and flat field for Dynamic Flat Field correction
Author: Egor Sobolev
Computation of dark offsets and flat-field principal components
%% Cell type:code id: tags:
``` python
in_folder = "/gpfs/exfel/exp/SPB/202430/p900425/raw" # input folder, required
out_folder = '/gpfs/exfel/data/scratch/esobolev/test/shimadzu' # output folder, required
metadata_folder = "" # Directory containing calibration_metadata.yml when run by xfel-calibrate
runs = [1, 2] # list of two run numbers: dark field and flat field
run_high = 1 # run number in which dark data was recorded, required
run_low = 2 # run number in which flat-field data was recorded, required
operation_mode = "TI_DynamicFF" # Detector operation mode, optional (defaults to "TI_DynamicFF")
# Data files parameters.
karabo_da = ['-1'] # data aggregators
karabo_id = "SPB_MIC_HPVX2" # karabo prefix of Shimadzu HPV-X2 devices
#receiver_id = "PNCCD_FMT-0" # inset for receiver devices
#path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data
#instrument_source_template = 'SPB_EHD_MIC/CAM/HPVX2_{module}:daqOutput' # data source path in h5file.
#instrument_source_template = 'SPB_EHD_HPVX2_{module}/CAM/CAMERA:daqOutput'
#image_key = "data.image.pixels" # image data key in Karabo or exdf notation
#db_module_template = "Shimadzu_HPVX2_{}"
# Database access parameters.
use_dir_creation_date = True # use dir creation date as data production reference date
cal_db_interface = "tcp://max-exfl-cal001:8021" # calibration DB interface to use
cal_db_timeout = 300000 # timeout on caldb requests
db_output = True # if True, the notebook sends dark constants to the calibration database
local_output = True # if True, the notebook saves dark constants locally
creation_time = "" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. 2019-07-04 11:02:41.00
n_components = 50 # Number of principal components to compute
# Calibration constants parameters
n_components = 50 # Number of principal components of flat-field to compute (default: 50)
```
%% Cell type:code id: tags:
``` python
import datetime
import os
import warnings
warnings.filterwarnings('ignore')
import time
import numpy as np
import matplotlib.pyplot as plt
from IPython.display import display, Markdown
from extra_data import RunDirectory
%matplotlib inline
from cal_tools.step_timing import StepTimer
from cal_tools.tools import (
get_dir_creation_date,
# get_pdu_from_db,
get_random_db_interface,
get_report,
# save_const_to_h5,
save_dict_to_hdf5,
# send_to_db,
run_prop_seq_from_path,
)
from cal_tools.restful_config import calibration_client
from cal_tools.shimadzu import ShimadzuHPVX2
import dynflatfield as dffc
from dynflatfield.draw import plot_images, plot_camera_image
```
%% Cell type:code id: tags:
``` python
creation_time=None
if use_dir_creation_date:
creation_time = get_dir_creation_date(in_folder, max(runs))
print(f"Using {creation_time} as creation time of constant.")
run, proposal, seq = run_prop_seq_from_path(in_folder)
#file_loc = f'proposal: {prop}, runs: {dark_run} {flat_run}'
# Read report path and create file location tuple to add with the injection
#file_loc = f"proposal:{proposal} runs:" + ' '.join(str(run) for run in runs)
report = get_report(metadata_folder)
cal_db_interface = get_random_db_interface(cal_db_interface)
print(f'Calibration database interface: {cal_db_interface}')
print()
cc = calibration_client()
pdus = cc.get_all_phy_det_units_from_detector(
{"detector_identifier": karabo_id})
if not pdus["success"]:
print("exception")
detector_info = pdus['data'][0]['detector']
detector = ShimadzuHPVX2(detector_info["source_name_pattern"])
print(f"Instrument {detector.instrument}")
print(f"Detector in use is {karabo_id}")
modules = {}
for pdu_no, pdu in enumerate(pdus["data"]):
db_module = pdu["physical_name"]
module = pdu["module_number"]
da = pdu["karabo_da"]
if karabo_da[0] != "-1" and da not in karabo_da:
continue
instrument_source_name = detector.instrument_source(module)
print('-', da, db_module, module, instrument_source_name)
modules[da] = dict(
db_module=db_module,
module=module,
raw_source_name=instrument_source_name,
pdu_no=pdu_no,
)
constants = {}
step_timer = StepTimer()
```
%% Cell type:markdown id: tags:
# Offset map
%% Cell type:code id: tags:
``` python
dark_run = runs[0]
dark_run = run_high
dark_creation_time = get_dir_creation_date(in_folder, dark_run)
print(f"Using {dark_creation_time} as creation time of Offset constant.")
for da, meta in modules.items():
source_name = detector.instrument_source(meta["module"])
image_key = detector.image_key
display(Markdown(f"## {source_name}"))
# read
step_timer.start()
file_da, _, _ = da.partition('/')
dark_dc = RunDirectory(f"{in_folder}/r{dark_run:04d}",
include=f"RAW-R{dark_run:04d}-{file_da}-S*.h5")
dark_dc = dark_dc.select([(source_name, image_key)])
key_data = dark_dc[source_name, image_key]
images_dark = key_data.ndarray()
ntrain, npulse, ny, nx = images_dark.shape
print(f"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})")
print(f"Image size: {ny} x {nx} px")
step_timer.done_step("Read dark images")
# process
step_timer.start()
dark = dffc.process_dark(images_dark)
# put results in the dict
conditions = detector.conditions(dark_dc, meta["module"])
module_constants = constants.setdefault(meta["db_module"], {})
module_constants["Offset"] = dict(
conditions=conditions, data=dark, pdu_no=meta["pdu_no"])
conditions=conditions, data=dark, pdu_no=meta["pdu_no"],
creation_time=dark_creation_time
)
step_timer.done_step("Process dark images")
display()
# draw plots
step_timer.start()
plot_camera_image(dark)
plt.show()
step_timer.done_step("Draw offsets")
```
%% Cell type:markdown id: tags:
# Flat-field PCA decomposition
%% Cell type:code id: tags:
``` python
flat_run = runs[1]
flat_run = run_low
flat_creation_time = get_dir_creation_date(in_folder, flat_run)
print(f"Using {flat_creation_time} as creation time of DynamicFF constant.")
for da, meta in modules.items():
source_name = detector.instrument_source(meta["module"])
image_key = detector.image_key
display(Markdown(f"## {source_name}"))
# read
step_timer.start()
file_da, _, _ = da.partition('/')
flat_dc = RunDirectory(f"{in_folder}/r{flat_run:04d}",
include=f"RAW-R{flat_run:04d}-{file_da}-S*.h5")
flat_dc = flat_dc.select([(source_name, image_key)])
key_data = flat_dc[source_name][image_key]
images_flat = key_data.ndarray()
ntrain, npulse, ny, nx = images_flat.shape
print(f"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})")
print(f"Image size: {ny} x {nx} px")
step_timer.done_step("Read flat-field images")
# process
step_timer.start()
flat, components, explained_variance_ratio = dffc.process_flat(
images_flat, dark, n_components)
flat_data = np.concatenate([flat[None, ...], components])
# put results in the dict
conditions = detector.conditions(flat_dc, meta["module"])
module_constants = constants.setdefault(meta["db_module"], {})
module_constants["DynamicFF"] = dict(
conditions=conditions, data=flat_data, pdu_no=meta["pdu_no"]
conditions=conditions, data=flat_data, pdu_no=meta["pdu_no"],
creation_time=flat_creation_time
)
step_timer.done_step("Process flat-field images")
# draw plots
step_timer.start()
display(Markdown("### Average flat-field"))
plot_camera_image(flat)
plt.show()
display(Markdown("### Explained variance ratio"))
fig, ax = plt.subplots(1, 1, figsize=(10,4), tight_layout=True)
ax.semilogy(explained_variance_ratio, 'o')
ax.set_xticks(np.arange(len(explained_variance_ratio)))
ax.set_xlabel("Component no.")
ax.set_ylabel("Variance fraction")
plt.show()
display(Markdown("### The first principal components (up to 20)"))
plot_images(components[:20], figsize=(13, 8))
plt.show()
step_timer.done_step("Draw flat-field")
```
%% Cell type:markdown id: tags:
## Calibration constants
%% Cell type:code id: tags:
``` python
step_timer.start()
# Output Folder Creation:
os.makedirs(out_folder, exist_ok=True)
def inject_ccv(metadata_folder, calibration, cond, pdu, proposal, runs, const_file, begin_at):
print("* Send to db:", const_file)
print(" -", metadata_folder)
def inject_ccv(in_folder, metadata_folder, runs, calibration, cond, pdu, const_input, begin_at):
print("* Send to db:", const_input)
print(" - in folder:", in_folder)
print(" - metadata folder:", metadata_folder)
print(" - runs:", runs)
print(" -", calibration)
print(" -", cond)
print(" - proposal", proposal)
print(" - runs", runs)
print(" -", begin_at)
for db_module, module_constants in constants.items():
for constant_name, constant in module_constants.items():
conditions = constant["conditions"]
conditions_dict = conditions.make_dict(
conditions.calibration_types[constant_name])
data_to_store = {db_module: {constant_name: {'0': {
'conditions': conditions_dict,
'data': constant["data"],
}}}}
ofile = f"{out_folder}/const_{constant_name}_{db_module}.h5"
if os.path.isfile(ofile):
print(f'File {ofile} already exists and will be overwritten')
save_dict_to_hdf5(data_to_store, ofile)
if db_output:
inject_ccv(
metadata_folder, constant_name, conditions,
pdus["data"][constant["pdu_no"]],
proposal, runs, ofile, creation_time
in_folder, metadata_folder, [dark_run, flat_run],
constant_name, conditions, pdus["data"][constant["pdu_no"]],
ofile, constant["creation_time"]
)
if not local_output:
os.unlink(ofile)
```
%% Cell type:code id: tags:
``` python
print(f"Total processing time {step_timer.timespan():.01f} s")
step_timer.print_summary()
```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts