Skip to content
Snippets Groups Projects
Commit c535c074 authored by Karim Ahmed's avatar Karim Ahmed
Browse files

add pdu name in plot titles

parent aed3823f
1 merge request!684[Gotthard-II] Dark Processing.
Hide whitespace changes
Inline Side-by-side
notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
+ 19
− 22
View file @ c535c074
%% Cell type:markdown id:49b6577f-96a5-4dd2-bdd9-da661b2c4619 tags:
# Gotthard2 Dark Image Characterization
Author: European XFEL Detector Group, Version: 1.0
%% Cell type:code id:818e24e8 tags:
``` python
in_folder = "/gpfs/exfel/exp/FXE/202221/p003225/raw" # the folder to read data from, required
out_folder = "/gpfs/exfel/data/scratch/ahmedk/test/gotthard2/darks" # the folder to output to, required
run_high = 50 # run number for G0 dark run, required
run_med = 51 # run number for G1 dark run, required
run_low = 52 # run number for G2 dark run, required
sequences = [-1] # sequences to correct, set to [-1] for all, range allowed
sequences_per_node = 1 # number of sequence files per node if notebook executed through xfel-calibrate, set to 0 to not run SLURM parallel
# Parameters used to access raw data.
karabo_id = "FXE_XAD_G2XES" # karabo prefix of Gotthard-II devices
karabo_da = ["GH201"] # data aggregators
receiver_template = "RECEIVER" # receiver template used to read INSTRUMENT keys.
control_template = "CONTROL" # control template used to read CONTROL keys.
instrument_source_template = '{}/DET/{}:daqOutput' # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput'
ctrl_source_template = '{}/DET/{}' # template for control source name (filled with karabo_id_control)
karabo_id_control = "" # Control karabo ID. Set to empty string to use the karabo-id
# Parameters for the calibration database.
use_dir_creation_date = True
cal_db_interface = "tcp://max-exfl017:8020" # calibration DB interface to use
cal_db_timeout = 300000 # timeout on caldb requests
db_output = False # Output constants to the calibration database
local_output = True # Output constants locally
# Conditions used for injected calibration constants.
bias_voltage = -1 # Detector bias voltage, set to -1 to use value in raw file.
exposure_time = -1 # Detector exposure time, set to -1 to use value in raw file.
exposure_period = -1 # Detector exposure period, set to -1 to use value in raw file.
operation_mode = -1 # Detector operation mode (1.1/4.5), set to -1 to use value in raw file.
single_photon = -1 # Detector single photon mode (High/Low CDS), set to -1 to use value in raw file.
# Parameters used during selecting raw data trains.
min_trains = 1 # Minimum number of trains that should be available to process dark constants. Default 1.
max_trains = 1000 # Maximum number of trains to use for processing dark constants. Set to 0 to use all available trains.
badpixel_threshold_sigma = 5. # bad pixels defined by values outside n times this std from median
# Don't delete! myMDC sends this by default.
operation_mode = '' # Detector dark run acquiring operation mode, optional
```
%% Cell type:code id:8085f9aa tags:
``` python
import numpy as np
import matplotlib.pyplot as plt
import multiprocessing
import pasha as psh
from IPython.display import Markdown, display
from extra_data import RunDirectory
from pathlib import Path
from cal_tools.enums import BadPixels
from cal_tools.gotthard2 import gotthard2algs, gotthard2lib
from cal_tools.step_timing import StepTimer
from cal_tools.tools import (
get_dir_creation_date,
get_from_db,
get_constant_from_db_and_time,
get_pdu_from_db,
get_report,
save_const_to_h5,
send_to_db,
)
from iCalibrationDB import Conditions, Constants
from XFELDetAna.plotting.heatmap import heatmapPlot
```
%% Cell type:code id:18fe4379 tags:
``` python
run_nums = [run_high, run_med, run_low]
in_folder = Path(in_folder)
out_folder = Path(out_folder)
out_folder.mkdir(exist_ok=True)
print(f"Process modules: {karabo_da}")
run_dc = RunDirectory(in_folder / f"r{run_high:04d}")
file_loc = f"proposal:{run_dc.run_metadata()['proposalNumber']} runs:{run_high} {run_med} {run_low}" # noqa
instrument_src = instrument_source_template.format(
karabo_id, receiver_template)
ctrl_src = ctrl_source_template.format(
karabo_id_control, control_template)
# Read report path to associate it later with injected constants.
report = get_report(out_folder)
if use_dir_creation_date:
creation_time = get_dir_creation_date(in_folder, run_high)
print(f"Using {creation_time.isoformat()} as creation time")
# TODO: Remove later
import datetime
creation_time=datetime.datetime.strptime(
"2022-06-28 13:00:00.00",
"%Y-%m-%d %H:%M:%S.%f")
if not karabo_id_control:
karabo_id_control = karabo_id
```
%% Cell type:code id:108be688 tags:
``` python
step_timer = StepTimer()
```
%% Cell type:code id:ff9149fc tags:
``` python
# Read parameter conditions and validate the values for the three runs.
step_timer.start()
run_dcs_dict = dict()
ctrl_src = ctrl_source_template.format(karabo_id_control, control_template)
conditions = {
"bias_voltage": set(),
"exposure_time": set(),
"exposure_period": set(),
#"operation_mode": set(),
#"single_photon": set(),
}
for gain, run in enumerate(run_nums):
run_dc = RunDirectory(in_folder / f"r{run:04d}/")
run_dcs_dict[run] = [gain, run_dc]
# Read slow data.
g2ctrl = gotthard2lib.Gotthard2Ctrl(
run_dc=run_dc, ctrl_src=ctrl_src)
conditions["bias_voltage"].add(g2ctrl.get_bias_voltage() if bias_voltage == -1 else bias_voltage) # noqa
conditions["exposure_time"].add(g2ctrl.get_exposure_time() if exposure_time == -1 else exposure_time) # noqa
conditions["exposure_period"].add(g2ctrl.get_exposure_period() if exposure_period == -1 else exposure_period) # noqa
# conditions["single_photon"].add(g2ctrl.get_single_photon() if single_photon == -1 else single_photon) # noqa
# conditions["operation_mode"].add(g2ctrl.get_operation_mode() if operation_mode == -1 else operation_mode) # noqa
for c, v in conditions.items():
assert len(v) == 1, f"{c} value is not the same for the three runs."
bias_voltage = conditions["bias_voltage"].pop()
print("Bias voltage: ", bias_voltage)
exposure_time = conditions["exposure_time"].pop()
print("Exposure time: ", exposure_time)
exposure_period = conditions["exposure_period"].pop()
print("Exposure period: ", exposure_period)
# single_photon = conditions["single_photon"].pop()
# operation_mode = conditions["operation_mode"].pop()
```
%% Cell type:code id:ac9c5dc3-bc66-4e7e-b6a1-360259be535c tags:
``` python
def specify_trains_to_process(
img_key_data: "extra_data.KeyData", # noqa
max_trains: int = 0,
min_trains: int = 0,
):
"""Specify total number of trains to process.
Based on given min_trains and max_trains, if given.
Print number of trains to process and number of empty trains.
Raise ValueError if specified trains are less than min_trains.
"""
# Specifies total number of trains to proccess.
n_trains = img_key_data.shape[0]
all_trains = len(img_key_data.train_ids)
print(f"{mod} has {all_trains - n_trains} "
f"trains with empty frames out of {all_trains} trains")
if n_trains < min_trains:
raise ValueError(
f"Less than {min_trains} trains are available in RAW data."
" Not enough data to process darks.")
if max_trains > 0:
n_trains = min(n_trains, max_trains)
print(f"Processing {n_trains} trains.")
return n_trains
```
%% Cell type:code id:3c59c11d tags:
``` python
# set the operating condition
# TODO: add the final conditions for constants.
condition = Conditions.Dark.Gotthard2(
bias_voltage=bias_voltage,
exposure_time=exposure_time,
exposure_period=exposure_period,
#operation_mode=operation_mode,
#single_photon=single_photon,
)
db_modules = get_pdu_from_db(
karabo_id=karabo_id,
karabo_da=karabo_da,
constant=Constants.Gotthard2.LUT(),
condition=condition,
cal_db_interface=cal_db_interface,
snapshot_at=creation_time)
```
%% Cell type:code id:e2eb2fc0-df9c-4887-9691-f81474f8c131 tags:
``` python
def convert_train(wid, index, tid, d):
"""Convert a Gotthard2 train from 12bit to 10bit."""
gotthard2algs.convert_to_10bit(d[instr_mod_src]["data.adc"], lut, data_10bit[index, ...])
```
%% Cell type:code id:4e8ffeae tags:
``` python
# Calculate noise and offset per pixel and global average, std and median
noise_map = dict()
offset_map = dict()
badpixels_map = dict()
context = psh.context.ProcessContext(num_workers=multiprocessing.cpu_count())
for mod in karabo_da:
# Retrieve LUT constant
lut, time = get_constant_from_db_and_time(
constant=Constants.Gotthard2.LUT(),
condition=condition,
empty_constant=None,
karabo_id=karabo_id,
karabo_da=mod,
cal_db_interface=cal_db_interface,
creation_time=creation_time,
timeout=cal_db_timeout,
print_once=False,
)
# Path to pixels ADC values
instr_mod_src = instrument_src.format(int(mod[-2:]))
# TODO: Validate the final shape to store constants.
cshape = (1280, 2, 3)
offset_map[mod] = context.alloc(shape=cshape, dtype=np.float32)
noise_map[mod] = context.alloc(like=offset_map[mod])
badpixels_map[mod] = context.alloc(like=offset_map[mod], dtype=np.uint32)
for run_num, [gain, run_dc] in run_dcs_dict.items():
step_timer.start()
n_trains = specify_trains_to_process(run_dc[instr_mod_src, "data.adc"])
# Select requested number of trains to process.
dc = run_dc.select(
instr_mod_src, require_all=True).select_trains(np.s_[:n_trains]) # noqa
step_timer.done_step("preparing raw data")
step_timer.start()
# Convert 12bit data to 10bit
data_10bit = context.alloc(
shape=dc[instr_mod_src, "data.adc"].shape,
dtype=np.float32
)
context.map(convert_train, dc)
step_timer.done_step("convert to 10bit")
step_timer.start()
# Split even and odd data to calculate the two storage cell constants.
# Detector always operates in burst mode.
even_data = data_10bit[:, ::2, :]
odd_data = data_10bit[:, 1::2, :]
data_gain = dc[instr_mod_src, "data.gain"].ndarray()
even_gain = data_gain[:, ::2, :]
odd_gain = data_gain[:, 1::2, :]
def offset_noise_cell(wid, index, d):
offset_map[mod][:, index, gain] = np.mean(d, axis=(0, 1))
noise_map[mod][:, index, gain] = np.std(d, axis=(0, 1))
context.map(offset_noise_cell, (even_data, odd_data))
raw_g = 3 if gain == 2 else gain
def badpixels_cell(wid, index, g):
"""Check if there are wrong bad gain values.
Indicate pixels with wrong gain value across all trains for each cell."""
badpixels_map[mod][np.mean(g, axis=(0, 1)) != raw_g, index,
gain] |= BadPixels.WRONG_GAIN_VALUE.value
context.map(badpixels_cell, (even_gain, odd_gain))
step_timer.done_step("Processing darks")
```
%% Cell type:code id:3fc17e05-17ab-4ac4-9e79-c95399278bb9 tags:
``` python
def print_bp_entry(bp):
print(f"{bp.name:<30s} {bp.value:032b} -> {int(bp.value)}")
print_bp_entry(BadPixels.NOISE_OUT_OF_THRESHOLD)
print_bp_entry(BadPixels.OFFSET_NOISE_EVAL_ERROR)
print_bp_entry(BadPixels.WRONG_GAIN_VALUE)
def eval_bpidx(d):
mdn = np.nanmedian(d, axis=(0))[None, :, :]
std = np.nanstd(d, axis=(0))[None, :, :]
idx = (d > badpixel_threshold_sigma*std+mdn) | (d < (-badpixel_threshold_sigma)*std+mdn)
idx = (
d > badpixel_threshold_sigma * std + mdn
) | (
d < (-badpixel_threshold_sigma) * std + mdn
)
return idx
```
%% Cell type:code id:9409d10f tags:
``` python
badpixels_map[mod][~np.isfinite(noise_map[mod])]
```
%% Cell type:code id:40c34cc5-fe93-4b83-bf39-f465f37c40b4 tags:
``` python
step_timer.start()
g_name = ['G0', 'G1', 'G2']
for mod in karabo_da:
for mod, pdu in zip(karabo_da, db_modules):
display(Markdown(f"### Badpixels for module {mod}:"))
badpixels_map[mod][~np.isfinite(offset_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value
badpixels_map[mod][~np.isfinite(offset_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value # npqa
badpixels_map[mod][eval_bpidx(noise_map[mod])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value
badpixels_map[mod][~np.isfinite(noise_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value
badpixels_map[mod][~np.isfinite(noise_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value # noqa
for cell in [0, 1]:
fig, ax = plt.subplots(figsize=(10, 5))
for g_idx in [0, 1, 2]:
ax.plot(badpixels_map[mod][:, cell, g_idx], label=f"G{g_idx} Bad pixel map")
ax.set_xticks(np.arange(0, 1281, 80))
ax.set_xlabel("Stripes #")
ax.set_xlabel("BadPixels")
ax.set_title(f'Cell {cell} - Module {mod}')
ax.set_title(f'Cell {cell} - Module {mod} ({pdu})')
ax.set_ylim([0, BadPixels.WRONG_GAIN_VALUE.value])
ax.set_yticks([0,
BadPixels.NOISE_OUT_OF_THRESHOLD.value,
BadPixels.OFFSET_NOISE_EVAL_ERROR.value,
BadPixels.WRONG_GAIN_VALUE.value])
ax.legend()
pass
step_timer.done_step(f'Creating bad pixels constant and plotting it.')
```
%% Cell type:code id:e5131112 tags:
``` python
step_timer.start()
g_name = ['G0', 'G1', 'G2']
for mod in karabo_da:
for mod, pdu in zip(karabo_da, db_modules):
display(Markdown(f"### Badpixels for module {mod}:"))
badpixels_map[mod][~np.isfinite(offset_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value
badpixels_map[mod][eval_bpidx(noise_map[mod])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value
badpixels_map[mod][~np.isfinite(noise_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value
for g_idx in [0, 1, 2]:
# for cell in [0, 1]:
heatmapPlot(
badpixels_map[mod][:, :, g_idx],
y_label="Stripes",
x_label="Cells",
# lut_label=unit,
vmin=0, vmax=5,
title=f"Even / Odd Offset map G{g_idx} - Module {mod}", # TODO: add PDU name({pdu})',
title=f"Even / Odd Offset map G{g_idx} - Module {mod} ({pdu})",
)
pass
step_timer.done_step(f'Creating bad pixels constant and plotting it.')
```
%% Cell type:code id:fde8e1cf-bc74-462f-b6e5-cfee8279090d tags:
``` python
from XFELDetAna.plotting.heatmap import heatmapPlot
unit = '[ADCu]'
for mod in karabo_da:
for mod, pdu in zip(karabo_da, db_modules):
for _, [gain, _] in run_dcs_dict.items():
heatmapPlot(
offset_map[mod][:, :, gain],
y_label="Stripes",
x_label="Cells",
lut_label=unit,
x_ticks=[0.5, 1.5],
x_ticklabels=["EVEN", "ODD"],
x_tick_rotation=0,
title=f"Even / Odd Offset map G{gain} - Module {mod}", # TODO: add PDU name({pdu})',
title=f"Even / Odd Offset map G{gain} - Module {mod} ({pdu})",
)
heatmapPlot(
noise_map[mod][:, :, gain],
y_label="Stripes",
x_label="Cells",
lut_label=unit,
x_ticks=[0.5, 1.5],
x_ticklabels=["EVEN", "ODD"],
x_tick_rotation=0,
title=f"Even / Odd noise map G{gain} - Module {mod}", # TODO: add PDU name({pdu})',
title=f"Even / Odd noise map G{gain} - Module {mod} ({pdu})",
)
pass
```
%% Cell type:code id:c8777cfe tags:
``` python
for mod in karabo_da:
for mod, pdu in zip(karabo_da, db_modules):
for cons, cname in zip([offset_map, noise_map], ["Offset", "Noise"]):
for cell in [0, 1]:
fig, ax = plt.subplots(figsize=(10, 5))
for g_idx in [0, 1, 2]:
ax.plot(cons[mod][:, cell, g_idx], label=f"G{g_idx} {cname} map")
ax.set_xlabel("Stripes #")
ax.set_xlabel(cname)
ax.set_title(f"{cname} map - Cell {cell} - Module {mod}"), # TODO: add PDU name({pdu})'
ax.set_title(f"{cname} map - Cell {cell} - Module {mod} ({pdu})"),
ax.legend()
pass
```
%% Cell type:code id:1c4eddf7-7d6e-49f4-8cbb-12d2bc496a8f tags:
``` python
step_timer.start()
for mod, db_mod in zip(karabo_da, db_modules):
constants = {
'Offset': offset_map[mod],
'Noise': noise_map[mod],
"Offset": offset_map[mod],
"Noise": noise_map[mod],
"BadPixelsDark": badpixels_map[mod]
}
md = None
for key, const_data in constants.items():
const = getattr(Constants.Gotthard2, key)()
const.data = const_data
if db_output:
md = send_to_db(
db_module=db_mod,
karabo_id=karabo_id,
constant=const,
condition=condition,
file_loc=file_loc,
report_path=report,
cal_db_interface=cal_db_interface,
creation_time=creation_time,
timeout=cal_db_timeout,
)
if local_output:
md = save_const_to_h5(
db_module=db_mod,
karabo_id=karabo_id,
constant=const,
condition=condition,
data=const.data,
file_loc=file_loc,
report=report,
creation_time=creation_time,
out_folder=out_folder,
)
print(f"Calibration constant {key} is stored locally at {out_folder}.\n")
print("Constants parameter conditions are:\n")
# TODO: add the final conditions for constants.
print(
f"• Bias voltage: {bias_voltage}\n"
f"• Exposure time: {exposure_time}\n"
f"• Exposure period: {exposure_period}\n"
f"• Operation mode: {operation_mode}\n"
f"• Single photon: {single_photon}\n"
f"• Creation time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\n") # noqa
step_timer.done_step("Injecting constants.")
```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts