Skip to content
Snippets Groups Projects
Commit 28d81f91 authored by Karim Ahmed's avatar Karim Ahmed
Browse files

avoid plotting if prev. constant missing for all detector

parent 48229114
No related branches found
No related tags found
1 merge request!730[JUNGFRAU][DARK] New notebook for summarizing processed darks
Hide whitespace changes
Inline Side-by-side
notebooks/Jungfrau/Jungfrau_darks_Summary_NBC.ipynb
+ 61
− 34
View file @ 28d81f91
%% Cell type:markdown id: tags:
# Jungfrau Dark Summary
Author: European XFEL Detector Department, Version: 1.0
Summary for process dark constants and a comparison with previously injected constants with the same conditions.
%% Cell type:code id: tags:
``` python
out_folder = "/gpfs/exfel/data/scratch/ahmedk/test/jungfrau_assembeled_dark" # path to output to, required
metadata_folder = "" # Directory containing calibration_metadata.yml when run by xfel-calibrate.
# Parameters used to access raw data.
karabo_da = [] # list of data aggregators, which corresponds to different JF modules. This is only needed for the detectors of one module.
karabo_id = "FXE_XAD_JF1M" # detector identifier.
# Parameters to be used for injecting dark calibration constants.
local_output = True # Boolean indicating that local constants were stored in the out_folder
# Skip the whole notebook if local_output is false in the preceding notebooks.
if not local_output:
print('No local constants saved. Skipping summary plots')
import sys
sys.exit(0)
```
%% Cell type:code id: tags:
``` python
import warnings
from collections import OrderedDict
from pathlib import Path
warnings.filterwarnings('ignore')
import h5py
import matplotlib
import matplotlib.gridspec as gridspec
import matplotlib.pyplot as plt
import numpy as np
import yaml
from IPython.display import Markdown, display
matplotlib.use("agg")
%matplotlib inline
from cal_tools.enums import BadPixels, JungfrauSettings
from cal_tools.ana_tools import get_range
from cal_tools.plotting import init_jungfrau_geom, show_processed_modules_jungfrau
from cal_tools.tools import CalibrationMetadata
from XFELDetAna.plotting.simpleplot import simplePlot
```
%% Cell type:code id: tags:
``` python
# Prepare paths and load previous constants' metadata.
out_folder = Path(out_folder)
metadata = CalibrationMetadata(metadata_folder or out_folder)
mod_mapping = metadata.setdefault("modules-mapping", {})
dark_constants = ["Offset", "Noise", "BadPixelsDark"]
prev_const_metadata = {}
for fn in Path(metadata_folder or out_folder).glob("module_metadata_*.yml"):
with fn.open("r") as fd:
fdict = yaml.safe_load(fd)
module = fdict["module"]
mod_mapping[module] = fdict["pdu"]
prev_const_metadata[module] = fdict["old-constants"]
metadata.save()
```
%% Cell type:code id: tags:
``` python
expected_modules, geom = init_jungfrau_geom(
karabo_id=karabo_id, karabo_da=karabo_da)
nmods = len(expected_modules)
```
%% Cell type:markdown id: tags:
Preparing newly injected and previous constants from produced local folder in out_folder.
%% Cell type:code id: tags:
``` python
fixed_gain = False # constant is adaptive by default.
# Get the constant shape from one of the local constants.
# This is one way to realize the number of memory cells.
with h5py.File(list(out_folder.glob("const_Offset_*"))[0], 'r') as f:
const_shape = f["data"][()].shape
# Get fixed gain value to decide offset vmin, vmax
# for later constant map plots.
gain_mode = "condition/Gain mode/value"
if gain_mode in f:
fixed_gain = f[gain_mode][()]
initial_stacked_constants = np.zeros(((nmods,)+const_shape))
initial_stacked_constants = np.full(((nmods,)+const_shape), np.nan)
curr_constants = { c: initial_stacked_constants.copy() for c in dark_constants}
prev_constants = { c: initial_stacked_constants.copy() for c in dark_constants}
execulded_constants = [] # constants excluded from comparison plots.
# Loop over modules
for i, mod in enumerate(sorted(expected_modules)):
# Loop over expected dark constants in out_folder.
# Some constants can be missing in out_folder.
pdu = mod_mapping[mod]
for cname in dark_constants:
for cname in dark_constants:
excluded_modules = [] # modules with no previous constants.
for i, mod in enumerate(sorted(expected_modules)):
# Loop over expected dark constants in out_folder.
# Some constants can be missing in out_folder.
pdu = mod_mapping[mod]
# first load new constant
fpath = out_folder / f"const_{cname}_{pdu}.h5"
with h5py.File(fpath, 'r') as f:
curr_constants[cname][i, ...] = f["data"][()]
# Load previous constants.
old_mod_mdata = prev_const_metadata[mod]
if cname not in old_mod_mdata:
continue
filepath = old_mod_mdata[cname]["filepath"]
h5path = old_mod_mdata[cname]["h5path"]
if cname in old_mod_mdata: # a module can be missing from detector dark processing.
filepath = old_mod_mdata[cname]["filepath"]
h5path = old_mod_mdata[cname]["h5path"]
if not filepath or not h5path:
excluded_modules.append(mod)
prev_constants[cname][i, ...].fill(np.nan)
else:
with h5py.File(filepath, "r") as fd:
prev_constants[cname][i, ...] = fd[f"{h5path}/data"][()]
if not filepath or not h5path:
continue
with h5py.File(filepath, "r") as fd:
prev_constants[cname][i, ...] = fd[f"{h5path}/data"][()]
if excluded_modules:
print(f"Previous {cname} constants for {excluded_modules} are not available.\n.")
# Exclude constants from comparison plots, if the corresponding
# previous constants are not available for all modules.
if len(excluded_modules) == nmods:
execulded_constants.append(cname)
print(f"No comparison plots for {cname}.\n")
```
%% Cell type:code id: tags:
``` python
display(Markdown('## Processed modules'))
processed_modules = list(mod_mapping.keys())
processed_pdus = list(mod_mapping.values())
show_processed_modules_jungfrau(
jungfrau_geom=geom,
constants=curr_constants,
processed_modules=processed_modules,
expected_modules=expected_modules,
display_module_names=processed_pdus,
)
```
%% Cell type:code id: tags:
``` python
gainstages = 3
gain_names = ["High Gain", "Medium Gain", "Low Gain" ]
const_range = {
"Offset": [(0, 8000), (8000, 16000), (8000, 16000)],
"Noise": [(0., 50.), (0., 50.), (0., 50.)],
"BadPixelsDark": [(0., 5.), (0., 5.), (0., 5.)],
}
# vmin and vmax are different for Offset for fixed gain constants.
if fixed_gain:
const_range["Offset"] = [(0, 8000), (0, 8000), (0, 8000)]
diff_const_range = {
"Offset": [(0, 500), (0, 500), (0, 500)],
"Noise": [(0., 5.), (0., 5.), (0., 5.)],
"BadPixelsDark": [(0., 5.), (0., 5.), (0., 5.)],
}
gs = gridspec.GridSpec(2, 4)
axis = OrderedDict({
"ax0": {
"gs": gs[0, 1:3],
"shrink": 0.9,
"shrink": 0.7,
"pad": 0.05,
"label": "ADCu",
"title": '{}',
'location': 'right',
"title": "{}",
"location": "right",
},
"ax1": {
"gs": gs[1, :2],
"shrink": 0.6,
"pad": 0.05,
"shrink": 0.7,
"pad": 0.02,
"label": "ADCu",
'location': 'left',
"title": 'Difference with previous {}',
"location": "left",
"title": "Difference with previous {}",
},
"ax2": {
"gs": gs[1, 2:],
"shrink": 0.6,
"pad": 0.05,
"shrink": 0.7,
"pad": 0.02,
"label": "%",
'location': 'right',
"title": 'Difference with previous {} %'},
"location": "right",
"title": "Difference with previous {} %"},
})
```
%% Cell type:markdown id: tags:
## Summary figures across pixels and memory cells.
The following plots give an overview of calibration constants averaged across pixels and memory cells. A bad pixel mask is applied.
%% Cell type:code id: tags:
``` python
for cname, const in curr_constants.items():
# Prepare the stacked mean of constant,
# the difference with the previous constant
# and the fraction of that difference.
mean_const = np.nanmean(const, axis=3)
mean_diff = np.abs(np.nanmean(const, axis=3) - np.nanmean(prev_constants[cname], axis=3)) # noqa
mean_frac = np.abs(mean_diff / mean_const) * 100
for gain in range(gainstages):
plot_data = {
data_to_plot = {
f'ax0': mean_const[..., gain],
f'ax1': mean_diff[..., gain],
f'ax2': mean_frac[..., gain],
}
# Plotting constant overall modules.
display(Markdown(f'### {cname} - {gain_names[gain]}'))
if nmods > 1:
fig = plt.figure(figsize=(30, 30))
fig = plt.figure(figsize=(20, 20))
else:
fig = plt.figure(figsize=(31, 18))
fig = plt.figure(figsize=(20, 10))
for axname, axv in axis.items():
# Add the min and max plot values for each axis.
if axname == "ax0":
vmin, vmax = const_range[cname][gain]
elif axname == "ax2":
vmin, vmax = 0, 100
else:
vmin, vmax = diff_const_range[cname][gain]
# Avoid difference plots if previous constants
# are missing for the detector.
if cname in execulded_constants and axname != "ax0":
break
ax = fig.add_subplot(axv["gs"])
geom.plot_data_fast(
plot_data[axname],
geom.plot_data(
data_to_plot[axname],
vmin=vmin, vmax=vmax, ax=ax,
colorbar={
"shrink": axv["shrink"],
"pad": axv["pad"],
"location": axv["location"]},
)
"location": axv["location"],
}
)
colorbar = ax.images[0].colorbar
colorbar.set_label(axv["label"], fontsize=15)
colorbar.ax.tick_params(labelsize=15)
ax.tick_params(labelsize=1)
ax.set_title(axv["title"].format(
f"{cname} {gain_names[gain]}"), fontsize=15)
if axname == "ax0":
ax.set_xlabel('Columns', fontsize=15)
ax.set_ylabel('Rows', fontsize=15)
ax.tick_params(labelsize=15)
else:
ax.tick_params(labelsize=0)
# Remove axes labels for comparison plots.
ax.set_xlabel('')
ax.set_ylabel('')
plt.show()
```
%% Cell type:code id: tags:
``` python
if curr_constants["Offset"].shape[-2] > 1:
display(Markdown("## ## Summary across pixels per memory cells"))
# Loop over modules and constants
for const_name, const in curr_constants.items():
display(Markdown(f'### {const_name}'))
for gain in range(gainstages):
data = np.copy(const[..., gain])
if const_name == 'BadPixelsDark':
data[data > 0] = 1.0
datamean = np.nanmean(data, axis=(1, 2))
datamean[datamean == 1.0] = np.nan
fig = plt.figure(
figsize=(15, 6),
tight_layout={'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})
label = 'Fraction of bad pixels'
ax = fig.add_subplot(111)
else:
datamean = np.nanmean(data, axis=(1, 2))
fig = plt.figure(
figsize=(15, 6),
tight_layout={'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})
label = f'{const_name} value [ADU], good pixels only'
ax = fig.add_subplot(121)
d = []
for i, mod in enumerate(datamean):
d.append({
'x': np.arange(mod.shape[0]),
'y': mod,
'drawstyle': 'steps-pre',
'label': processed_modules[i],
})
simplePlot(
d, figsize=(10, 10), xrange=(-12, 510),
x_label='Memory Cell ID',
y_label=label,
use_axis=ax,
title=f'{gain_names[gain]}',
title_position=[0.5, 1.18],
legend='outside-top-ncol6-frame',
legend_size='18%',
legend_pad=0.00,
)
# Extra Sigma plot for Offset and Noise constants.
if const_name != 'BadPixelsDark':
ax = fig.add_subplot(122)
label = f'$\sigma$ {const_name} [ADU], good pixels only'
d = []
for i, mod in enumerate(np.nanstd(data, axis=(1, 2))):
d.append({
'x': np.arange(mod.shape[0]),
'y': mod,
'drawstyle': 'steps-pre',
'label': processed_modules[i],
})
simplePlot(
d, figsize=(10, 10), xrange=(-12, 510),
x_label='Memory Cell ID',
y_label=label,
use_axis=ax,
title=f'{gain_names[gain]} $\sigma$',
title_position=[0.5, 1.18],
legend='outside-top-ncol6-frame',
legend_size='18%',
legend_pad=0.00,
)
plt.show()
```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts