Skip to content
Snippets Groups Projects
Commit 8ae30982 authored by Karim Ahmed's avatar Karim Ahmed
Browse files

rename axnames and improve badpixel plots slightly

parent dc9883f3
No related branches found
No related tags found
1 merge request!947[Jungfrau][Dark][Correct] Improve Jungfrau plots
Hide whitespace changes
Inline Side-by-side
notebooks/Jungfrau/Jungfrau_darks_Summary_NBC.ipynb
+ 22
− 14
View file @ 8ae30982
%% Cell type:markdown id: tags:
# Jungfrau Dark Summary
Author: European XFEL Detector Department, Version: 1.0
Summary for process dark constants and a comparison with previously injected constants with the same conditions.
%% Cell type:code id: tags:
``` python
out_folder = "/gpfs/exfel/data/scratch/ahmedk/test/jungfrau_assembeled_dark" # path to output to, required
metadata_folder = "" # Directory containing calibration_metadata.yml when run by xfel-calibrate.
# Parameters used to access raw data.
karabo_da = [] # list of data aggregators, which corresponds to different JF modules. This is only needed for the detectors of one module.
karabo_id = "FXE_XAD_JF1M" # detector identifier.
# Parameters to be used for injecting dark calibration constants.
local_output = True # Boolean indicating that local constants were stored in the out_folder
# Skip the whole notebook if local_output is false in the preceding notebooks.
if not local_output:
print('No local constants saved. Skipping summary plots')
import sys
sys.exit(0)
```
%% Cell type:code id: tags:
``` python
import warnings
from pathlib import Path
warnings.filterwarnings('ignore')
import h5py
import matplotlib
import matplotlib.gridspec as gridspec
import matplotlib.pyplot as plt
import numpy as np
import yaml
from IPython.display import Markdown, display
matplotlib.use("agg")
%matplotlib inline
import tabulate
from IPython.display import Latex, Markdown, display
from XFELDetAna.plotting.simpleplot import simplePlot
from cal_tools.enums import BadPixels
from cal_tools.plotting import init_jungfrau_geom, show_processed_modules_jungfrau
from cal_tools.tools import CalibrationMetadata
```
%% Cell type:code id: tags:
``` python
# Prepare paths and load previous constants' metadata.
out_folder = Path(out_folder)
metadata = CalibrationMetadata(metadata_folder or out_folder)
mod_mapping = metadata.setdefault("modules-mapping", {})
dark_constants = ["Offset", "Noise", "BadPixelsDark"]
prev_const_metadata = {}
for fn in Path(metadata_folder or out_folder).glob("module_metadata_*.yml"):
with fn.open("r") as fd:
fdict = yaml.safe_load(fd)
module = fdict["module"]
mod_mapping[module] = fdict["pdu"]
prev_const_metadata[module] = fdict["old-constants"]
metadata.save()
```
%% Cell type:code id: tags:
``` python
expected_modules, geom = init_jungfrau_geom(
karabo_id=karabo_id, karabo_da=karabo_da)
nmods = len(expected_modules)
```
%% Cell type:markdown id: tags:
Preparing newly injected and previous constants from produced local folder in out_folder.
%% Cell type:code id: tags:
``` python
fixed_gain = False # constant is adaptive by default.
# Get the constant shape from one of the local constants.
# This is one way to realize the number of memory cells.
with h5py.File(list(out_folder.glob("const_Offset_*"))[0], 'r') as f:
const_shape = f["data"][()].shape
# Get fixed gain value to decide offset vmin, vmax
# for later constant map plots.
gain_mode = "condition/Gain mode/value"
if gain_mode in f:
fixed_gain = f[gain_mode][()]
initial_stacked_constants = np.full(((nmods,)+const_shape), np.nan)
curr_constants = { c: initial_stacked_constants.copy() for c in dark_constants}
prev_constants = { c: initial_stacked_constants.copy() for c in dark_constants}
exculded_constants = [] # constants excluded from comparison plots.
# Loop over modules
for cname in dark_constants:
excluded_modules = [] # modules with no previous constants.
for i, mod in enumerate(sorted(expected_modules)):
# Loop over expected dark constants in out_folder.
# Some constants can be missing in out_folder.
pdu = mod_mapping[mod]
# first load new constant
fpath = out_folder / f"const_{cname}_{pdu}.h5"
with h5py.File(fpath, 'r') as f:
curr_constants[cname][i, ...] = f["data"][()]
# Load previous constants.
old_mod_mdata = prev_const_metadata[mod]
if cname in old_mod_mdata: # a module can be missing from detector dark processing.
filepath = old_mod_mdata[cname]["filepath"]
h5path = old_mod_mdata[cname]["h5path"]
if not filepath or not h5path:
excluded_modules.append(mod)
prev_constants[cname][i, ...].fill(np.nan)
else:
with h5py.File(filepath, "r") as fd:
prev_constants[cname][i, ...] = fd[f"{h5path}/data"][()]
if excluded_modules:
print(f"Previous {cname} constants for {excluded_modules} are not available.\n.")
# Exclude constants from comparison plots, if the corresponding
# previous constants are not available for all modules.
if len(excluded_modules) == nmods:
exculded_constants.append(cname)
print(f"No comparison plots for {cname}.\n")
```
%% Cell type:code id: tags:
``` python
display(Markdown('## Processed modules'))
processed_modules = list(mod_mapping.keys())
processed_pdus = list(mod_mapping.values())
show_processed_modules_jungfrau(
jungfrau_geom=geom,
constants=curr_constants,
processed_modules=processed_modules,
expected_modules=expected_modules,
display_module_names=processed_pdus,
)
```
%% Cell type:code id: tags:
``` python
gainstages = 3
gain_names = ["High Gain", "Medium Gain", "Low Gain" ]
gs = gridspec.GridSpec(2, 4)
axes = {
"ax0": {
"map": {
"gs": gs[0, 1:3],
"shrink": 0.7,
"pad": 0.05,
"label": "ADCu",
"title": "{}",
"location": "right",
},
"ax1": {
"diff": {
"gs": gs[1, :2],
"shrink": 0.7,
"pad": 0.02,
"label": "ADCu",
"location": "left",
"title": "Difference with previous {}",
},
"ax2": {
"diff_frac": {
"gs": gs[1, 2:],
"shrink": 0.7,
"pad": 0.02,
"label": "%",
"location": "right",
"title": "Difference with previous {} %",
},
}
```
%% Cell type:code id: tags:
``` python
def badpx(constant_name):
return True if "bad" in constant_name.lower() else False
def bp_entry(bp):
return [f"{bp.name:<30s}", f"{bp.value:032b}", f"{int(bp.value)}"]
badpixels = [
BadPixels.OFFSET_OUT_OF_THRESHOLD,
BadPixels.NOISE_OUT_OF_THRESHOLD,
BadPixels.OFFSET_NOISE_EVAL_ERROR,
BadPixels.NO_DARK_DATA,
BadPixels.WRONG_GAIN_VALUE,
]
```
%% Cell type:markdown id: tags:
## Summary figures across pixels and memory cells.
The following plots give an overview of calibration constants averaged across pixels and memory cells. A bad pixel mask is applied.
%% Cell type:code id: tags:
``` python
for cname, const in curr_constants.items():
if badpx(cname):
table = [bp_entry(bp) for bp in badpixels]
display(Markdown("""**The bad pixel** mask is encoded as a bit mask."""))
display(Latex(
tabulate.tabulate(
table,
tablefmt='latex',
headers=["Name", "bit value", "integer value"]
)))
# Prepare the stacked mean of constant,
# the difference with the previous constant
# and the fraction of that difference.
mean_const = np.nanmean(const, axis=3)
mean_diff = np.abs(np.nanmean(const, axis=3) - np.nanmean(prev_constants[cname], axis=3)) # noqa
mean_frac = np.abs(mean_diff / mean_const) * 100
mean_diff = np.abs(
np.nanmean(const, axis=3) - np.nanmean(
prev_constants[cname],
axis=3)
)
mean_frac = np.divide(
mean_diff,
mean_const,
out=np.zeros_like(mean_const),
where=(mean_const != 0)
) * 100
for gain in range(gainstages):
data_to_plot = {
f'ax0': mean_const[..., gain],
f'ax1': mean_diff[..., gain],
f'ax2': mean_frac[..., gain],
f'map': mean_const[..., gain],
f'diff': mean_diff[..., gain],
f'diff_frac': mean_frac[..., gain],
}
# Plotting constant overall modules.
display(Markdown(f'### {cname} - {gain_names[gain]}'))
if nmods > 1:
fig = plt.figure(figsize=(20, 20))
else:
fig = plt.figure(figsize=(20, 10))
for axname, axv in axes.items():
# Avoid difference plots if previous constants
# are missing for the detector.
if cname in exculded_constants and axname != "ax0":
if cname in exculded_constants and axname != "map":
break
ax = fig.add_subplot(axv["gs"])
if axname == "ax2": # Difference in percentage
vmin, vmax = (0, 100)
elif badpx(cname):
if badpx(cname):
vmin, vmax = (0, sorted([bp.value for bp in badpixels])[-2])
else:
vmin, vmax = np.percentile(data_to_plot[axname], [5, 95])
geom.plot_data(
data_to_plot[axname],
vmin=vmin,
vmax=vmax,
ax=ax,
colorbar={
"shrink": axv["shrink"],
"pad": axv["pad"],
"location": axv["location"],
},
)
colorbar = ax.images[0].colorbar
colorbar.set_label(axv["label"], fontsize=15)
colorbar.ax.tick_params(labelsize=15)
ax.tick_params(labelsize=1)
ax.set_title(axv["title"].format(
f"{cname} {gain_names[gain]}"), fontsize=15)
if axname == "ax0":
if axname == "map":
ax.set_xlabel('Columns', fontsize=15)
ax.set_ylabel('Rows', fontsize=15)
ax.tick_params(labelsize=15)
else:
ax.tick_params(labelsize=0)
# Remove axes labels for comparison plots.
ax.set_xlabel('')
ax.set_ylabel('')
plt.show()
```
%% Cell type:code id: tags:
``` python
if curr_constants["Offset"].shape[-2] > 1:
display(Markdown("## Summary across pixels per memory cells"))
# Plot mean and std of memcells for each module, gain, and constant
# across trains.
for const_name, const in curr_constants.items():
display(Markdown(f'### {const_name}'))
for gain in range(gainstages):
data = np.copy(const[..., gain])
if const_name == 'BadPixelsDark':
data[data > 0] = 1.0
datamean = np.nanmean(data, axis=(1, 2))
datamean[datamean == 1.0] = np.nan
fig = plt.figure(
figsize=(15, 6),
tight_layout={'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})
label = 'Fraction of bad pixels'
ax = fig.add_subplot(1, 1, 1)
else:
datamean = np.nanmean(data, axis=(1, 2))
fig = plt.figure(
figsize=(15, 6),
tight_layout={'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})
label = f'{const_name} value [ADU], good pixels only'
ax = fig.add_subplot(1, 2, 1)
d = []
for i, mod in enumerate(datamean):
d.append({
'x': np.arange(mod.shape[0]),
'y': mod,
'drawstyle': 'steps-pre',
'label': processed_modules[i],
})
simplePlot(
d, figsize=(10, 10), xrange=(-12, 510),
x_label='Memory Cell ID',
y_label=label,
use_axis=ax,
title=f'{gain_names[gain]}',
title_position=[0.5, 1.18],
legend='outside-top-ncol6-frame',
legend_size='18%',
legend_pad=0.00,
)
# Extra Sigma plot for Offset and Noise constants.
if const_name != 'BadPixelsDark':
ax = fig.add_subplot(1, 2, 2)
label = f'$\sigma$ {const_name} [ADU], good pixels only'
d = []
for i, mod in enumerate(np.nanstd(data, axis=(1, 2))):
d.append({
'x': np.arange(mod.shape[0]),
'y': mod,
'drawstyle': 'steps-pre',
'label': processed_modules[i],
})
simplePlot(
d, figsize=(10, 10), xrange=(-12, 510),
x_label='Memory Cell ID',
y_label=label,
use_axis=ax,
title=f'{gain_names[gain]} $\sigma$',
title_position=[0.5, 1.18],
legend='outside-top-ncol6-frame',
legend_size='18%',
legend_pad=0.00,
)
plt.show()
```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts