Skip to content
Snippets Groups Projects

Fix: LPD DARK Reading of saved constants

Merged Fix: LPD DARK Reading of saved constants
fix/file_reading into master
1 unresolved thread
Merged Mikhail Karnevskiy requested to merge fix/file_reading into master
1 unresolved thread
Compare
and
1 file
+ 23
− 17
Compare changes
  • Side-by-side
  • Inline
notebooks/LPD/LPDChar_Darks_Summary_NBC.ipynb
+ 23
− 17
%% Cell type:markdown id: tags:
# Summary of LPD dark characterization #
%% Cell type:code id: tags:
``` python
cluster_profile = "noDB" # The ipcluster profile to use
out_folder = "/gpfs/exfel/data/scratch/karnem/LPD/" # path to output to, required
out_folder = "/gpfs/exfel/data/scratch/karnem/test/LPD_dark_004/" # path to output to, required
```
%% Cell type:code id: tags:
``` python
from collections import OrderedDict
import copy
from datetime import datetime
import os
import warnings
warnings.filterwarnings('ignore')
from iCalibrationDB import Detectors
import glob
import h5py
from IPython.display import display, Markdown, Latex
import numpy as np
import matplotlib
matplotlib.use("agg")
import matplotlib.patches as patches
import matplotlib.pyplot as plt
%matplotlib inline
import tabulate
from XFELDetAna.plotting.heatmap import heatmapPlot
from XFELDetAna.plotting.simpleplot import simplePlot
gain_names = ['High', 'Medium', 'Low']
```
%% Cell type:code id: tags:
``` python
# Load constants from local files
files = glob.glob('{}/*h5'.format(out_folder))
# TODO: After changes in the Cal DB interface read files from cal repositofy
# Load constants from local files
data = OrderedDict()
mod_names = []
# Loop over files
for filename in files:
with h5py.File(filename, 'r') as f:
# Loop over modules
for mKey in f.keys():
if mKey not in data:
mod_names.append(mKey)
data[mKey] = OrderedDict()
# Loop over constants
for cKey in f.get(mKey):
if cKey not in data[mKey]:
#print("/".join((mKey, cKey, '0', 'data')))
data[mKey][cKey] = f.get(
"/".join((mKey, cKey, '0', 'data'))).value
# Loop over modules
for i in range(16):
qm = "Q{}M{}".format(i//4 + 1, i % 4 + 1)
# loop over constants
for const in ['Offset', 'Noise', 'BadPixelsDark']:
det = getattr(Detectors.LPD1M1, qm, None)
if det is None:
continue
det_name = det.device_name
fpath = '{}/const_{}_{}.h5'.format(out_folder, const, det_name)
if not os.path.isfile(fpath):
continue
with h5py.File(fpath, 'r') as f:
if qm not in data:
mod_names.append(qm)
data[qm] = OrderedDict()
data[qm][const] = f["data"][()]
```
%% Cell type:code id: tags:
``` python
mod_idx = np.argsort(mod_names)
constants = {'Offset': np.zeros((len(mod_names), 256, 256, 512, 3)),
'Noise': np.zeros((len(mod_names), 256, 256, 512, 3)),
'BadPixelsDark': np.zeros((len(mod_names), 256, 256, 512, 3))}
for i, idx in enumerate(mod_idx):
for key, item in constants.items():
item[i] = data[mod_names[idx]][key]
mod_names = np.array(mod_names)[mod_idx]
```
%% Cell type:code id: tags:
``` python
display(Markdown('## Processed modules ##'))
fig, ax = plt.subplots(1, figsize=(10, 10))
ax.set_axis_off()
ax.set_xlim(0, 97)
ax.set_ylim(0, 97)
q_poses = np.array([[51, 47], [47, 1], [1, 5], [5, 51]])
m_poses = np.array([[22.5, 20.5], [22.5, 0.5], [0.5, 0.5], [0.5, 20.5]])
counter = 0
for iq, q_pos in enumerate(q_poses):
ax.add_patch(patches.Rectangle(q_pos, 45, 45, linewidth=2, edgecolor='dodgerblue',
facecolor='y', fill=True))
ax.text(q_pos[0]+20, q_pos[1]+41.5, 'Q{}'.format(iq+1), fontsize=22)
for im, m_pos in enumerate(m_poses):
color = 'gray'
if 'Q{}M{}'.format(iq+1, im+1) in mod_names:
color = 'green'
if np.nanmean(constants['Noise'][counter, :, :, :, 0]) == 0:
color = 'red'
counter += 1
ax.add_patch(patches.Rectangle(q_pos+m_pos, 22, 20, linewidth=3, edgecolor='dodgerblue',
facecolor=color, fill=True))
pos = q_pos+m_pos+np.array([5, 8])
ax.text(pos[0], pos[1], 'Q{}M{}'.format(
iq+1, im+1), fontsize=24, color='yellow')
_ = ax.legend(handles=[patches.Patch(facecolor='red', label='No data'),
patches.Patch(facecolor='gray', label='Not processed'),
patches.Patch(facecolor='green', label='Processed')],
loc='outside-top', ncol=3, bbox_to_anchor=(0.1, 0.25, 0.7, 0.8))
```
%% Cell type:markdown id: tags:
## Summary figures across Modules ##
Plots give an overview of calibration constants averaged across pixels. A bad pixel mask is applied. Constants are averaged across pixels.
%% Cell type:code id: tags:
``` python
q_pad = 15
m_pad = 5
m_size = 256
q_size = m_size*2+m_pad*2
image = np.zeros((m_size*4+q_pad+m_pad*3, m_size*4+q_pad+m_pad*3))
q_poses = [[q_size+q_pad, q_size],
[q_size, 0],
[0, q_pad], [q_pad, q_size+q_pad]]
m_poses = [[m_size+m_pad, m_size+m_pad], [m_size+m_pad, 0], [0, 0],
[0, m_size+m_pad]]
# Loop over capacitor settings, modules, constants
for const_name, const in constants.items():
if const_name == 'BadPixelsDark':
continue
display(Markdown('### {}'.format(const_name)))
for gain in range(3):
image[:] = np.nan
counter = 0
for iq, q_pos in enumerate(q_poses):
for im, m_pos in enumerate(m_poses):
if 'Q{}M{}'.format(iq+1, im+1) in mod_names:
values = np.nanmean(const[counter, :, :, :, gain], axis=2)
values[values == 0] = np.nan
image[q_pos[1]+m_pos[1]: q_pos[1]+m_pos[1]+m_size,
q_pos[0]+m_pos[0]: q_pos[0]+m_pos[0] + m_size] = values
counter += 1
std = np.nanstd(image)
mean = np.nanmedian(image)
if const_name == 'Noise':
std = mean/4.
_ = heatmapPlot(image, add_panels=False, figsize=(20, 20),
vmin=mean-std*2, vmax=mean+std*2,
x_label='columns', y_label='rows',
cb_label='{}, mean over memory cells [ADU]'.format(
const_name),
cmap='viridis',
title='{}. {} gain'.format(const_name, gain_names[gain]))
plt.show()
```
%% Cell type:code id: tags:
``` python
# Loop over capacitor settings, modules, constants
for const_name, const in constants.items():
display(Markdown('### Summary across Modules - {}'.format(const_name)))
for gain in range(3):
data = np.copy(const[:, :, :, :, gain])
if const_name != 'BadPixelsDark':
label = '{} value [ADU], good pixels only'.format(const_name)
data[constants['BadPixelsDark'][:, :, :, :, gain] > 0] = np.nan
datamean = np.nanmean(data, axis=(1, 2))
fig = plt.figure(figsize=(15, 6), tight_layout={
'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})
ax = fig.add_subplot(121)
else:
label = 'Fraction of bad pixels'
data[data > 0] = 1.0
datamean = np.nanmean(data, axis=(1, 2))
datamean[datamean == 1.0] = np.nan
fig = plt.figure(figsize=(15, 6), tight_layout={
'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})
ax = fig.add_subplot(111)
d = []
for im, mod in enumerate(datamean):
d.append({'x': np.arange(mod.shape[0]),
'y': mod,
'drawstyle': 'steps-pre',
'label': mod_names[im],
})
_ = simplePlot(d, figsize=(10, 10), xrange=(-12, 510),
x_label='Memory Cell ID',
y_label=label,
use_axis=ax,
title='{} gain'.format(gain_names[gain]),
title_position=[0.5, 1.18],
legend='outside-top-ncol6-frame', legend_size='18%',
legend_pad=0.00)
if const_name != 'BadPixelsDark':
ax = fig.add_subplot(122)
label = '$\sigma$ {} [ADU], good pixels only'.format(const_name)
d = []
for im, mod in enumerate(np.nanstd(data, axis=(1, 2))):
d.append({'x': np.arange(mod.shape[0]),
'y': mod,
'drawstyle': 'steps-pre',
'label': mod_names[im],
})
_ = simplePlot(d, figsize=(10, 10), xrange=(-12, 510),
x_label='Memory Cell ID',
y_label=label,
use_axis=ax,
title='{} gain'.format(gain_names[gain]),
title_position=[0.5, 1.18],
legend='outside-top-ncol6-frame', legend_size='18%',
legend_pad=0.00)
plt.show()
```
%% Cell type:markdown id: tags:
## Summary tables across Modules ##
Tables show values averaged across all pixels and memory cells of a given detector module.
%% Cell type:code id: tags:
``` python
if u'$' in tabulate.LATEX_ESCAPE_RULES:
del(tabulate.LATEX_ESCAPE_RULES[u'$'])
if u'\\' in tabulate.LATEX_ESCAPE_RULES:
del(tabulate.LATEX_ESCAPE_RULES[u'\\'])
```
%% Cell type:code id: tags:
``` python
header = ['Module', 'High gain', 'Medium gain', 'Low gain']
for const_name, const in constants.items():
table = []
for i_mod, mod in enumerate(mod_names):
t_line = [mod]
for gain in range(3):
data = np.copy(const[i_mod, :, :, :, gain])
if const_name == 'BadPixelsDark':
data[data > 0] = 1.0
datasum = np.nansum(data)
datamean = np.nanmean(data)
if datamean == 1.0:
datamean = np.nan
datasum = np.nan
t_line.append('{:6.0f} ({:6.3f}) '.format(
datasum, datamean))
label = '## Number (fraction) of bad pixels'
else:
data[constants['BadPixelsDark']
[i_mod, :, :, :, gain] > 0] = np.nan
t_line.append('{:6.1f} $\\pm$ {:6.1f}'.format(
np.nanmean(data), np.nanstd(data)))
label = '## Average {} [ADU], good pixels only ##'.format(const_name)
table.append(t_line)
display(Markdown(label))
md = display(Latex(tabulate.tabulate(
table, tablefmt='latex', headers=header)))
```

Legal Notice | Contacts