Skip to content
Snippets Groups Projects

[JUNGFRAU][DARK] New notebook for summarizing processed darks

Merged [JUNGFRAU][DARK] New notebook for summarizing processed darks
fix/jungfrau_dark_summary_across_modules into master
Merged Karim Ahmed requested to merge fix/jungfrau_dark_summary_across_modules into master
Compare
and
5 files
+ 663
− 91
Compare changes
  • Side-by-side
  • Inline
Files
5
notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
+ 176
− 88
# Parameters for calibration database and storing constants.
# Parameters for calibration database and storing constants.
use_dir_creation_date = True # use dir creation date
use_dir_creation_date = True # use dir creation date
cal_db_interface = 'tcp://max-exfl016:8016' # calibrate db interface to connect to
cal_db_interface = 'tcp://max-exfl016:8016#8045' # calibrate db interface to connect to
cal_db_timeout = 300000 # timeout on caldb requests
cal_db_timeout = 300000 # timeout on caldb requests
local_output = True # output constants locally
local_output = True # output constants locally
db_output = False # output constants to database
db_output = False # output constants to database
bias_voltage = 90 # sensor bias voltage in V, will be overwritten by value in file
bias_voltage = 90 # sensor bias voltage in V, will be overwritten by value in file
memory_cells = 16 # number of memory cells
memory_cells = 16 # number of memory cells
 
# Parameters used for plotting
 
detailed_report = False
 
# TODO: this is used for only Warning check at AGIPD dark.
# TODO: this is used for only Warning check at AGIPD dark.
# Need to rethink if it makes sense to use it here as well.
# Need to rethink if it makes sense to use it here as well.
operation_mode = 'ADAPTIVE_GAIN' # Detector operation mode, optional
operation_mode = 'ADAPTIVE_GAIN' # Detector operation mode, optional
import multiprocessing
import multiprocessing
import numpy as np
import numpy as np
import pasha as psh
import pasha as psh
 
import yaml
from IPython.display import Markdown, display
from IPython.display import Markdown, display
from extra_data import RunDirectory
from extra_data import RunDirectory
snapshot_at=creation_time)
snapshot_at=creation_time)
%% Cell type:code id: tags:
%% Cell type:code id: tags:
``` python
``` python
 
# Start retrieving existing constants for comparison
 
mod_x_const = [(mod, const) for const in ["Offset", "Noise", "BadPixelsDark"] for mod in karabo_da]
 
 
from cal_tools.tools import get_from_db
 
from datetime import timedelta
 
 
def retrieve_old_constant(mod, const):
 
dconst = getattr(Constants.jungfrau, const)()
 
 
data, mdata = get_from_db(
 
karabo_id=karabo_id,
 
karabo_da=mod,
 
constant=dconst,
 
condition=condition,
 
empty_constant=None,
 
cal_db_interface=cal_db_interface,
 
creation_time=creation_time-timedelta(seconds=60) if creation_time else None,
 
strategy="pdu_prior_in_time",
 
verbosity=1,
 
timeout=cal_db_timeout
 
)
 
 
if mdata is None or data is None:
 
timestamp = "Not found"
 
filepath = None
 
h5path = None
 
else:
 
timestamp = mdata.calibration_constant_version.begin_at.isoformat()
 
filepath = os.path.join(
 
mdata.calibration_constant_version.hdf5path,
 
mdata.calibration_constant_version.filename
 
)
 
h5path = mdata.calibration_constant_version.h5path
 
 
return data, timestamp, filepath, h5path
 
 
 
old_retrieval_pool = multiprocessing.Pool()
 
old_retrieval_res = old_retrieval_pool.starmap_async(
 
retrieve_old_constant, mod_x_const
 
)
 
old_retrieval_pool.close()
 
%% Cell type:code id: tags:
step_timer.done_step(f'Creating Offset and noise constants for a module.')
step_timer.done_step(f'Creating Offset and noise constants for a module.')
%% Cell type:markdown id: tags:
## Offset and Noise Maps ##
Below offset and noise maps for the high ($g_0$) gain stage are shown, alongside the distribution of these values. One expects block-like structures mapping to the ASICs of the detector
%% Cell type:code id: tags:
``` python
g_name = ['G0', 'G1', 'G2']
g_range = [(0, 8000), (8000, 16000), (8000, 16000)]
n_range = [(0., 50.), (0., 50.), (0., 50.)]
unit = '[ADCu]'
%% Cell type:code id: tags:
%% Cell type:code id: tags:
``` python
``` python
# TODO: Fix plots arrangment and speed for Jungfrau burst mode.
if detailed_report:
step_timer.start()
display(Markdown("## Offset and Noise Maps:"))
for pdu, mod in zip(db_modules, karabo_da):
display(Markdown(
for g_idx in gains:
"Below offset and noise maps for the high ($g_0$) gain stage are shown, "
for cell in range(0, memory_cells):
"alongside the distribution of these values. One expects block-like "
f_o0 = heatmapPlot(
"structures mapping to the ASICs of the detector"))
np.swapaxes(offset_map[mod][..., cell, g_idx], 0, 1),
g_name = ['G0', 'G1', 'G2']
y_label="Row",
g_range = [(0, 8000), (8000, 16000), (8000, 16000)]
x_label="Column",
n_range = [(0., 50.), (0., 50.), (0., 50.)]
lut_label=unit,
aspect=1.,
vmin=g_range[g_idx][0],
vmax=g_range[g_idx][1],
title=f'Pedestal {g_name[g_idx]} - Cell {cell:02d} - Module {mod} ({pdu})')
fo0, ax_o0 = plt.subplots()
res_o0 = histPlot(
ax_o0, offset_map[mod][..., cell, g_idx],
bins=800,
range=g_range[g_idx],
facecolor='b',
histotype='stepfilled',
)
ax_o0.tick_params(axis='both',which='major',labelsize=15)
ax_o0.set_title(
f'Module pedestal distribution - Cell {cell:02d} - Module {mod} ({pdu})',
fontsize=15)
ax_o0.set_xlabel(f'Pedestal {g_name[g_idx]} {unit}',fontsize=15)
ax_o0.set_yscale('log')
f_n0 = heatmapPlot(
np.swapaxes(noise_map[mod][..., cell, g_idx], 0, 1),
y_label="Row",
x_label="Column",
lut_label= unit,
aspect=1.,
vmin=n_range[g_idx][0],
vmax=n_range[g_idx][1],
title=f"RMS noise {g_name[g_idx]} - Cell {cell:02d} - Module {mod} ({pdu})",
)
fn0, ax_n0 = plt.subplots()
unit = '[ADCu]'
res_n0 = histPlot(
# TODO: Fix plots arrangment and speed for Jungfrau burst mode.
ax_n0,
step_timer.start()
noise_map[mod][..., cell, g_idx],
for pdu, mod in zip(db_modules, karabo_da):
bins=100,
for g_idx in gains:
range=n_range[g_idx],
for cell in range(0, memory_cells):
facecolor='b',
f_o0 = heatmapPlot(
histotype='stepfilled',
np.swapaxes(offset_map[mod][..., cell, g_idx], 0, 1),
)
y_label="Row",
x_label="Column",
ax_n0.tick_params(axis='both', which='major', labelsize=15)
lut_label=unit,
ax_n0.set_title(
aspect=1.,
f'Module noise distribution - Cell {cell:02d} - Module {mod} ({pdu})',
vmin=g_range[g_idx][0],
fontsize=15)
vmax=g_range[g_idx][1],
ax_n0.set_xlabel(
title=f'Pedestal {g_name[g_idx]} - Cell {cell:02d} - Module {mod} ({pdu})')
f'RMS noise {g_name[g_idx]} ' + unit, fontsize=15)
plt.show()
fo0, ax_o0 = plt.subplots()
step_timer.done_step(f'Plotting offset and noise maps.')
res_o0 = histPlot(
 
ax_o0, offset_map[mod][..., cell, g_idx],
 
bins=800,
 
range=g_range[g_idx],
 
facecolor='b',
 
histotype='stepfilled',
 
)
 
 
ax_o0.tick_params(axis='both',which='major',labelsize=15)
 
ax_o0.set_title(
 
f'Module pedestal distribution - Cell {cell:02d} - Module {mod} ({pdu})',
 
fontsize=15)
 
ax_o0.set_xlabel(f'Pedestal {g_name[g_idx]} {unit}',fontsize=15)
 
ax_o0.set_yscale('log')
 
 
f_n0 = heatmapPlot(
 
np.swapaxes(noise_map[mod][..., cell, g_idx], 0, 1),
 
y_label="Row",
 
x_label="Column",
 
lut_label= unit,
 
aspect=1.,
 
vmin=n_range[g_idx][0],
 
vmax=n_range[g_idx][1],
 
title=f"RMS noise {g_name[g_idx]} - Cell {cell:02d} - Module {mod} ({pdu})",
 
)
 
 
fn0, ax_n0 = plt.subplots()
 
res_n0 = histPlot(
 
ax_n0,
 
noise_map[mod][..., cell, g_idx],
 
bins=100,
 
range=n_range[g_idx],
 
facecolor='b',
 
histotype='stepfilled',
 
)
 
 
ax_n0.tick_params(axis='both', which='major', labelsize=15)
 
ax_n0.set_title(
 
f'Module noise distribution - Cell {cell:02d} - Module {mod} ({pdu})',
 
fontsize=15)
 
ax_n0.set_xlabel(
 
f'RMS noise {g_name[g_idx]} ' + unit, fontsize=15)
 
plt.show()
 
step_timer.done_step(f'Plotting offset and noise maps.')
%% Cell type:markdown id: tags:
%% Cell type:markdown id: tags:
bad_pixels_map[mod][(offset_map[mod] < offset_abs_threshold[0][None, None, None, :]) | (offset_map[mod] > offset_abs_threshold[1][None, None, None, :])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value # noqa
bad_pixels_map[mod][(offset_map[mod] < offset_abs_threshold[0][None, None, None, :]) | (offset_map[mod] > offset_abs_threshold[1][None, None, None, :])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value # noqa
for g_idx in gains:
if detailed_report:
for cell in range(memory_cells):
bad_pixels = bad_pixels_map[mod][:, :, cell, g_idx]
for g_idx in gains:
fn_0 = heatmapPlot(
for cell in range(memory_cells):
np.swapaxes(bad_pixels, 0, 1),
bad_pixels = bad_pixels_map[mod][:, :, cell, g_idx]
y_label="Row",
fn_0 = heatmapPlot(
x_label="Column",
np.swapaxes(bad_pixels, 0, 1),
lut_label=f"Badpixels {g_name[g_idx]} [ADCu]",
y_label="Row",
aspect=1.,
x_label="Column",
vmin=0, vmax=5,
lut_label=f"Badpixels {g_name[g_idx]} [ADCu]",
title=f'G{g_idx} Bad pixel map - Cell {cell:02d} - Module {mod} ({pdu})')
aspect=1.,
step_timer.done_step(f'Creating bad pixels constant and plotting it for a module.')
vmin=0, vmax=5,
 
title=f'G{g_idx} Bad pixel map - Cell {cell:02d} - Module {mod} ({pdu})')
 
step_timer.done_step(f'Creating bad pixels constant')
%% Cell type:markdown id: tags:
%% Cell type:markdown id: tags:
print(f"Total processing time {step_timer.timespan():.01f} s")
print(f"Total processing time {step_timer.timespan():.01f} s")
step_timer.print_summary()
step_timer.print_summary()
 
%% Cell type:code id: tags:
 
``` python
 
# now we need the old constants
 
old_const = {}
 
old_mdata = {}
 
old_retrieval_res.wait()
 
 
for (mod, const), (data, timestamp, filepath, h5path) in zip(
 
mod_x_const, old_retrieval_res.get()):
 
old_const.setdefault(mod, {})[const] = data
 
old_mdata.setdefault(mod, {})[const] = {
 
"timestamp": timestamp,
 
"filepath": filepath,
 
"h5path": h5path,
 
}
 
%% Cell type:code id: tags:
 
``` python
 
display(Markdown("## The following pre-existing constants are used for comparison:"))
 
 
for mod, consts in old_mdata.items():
 
pdu = db_modules[karabo_da.index(mod)]
 
display(Markdown(f"- {mod} ({pdu})"))
 
for const in consts:
 
display(Markdown(f" - {const} at {consts[const]['timestamp']}"))
 
# saving locations of old constants for summary notebook
 
with open(f"{metadata_folder or out_folder}/module_metadata_{mod}.yml", "w") as fd:
 
yaml.safe_dump(
 
{
 
"module": mod,
 
"pdu": pdu,
 
"old-constants": old_mdata[mod],
 
},
 
fd,
 
)

Legal Notice | Contacts