diff --git a/cal_tools/cal_tools/plotting.py b/cal_tools/cal_tools/plotting.py
index 0e9d9983578db0a1f2f8598be8adcb2199220783..5f37145f401a95c5af9aee1ee2e7fb5913ae9450 100644
--- a/cal_tools/cal_tools/plotting.py
+++ b/cal_tools/cal_tools/plotting.py
@@ -1,15 +1,27 @@
import matplotlib.pyplot as plt
+from matplotlib.patches import Patch, Rectangle
from mpl_toolkits.axes_grid1 import AxesGrid
from mpl_toolkits.mplot3d import Axes3D
import numpy as np
def show_overview(d, cell_to_preview, gain_to_preview, out_folder=None, infix=None):
-
+ """
+ Show an overview
+ :param d: A dict with the number of modules and
+ a dict for the constant names and it's data.
+ :param cell_to_preview: Integer number of memory cells to preview.
+ :param gain_to_preview: Integer number for the gain stages to preview.
+ :param out_folder: Output folder for saving the plotted.png image.
+ :param infix: infix to include in the png file name.
+ :return:
+ """
for module, data in d.items():
- fig = plt.figure(figsize=(20,20))
+ # Adapt number of columns to number of constants in d.
+ ncols = (len(d[module]) + (2 - 1)) // 2
+ fig = plt.figure(figsize=(20, 20))
grid = AxesGrid(fig, 111,
- nrows_ncols=(2, 2),
+ nrows_ncols=(2, ncols),
axes_pad=(0.9, 0.15),
label_mode="1",
share_all=True,
@@ -24,36 +36,49 @@ def show_overview(d, cell_to_preview, gain_to_preview, out_folder=None, infix=No
if "ThresholdsDark" in key:
cf = -1
if len(item.shape) == 4:
- med = np.nanmedian(item[...,cell_to_preview, gain_to_preview + cf])
+ med = np.nanmedian(item[..., cell_to_preview,
+ gain_to_preview + cf])
else:
- med = np.nanmedian(item[...,cell_to_preview])
-
+ med = np.nanmedian(item[..., cell_to_preview])
+
bound = 0.2
- while(np.count_nonzero((item[...,cell_to_preview, gain_to_preview + cf] < med-np.abs(bound*med)) |
- (item[...,cell_to_preview, gain_to_preview + cf] > med+np.abs(bound*med)))/item[...,cell_to_preview, gain_to_preview + cf].size > 0.01):
- bound *=2
-
+ while (np.count_nonzero((item[..., cell_to_preview, gain_to_preview + cf] < med - np.abs(bound * med)) | # noqa
+ (item[..., cell_to_preview, gain_to_preview + cf] > med + np.abs(bound * med))) / # noqa
+ item[..., cell_to_preview, gain_to_preview + cf].size > 0.01): # noqa
+ bound *= 2
+
if "BadPixelsDark" in key:
- im = grid[i].imshow(np.log2(item[...,cell_to_preview, gain_to_preview + cf]), interpolation="nearest",
- vmin=0, vmax=8, aspect='auto')
+ im = grid[i].imshow(np.log2(item[..., cell_to_preview,
+ gain_to_preview + cf]),
+ interpolation="nearest", vmin=0, vmax=8,
+ aspect='auto')
else:
+
if len(item.shape) == 4:
- im = grid[i].imshow(item[...,cell_to_preview, gain_to_preview + cf], interpolation="nearest",
- vmin=med-np.abs(bound*med), vmax=np.abs(med+bound*med), aspect='auto')
+ im_prev = item[..., cell_to_preview, gain_to_preview + cf]
+ vmax = np.abs(med + bound * med)
else:
- im = grid[i].imshow(item[...,cell_to_preview], interpolation="nearest",
- vmin=med-np.abs(bound*med), vmax=med+np.abs(bound*med), aspect='auto')
+ im_prev = item[..., cell_to_preview]
+ # move the axis of the image to show horizontally
+ # on the output report.
+ if im_prev.shape[0] > im_prev.shape[1]:
+ im_prev = np.moveaxis(item[..., cell_to_preview], 0, 1)
+ vmax = med + np.abs(bound * med)
+
+ im = grid[i].imshow(im_prev, interpolation="nearest",
+ vmin=med - np.abs(bound * med),
+ vmax=vmax, aspect='auto')
+
cb = grid.cbar_axes[i].colorbar(im)
cb.set_label_text("ADU" if key != "BadPixels" else "Bad Pixel Code")
- grid[i].text(5, 20, key, color="w" if key != "BadPixels" else "k", fontsize=20)
+ grid[i].text(5, 20, key, color="w" if key != "BadPixels" else "k", fontsize=20) # noqa
i += 1
- grid[0].text(5, 50, module, color="r" if key != "BadPixels" else "k", fontsize=20)
+ grid[0].text(5, 50, module, color="r" if key != "BadPixels" else "k", fontsize=20) # noqa
if out_folder and infix:
- fig.savefig("{}/dark_analysis_{}_module_{}.png".format(out_folder,
- infix,
- module))
+ fig.savefig(f"{out_folder}/"
+ f"dark_analysis_{infix}_module_{module}.png")
def rebin(a, *args):
@@ -67,19 +92,18 @@ def rebin(a, *args):
'''
shape = a.shape
lenShape = len(shape)
- factor = np.asarray(shape)//np.asarray(args)
+ factor = np.asarray(shape) // np.asarray(args)
evList = ['a.reshape('] + \
- ['args[%d],factor[%d],'%(i,i) for i in range(lenShape)] + \
- [')'] + ['.sum(%d)'%(i+1) for i in range(lenShape)] + \
- ['/factor[%d]'%i for i in range(lenShape-1)]
-
+ ['args[%d],factor[%d],' % (i, i) for i in range(lenShape)] + \
+ [')'] + ['.sum(%d)' % (i + 1) for i in range(lenShape)] + \
+ ['/factor[%d]' % i for i in range(lenShape - 1)]
ta = eval(''.join(evList))
return ta.astype(np.uint32), np.indices([s + 1 for s in ta.shape])
def plot_badpix_3d(data, definitions, title=None, rebin_fac=2, azim=22.5):
od = data
- d, dims = rebin(od.astype(np.uint32), od.shape[0]//rebin_fac, od.shape[1]//rebin_fac, od.shape[2])
+ d, dims = rebin(od.astype(np.uint32), od.shape[0] // rebin_fac, od.shape[1] // rebin_fac, od.shape[2])
xx, yy, zz = dims
voxels = d.astype(np.bool)
colors = np.full(voxels.shape, '#FFFFFF')
@@ -88,15 +112,15 @@ def plot_badpix_3d(data, definitions, title=None, rebin_fac=2, azim=22.5):
for k, c in cols.items():
colors[d == k] = c[1]
- fig = plt.figure(figsize=(15,10))
+ fig = plt.figure(figsize=(15, 10))
ax = fig.gca(projection="3d")
ax.voxels(xx*rebin_fac, yy*rebin_fac, zz, voxels, facecolors=colors)
ax.view_init(elev=25., azim=azim)
ax.set_xlabel("pixels")
ax.set_ylabel("pixels")
ax.set_zlabel("memory cell")
- ax.set_xlim(0, np.max(xx)*rebin_fac)
- ax.set_ylim(0, np.max(yy)*rebin_fac)
+ ax.set_xlim(0, np.max(xx) * rebin_fac)
+ ax.set_ylim(0, np.max(yy) * rebin_fac)
ax.set_zlim(0, np.max(zz))
for k, c in cols.items():
@@ -105,9 +129,10 @@ def plot_badpix_3d(data, definitions, title=None, rebin_fac=2, azim=22.5):
if title:
ax.set_title(title)
+
def create_constant_overview(constant, name, cells, vmin=None, vmax=None,
- entries=3, out_folder=None, infix=None,
- badpixels=None, gmap=None, marker=None):
+ entries=3, badpixels=None, gmap=None,
+ marker=None):
"""
Create a step plot for constant data across memory cells for requested
gain entries
@@ -135,7 +160,7 @@ def create_constant_overview(constant, name, cells, vmin=None, vmax=None,
if gmap is None:
gmap = ['High gain', 'Medium gain', 'Low gain']
if marker is None:
- marker = ['']*entries
+ marker = [''] * entries
fig = plt.figure(figsize=(10, 5))
ax = fig.add_subplot(111)
for g in range(entries):
@@ -159,7 +184,7 @@ def create_constant_overview(constant, name, cells, vmin=None, vmax=None,
table.append([name, qm, gmap[g], np.nanmean(d), np.nanmedian(d),
np.nanstd(d)])
- ax.step(np.arange(cells), np.nanmean(d, axis=(0,1)),
+ ax.step(np.arange(cells), np.nanmean(d, axis=(0, 1)),
label=gmap[g], color=f'C{g}', marker=marker[g])
# Plotting good pixels only if bad-pixels were given
if dbp is not None:
@@ -169,6 +194,223 @@ def create_constant_overview(constant, name, cells, vmin=None, vmax=None,
ax.set_xlabel("Memory cell")
ax.set_ylabel(name)
ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
- ax.set_title(f"{name} Median per Cell".format(gmap[g]))
+ ax.set_title(f"{name} Median per Cell")
if vmin and vmax:
ax.set_ylim(vmin, vmax)
+
+
+def show_processed_modules(dinstance, constants, mnames, mode):
+ """
+ Show the status of the processed modules.
+ Green: Processed. Gray: Not Processed. Red: No data available.
+ :param dinstance: The detector instance (e.g. AGIPD1M1 or LPD1M1)
+ :param constants: A dict of the plotted constants data.
+ {"const_name":constant_data}. Can be None in case of position mode.
+ :param mnames: A list of available module names.
+ :param mode: String selecting on of the two modes of operation.
+ "position": To just show the position of the processed modules.
+ "processed": To show the modules successfully processed.
+ :return
+ """
+ fig, ax = plt.subplots(1, figsize=(10, 10))
+ ax.set_axis_off()
+ counter = 0
+
+ if 'AGIPD' in dinstance:
+
+ ax.set_xlim(0, 90)
+ ax.set_ylim(0, 75)
+ asic_pos = 5
+ q_st = 8
+ # quad_pos = [Q1[x, y], Q2[x, y], Q3[x, y], Q4[x, y]]
+ quad_pos = [[43, 66], [45, 34], [4, 32], [2, 64]]
+ l_y = 6
+ l_x = 5
+ for iq, q_x in enumerate(quad_pos):
+ for im in range(4):
+ if mode == "processed":
+ color = 'gray'
+ ecolor = 'darkgreen'
+ if f'Q{iq+1}M{im+1}' in mnames:
+ color = 'green'
+ if 'Noise' not in constants.keys() or \
+ np.nanmean(constants['Noise'][counter, :, :, :, 0]) == 0: # noqa
+ color = 'red'
+ counter += 1
+ else:
+ color = 'lightblue'
+ ecolor = 'c'
+ x = q_x[0]
+ for _ in range(8):
+ # adding modules patches
+ ax.add_patch(Rectangle((x, q_x[1] - q_st * im), 5, 6,
+ linewidth=2, edgecolor=ecolor,
+ facecolor=color, fill=True))
+ if mode == "position":
+ if f"Q{iq+1}M{im+1}" in mnames:
+ ax.add_patch(
+ Rectangle((x, q_x[1] - q_st * im),
+ l_x, l_y / 2, linewidth=2,
+ edgecolor='c',
+ facecolor='sandybrown',
+ fill=True))
+ ax.add_patch(
+ Rectangle((x, (q_x[1] - q_st * im + 3)),
+ l_x, l_y / 2, linewidth=2,
+ edgecolor='c',
+ facecolor='sandybrown',
+ fill=True))
+ x += asic_pos
+ if mode == "position":
+ if f"Q{iq+1}M{im+1}" in mnames:
+ # Change Text for current processed module.
+ ax.text(q_x[0] + 13, q_x[1] - q_st * im + 1.5,
+ f'Q{iq+1}M{im+1}', fontsize=28,
+ color='mediumblue', weight='bold')
+ else:
+ ax.text(q_x[0] + 14, q_x[1] - q_st * im + 1.5,
+ f'Q{iq+1}M{im+1}', fontsize=25, color='k')
+ elif mode == "processed":
+ ax.text(q_x[0] + 14.5, q_x[1] - q_st * im + 1.5,
+ f'Q{iq+1}M{im+1}', fontsize=24, color='gold')
+ elif 'LPD' in dinstance:
+
+ ax.set_xlim(0, 97)
+ ax.set_ylim(0, 97)
+ q_poses = np.array([[51, 47], [47, 1], [1, 5], [5, 51]])
+ m_poses = np.array([[22.5, 20.5], [22.5, 0.5],
+ [0.5, 0.5], [0.5, 20.5]])
+
+ for iq, q_pos in enumerate(q_poses):
+ ax.add_patch(Rectangle(q_pos, 45, 45, linewidth=2,
+ edgecolor='darkgreen',
+ facecolor='cornsilk', fill=True))
+ ax.text(q_pos[0] + 20, q_pos[1] + 41.5, f'Q{iq+1}', fontsize=22)
+
+ for im, m_pos in enumerate(m_poses):
+ if mode == "processed":
+ color = 'gray'
+ ecolor = 'darkgreen'
+ if f'Q{iq+1}M{im+1}' in mnames:
+ color = 'green'
+ if 'Noise' not in constants.keys() or \
+ np.nanmean(constants['Noise'][counter, :, :, :, 0]) == 0: # noqa
+ color = 'red'
+ counter += 1
+ pos = q_pos + m_pos + np.array([5, 8])
+ ax.text(pos[0], pos[1], f'Q{iq+1}M{im+1}',
+ fontsize=24, color='gold')
+ elif mode == "position":
+ color = 'lightblue'
+ ecolor = 'c'
+ # adding modules patches
+ ax.add_patch(Rectangle(q_pos + m_pos, 22, 20, linewidth=3,
+ edgecolor=ecolor, facecolor=color,
+ fill=True))
+ if mode == "position":
+ if f"Q{iq+1}M{im+1}" in mnames:
+ for a_posx in range(2):
+ for a_posy in range(8):
+ a_pos = np.array([a_posx*11., a_posy*20/8.])
+ pos = q_pos + m_pos + a_pos
+ # Top Asics patches
+ ax.add_patch(Rectangle(q_pos+m_pos+a_pos, 11,
+ 20/8., linewidth=1,
+ edgecolor='k',
+ facecolor='sandybrown',
+ fill=True))
+ if a_posx == 0:
+ label = str(a_posy + 9)
+ else:
+ label = str(-a_posy + (a_posx * 8))
+ ax.text(pos[0] + 4, pos[1] + 0.3, label,
+ fontsize=14)
+ else:
+ pos = q_pos + m_pos + np.array([5, 8])
+ ax.text(pos[0], pos[1], f'Q{iq+1}M{im+1}',
+ fontsize=22, color='k')
+
+ if mode == "position":
+ ax.add_patch(Rectangle([65, 93], 30, 4, linewidth=1,
+ edgecolor='k', facecolor='sandybrown',
+ fill=True))
+ ax.text(52, 94, 'ASICs:', fontsize=22, color='k')
+
+ for i_pos in range(8):
+ pos = np.array([65, 93]) + np.array([i_pos*30/8.+0.3, 0.3]) # noqa
+
+ ax.add_patch(Rectangle(pos, 24/8., 3.4, linewidth=1,
+ edgecolor='k',
+ facecolor='sandybrown',
+ fill=True))
+
+ ax.text(pos[0]+0.5, pos[1]+0.5, f'{i_pos+1}',
+ fontsize=18, color='k')
+
+ elif 'DSSC' in dinstance:
+ ax.set_xlim(0, 85)
+ ax.set_ylim(0, 79)
+ asic_pos = 20
+ q_st = 8
+
+ # quad_pos = [Q1[x, y], Q2[x, y], Q3[x, y], Q4[x, y]]
+ quad_pos = [[45, 64], [43, 24], [0, 28], [0, 66]]
+ mod_ind = [[4, 3, 2, 1], [4, 3, 2, 1], [1, 2, 3, 4], [1, 2, 3, 4]]
+
+ for iq, q_x in enumerate(quad_pos):
+ for im in range(4):
+ if mode == "processed":
+ color = 'gray'
+ ecolor = 'darkgreen'
+ if f'Q{iq+1}M{mod_ind[iq][im]}' in mnames:
+ color = 'green'
+ if 'Noise' not in constants.keys() or \
+ np.nanmean(constants['Noise'][counter, :, :, :]) == 0: # noqa
+ color = 'red'
+ counter += 1
+ else:
+ color = 'lightblue'
+ ecolor = 'c'
+ x = q_x[0]
+ for _ in range(2):
+ # adding modules patches
+ ax.add_patch(Rectangle((x, q_x[1] - q_st * im), 19, 10,
+ linewidth=2, edgecolor=ecolor,
+ facecolor=color, fill=True))
+ if mode == "position":
+ # Check if it is the processed module to highlight.
+ if f"Q{iq+1}M{mod_ind[iq][im]}" in mnames:
+ ax.add_patch(Rectangle((x, q_x[1] - q_st * im),
+ 19, 10, linewidth=2,
+ edgecolor='c',
+ facecolor="sandybrown",
+ fill=True))
+
+ ax.text(q_x[0] + 2, q_x[1] - q_st * im + 4.5,
+ 'T1 T2', fontsize=18, # noqa
+ color='grey')
+ x += asic_pos
+ if mode == "position":
+ # Check if it is the processed module to highlight.
+ if f'Q{iq+1}M{mod_ind[iq][im]}' in mnames:
+ # Change Text for current processed module.
+ ax.text(q_x[0] + 12.7, q_x[1] - q_st * im + 4.5,
+ f'Q{iq + 1} M{mod_ind[iq][im]}', fontsize=26,
+ color='mediumblue', weight='bold')
+ else:
+ ax.text(q_x[0] + 13.5, q_x[1] - q_st * im + 4.5,
+ f'Q{iq+1} M{mod_ind[iq][im]}',
+ fontsize=25, color='k')
+ elif mode == "processed":
+ ax.text(q_x[0] + 13.5, q_x[1] - q_st * im + 4.5,
+ f'Q{iq+1} M{mod_ind[iq][im]}', fontsize=24,
+ color='gold')
+ ax.text(q_x[0] + 2, q_x[1] - q_st * im + 4.5,
+ 'T1 T2', fontsize=18,
+ color='y')
+ if mode == "processed":
+ _ = ax.legend(handles=[Patch(facecolor='red', label='No data'),
+ Patch(facecolor='gray', label='Not processed'),
+ Patch(facecolor='green', label='Processed')],
+ loc='outside-top', ncol=3, bbox_to_anchor=(0.1, 0.25, 0.7, 0.8))
+ plt.show()
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index ba4e90eba0e4fabbf2aa17323e3e678305f5d9a1..69bbdc18336c68c6ed3844bf265e81708991b4a2 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -390,8 +390,8 @@ def get_from_db(device, constant, condition, empty_constant,
if constant.name not in already_printed or verbosity > 1:
already_printed[constant.name] = True
begin_at = metadata.calibration_constant_version.begin_at
- print("{} was injected on: {}".format(constant.name,
- begin_at))
+ print(f"Retrieved {constant.name} "
+ f"with creation time: {begin_at}")
return constant.data, metadata
else:
return empty_constant, metadata
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index a10165106967ae33e7c44eac14e902df68af3ba2..22b5d9dc80c4a0ae7abe889e517f5f324cd6a840 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -107,7 +107,9 @@
" get_random_db_interface, get_from_db)\n",
"from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.plotting import show_overview, plot_badpix_3d, create_constant_overview\n",
+ "from cal_tools.plotting import (show_overview, plot_badpix_3d,\n",
+ " create_constant_overview,\n",
+ " show_processed_modules)\n",
"from cal_tools.agipdlib import get_gain_setting\n",
"\n",
"# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
@@ -161,7 +163,8 @@
"source": [
"control_fname = '{}/r{:04d}/RAW-R{:04d}-{}-S00000.h5'.format(in_folder, run_high, \n",
" run_high, karabo_da_control)\n",
- "if '{' in h5path_ctrl:\n",
+ "gain_names = ['High', 'Medium', 'Low']\n",
+ "if \"{\" in h5path_ctrl:\n",
" h5path_ctrl = h5path_ctrl.format(karabo_id_control)\n",
"\n",
"if gain_setting == 0.1:\n",
@@ -234,7 +237,7 @@
"os.makedirs(out_folder, exist_ok=True)\n",
"gmf = map_gain_stages(in_folder, offset_runs, path_template, karabo_da, sequences)\n",
"gain_mapped_files, total_sequences, total_file_size = gmf\n",
- "print(f\"Will process a total of {total_sequences} sequences.\")"
+ "print(f\"Will process a total of {total_sequences} files.\")"
]
},
{
@@ -382,6 +385,7 @@
" qm = f\"Q{i//4+1}M{i%4+1}\"\n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
" fname_in = mapped_files[qm].get()\n",
+ " print(\"Process file: \", fname_in)\n",
" dones.append(mapped_files[qm].empty())\n",
" else:\n",
" continue\n",
@@ -402,25 +406,26 @@
" offset, noise, gains, gains_std, gg, bp, thiscell, thisacq = r\n",
" all_cells.append(thiscell)\n",
" all_acq_rate.append(thisacq)\n",
- " qm = \"Q{}M{}\".format(i//4 + 1, i % 4 + 1)\n",
- " if qm not in offset_g:\n",
- " offset_g[qm] = np.zeros((offset.shape[0], offset.shape[1], offset.shape[2], 3))\n",
- " noise_g[qm] = np.zeros_like(offset_g[qm])\n",
- " gain_g[qm] = np.zeros_like(offset_g[qm])\n",
- " gainstd_g[qm] = np.zeros_like(offset_g[qm])\n",
- " badpix_g[qm] = np.zeros_like(offset_g[qm], np.uint32)\n",
- "\n",
- " offset_g[qm][...,gg] = offset\n",
- " noise_g[qm][...,gg] = noise\n",
- " gain_g[qm][...,gg] = gains\n",
- " gainstd_g[qm][..., gg] = gains_std\n",
- " badpix_g[qm][...,gg] = bp\n",
+ " for i in modules:\n",
+ " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ " if qm not in offset_g:\n",
+ " offset_g[qm] = np.zeros((offset.shape[0], offset.shape[1], offset.shape[2], 3))\n",
+ " noise_g[qm] = np.zeros_like(offset_g[qm])\n",
+ " gain_g[qm] = np.zeros_like(offset_g[qm])\n",
+ " gainstd_g[qm] = np.zeros_like(offset_g[qm])\n",
+ " badpix_g[qm] = np.zeros_like(offset_g[qm], np.uint32)\n",
+ "\n",
+ " offset_g[qm][...,gg] = offset\n",
+ " noise_g[qm][...,gg] = noise\n",
+ " gain_g[qm][...,gg] = gains\n",
+ " gainstd_g[qm][..., gg] = gains_std\n",
+ " badpix_g[qm][...,gg] = bp\n",
" \n",
"\n",
- "duration = (datetime.now()-start).total_seconds()\n",
+ "duration = (datetime.now() - start).total_seconds()\n",
"logger.runtime_summary_entry(success=True, runtime=duration,\n",
- " total_sequences=total_sequences,\n",
- " filesize=total_file_size)\n",
+ " total_sequences=total_sequences,\n",
+ " filesize=total_file_size)\n",
"logger.send()\n",
"max_cells = np.max(all_cells)\n",
"print(f\"Using {max_cells} memory cells\")\n",
@@ -569,12 +574,6 @@
"source": [
"for qm in res:\n",
"\n",
- " print(f\"Injecting constants {const} with conditions:\\n\")\n",
- " print(\"1. memory_cells: {}\\n2. bias_voltage: {}\\n\"\n",
- " \"3. acquisition_rate: {}\\n4. gain_setting: {}\\n\"\n",
- " \"5. creation_time: {}\\n\".format(max_cells, bias_voltage,\n",
- " acq_rate, gain_setting,\n",
- " creation_time))\n",
" for const in res[qm]:\n",
" metadata = ConstantMetaData()\n",
" dconst = getattr(Constants.AGIPD, const)()\n",
@@ -603,15 +602,19 @@
" if db_output:\n",
" try:\n",
" metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
- " msg = 'Const {} for module {} was injected to the calibration DB. Begin at: {}'\n",
- " print(msg.format(const, qm,\n",
- " metadata.calibration_constant_version.begin_at))\n",
+ " print(f'Const {const} for module {qm} was injected to the calibration DB. '\n",
+ " f'Begin at: {metadata.calibration_constant_version.begin_at}')\n",
" except Exception as e:\n",
" print(\"Error:\", e)\n",
"\n",
" if local_output:\n",
" save_const_to_h5(metadata, out_folder)\n",
- " print(\"Calibration constant {} is stored locally.\\n\".format(const))"
+ " print(f\"Calibration constant {const} is stored locally.\\n\")\n",
+ " \n",
+ " print(\"Generated constants with conditions:\\n\")\n",
+ " print(f\"• memory_cells: {max_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
+ " f\"• acquisition_rate: {acq_rate}\\n• gain_setting: {gain_setting}\\n\"\n",
+ " f\"• creation_time: {creation_time}\\n\")"
]
},
{
@@ -622,43 +625,12 @@
},
"outputs": [],
"source": [
- "qm = \"Q{}M{}\".format(modules[0]//4+1, modules[0] % 4+1)\n",
- "display(Markdown('## Position of the module {} and it\\'s ASICs##'.format(qm)))\n",
- "\n",
- "fig, ax = plt.subplots(1, figsize=(10, 10))\n",
- "ax.set_axis_off()\n",
- "\n",
- "ax.set_xlim(0, 90)\n",
- "ax.set_ylim(0, 75)\n",
- "\n",
- "asic_pos = 5\n",
- "\n",
- "q_st = 8\n",
- "\n",
- "l_y = 6\n",
- "l_x = 5\n",
- "\n",
- "for iq, q_x in enumerate([[43,66],[45,34],[4,32],[2,64]]):\n",
- " for im in range(4):\n",
- " x = q_x[0]\n",
- " for i_as in range(8):\n",
- " ax.add_patch(matplotlib.patches.Rectangle((x,q_x[1]-q_st*im), l_x, l_y, linewidth=2, edgecolor='c',\n",
- " facecolor='lightblue', fill=True))\n",
- "\n",
- " if iq*4+im == modules[0]:\n",
- " ax.add_patch(matplotlib.patches.Rectangle((x,q_x[1]-q_st*im), l_x, l_y/2, linewidth=2,edgecolor='c',\n",
- " facecolor='sandybrown', fill=True))\n",
- " ax.add_patch(matplotlib.patches.Rectangle((x,(q_x[1]-q_st*im+3)), l_x, l_y/2, linewidth=2,edgecolor='c',\n",
- " facecolor='sandybrown', fill=True))\n",
- " x += asic_pos\n",
- "\n",
- " if iq*4+im == modules[0]:\n",
- " # Change Text for current processed module.\n",
- " ax.text(q_x[0]+13, q_x[1]-q_st*im+1.5, 'Q{}M{}'.format(\n",
- " iq+1, im+1), fontsize=28, color='mediumblue',weight='bold')\n",
- " else:\n",
- " ax.text(q_x[0]+14, q_x[1]-q_st*im+1.5, 'Q{}M{}'.format(\n",
- " iq+1, im+1), fontsize=25, color='k')\n"
+ "mnames=[]\n",
+ "for i in modules:\n",
+ " qm = f\"Q{i//4+1}M{i % 4+1}\"\n",
+ " mnames.append(qm)\n",
+ " display(Markdown(f'## Position of the module {qm} and it\\'s ASICs##'))\n",
+ "show_processed_modules(dinstance, constants=None, mnames=mnames, mode=\"position\")"
]
},
{
@@ -789,8 +761,7 @@
},
"outputs": [],
"source": [
- "create_constant_overview(offset_g, \"Offset (ADU)\", max_cells, 4000, 10000,\n",
- " out_folder=out_folder, infix=\"{}-{}-{}\".format(*offset_runs.values()),\n",
+ "create_constant_overview(offset_g, \"Offset (ADU)\", max_cells, 4000, 8000,\n",
" badpixels=[badpix_g, np.nan])"
]
},
@@ -803,7 +774,6 @@
"outputs": [],
"source": [
"create_constant_overview(noise_g, \"Noise (ADU)\", max_cells, 0, 100,\n",
- " out_folder=out_folder, infix=\"{}-{}-{}\".format(*offset_runs.values()),\n",
" badpixels=[badpix_g, np.nan])"
]
},
@@ -824,7 +794,6 @@
"\n",
"\n",
"create_constant_overview(thresholds_g, \"Threshold (ADU)\", max_cells, 4000, 10000, 5,\n",
- " out_folder=out_folder, infix=\"{}-{}-{}\".format(*offset_runs.values()),\n",
" badpixels=[bp_thresh, np.nan],\n",
" gmap=['HG-MG Threshold', 'MG-LG Threshold', 'High gain', 'Medium gain', 'low gain'],\n",
" marker=['d','d','','','']\n",
@@ -840,8 +809,7 @@
"bad_pixel_aggregate_g = OrderedDict()\n",
"for m, d in badpix_g.items():\n",
" bad_pixel_aggregate_g[m] = d.astype(np.bool).astype(np.float)\n",
- "create_constant_overview(bad_pixel_aggregate_g, \"Bad pixel fraction\", max_cells, 0, 0.10, 3,\n",
- " out_folder=out_folder, infix=\"{}-{}-{}\".format(*offset_runs.values()))"
+ "create_constant_overview(bad_pixel_aggregate_g, \"Bad pixel fraction\", max_cells, 0, 0.10, 3)"
]
},
{
diff --git a/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb b/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
index 51a27c7409880050a615b4e16206375f2fa013c1..0b4cc8a538796ac17c48a65a9aef3e24cf03b8a3 100644
--- a/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
+++ b/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
@@ -8,7 +8,7 @@
"\n",
"Author: S. Hauf, Version: 0.1\n",
"\n",
- "The following code analyzes a set of dark images taken with the DSSC detector to deduce detector offsets and noise. Data for the detector is presented in one run and don't acquire multiple gain stages.\n",
+ "The following code analyzes a set of dark images taken with the DSSC detector to deduce detector offsets and noise. Data for the detector is presented in one run and don't acquire multiple gain stages. \n",
"\n",
"The notebook explicitely does what pyDetLib provides in its offset calculation method for streaming data."
]
@@ -25,11 +25,11 @@
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
- "in_folder = \"/gpfs/exfel/exp/SCS/201931/p900095/raw\" # path to input data, required\n",
+ "in_folder = \"/gpfs/exfel/exp/SCS/202030/p900125/raw\" # path to input data, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/DSSC\" # path to output to, required\n",
"sequences = [0] # sequence files to evaluate.\n",
"modules = [-1] # modules to run for\n",
- "run = 1497 # run number in which data was recorded, required\n",
+ "run = 222 # run numbr in which data was recorded, required\n",
"\n",
"karabo_id = \"SCS_DET_DSSC1M-1\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
@@ -38,7 +38,7 @@
"h5path = '/INSTRUMENT/{}/DET/{}:xtdf/image' # path in the HDF5 file to images\n",
"h5path_idx = '/INDEX/{}/DET/{}:xtdf/image' # path in the HDF5 file to images\n",
"\n",
- "dont_use_dir_date = True # don't use the dir creation date for determining the creation time\n",
+ "use_dir_creation_date = True # use the dir creation date for determining the creation time\n",
"cal_db_interface = \"tcp://max-exfl016:8020\" # the database interface to use\n",
"cal_db_timeout = 3000000 # timeout on caldb requests\"\n",
"local_output = True # output constants locally\n",
@@ -84,13 +84,17 @@
"\n",
"from cal_tools.tools import (map_gain_stages, parse_runs, run_prop_seq_from_path, \n",
" get_notebook_name, get_dir_creation_date,\n",
- " get_random_db_interface)\n",
+ " get_random_db_interface, get_from_db, save_const_to_h5)\n",
"from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.plotting import show_overview, plot_badpix_3d, create_constant_overview\n",
+ "from cal_tools.plotting import (show_overview, plot_badpix_3d,\n",
+ " create_constant_overview,\n",
+ " show_processed_modules)\n",
"\n",
"# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
"from ipyparallel import Client\n",
+ "from IPython.display import display, Markdown, Latex\n",
+ "import tabulate\n",
"\n",
"view = Client(profile=cluster_profile)[:]\n",
"view.use_dill()\n",
@@ -100,7 +104,7 @@
"\n",
"h5path = h5path.format(karabo_id, receiver_id)\n",
"h5path_idx = h5path_idx.format(karabo_id, receiver_id)\n",
- "\n",
+ "gain_names = ['High', 'Medium', 'Low']\n",
"\n",
"if karabo_da[0] == '-1':\n",
" if modules[0] == -1:\n",
@@ -115,16 +119,14 @@
"offset_runs[\"high\"] = run\n",
"\n",
"creation_time=None\n",
- "if not dont_use_dir_date:\n",
+ "if use_dir_creation_date:\n",
" creation_time = get_dir_creation_date(in_folder, run)\n",
- "\n",
+ " print(f\"Using {creation_time} as creation time of constant.\")\n",
"\n",
"run, prop, seq = run_prop_seq_from_path(in_folder)\n",
"logger = InfluxLogger(detector=\"DSSC\", instrument=instrument, mem_cells=mem_cells,\n",
" notebook=get_notebook_name(), proposal=prop)\n",
"\n",
- "print(f\"Using {creation_time} as creation time of constant.\")\n",
- " \n",
"dinstance = \"DSSC1M1\"\n",
"\n",
"print(f\"Detector in use is {karabo_id}\") \n",
@@ -176,7 +178,7 @@
"os.makedirs(out_folder, exist_ok=True)\n",
"gmf = map_gain_stages(in_folder, offset_runs, path_template, karabo_da, sequences)\n",
"gain_mapped_files, total_sequences, total_file_size = gmf\n",
- "print(f\"Will process a total of {total_sequences} sequences.\")"
+ "print(f\"Will process a total of {total_sequences} file.\")"
]
},
{
@@ -221,7 +223,7 @@
" dists[dists<0] = 10000 # assure to always go higher\n",
" return options[np.argmin(dists)]\n",
" \n",
- " filename, filename_out, channel = inp\n",
+ " filename, channel = inp\n",
" \n",
" h5path = h5path.format(channel)\n",
" h5path_idx = h5path_idx.format(channel)\n",
@@ -229,6 +231,8 @@
" if cells == 0:\n",
" cells = get_num_cells(filename, h5path)\n",
"\n",
+ " print(f\"Using {cells} memory cells\")\n",
+ " \n",
" pulseid_checksum = None\n",
"\n",
" thresholds_offset_hard, thresholds_offset_sigma, thresholds_noise_hard, thresholds_noise_sigma = bp_thresh \n",
@@ -306,19 +310,18 @@
"checksums = {}\n",
"\n",
"for gain, mapped_files in gain_mapped_files.items():\n",
- " \n",
" inp = []\n",
" dones = []\n",
" for i in modules:\n",
" qm = \"Q{}M{}\".format(i//4 +1, i % 4 + 1) \n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
- " fname_in = mapped_files[qm].get() \n",
+ " fname_in = mapped_files[qm].get()\n",
+ " print(\"Process file: \", fname_in)\n",
" dones.append(mapped_files[qm].empty())\n",
" else:\n",
" continue\n",
- " fout = os.path.abspath(\"{}/{}\".format(out_folder, (os.path.split(fname_in)[-1]).replace(\"RAW\", \"CORR\")))\n",
- " inp.append((fname_in, fout, i))\n",
- " first = False\n",
+ " inp.append((fname_in, i))\n",
+ "\n",
" p = partial(characterize_module, max_cells,\n",
" (thresholds_offset_hard, thresholds_offset_sigma,\n",
" thresholds_noise_hard, thresholds_noise_sigma), rawversion, karabo_id, h5path, h5path_idx)\n",
@@ -346,8 +349,49 @@
" total_sequences=total_sequences,\n",
" filesize=total_file_size)\n",
"logger.send()\n",
- "max_cells = np.max(all_cells)\n",
- "print(f\"Using {max_cells} memory cells\")"
+ "if len(all_cells) > 0:\n",
+ " max_cells = np.max(all_cells)\n",
+ " print(f\"Using {max_cells} memory cells\")\n",
+ "else:\n",
+ " raise ValueError(\"0 processed memory cells. No raw data available.\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Retrieve existing constants for comparison\n",
+ "clist = [\"Offset\", \"Noise\"]\n",
+ "old_const = {}\n",
+ "old_mdata = {}\n",
+ "\n",
+ "print('Retrieve pre-existing constants for comparison.')\n",
+ "detinst = getattr(Detectors, dinstance)\n",
+ "for qm in offset_g.keys():\n",
+ " device = getattr(detinst, qm)\n",
+ " for const in clist:\n",
+ " condition = Conditions.Dark.DSSC(memory_cells=max_cells,\n",
+ " bias_voltage=bias_voltage,\n",
+ " pulseid_checksum=checksums[qm])\n",
+ "\n",
+ " data, mdata = get_from_db(device,\n",
+ " getattr(Constants.DSSC, const)(),\n",
+ " condition,\n",
+ " None,\n",
+ " cal_db_interface, creation_time=creation_time,\n",
+ " verbosity=2, timeout=cal_db_timeout)\n",
+ "\n",
+ " old_const[const] = data\n",
+ "\n",
+ " if mdata is not None and data is not None:\n",
+ " time = mdata.calibration_constant_version.begin_at\n",
+ " old_mdata[const] = time.isoformat()\n",
+ " os.makedirs(f'{out_folder}/old/', exist_ok=True)\n",
+ " save_const_to_h5(mdata, f'{out_folder}/old/')\n",
+ " else:\n",
+ " old_mdata[const] = \"Not found\""
]
},
{
@@ -363,20 +407,15 @@
"source": [
"res = OrderedDict()\n",
"for i in modules:\n",
- " qm = \"Q{}M{}\".format(i//4+1, i%4+1)\n",
- " res[qm] = {'Offset': offset_g[qm],\n",
- " 'Noise': noise_g[qm],\n",
- " 'BadPixels': badpix_g[qm] \n",
- " }\n",
- " \n",
- "if local_output:\n",
- " for qm in offset_g.keys():\n",
- " ofile = \"{}/dssc_offset_store_{}_{}.h5\".format(out_folder, run, qm)\n",
- " store_file = h5py.File(ofile, \"w\")\n",
- " store_file[\"{}/Offset/0/data\".format(qm)] = offset_g[qm]\n",
- " store_file[\"{}/Noise/0/data\".format(qm)] = noise_g[qm]\n",
- " store_file[\"{}/BadPixels/0/data\".format(qm)] = badpix_g[qm]\n",
- " store_file.close()"
+ " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ " try:\n",
+ " res[qm] = {'Offset': offset_g[qm],\n",
+ " 'Noise': noise_g[qm],\n",
+ " #TODO: No badpixelsdark, yet.\n",
+ " #'BadPixelsDark': badpix_g[qm] \n",
+ " }\n",
+ " except Exception as e:\n",
+ " print(f\"Error: No constants for {qm}: {e}\")"
]
},
{
@@ -390,87 +429,75 @@
},
"outputs": [],
"source": [
- "if db_output:\n",
- " for dont_use_pulseIds in [True, False]:\n",
- " for qm in offset_g.keys():\n",
- " try:\n",
- " metadata = ConstantMetaData()\n",
- " offset = Constants.DSSC.Offset()\n",
- " offset.data = offset_g[qm]\n",
- " metadata.calibration_constant = offset\n",
- " pidsum = None if dont_use_pulseIds else checksums[qm]\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.DSSC(memory_cells=max_cells, bias_voltage=bias_voltage,\n",
- " pulseid_checksum=pidsum)\n",
- " detinst = getattr(Detectors, dinstance)\n",
- "\n",
- " device = getattr(detinst, qm)\n",
- "\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
- "\n",
- "\n",
- " metadata = ConstantMetaData()\n",
- " noise = Constants.DSSC.Noise()\n",
- " noise.data = noise_g[qm]\n",
- " metadata.calibration_constant = noise\n",
- "\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.DSSC(memory_cells=max_cells, bias_voltage=bias_voltage,\n",
- " pulseid_checksum=pidsum)\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
- "\n",
- " continue # no bad pixels yet\n",
- " metadata = ConstantMetaData()\n",
- " badpixels = Constants.DSSC.BadPixelsDark()\n",
- " badpixels.data = badpix_g[qm]\n",
- " metadata.calibration_constant = badpixels\n",
- "\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.DSSC(memory_cells=max_cells, bias_voltage=bias_voltage,\n",
- " pulseid_checksum=pidsum)\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
- " except Exception as e:\n",
- " print(e)"
+ "# Push the same constant two different times.\n",
+ "# One with the generated pulseID check sum setting for the offline calibration.\n",
+ "# And another for the online calibration as it doesn't have this pulseID checksum, yet.\n",
+ "for dont_use_pulseIds in [True, False]:\n",
+ " for qm in res.keys():\n",
+ " detinst = getattr(Detectors, dinstance)\n",
+ " device = getattr(detinst, qm)\n",
+ "\n",
+ " for const in res[qm].keys():\n",
+ "\n",
+ " metadata = ConstantMetaData()\n",
+ " dconst = getattr(Constants.DSSC, const)()\n",
+ " dconst.data = res[qm][const]\n",
+ " metadata.calibration_constant = dconst\n",
+ "\n",
+ " pidsum = None if dont_use_pulseIds else checksums[qm]\n",
+ " # set the operating condition\n",
+ " condition = Conditions.Dark.DSSC(memory_cells=max_cells,\n",
+ " bias_voltage=bias_voltage,\n",
+ " pulseid_checksum=pidsum)\n",
+ "\n",
+ " metadata.detector_condition = condition\n",
+ "\n",
+ " # specify the a version for this constant\n",
+ " if creation_time is None:\n",
+ " metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ " else:\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
+ "\n",
+ " if db_output:\n",
+ " try:\n",
+ " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
+ " except Exception as e:\n",
+ " print(\"Error\", e)\n",
+ "\n",
+ " if local_output:\n",
+ " # Don't save constant localy two times.\n",
+ " if dont_use_pulseIds:\n",
+ " save_const_to_h5(metadata, out_folder)\n",
+ " print(f\"Calibration constant {const} is stored locally.\\n\")\n",
+ " \n",
+ " if not dont_use_pulseIds:\n",
+ " print(\"Generated constants with conditions:\\n\")\n",
+ " print(f\"• memory_cells: {max_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
+ " f\"• pulseid_checksum: {pidsum}\\n• creation_time: {creation_time}\\n\")"
]
},
{
- "cell_type": "markdown",
+ "cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
- "## Single-Cell Overviews ##\n",
- "\n",
- "Single cell overviews allow to identify potential effects on all memory cells, e.g. on sensor level. Additionally, they should serve as a first sanity check on expected behaviour, e.g. if structuring on the ASIC level is visible in the offsets, but otherwise no immediate artifacts are visible."
+ "mnames = []\n",
+ "for i in modules:\n",
+ " qm = f\"Q{i//4+1}M{i % 4+1}\"\n",
+ " display(Markdown(f'## Position of the module {mnames} and it\\'s ASICs##'))\n",
+ " mnames.append(qm)\n",
+ " \n",
+ "show_processed_modules(dinstance=dinstance, constants=None, mnames=mnames, mode=\"position\")\n"
]
},
{
- "cell_type": "code",
- "execution_count": null,
+ "cell_type": "markdown",
"metadata": {},
- "outputs": [],
"source": [
- "for r in res.values():\n",
- " del r[\"BadPixels\"]"
+ "## Single-Cell Overviews ##\n",
+ "\n",
+ "Single cell overviews allow to identify potential effects on all memory cells, e.g. on sensor level. Additionally, they should serve as a first sanity check on expected behaviour, e.g. if structuring on the ASIC level is visible in the offsets, but otherwise no immediate artifacts are visible."
]
},
{
@@ -536,13 +563,10 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
- "create_constant_overview(offset_g, \"Offset (ADU)\", max_cells,\n",
- " out_folder=out_folder, infix=\"_{}\".format(run), entries=1)"
+ "create_constant_overview(offset_g, \"Offset (ADU)\", max_cells, entries=1)"
]
},
{
@@ -553,8 +577,7 @@
},
"outputs": [],
"source": [
- "create_constant_overview(noise_g, \"Noise (ADU)\", max_cells, 0, 100,\n",
- " out_folder=out_folder, infix=\"_{}\".format(run), entries=1)"
+ "create_constant_overview(noise_g, \"Noise (ADU)\", max_cells, 0, 100, entries=1)"
]
},
{
@@ -566,8 +589,53 @@
"bad_pixel_aggregate_g = OrderedDict()\n",
"for m, d in badpix_g.items():\n",
" bad_pixel_aggregate_g[m] = d.astype(np.bool).astype(np.float)\n",
- "create_constant_overview(bad_pixel_aggregate_g, \"Bad pixel fraction\", max_cells, entries=1,\n",
- " out_folder=out_folder, infix=\"_{}\".format(run))"
+ "create_constant_overview(bad_pixel_aggregate_g, \"Bad pixel fraction\", max_cells, entries=1)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Summary tables ##\n",
+ "\n",
+ "The following tables show summary information for the evaluated module. Values for currently evaluated constants are compared with values for pre-existing constants retrieved from the calibration database."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "header = ['Parameter', \n",
+ " \"New constant\", \"Old constant \", \n",
+ " \"New constant\", \"Old constant \", \n",
+ " \"New constant\", \"Old constant \"]\n",
+ "\n",
+ "for const in ['Offset', 'Noise']:\n",
+ " table = [['','High gain', 'High gain']]\n",
+ " for qm in res.keys():\n",
+ "\n",
+ " data = np.copy(res[qm][const])\n",
+ "\n",
+ " if old_const[const] is not None:\n",
+ " dataold = np.copy(old_const[const])\n",
+ "\n",
+ " f_list = [np.nanmedian, np.nanmean, np.nanstd, np.nanmin, np.nanmax]\n",
+ " n_list = ['Median', 'Mean', 'Std', 'Min', 'Max']\n",
+ "\n",
+ " for i, f in enumerate(f_list):\n",
+ " line = [n_list[i]]\n",
+ " line.append('{:6.1f}'.format(f(data[...,gain])))\n",
+ " if old_const[const] is not None:\n",
+ " line.append('{:6.1f}'.format(f(dataold[...,gain])))\n",
+ " else:\n",
+ " line.append('-')\n",
+ "\n",
+ " table.append(line)\n",
+ "\n",
+ " display(Markdown('### {} [ADU], good and bad pixels ###'.format(const)))\n",
+ " md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=header))) "
]
},
{
diff --git a/notebooks/LPD/LPDChar_Darks_NBC.ipynb b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
index 0b9c384fb00ff7fba60f53a2425e9676d520f572..1a0b3fbc6d7cdb174e3c6911c37ac1ab50e08adb 100644
--- a/notebooks/LPD/LPDChar_Darks_NBC.ipynb
+++ b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
@@ -13,40 +13,7 @@
"The notebook will correctly handle veto settings, but note that if you veto cells you will not be able to use these offsets for runs with different veto settings - vetoed cells will have zero offset.\n",
"\n",
"The evaluated calibration constants are stored locally and injected in the calibration data base.\n",
- "\n",
- "**The offset** ($O$) is defined as the median ($M$) of the dark signal ($Ds$) over trains ($t$) for a given pixel ($x,y$) and memory cell ($c$). \n",
- "\n",
- "**The noise** $N$ is the standard deviation $\\sigma$ of the dark signal.\n",
- "\n",
- "$$ O_{x,y,c} = M(Ds)_{t} ,\\,\\,\\,\\,\\,\\, N_{x,y,c} = \\sigma(Ds)_{t}$$\n",
- "\n",
- "**The bad pixel** mask is encoded as a bit mask.\n",
- "\n",
- "**\"OFFSET_OUT_OF_THRESHOLD\":**\n",
- "\n",
- "Offset outside of bounds:\n",
- "\n",
- "$$M(O)_{x,y} - \\sigma(O)_{x,y} * \\mathrm{thresholds\\_offset\\_sigma} < O < M(O)_{x,y} + \\sigma(O)_{x,y} * \\mathrm{thresholds\\_offset\\_sigma} $$\n",
- "\n",
- "or offset outside of hard limits\n",
- "\n",
- "$$ \\mathrm{thresholds\\_offset\\_hard}_\\mathrm{low} < O < \\mathrm{thresholds\\_offset\\_hard}_\\mathrm{high} $$\n",
- "\n",
- "**\"NOISE_OUT_OF_THRESHOLD\":**\n",
- "\n",
- "Noise outside of bounds:\n",
- "\n",
- "$$M(N)_{x,y} - \\sigma(N)_{x,y} * \\mathrm{thresholds\\_noise\\_sigma} < N < M(N)_{x,y} + \\sigma(N)_{x,y} * \\mathrm{thresholds\\_noise\\_sigma} $$\n",
- "\n",
- "or noise outside of hard limits\n",
- "\n",
- "$$\\mathrm{thresholds\\_noise\\_hard}_\\mathrm{low} < N < \\mathrm{thresholds\\_noise\\_hard}_\\mathrm{high} $$\n",
- "\n",
- "**\"OFFSET_NOISE_EVAL_ERROR\":**\n",
- "\n",
- "Offset and Noise both not $nan$ values \n",
- "\n",
- "Values: $\\mathrm{thresholds\\_offset\\_sigma}$, $\\mathrm{thresholds\\_offset\\_hard}$, $\\mathrm{thresholds\\_noise\\_sigma}$, $\\mathrm{thresholds\\_noise\\_hard}$ are given as parameters."
+ "\n"
]
},
{
@@ -56,13 +23,13 @@
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
- "in_folder = \"/gpfs/exfel/exp/FXE/201931/p900088/raw\" # path to input data, required\n",
+ "in_folder = \"/gpfs/exfel/exp/FXE/202030/p900121/raw\" # path to input data, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/LPD/\" # path to output to, required\n",
"sequence = 0 # sequence files to evaluate\n",
"modules = [-1] # list of modules to evaluate, RANGE ALLOWED\n",
- "run_high = 112 # run number in which high gain data was recorded, required\n",
- "run_med = 113 # run number in which medium gain data was recorded, required\n",
- "run_low = 114 # run number in which low gain data was recorded, required\n",
+ "run_high = 120 # run number in which high gain data was recorded, required\n",
+ "run_med = 121 # run number in which medium gain data was recorded, required\n",
+ "run_low = 122 # run number in which low gain data was recorded, required\n",
"\n",
"karabo_id = \"FXE_DET_LPD1M-1\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
@@ -132,7 +99,8 @@
"from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import (show_overview, plot_badpix_3d, \n",
- " create_constant_overview)"
+ " create_constant_overview,\n",
+ " show_processed_modules)"
]
},
{
@@ -204,7 +172,7 @@
"# set everything up filewise\n",
"gmf = map_gain_stages(in_folder, gain_runs, path_template, karabo_da, [sequence])\n",
"gain_mapped_files, total_sequences, total_file_size = gmf\n",
- "print(f\"Will process a total of {total_sequences} sequences.\")"
+ "print(f\"Will process a total of {total_sequences} files.\")"
]
},
{
@@ -245,7 +213,6 @@
" \n",
" h5path = h5path.format(channel)\n",
" h5path_idx = h5path_idx.format(channel)\n",
- " print(h5path_idx)\n",
" count = infile[f\"{h5path_idx}/count\"][()]\n",
" first = infile[f\"{h5path_idx}/first\"][()]\n",
" valid = count != 0\n",
@@ -338,6 +305,9 @@
" thresholds_noise_hard, thresholds_noise_sigma),\n",
" skip_first_ntrains, ntrains, test_for_normality,\n",
" h5path, h5path_idx)\n",
+ "\n",
+ "# Don't remove. Used for Debugging.\n",
+ "#results = list(map(p, inp))\n",
"results = view.map_sync(p, inp)\n",
"\n",
"for ir, r in enumerate(results):\n",
@@ -358,8 +328,8 @@
" ntest_g[cap][qm][..., gg] = normal\n",
"\n",
" hn, cn = np.histogram(data, bins=20)\n",
- " print(\"{} gain, Capacitor {}, Module: {}. Number of processed trains per cell: {}.\\n\".format(\n",
- " gain_names[gg], cap, qm, data.shape[0]))\n",
+ " print(f\"{gain_names[gg]} gain, Capacitor {cap}, Module: {qm}. \"\n",
+ " f\"Number of processed trains per cell: {data.shape[0]}.\")\n",
"\n",
"duration = (datetime.now()-start).total_seconds()\n",
"logger.runtime_summary_entry(success=True, runtime=duration,\n",
@@ -379,6 +349,8 @@
"old_const = {}\n",
"old_mdata = {}\n",
"\n",
+ "dinstance = \"LPD1M1\"\n",
+ "detinst = getattr(Detectors, dinstance)\n",
"print('Retrieve pre-existing constants for comparison.')\n",
"for cap in capacitor_settings:\n",
" for qm in offset_g[cap].keys():\n",
@@ -387,7 +359,7 @@
" condition = Conditions.Dark.LPD(memory_cells=max_cells, bias_voltage=bias_voltage,\n",
" capacitor=cap)\n",
"\n",
- " data, mdata = get_from_db(getattr(Detectors.LPD1M1, qm),\n",
+ " data, mdata = get_from_db(getattr(detinst, qm),\n",
" getattr(Constants.LPD, const)(),\n",
" condition,\n",
" None,\n",
@@ -473,15 +445,18 @@
" if db_output:\n",
" try:\n",
" metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
- " msg = 'Const {} for module {} was injected to the calibration DB. Begin at: {}'\n",
- " print(msg.format(const, qm,\n",
- " metadata.calibration_constant_version.begin_at))\n",
+ " print(f'Const {const} for module {qm} was injected to the calibration DB. '\n",
+ " f'Begin at: {metadata.calibration_constant_version.begin_at}')\n",
" except Exception as e:\n",
- " print(e)\n",
+ " print(\"Error:\", e)\n",
"\n",
" if local_output:\n",
" save_const_to_h5(metadata, out_folder)\n",
- " print(\"Calibration constant {} is stored locally.\".format(const))"
+ " print(f\"Calibration constant {const} is stored locally.\\n\")\n",
+ "\n",
+ " print(\"Generated constants with conditions:\\n\")\n",
+ " print(f\"• memory_cells: {max_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
+ " f\"• capacitor: {cap}\\n• creation_time: {creation_time}\\n\")"
]
},
{
@@ -490,61 +465,11 @@
"metadata": {},
"outputs": [],
"source": [
- "qm = \"Q{}M{}\".format(modules[0]//4+1, modules[0] % 4+1)\n",
- "display(Markdown('## Position of the module {}, it tiles and ASICs of tile ##'.format(qm)))\n",
- "\n",
- "fig, ax = plt.subplots(1, figsize=(10, 10))\n",
- "ax.set_axis_off()\n",
- "\n",
- "ax.set_xlim(0, 97)\n",
- "ax.set_ylim(0, 97)\n",
- "\n",
- "q_poses = np.array([[51, 47], [47, 1], [1, 5], [5, 51]])\n",
- "m_poses = np.array([[22.5, 20.5], [22.5, 0.5], [0.5, 0.5], [0.5, 20.5]])\n",
- "\n",
- "for iq, q_pos in enumerate(q_poses):\n",
- " ax.add_patch(patches.Rectangle(q_pos, 45, 45, linewidth=2, edgecolor='r',\n",
- " facecolor='y', fill=True))\n",
- "\n",
- " ax.text(q_pos[0]+20, q_pos[1]+41.5, 'Q{}'.format(iq+1), fontsize=22)\n",
- " for im, m_pos in enumerate(m_poses):\n",
- " ax.add_patch(patches.Rectangle(q_pos+m_pos, 22, 20, linewidth=3, edgecolor='r',\n",
- " facecolor='g', fill=True))\n",
- "\n",
- " if iq*4+im == modules[0]:\n",
- " for a_posx in range(2):\n",
- " for a_posy in range(8):\n",
- " a_pos = np.array([a_posx*11., a_posy*20/8.])\n",
- " pos = q_pos+m_pos+a_pos\n",
- "\n",
- " ax.add_patch(patches.Rectangle(q_pos+m_pos+a_pos, 11, 20/8., \n",
- " linewidth=1, edgecolor='black',\n",
- " facecolor='r', fill=True))\n",
- "\n",
- " if a_posx == 0:\n",
- " label = str(a_posy+9)\n",
- " else:\n",
- " label = str(-a_posy+(a_posx*8))\n",
- "\n",
- " ax.text(pos[0]+4, pos[1]+0.3, label, fontsize=14)\n",
- " else:\n",
- " pos = q_pos+m_pos+np.array([5, 8])\n",
- " ax.text(pos[0], pos[1], 'Q{}M{}'.format(\n",
- " iq+1, im+1), fontsize=22, color='y')\n",
- "\n",
- "ax.add_patch(patches.Rectangle([65, 93], 30, 4, linewidth=1, edgecolor='black',\n",
- " facecolor='r', fill=True))\n",
- "\n",
- "ax.text(52, 94, 'ASICs:', fontsize=22, color='black')\n",
- "\n",
- "for i_pos in range(8):\n",
- " pos = np.array([65, 93]) + np.array([i_pos*30/8.+0.3, 0.3])\n",
- "\n",
- " ax.add_patch(patches.Rectangle(pos, 24/8., 3.4, linewidth=1, edgecolor='black',\n",
- " facecolor='deepskyblue', fill=True))\n",
- "\n",
- " ax.text(pos[0]+0.5, pos[1]+0.5, '{}'.format(i_pos + 1),\n",
- " fontsize=18, color='black')"
+ "mnames = []\n",
+ "for i in modules:\n",
+ " qm = f\"Q{modules[0]//4+1}M{modules[0]%4+1}\"\n",
+ " mnames.append(qm)\n",
+ " show_processed_modules(dinstance=dinstance, constants=None, mnames=mnames, mode=\"position\")"
]
},
{
@@ -841,8 +766,8 @@
" Colors encode the bad pixel type, or mixed type.\n",
"\n",
" \"\"\"))\n",
- " # set rebin_fac to 1 for avoiding rebining and\n",
- " # losing real values of badpixels(High resolution).\n",
+ " # Switch rebin to 1 for full resolution and \n",
+ " # no interpolation for badpixel values.\n",
" rebin = 2\n",
" for gain in range(3):\n",
" display(Markdown('### Bad pixel behaviour - {} gain ###'.format(gain_names[gain])))\n",
@@ -1256,13 +1181,6 @@
" display(Markdown('### {} [ADU], good pixels only ###'.format(const)))\n",
" md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=header))) "
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
diff --git a/notebooks/LPD/LPDChar_Darks_Summary_NBC.ipynb b/notebooks/LPD/LPDChar_Darks_Summary_NBC.ipynb
deleted file mode 100644
index 073706cdab625a4267562718578582a2b80f2dfb..0000000000000000000000000000000000000000
--- a/notebooks/LPD/LPDChar_Darks_Summary_NBC.ipynb
+++ /dev/null
@@ -1,371 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Summary of LPD dark characterization #\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "cluster_profile = \"noDB\" # The ipcluster profile to use\n",
- "out_folder = \"/gpfs/exfel/data/scratch/karnem/test/LPD_dark_004/\" # path to output to, required"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from collections import OrderedDict\n",
- "import copy\n",
- "from datetime import datetime\n",
- "import os\n",
- "import warnings\n",
- "warnings.filterwarnings('ignore')\n",
- "\n",
- "from iCalibrationDB import Detectors\n",
- "import glob\n",
- "import h5py\n",
- "from IPython.display import display, Markdown, Latex\n",
- "import numpy as np\n",
- "import matplotlib\n",
- "matplotlib.use(\"agg\")\n",
- "import matplotlib.patches as patches\n",
- "import matplotlib.pyplot as plt\n",
- "%matplotlib inline\n",
- "import tabulate\n",
- "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
- "from XFELDetAna.plotting.simpleplot import simplePlot\n",
- "\n",
- "gain_names = ['High', 'Medium', 'Low']"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# TODO: After changes in the Cal DB interface read files from cal repositofy\n",
- "\n",
- "# Load constants from local files\n",
- "data = OrderedDict()\n",
- "mod_names = []\n",
- "# Loop over modules\n",
- "for i in range(16):\n",
- " qm = \"Q{}M{}\".format(i//4 + 1, i % 4 + 1)\n",
- " # loop over constants\n",
- " for const in ['Offset', 'Noise', 'BadPixelsDark']:\n",
- " det = getattr(Detectors.LPD1M1, qm, None)\n",
- " if det is None:\n",
- " continue\n",
- " det_name = det.device_name\n",
- "\n",
- " fpath = '{}/const_{}_{}.h5'.format(out_folder, const, det_name)\n",
- " if not os.path.isfile(fpath):\n",
- " continue\n",
- " with h5py.File(fpath, 'r') as f:\n",
- " if qm not in data:\n",
- " mod_names.append(qm)\n",
- " data[qm] = OrderedDict()\n",
- "\n",
- " data[qm][const] = f[\"data\"][()]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "mod_idx = np.argsort(mod_names)\n",
- "\n",
- "constants = {'Offset': np.zeros((len(mod_names), 256, 256, 512, 3)),\n",
- " 'Noise': np.zeros((len(mod_names), 256, 256, 512, 3)),\n",
- " 'BadPixelsDark': np.zeros((len(mod_names), 256, 256, 512, 3))}\n",
- "\n",
- "for i, idx in enumerate(mod_idx):\n",
- " for key, item in constants.items():\n",
- " item[i] = data[mod_names[idx]][key]\n",
- "\n",
- "mod_names = np.array(mod_names)[mod_idx]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "display(Markdown('## Processed modules ##'))\n",
- "\n",
- "fig, ax = plt.subplots(1, figsize=(10, 10))\n",
- "ax.set_axis_off()\n",
- "\n",
- "ax.set_xlim(0, 97)\n",
- "ax.set_ylim(0, 97)\n",
- "\n",
- "q_poses = np.array([[51, 47], [47, 1], [1, 5], [5, 51]])\n",
- "m_poses = np.array([[22.5, 20.5], [22.5, 0.5], [0.5, 0.5], [0.5, 20.5]])\n",
- "\n",
- "counter = 0\n",
- "for iq, q_pos in enumerate(q_poses):\n",
- " ax.add_patch(patches.Rectangle(q_pos, 45, 45, linewidth=2, edgecolor='dodgerblue',\n",
- " facecolor='y', fill=True))\n",
- "\n",
- " ax.text(q_pos[0]+20, q_pos[1]+41.5, 'Q{}'.format(iq+1), fontsize=22)\n",
- " for im, m_pos in enumerate(m_poses):\n",
- " color = 'gray'\n",
- " if 'Q{}M{}'.format(iq+1, im+1) in mod_names:\n",
- " color = 'green'\n",
- " if np.nanmean(constants['Noise'][counter, :, :, :, 0]) == 0:\n",
- " color = 'red'\n",
- " counter += 1\n",
- " ax.add_patch(patches.Rectangle(q_pos+m_pos, 22, 20, linewidth=3, edgecolor='dodgerblue',\n",
- " facecolor=color, fill=True))\n",
- "\n",
- " pos = q_pos+m_pos+np.array([5, 8])\n",
- " ax.text(pos[0], pos[1], 'Q{}M{}'.format(\n",
- " iq+1, im+1), fontsize=24, color='yellow')\n",
- "\n",
- "\n",
- "_ = ax.legend(handles=[patches.Patch(facecolor='red', label='No data'),\n",
- " patches.Patch(facecolor='gray', label='Not processed'),\n",
- " patches.Patch(facecolor='green', label='Processed')],\n",
- " loc='outside-top', ncol=3, bbox_to_anchor=(0.1, 0.25, 0.7, 0.8))"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Summary figures across Modules ##\n",
- "\n",
- "Plots give an overview of calibration constants averaged across pixels. A bad pixel mask is applied. Constants are averaged across pixels."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "q_pad = 15\n",
- "m_pad = 5\n",
- "\n",
- "m_size = 256\n",
- "q_size = m_size*2+m_pad*2\n",
- "\n",
- "image = np.zeros((m_size*4+q_pad+m_pad*3, m_size*4+q_pad+m_pad*3))\n",
- "\n",
- "q_poses = [[q_size+q_pad, q_size],\n",
- " [q_size, 0],\n",
- " [0, q_pad], [q_pad, q_size+q_pad]]\n",
- "m_poses = [[m_size+m_pad, m_size+m_pad], [m_size+m_pad, 0], [0, 0],\n",
- " [0, m_size+m_pad]]\n",
- "\n",
- "# Loop over capacitor settings, modules, constants\n",
- "for const_name, const in constants.items():\n",
- "\n",
- " if const_name == 'BadPixelsDark':\n",
- " continue\n",
- "\n",
- " display(Markdown('### {}'.format(const_name)))\n",
- " for gain in range(3):\n",
- "\n",
- " image[:] = np.nan\n",
- " counter = 0\n",
- " for iq, q_pos in enumerate(q_poses):\n",
- " for im, m_pos in enumerate(m_poses):\n",
- " if 'Q{}M{}'.format(iq+1, im+1) in mod_names:\n",
- " values = np.nanmean(const[counter, :, :, :, gain], axis=2)\n",
- " values[values == 0] = np.nan\n",
- " image[q_pos[1]+m_pos[1]: q_pos[1]+m_pos[1]+m_size,\n",
- " q_pos[0]+m_pos[0]: q_pos[0]+m_pos[0] + m_size] = values\n",
- " counter += 1\n",
- "\n",
- " std = np.nanstd(image)\n",
- " mean = np.nanmedian(image)\n",
- " if const_name == 'Noise':\n",
- " std = mean/4.\n",
- " _ = heatmapPlot(image, add_panels=False, figsize=(20, 20),\n",
- " vmin=mean-std*2, vmax=mean+std*2,\n",
- " x_label='columns', y_label='rows',\n",
- " cb_label='{}, mean over memory cells [ADU]'.format(\n",
- " const_name),\n",
- " cmap='viridis',\n",
- " title='{}. {} gain'.format(const_name, gain_names[gain]))\n",
- " plt.show()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "scrolled": true
- },
- "outputs": [],
- "source": [
- "# Loop over capacitor settings, modules, constants\n",
- "for const_name, const in constants.items():\n",
- "\n",
- " display(Markdown('### Summary across Modules - {}'.format(const_name)))\n",
- " for gain in range(3):\n",
- " data = np.copy(const[:, :, :, :, gain])\n",
- "\n",
- " if const_name != 'BadPixelsDark':\n",
- " label = '{} value [ADU], good pixels only'.format(const_name)\n",
- " data[constants['BadPixelsDark'][:, :, :, :, gain] > 0] = np.nan\n",
- " datamean = np.nanmean(data, axis=(1, 2))\n",
- "\n",
- " fig = plt.figure(figsize=(15, 6), tight_layout={\n",
- " 'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})\n",
- " ax = fig.add_subplot(121)\n",
- " else:\n",
- " label = 'Fraction of bad pixels'\n",
- " data[data > 0] = 1.0\n",
- " datamean = np.nanmean(data, axis=(1, 2))\n",
- " datamean[datamean == 1.0] = np.nan\n",
- "\n",
- " fig = plt.figure(figsize=(15, 6), tight_layout={\n",
- " 'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})\n",
- " ax = fig.add_subplot(111)\n",
- "\n",
- " d = []\n",
- " for im, mod in enumerate(datamean):\n",
- " d.append({'x': np.arange(mod.shape[0]),\n",
- " 'y': mod,\n",
- " 'drawstyle': 'steps-pre',\n",
- " 'label': mod_names[im],\n",
- " })\n",
- "\n",
- " _ = simplePlot(d, figsize=(10, 10), xrange=(-12, 510),\n",
- " x_label='Memory Cell ID',\n",
- " y_label=label,\n",
- " use_axis=ax,\n",
- " title='{} gain'.format(gain_names[gain]),\n",
- " title_position=[0.5, 1.18],\n",
- " legend='outside-top-ncol6-frame', legend_size='18%',\n",
- " legend_pad=0.00)\n",
- "\n",
- " if const_name != 'BadPixelsDark':\n",
- " ax = fig.add_subplot(122)\n",
- " label = '$\\sigma$ {} [ADU], good pixels only'.format(const_name)\n",
- " d = []\n",
- " for im, mod in enumerate(np.nanstd(data, axis=(1, 2))):\n",
- " d.append({'x': np.arange(mod.shape[0]),\n",
- " 'y': mod,\n",
- " 'drawstyle': 'steps-pre',\n",
- " 'label': mod_names[im],\n",
- " })\n",
- "\n",
- " _ = simplePlot(d, figsize=(10, 10), xrange=(-12, 510),\n",
- " x_label='Memory Cell ID',\n",
- " y_label=label,\n",
- " use_axis=ax,\n",
- " title='{} gain'.format(gain_names[gain]),\n",
- " title_position=[0.5, 1.18],\n",
- " legend='outside-top-ncol6-frame', legend_size='18%',\n",
- " legend_pad=0.00)\n",
- "\n",
- " plt.show()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Summary tables across Modules ##\n",
- "\n",
- "Tables show values averaged across all pixels and memory cells of a given detector module."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "if u'$' in tabulate.LATEX_ESCAPE_RULES:\n",
- " del(tabulate.LATEX_ESCAPE_RULES[u'$'])\n",
- " \n",
- "if u'\\\\' in tabulate.LATEX_ESCAPE_RULES:\n",
- " del(tabulate.LATEX_ESCAPE_RULES[u'\\\\'])"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "header = ['Module', 'High gain', 'Medium gain', 'Low gain']\n",
- "\n",
- "for const_name, const in constants.items():\n",
- " table = []\n",
- "\n",
- " for i_mod, mod in enumerate(mod_names):\n",
- "\n",
- " t_line = [mod]\n",
- " for gain in range(3):\n",
- "\n",
- " data = np.copy(const[i_mod, :, :, :, gain])\n",
- " if const_name == 'BadPixelsDark':\n",
- " data[data > 0] = 1.0\n",
- " datasum = np.nansum(data)\n",
- " datamean = np.nanmean(data)\n",
- " if datamean == 1.0:\n",
- " datamean = np.nan\n",
- " datasum = np.nan\n",
- "\n",
- " t_line.append('{:6.0f} ({:6.3f}) '.format(\n",
- " datasum, datamean))\n",
- "\n",
- " label = '## Number (fraction) of bad pixels'\n",
- " else:\n",
- "\n",
- " data[constants['BadPixelsDark']\n",
- " [i_mod, :, :, :, gain] > 0] = np.nan\n",
- "\n",
- " t_line.append('{:6.1f} $\\\\pm$ {:6.1f}'.format(\n",
- " np.nanmean(data), np.nanstd(data)))\n",
- "\n",
- " label = '## Average {} [ADU], good pixels only ##'.format(const_name)\n",
- "\n",
- " table.append(t_line)\n",
- "\n",
- " display(Markdown(label))\n",
- " md = display(Latex(tabulate.tabulate(\n",
- " table, tablefmt='latex', headers=header)))"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.6.7"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb b/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
index 3509955b32c9969afd80d19efe7c55b63e1e18a1..897505641d6ceff13e9d4dc8119817e1e16725a5 100644
--- a/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
+++ b/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
@@ -1,9 +1,67 @@
{
"cells": [
{
- "cell_type": "markdown",
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#Author: K. Ahmed, M. Karnevsky, Version: 0.1\n",
+ "#The following is a summary for the processing of dark images and calibration constants production.\n",
+ "\n",
+ "cluster_profile = \"noDB\" # The ipcluster profile to use\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/agipd-test-report\" # path to output to, required\n",
+ "karabo_id = \"SPB_DET_AGIPDM-1\" # detector instance\n",
+ "gain_names = ['High gain', 'Medium gain', 'Low gain'] # a list of gain names to be used in plotting\n",
+ "threshold_names = ['HG-MG threshold', 'MG_LG threshold'] # a list of gain names to be used in plotting"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
+ "from collections import OrderedDict\n",
+ "import copy\n",
+ "from datetime import datetime\n",
+ "import os\n",
+ "import warnings\n",
+ "warnings.filterwarnings('ignore')\n",
+ "\n",
+ "import glob\n",
+ "import h5py\n",
+ "from IPython.display import display, Markdown, Latex\n",
+ "import numpy as np\n",
+ "import matplotlib\n",
+ "matplotlib.use(\"agg\")\n",
+ "import matplotlib.gridspec as gridspec\n",
+ "import matplotlib.patches as patches\n",
+ "import matplotlib.pyplot as plt\n",
+ "%matplotlib inline\n",
+ "import tabulate\n",
+ "from cal_tools.ana_tools import get_range\n",
+ "from cal_tools.plotting import show_processed_modules\n",
+ "import extra_geom\n",
+ "from iCalibrationDB import Detectors\n",
+ "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
+ "from XFELDetAna.plotting.simpleplot import simplePlot"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "SHOW=True\n",
+ "if \"AGIPD\" in karabo_id:\n",
+ " if \"SPB\" in karabo_id:\n",
+ " dinstance = \"AGIPD1M1\"\n",
+ " elif \"MID\" in karabo_id:\n",
+ " dinstance = \"AGIPD1M2\"\n",
+ " display(Markdown(\"\"\"\n",
+ " \n",
"# Summary of AGIPD dark characterization #\n",
"\n",
"The following report shows a set of dark images taken with the AGIPD detector to deduce detector offsets, noise, bad-pixel maps and thresholding. All four types of constants are evaluated per-pixel and per-memory cell.\n",
@@ -39,60 +97,75 @@
"\n",
"**\"OFFSET_NOISE_EVAL_ERROR\":**\n",
"\n",
- "Offset and Noise both not $nan$ values \n",
+ "Offset and Noise both not $nan$ values\n",
"\n",
"Values: $\\mathrm{thresholds\\_offset\\_sigma}$, $\\mathrm{thresholds\\_offset\\_hard}$, $\\mathrm{thresholds\\_noise\\_sigma}$, $\\mathrm{thresholds\\_noise\\_hard}$ are given as parameters.\n",
"\n",
- "**\"GAIN_THRESHOLDING_ERROR\":**\n",
+ "\"**\\\"GAIN_THRESHOLDING_ERROR\\\":**\n",
"\n",
"Bad gain separated pixels with sigma separation less than gain_separation_sigma_threshold\n",
"\n",
- "$$ sigma\\_separation = \\frac{\\mathrm{gain\\_offset} - \\mathrm{previous\\_gain\\_offset}}{\\sqrt{\\mathrm{gain\\_offset_{std}}^\\mathrm{2} + \\mathrm{previuos\\_gain\\_offset_{std}}^\\mathrm{2}}}$$ \n",
- "$$ Bad\\_separation = sigma\\_separation < \\mathrm{gain\\_separation\\_sigma\\_threshold} $$"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "cluster_profile = \"noDB\" # The ipcluster profile to use\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_bad_gain_separation/\" # path to output to, required\n",
- "dinstance = \"AGIPD1M1\" # detector instance\n",
- "gain_names = ['High gain', 'Medium gain', 'Low gain'] # a list of gain names to be used in plotting\n",
- "threshold_names = ['HG-MG threshold', 'MG_LG threshold'] # a list of gain names to be used in plotting"
+ "$$ sigma\\_separation = \\\\frac{\\mathrm{gain\\_offset} - \\mathrm{previous\\_gain\\_offset}}{\\sqrt{\\mathrm{gain\\_offset_{std}}^\\mathrm{2} + \\mathrm{previuos\\_gain\\_offset_{std}}^\\mathrm{2}}}$$ \n",
+ "$$ Bad\\_separation = sigma\\_separation < \\mathrm{gain\\_separation\\_sigma\\_threshold} $$\n",
+ "\n",
+ "\"\"\"))\n",
+ " \n",
+ "elif \"LPD\" in karabo_id:\n",
+ " dinstance = \"LPD1M1\"\n",
+ " display(Markdown(\"\"\"\n",
+ " \n",
+ "# Summary of LPD dark characterization #\n",
+ "\n",
+ "The following report shows a set of dark images taken with the LPD detector to deduce detector offsets, noise, bad-pixel maps. All three types of constants are evaluated per-pixel and per-memory cell.\n",
+ "\n",
+ "**The offset** ($O$) is defined as the median ($M$) of the dark signal ($Ds$) over trains ($t$) for a given pixel ($x,y$) and memory cell ($c$). \n",
+ "\n",
+ "**The noise** $N$ is the standard deviation $\\sigma$ of the dark signal.\n",
+ "\n",
+ "$$ O_{x,y,c} = M(Ds)_{t} ,\\,\\,\\,\\,\\,\\, N_{x,y,c} = \\sigma(Ds)_{t}$$\n",
+ "\n",
+ "**The bad pixel** mask is encoded as a bit mask.\n",
+ "\n",
+ "**\"OFFSET_OUT_OF_THRESHOLD\":**\n",
+ "\n",
+ "Offset outside of bounds:\n",
+ "\n",
+ "$$M(O)_{x,y} - \\sigma(O)_{x,y} * \\mathrm{thresholds\\_offset\\_sigma} < O < M(O)_{x,y} + \\sigma(O)_{x,y} * \\mathrm{thresholds\\_offset\\_sigma} $$\n",
+ "\n",
+ "or offset outside of hard limits\n",
+ "\n",
+ "$$ \\mathrm{thresholds\\_offset\\_hard}_\\mathrm{low} < O < \\mathrm{thresholds\\_offset\\_hard}_\\mathrm{high} $$\n",
+ "\n",
+ "**\"NOISE_OUT_OF_THRESHOLD\":**\n",
+ "\n",
+ "Noise outside of bounds:\n",
+ "\n",
+ "$$M(N)_{x,y} - \\sigma(N)_{x,y} * \\mathrm{thresholds\\_noise\\_sigma} < N < M(N)_{x,y} + \\sigma(N)_{x,y} * \\mathrm{thresholds\\_noise\\_sigma} $$\n",
+ "\n",
+ "or noise outside of hard limits\n",
+ "\n",
+ "$$\\mathrm{thresholds\\_noise\\_hard}_\\mathrm{low} < N < \\mathrm{thresholds\\_noise\\_hard}_\\mathrm{high} $$\n",
+ "\n",
+ "**\"OFFSET_NOISE_EVAL_ERROR\":**\n",
+ "\n",
+ "Offset and Noise both not $nan$ values \n",
+ "\n",
+ "\"Values: $\\\\mathrm{thresholds\\\\_offset\\\\_sigma}$, $\\\\mathrm{thresholds\\\\_offset\\\\_hard}$, $\\\\mathrm{thresholds\\\\_noise\\\\_sigma}$, $\\\\mathrm{thresholds\\\\_noise\\\\_hard}$ are given as parameters.\\n\",\n",
+ "\"\"\"))\n",
+ "elif \"DSSC\" in karabo_id:\n",
+ " dinstance = \"DSSC1M1\"\n",
+ " display(Markdown(\"\"\"\n",
+ " \n",
+ "# Summary of DSSC dark characterization #\n",
+ " \n",
+ " \"\"\"))"
]
},
{
- "cell_type": "code",
- "execution_count": null,
+ "cell_type": "markdown",
"metadata": {},
- "outputs": [],
"source": [
- "from collections import OrderedDict\n",
- "import copy\n",
- "from datetime import datetime\n",
- "import os\n",
- "import warnings\n",
- "warnings.filterwarnings('ignore')\n",
- "\n",
- "import glob\n",
- "import h5py\n",
- "from IPython.display import display, Markdown, Latex\n",
- "import numpy as np\n",
- "import matplotlib\n",
- "matplotlib.use(\"agg\")\n",
- "import matplotlib.gridspec as gridspec\n",
- "import matplotlib.patches as patches\n",
- "import matplotlib.pyplot as plt\n",
- "%matplotlib inline\n",
- "import tabulate\n",
- "from cal_tools.ana_tools import get_range\n",
- "from extra_geom import AGIPD_1MGeometry\n",
- "from iCalibrationDB import Detectors\n",
- "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
- "from XFELDetAna.plotting.simpleplot import simplePlot"
+ "Preparing newly injected and previous constants from produced local folder in out_folder."
]
},
{
@@ -102,14 +175,13 @@
"outputs": [],
"source": [
"# TODO: After changes in the Cal DB interface read files from cal repository\n",
- "\n",
"# Load constants from local files\n",
"data = OrderedDict()\n",
"old_cons = OrderedDict()\n",
"mod_names = []\n",
"# Loop over modules\n",
"for i in range(16):\n",
- " qm = \"Q{}M{}\".format(i//4 + 1, i % 4 + 1)\n",
+ " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
" # loop over constants\n",
" detinst = getattr(Detectors, dinstance)\n",
" for const in ['Offset', 'Noise', 'ThresholdsDark', 'BadPixelsDark']:\n",
@@ -117,7 +189,6 @@
" if det is None:\n",
" continue\n",
" det_name = det.device_name\n",
- "\n",
" fpath = '{}/const_{}_{}.h5'.format(out_folder, const, det_name)\n",
" oldfpath = '{}/old/const_{}_{}.h5'.format(out_folder, const, det_name)\n",
" if not os.path.isfile(fpath):\n",
@@ -135,7 +206,7 @@
" with h5py.File(oldfpath, 'r') as oldf:\n",
" if qm not in old_cons:\n",
" old_cons[qm] = OrderedDict()\n",
- " old_cons[qm][const] = oldf[\"data\"][()]\n"
+ " old_cons[qm][const] = oldf[\"data\"][()]"
]
},
{
@@ -144,7 +215,6 @@
"metadata": {},
"outputs": [],
"source": [
- "modules = np.argsort(mod_names)\n",
"cons_shape = {}\n",
"# extracting constant shape.\n",
"for qm, constant in data.items():\n",
@@ -153,62 +223,32 @@
" if cname not in cons_shape:\n",
" cons_shape[cname] = shape\n",
" break\n",
+ "\n",
"constants = {}\n",
"prev_const = {}\n",
+ "\n",
"for cname, sh in cons_shape.items():\n",
" constants[cname]= np.zeros((len(mod_names),) + sh[:])\n",
" prev_const[cname]= np.zeros((len(mod_names),) + sh[:])\n",
- "\n",
- "for i, mod in enumerate(modules):\n",
+ "for i in range(len(mod_names)):\n",
" for cname, cval in constants.items():\n",
- " cval[i] = data[mod_names[mod]][cname]\n",
- " prev_const[cname][i] = old_cons[mod_names[mod]][cname]\n",
- "mod_names = np.array(mod_names)[modules]"
+ " cval[i] = data[mod_names[i]][cname]\n",
+ " if mod_names[i] in old_cons.keys():\n",
+ " prev_const[cname][i] = old_cons[mod_names[i]][cname]\n",
+ " else:\n",
+ " print(f\"No previous {cname} found for {mod_names[i]}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "scrolled": false
+ },
"outputs": [],
"source": [
"display(Markdown('## Processed modules ##'))\n",
- "\n",
- "fig, ax = plt.subplots(1, figsize=(10, 10))\n",
- "ax.set_axis_off()\n",
- "\n",
- "ax.set_xlim(0, 90)\n",
- "ax.set_ylim(0, 75)\n",
- "asic_pos = 5\n",
- "\n",
- "q_st = 8\n",
- "\n",
- "l_y = 6\n",
- "l_x = 5\n",
- "counter = 0\n",
- "\n",
- "for iq, q_x in enumerate([[43,66],[45,34],[4,32],[2,64]]):\n",
- " for im in range(4):\n",
- " color = 'gray'\n",
- " if 'Q{}M{}'.format(iq+1, im+1) in mod_names:\n",
- " color = 'green'\n",
- " if 'Noise' not in constants.keys() or \\\n",
- " np.nanmean(constants['Noise'][counter, :, :, :, 0]) == 0:\n",
- " color = 'red'\n",
- " if counter < len(mod_names)-1:\n",
- " counter += 1\n",
- " x = q_x[0]\n",
- " for i_as in range(8):\n",
- " ax.add_patch(matplotlib.patches.Rectangle((x,q_x[1]-q_st*im), l_x, l_y, linewidth=2,edgecolor='darkgreen',\n",
- " facecolor=color, fill=True))\n",
- " x += asic_pos\n",
- " ax.text(q_x[0]+14.5, q_x[1]-q_st*im+1.5, 'Q{}M{}'.format(\n",
- " iq+1, im+1), fontsize=24, color='gold')\n",
- "\n",
- "_ = ax.legend(handles=[patches.Patch(facecolor='red', label='No data'),\n",
- " patches.Patch(facecolor='gray', label='Not processed'),\n",
- " patches.Patch(facecolor='green', label='Processed')],\n",
- " loc='outside-top', ncol=3, bbox_to_anchor=(0.1, 0.25, 0.7, 0.8))"
+ "show_processed_modules(dinstance, constants, mod_names, mode=\"processed\")"
]
},
{
@@ -228,48 +268,68 @@
},
"outputs": [],
"source": [
- "#Initializing AGIPD geometry\n",
- "geom = AGIPD_1MGeometry.from_quad_positions(quad_pos=[\n",
- " (-525, 625),\n",
- " (-550, -10),\n",
- " (520, -160),\n",
- " (542.5, 475),\n",
- " ])\n",
- "\n",
+ "if \"LPD\" in dinstance:\n",
+ " geom = extra_geom.LPD_1MGeometry.from_quad_positions(quad_pos=[(11.4, 299),\n",
+ " (-11.5, 8),\n",
+ " (254.5, -16),\n",
+ " (278.5, 275)])\n",
+ " pixels_y = 256\n",
+ " pixels_x = 256\n",
+ "\n",
+ "elif \"AGIPD\" in dinstance:\n",
+ " geom = extra_geom.AGIPD_1MGeometry.from_quad_positions(quad_pos=[(-525, 625),\n",
+ " (-550, -10),\n",
+ " (520, -160),\n",
+ " (542.5, 475)])\n",
+ " pixels_y = 128\n",
+ " pixels_x = 512\n",
+ "\n",
+ "elif \"DSSC\" in dinstance:\n",
+ " pixels_y = 512\n",
+ " pixels_x = 128\n",
+ " quadpos = [(-130, 5), (-130, -125), (5, -125), (5, 5)]\n",
+ "\n",
+ " extrageom_pth = os.path.dirname(extra_geom.__file__)\n",
+ " geom = extra_geom.DSSC_1MGeometry.from_h5_file_and_quad_positions(\n",
+ " f\"{extrageom_pth}/tests/dssc_geo_june19.h5\", positions=quadpos)\n",
"\n",
"Mod_data=OrderedDict()\n",
- "for const_name, const in constants.items():\n",
+ "gainstages = 1\n",
"\n",
+ "for const_name, const in constants.items():\n",
" if const_name == 'BadPixelsDark':\n",
" continue\n",
- " \n",
- " Mod_data[const_name] = OrderedDict()\n",
- " Mod_data['d-{}'.format(const_name)] = OrderedDict()\n",
- " Mod_data['dpct-{}'.format(const_name)] = OrderedDict()\n",
- " display(Markdown('##### {}'.format(const_name)))\n",
- "\n",
- " for gain in range(3):\n",
- "\n",
- " # Avoid plotting for high gain offset in 3 index \n",
- " # for Threshold gain dimension.\n",
+ " # Check if constant gain available in constant e.g. AGIPD, LPD\n",
+ " if len(const.shape) == 5:\n",
+ " gainstages = 3\n",
+ " else:\n",
+ " gainstages = 1\n",
+ "\n",
+ " for dname in ['{}', 'd-{}', 'dpct-{}']:\n",
+ " Mod_data[dname.format(const_name)] = OrderedDict()\n",
+ " \n",
+ " display(Markdown(f'##### {const_name}'))\n",
+ " for gain in range(gainstages):\n",
" if const_name == 'ThresholdsDark':\n",
- " if gain == 2:\n",
+ " if gain > 2:\n",
" continue\n",
" glabel = threshold_names\n",
" else:\n",
" glabel = gain_names\n",
- "\n",
" for i in range(16):\n",
- " qm = \"Q{}M{}\".format(i//4 + 1, i % 4 + 1)\n",
- " if const.shape[0] > i:\n",
- " values = np.nanmean(const[i, :, :, :, gain], axis=2)\n",
- " dval = np.nanmean(prev_const[const_name][i, :, :, :, gain], axis=2)\n",
- "\n",
- "\n",
+ " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ " if qm in mod_names:\n",
+ " m_idx = mod_names.index(qm)\n",
+ " # Check if constant shape of 5 indices e.g. AGIPD, LPD\n",
+ " if len(const.shape) == 5:\n",
+ " values = np.nanmean(const[m_idx, :, :, :, gain], axis=2)\n",
+ " dval = np.nanmean(prev_const[const_name][m_idx, :, :, :, gain], axis=2)\n",
+ " else:\n",
+ " values = np.nanmean(const[m_idx, :, :, :], axis=2)\n",
+ " dval = np.nanmean(prev_const[const_name][m_idx, :, :, :], axis=2)\n",
" values[values == 0] = np.nan\n",
" dval[dval == 0] = np.nan\n",
" dval = values - dval\n",
- "\n",
" dval_pct = dval/values * 100\n",
"\n",
" values = np.moveaxis(values, 0, -1).reshape(1, values.shape[1], values.shape[0])\n",
@@ -277,74 +337,47 @@
" dval_pct = np.moveaxis(dval_pct, 0, -1).reshape(1, dval_pct.shape[1], dval_pct.shape[0])\n",
" else:\n",
" # if module not available fill arrays with nan\n",
- " values = np.zeros((1, 512, 128),dtype=np.float64)\n",
+ " values = np.zeros((1, pixels_x, pixels_y),dtype=np.float64)\n",
" values[values == 0] = np.nan\n",
" dval = values \n",
" dval_pct = dval\n",
- " try:\n",
- " Mod_data[const_name][gain_names[gain]] = np.concatenate((Mod_data[const_name][gain_names[gain]],\n",
- " values), axis=0)\n",
- " Mod_data['d-{}'.format(const_name)][gain_names[gain]] = \\\n",
- " np.concatenate((Mod_data['d-{}'.format(const_name)][gain_names[gain]],\n",
- " dval), axis=0)\n",
- " Mod_data['dpct-{}'.format(const_name)][gain_names[gain]] = \\\n",
- " np.concatenate((Mod_data['dpct-{}'.format(const_name)][gain_names[gain]],\n",
- " dval_pct), axis=0)\n",
- " except:\n",
- " Mod_data[const_name][gain_names[gain]] = values\n",
- " Mod_data['d-{}'.format(const_name)][gain_names[gain]] = dval\n",
- " Mod_data['dpct-{}'.format(const_name)][gain_names[gain]] = dval_pct\n",
- "\n",
- " vmin, vmax = get_range(Mod_data[const_name][gain_names[gain]], 2)\n",
- " dvmin, dvmax = get_range(Mod_data['d-{}'.format(const_name)][gain_names[gain]], 2)\n",
- " dpctvmin, dpctvmax = get_range(Mod_data['dpct-{}'.format(const_name)][gain_names[gain]], 2)\n",
+ " for k, v in {'{}': values, 'd-{}': dval , 'dpct-{}': dval_pct}.items():\n",
+ " try:\n",
+ " Mod_data[k.format(const_name)][gain_names[gain]] = \\\n",
+ " np.concatenate((Mod_data[k.format(const_name)][gain_names[gain]],\n",
+ " v), axis=0)\n",
+ " except:\n",
+ " Mod_data[k.format(const_name)][gain_names[gain]] = v\n",
" # Plotting constant overall modules.\n",
- " display(Markdown('###### {} ######'.format(glabel[gain])))\n",
- " \n",
+ " display(Markdown(f'###### {glabel[gain]} ######'))\n",
+ "\n",
" gs = gridspec.GridSpec(2, 2)\n",
" fig = plt.figure(figsize=(24, 32))\n",
- " \n",
- " ax0 = fig.add_subplot(gs[0, :])\n",
- " ax1 = fig.add_subplot(gs[1, 0])\n",
- " ax2 = fig.add_subplot(gs[1, 1])\n",
- " geom.plot_data_fast(Mod_data[const_name][gain_names[gain]], vmin=vmin, vmax=vmax,\n",
- " ax=ax0, colorbar={\n",
- " 'shrink': 0.9,\n",
- " 'pad': 0.05})\n",
- "\n",
- " colorbar = ax0.images[0].colorbar\n",
- " if const_name != 'BadPixelsDark':\n",
- " colorbar.set_label('ADUs')\n",
- "\n",
- " ax0.set_title('{} {}'.format(const_name, glabel[gain]), fontsize=15)\n",
- " ax0.set_xlabel('Columns', fontsize=15)\n",
- " ax0.set_ylabel('Rows', fontsize=15)\n",
- "\n",
- " # Plotting the difference with previous constants in ADUs and %\n",
- " geom.plot_data_fast(Mod_data['d-{}'.format(const_name)][gain_names[gain]],\n",
- " vmin=dvmin, vmax=dvmax, ax=ax1,colorbar={\n",
- " 'shrink': 0.6,\n",
- " 'pad': 0.1,\n",
- " 'orientation': 'horizontal'})\n",
- " colorbar = ax1.images[0].colorbar\n",
- " if const_name != 'BadPixelsDark':\n",
- " colorbar.set_label('ADUs')\n",
- " geom.plot_data_fast(Mod_data['dpct-{}'.format(const_name)][gain_names[gain]], \n",
- " vmin=dpctvmin, vmax=dpctvmax, ax=ax2,colorbar={\n",
- " 'shrink': 0.6,\n",
- " 'pad': 0.1,\n",
- " 'orientation': 'horizontal'})\n",
- " colorbar = ax2.images[0].colorbar\n",
- " colorbar.set_label('%')\n",
- " ax1.set_title('Difference with previous {} {}'.format(const_name, \n",
- " glabel[gain]), fontsize=15)\n",
- " ax1.set_xlabel('Columns', fontsize=15)\n",
- " ax1.set_ylabel('Rows', fontsize=15)\n",
- " ax2.set_title('Difference with previous {} {} (%)'.format(const_name, glabel[gain]), fontsize=15)\n",
- " ax2.set_xlabel('Columns', fontsize=15)\n",
- " ax2.set_ylabel('Rows', fontsize=15)\n",
- " plt.show()\n",
- " "
+ "\n",
+ " axis = OrderedDict()\n",
+ " axis = {\"ax0\": {\"cname\": \"{}\" ,\"gs\": gs[0, :], \"shrink\": 0.9, \"pad\": 0.05, \"label\": \"ADUs\"},\n",
+ " \"ax1\": {\"cname\": \"d-{}\",\"gs\": gs[1, 0], \"shrink\": 0.6, \"pad\": 0.1, \"label\": \"ADUs\"},\n",
+ " \"ax2\": {\"cname\": \"dpct-{}\", \"gs\": gs[1, 1], \"shrink\": 0.6, \"pad\": 0.1, \"label\": \"%\"}}\n",
+ "\n",
+ " for ax, axv in axis.items():\n",
+ " # Add the min and max plot values for each axis.\n",
+ " vmin, vmax = get_range(Mod_data[axv[\"cname\"].format(const_name)][gain_names[gain]], 2)\n",
+ " ax = fig.add_subplot(axv[\"gs\"])\n",
+ " geom.plot_data_fast(Mod_data[axv[\"cname\"].format(const_name)][gain_names[gain]],\n",
+ " vmin=vmin, vmax=vmax, ax=ax, \n",
+ " colorbar={'shrink': axv[\"shrink\"],\n",
+ " 'pad': axv[\"pad\"]\n",
+ " }\n",
+ " )\n",
+ "\n",
+ " colorbar = ax.images[0].colorbar\n",
+ " colorbar.set_label(axv[\"label\"])\n",
+ "\n",
+ " ax.set_title(f'{const_name} {glabel[gain]}', fontsize=15)\n",
+ " ax.set_xlabel('Columns', fontsize=15)\n",
+ " ax.set_ylabel('Rows', fontsize=15)\n",
+ "\n",
+ " plt.show()"
]
},
{
@@ -355,23 +388,31 @@
},
"outputs": [],
"source": [
- "# Loop over capacitor settings, modules, constants\n",
+ "# Loop over modules and constants\n",
"for const_name, const in constants.items():\n",
+ " display(Markdown(f'### Summary across Modules - {const_name}'))\n",
"\n",
- " display(Markdown('### Summary across Modules - {}'.format(const_name)))\n",
- " for gain in range(3):\n",
- "\n",
+ " for gain in range(gainstages):\n",
" if const_name == 'ThresholdsDark':\n",
" if gain == 2:\n",
" continue\n",
" glabel = threshold_names\n",
" else:\n",
" glabel = gain_names\n",
- " \n",
- " data = np.copy(const[:, :, :, :, gain])\n",
+ "\n",
+ " if len(const.shape) == 5:\n",
+ " data = np.copy(const[:, :, :, :, gain])\n",
+ " else:\n",
+ " data = np.copy(const[:, :, :, :])\n",
" if const_name != 'BadPixelsDark':\n",
- " label = '{} value [ADU], good pixels only'.format(const_name)\n",
- " data[constants['BadPixelsDark'][:, :, :, :, gain] > 0] = np.nan\n",
+ " if \"BadPixelsDark\" in constants.keys():\n",
+ " label = f'{const_name} value [ADU], good pixels only'\n",
+ " if len(const.shape) == 5:\n",
+ " data[constants['BadPixelsDark'][:, :, :, :, gain] > 0] = np.nan\n",
+ " else:\n",
+ " data[constants['BadPixelsDark'][:, :, :, :] > 0] = np.nan\n",
+ " else:\n",
+ " label = f'{const_name} value [ADU], good and bad pixels'\n",
" datamean = np.nanmean(data, axis=(1, 2))\n",
"\n",
" fig = plt.figure(figsize=(15, 6), tight_layout={\n",
@@ -403,10 +444,12 @@
" title_position=[0.5, 1.18],\n",
" legend='outside-top-ncol6-frame', legend_size='18%',\n",
" legend_pad=0.00)\n",
- "\n",
" if const_name != 'BadPixelsDark':\n",
" ax = fig.add_subplot(122)\n",
- " label = '## $\\sigma$ {} [ADU], good pixels only'.format(const_name)\n",
+ " if \"BadPixelsDark\" in constants.keys():\n",
+ " label = f'$\\sigma$ {const_name} [ADU], good pixels only'\n",
+ " else:\n",
+ " label = f'$\\sigma$ {const_name} [ADU], good and bad pixels'\n",
" d = []\n",
" for im, mod in enumerate(np.nanstd(data, axis=(1, 2))):\n",
" d.append({'x': np.arange(mod.shape[0]),\n",
@@ -452,20 +495,18 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"head = ['Module', 'High gain', 'Medium gain', 'Low gain']\n",
- "head_th = ['Module', 'High threshold', 'Medium threshold']\n",
+ "head_th = ['Module', 'HG_MG threshold', 'MG_LG threshold']\n",
"for const_name, const in constants.items():\n",
" table = []\n",
"\n",
" for i_mod, mod in enumerate(mod_names):\n",
"\n",
" t_line = [mod]\n",
- " for gain in range(3):\n",
+ " for gain in range(gainstages):\n",
" \n",
" if const_name == 'ThresholdsDark': \n",
" if gain == 2:\n",
@@ -473,8 +514,10 @@
" header = head_th\n",
" else:\n",
" header = head\n",
- "\n",
- " data = np.copy(const[i_mod, :, :, :, gain])\n",
+ " if len(const.shape) == 5: \n",
+ " data = np.copy(const[i_mod, :, :, :, gain])\n",
+ " else:\n",
+ " data = np.copy(const[i_mod, :, :, :])\n",
" if const_name == 'BadPixelsDark':\n",
" data[data > 0] = 1.0\n",
"\n",
@@ -487,9 +530,12 @@
" t_line.append(f'{datasum:6.0f} ({datamean:6.3f}) ')\n",
" label = '## Number(fraction) of bad pixels'\n",
" else:\n",
- "\n",
- " data[constants['BadPixelsDark']\n",
- " [i_mod, :, :, :, gain] > 0] = np.nan\n",
+ " if \"BadPixelsDark\" in constants.keys():\n",
+ " data[constants['BadPixelsDark']\n",
+ " [i_mod, :, :, :, gain] > 0] = np.nan\n",
+ " label = f'### Average {const_name} [ADU], good pixels only'\n",
+ " else:\n",
+ " label = f'### Average {const_name} [ADU], good and bad pixels'\n",
"\n",
" t_line.append(f'{np.nanmean(data):6.1f} $\\\\pm$ {np.nanstd(data):6.1f}')\n",
" label = f'## Average {const_name} [ADU], good pixels only'\n",
@@ -508,13 +554,6 @@
"outputs": [],
"source": []
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- },
{
"cell_type": "code",
"execution_count": null,
diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index 61e7a6b8b86cca8af898112618ae4c4dce9ae224..d8088e2607fc6d698d430c27a86030541b88fbfa 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -813,6 +813,8 @@ def run():
version = ""
title = title.rstrip()
+ if detector not in title:
+ title = f"{detector} {title}"
run_uuid = f"t{datetime.now().strftime('%y%m%d_%H%M%S')}"
@@ -843,12 +845,6 @@ def run():
caltype.upper(), datetime.now().isoformat())
if try_report_to_output:
if "out_folder" in args:
- if "run" in args and str(args["run"]) not in args["out_folder"]:
- rr = args["run"]
- # Only put run, when it is not in the out_folder
- if isinstance(rr, int):
- rr = "r{:04d}".format(rr)
- args["out_folder"] = "{}/{}/".format(args["out_folder"], rr)
out_path = os.path.abspath(args["out_folder"])
else:
print("No 'out_folder' defined as argument, outputting to '{}' instead.".format(
diff --git a/xfel_calibrate/finalize.py b/xfel_calibrate/finalize.py
index 86b198458480f726bd27ca32acdfa7f90002baab..c6573b036d48e0d103eadf3f2116b5659bb70e11 100644
--- a/xfel_calibrate/finalize.py
+++ b/xfel_calibrate/finalize.py
@@ -240,6 +240,18 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
move("{}/conf.py.tmp".format(run_path), "{}/conf.py".format(run_path))
direntries = listdir(run_path)
+ lead_rstfiles = ['InputParameters.rst', 'timing_summary.rst']
+
+ # Order rst files based on the known order(lead_rstfiles).
+ for f in direntries:
+ if f in lead_rstfiles:
+ direntries.insert(lead_rstfiles.index(f),
+ direntries.pop(direntries.index(f)))
+ # Move summary to the top, if it is exists,
+ # after the known leading rst files.
+ if "summary" in f.lower() and f not in lead_rstfiles:
+ direntries.insert(len(lead_rstfiles),
+ direntries.pop(direntries.index(f)))
files_to_handle = []
for entry in direntries:
if isfile("{}/{}".format(run_path, entry)):
diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py
index d831746da479589011250e580928cbefde180df1..03053917b1659fd1b781601e3e8909cf394cfa34 100644
--- a/xfel_calibrate/notebooks.py
+++ b/xfel_calibrate/notebooks.py
@@ -55,7 +55,7 @@ notebooks = {
"DARK": {
"notebook": "notebooks/LPD/LPDChar_Darks_NBC.ipynb",
"dep_notebooks": [
- "notebooks/LPD/LPDChar_Darks_Summary_NBC.ipynb"],
+ "notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb"],
"concurrency": {"parameter": "modules",
"default concurrency": list(range(16)),
"cluster cores": 8},
@@ -220,6 +220,8 @@ notebooks = {
"DSSC": {
"DARK": {
"notebook": "notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb",
+ "dep_notebooks": [
+ "notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb"],
"concurrency": {"parameter": "modules",
"default concurrency": list(range(16)),
"cluster cores": 8},