notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb

 %% Cell type:markdown id: tags:
 
 # AGIPD Characterize Dark Images #
 
 Author: S. Hauf, Version: 0.1
 
 The following code analyzes a set of dark images taken with the AGIPD detector to deduce detector offsets , noise, bad-pixel maps and thresholding. All four types of constants are evaluated per-pixel and per-memory cell. Data for the detector's three gain stages needs to be present, separated into separate runs.
 
 The evaluated calibration constants are stored locally and injected in the calibration data base.
 
 %% Cell type:code id: tags:
 
 ``` python
 cluster_profile = "noDB" # The ipcluster profile to use
-in_folder = "/gpfs/exfel/d/raw/SPB/202030/p900138" # path to input data, required
-out_folder = "/gpfs/exfel/data/scratch/ahmedk/test/AGIPDbad_sep64" # path to output to, required
+in_folder = "/gpfs/exfel/d/raw/DETLAB/202031/p900172/" # path to input data, required
+out_folder = "/gpfs/exfel/data/scratch/ahmedk/test/miniHalfAGIPD" # path to output to, required
 sequences = [0] # sequence files to evaluate.
 modules = [-1]  # list of modules to evaluate, RANGE ALLOWED
-run_high = 167 # run number in which high gain data was recorded, required
-run_med = 168 # run number in which medium gain data was recorded, required
-run_low = 169 # run number in which low gain data was recorded, required
+run_high = 84 # run number in which high gain data was recorded, required
+run_med = 87 # run number in which medium gain data was recorded, required
+run_low = 88 # run number in which low gain data was recorded, required
 
-karabo_id = "SPB_DET_AGIPD1M-1" # karabo karabo_id
+karabo_id = "DETLAB_DET_AGIPD500K2G" # karabo karabo_id
 karabo_da = ['-1']  # a list of data aggregators names, Default [-1] for selecting all data aggregators
 receiver_id = "{}CH0" # inset for receiver devices
 path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # the template to use to access data
 h5path = '/INSTRUMENT/{}/DET/{}:xtdf/image' # path in the HDF5 file to images
 h5path_idx = '/INDEX/{}/DET/{}:xtdf/image' # path in the HDF5 file to images
 h5path_ctrl = '/CONTROL/{}/MDL/FPGA_COMP_TEST' # path to control information
 karabo_id_control = "SPB_IRU_AGIPD1M1" # karabo-id for control device '
 karabo_da_control = "AGIPD1MCTRL00" # karabo DA for control infromation
 
 use_dir_creation_date = True  # use dir creation date as data production reference date
 cal_db_interface = "tcp://max-exfl016:8020" # the database interface to use
 cal_db_timeout = 3000000 # timeout on caldb requests"
 local_output = True # output constants locally
 db_output = False # output constants to database
 
 mem_cells = 0 # number of memory cells used, set to 0 to automatically infer
 bias_voltage = 300 # detector bias voltage
 gain_setting = 0.1 # the gain setting, use 0.1 to try to auto-determine
 acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine
 interlaced = False # assume interlaced data format, for data prior to Dec. 2017
 rawversion = 2 # RAW file format version
 
 thresholds_offset_sigma = 3. # offset sigma thresholds for offset deduced bad pixels
 thresholds_offset_hard = [0, 0] # For setting the same threshold offset for the 3 gains. Left for backcompatability. Default [0, 0] to take the following parameters.
 thresholds_offset_hard_hg = [3000, 7000] # High-gain thresholds in absolute ADU terms for offset deduced bad pixels
 thresholds_offset_hard_mg = [6000, 10000] # Medium-gain thresholds in absolute ADU terms for offset deduced bad pixels
 thresholds_offset_hard_lg = [6000, 10000] # Low-gain thresholds in absolute ADU terms for offset deduced bad pixels
 
 thresholds_noise_sigma = 5. # noise sigma thresholds for offset deduced bad pixels
 thresholds_noise_hard = [0, 0] # For setting the same threshold noise for the 3 gains. Left for backcompatability. Default [0, 0] to take the following parameters.
 thresholds_noise_hard_hg = [4, 20] # High-gain thresholds in absolute ADU terms for offset deduced bad pixels
 thresholds_noise_hard_mg = [4, 20] # Medium-gain thresholds in absolute ADU terms for offset deduced bad pixels
 thresholds_noise_hard_lg = [4, 20] # Low-gain thresholds in absolute ADU terms for offset deduced bad pixels
 
 thresholds_gain_sigma = 5. # Gain separation sigma threshold
 
 high_res_badpix_3d = False # set this to True if you need high-resolution 3d bad pixel plots. ~7mins extra time for 64 memory cells
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 # imports and things that do not usually need to be changed
 from datetime import datetime
 import dateutil.parser
 import warnings
 warnings.filterwarnings('ignore')
 from collections import OrderedDict
 import os
 import h5py
 import numpy as np
 import matplotlib
 import tabulate
 
 matplotlib.use('agg')
 import matplotlib.pyplot as plt
 from IPython.display import display, Markdown, Latex
 %matplotlib inline
 
 from cal_tools.tools import (get_from_db, get_dir_creation_date,
                              get_notebook_name, get_random_db_interface,
                              map_gain_stages, parse_runs,
                              run_prop_seq_from_path, save_const_to_h5,
                              send_to_db)
 from cal_tools.influx import InfluxLogger
 from cal_tools.enums import BadPixels
 from cal_tools.plotting import (create_constant_overview,
                                 plot_badpix_3d, show_processed_modules,
                                 show_overview)
 from cal_tools.agipdlib import get_gain_setting
 
 # make sure a cluster is running with ipcluster start --n=32, give it a while to start
 from ipyparallel import Client
 
 view = Client(profile=cluster_profile)[:]
 view.use_dill()
 
 from iCalibrationDB import Constants, Conditions, Detectors, Versions
 
 gains = np.arange(3)
 
 IL_MODE = interlaced
 max_cells = mem_cells
 
 offset_runs = OrderedDict()
 offset_runs["high"] = run_high
 offset_runs["med"] = run_med
 offset_runs["low"] = run_low
 
 creation_time=None
 if use_dir_creation_date:
     creation_time = get_dir_creation_date(in_folder, run_high)
 
 print(f"Using {creation_time} as creation time of constant.")
 
 run, prop, seq = run_prop_seq_from_path(in_folder)
 
 cal_db_interface = get_random_db_interface(cal_db_interface)
 print(f'Calibration database interface: {cal_db_interface}')
 
-loc = None
 instrument = karabo_id.split("_")[0]
+
 if instrument == "SPB":
     dinstance = "AGIPD1M1"
-else:
+    nmods = 16
+elif instrument == "MID":
     dinstance = "AGIPD1M2"
+    nmods = 16
+# TODO: Remove DETLAB
+elif instrument == "HED" or instrument == "DETLAB":
+    dinstance = "AGIPD500K"
+    nmods = 8
+
 print(f"Detector in use is {karabo_id}")
 print(f"Instrument {instrument}")
 print(f"Detector instance {dinstance}")
 
 logger = InfluxLogger(detector="AGIPD", instrument=instrument, mem_cells=mem_cells,
                       notebook=get_notebook_name(), proposal=prop)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
-gain_names = ['High', 'Medium', 'Low']
 runs = [run_high, run_med, run_low]
 
 if "{" in h5path_ctrl:
     h5path_ctrl = h5path_ctrl.format(karabo_id_control)
 
 if gain_setting == 0.1:
     if creation_time.replace(tzinfo=None) < dateutil.parser.parse('2020-01-31'):
         print("Set gain-setting to None for runs taken before 2020-01-31")
         gain_setting = None
     else:
         try:
             # extract gain setting and validate that all runs have the same setting
             gsettings = []
             for r in runs:
                 control_fname = '{}/r{:04d}/RAW-R{:04d}-{}-S00000.h5'.format(in_folder, r, r,
                                                                              karabo_da_control)
                 gsettings.append(get_gain_setting(control_fname, h5path_ctrl))
             if not all(g == gsettings[0] for g in gsettings):
                 raise ValueError(f"Different gain settings for the 3 input runs {gsettings}")
             gain_setting =  gsettings[0]
         except Exception as e:
             print(f'Error while reading gain setting from: \n{control_fname}')
             print(f'Error: {e}')
             if "component not found" in str(e):
                 print("Gain setting is not found in the control information")
             print("Data will not be processed")
             sequences = []
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 if karabo_da[0] == '-1':
     if modules[0] == -1:
-        modules = list(range(16))
+        modules = list(range(nmods))
     karabo_da = ["AGIPD{:02d}".format(i) for i in modules]
 else:
     modules = [int(x[-2:]) for x in karabo_da]
 h5path = h5path.format(karabo_id, receiver_id)
 h5path_idx = h5path_idx.format(karabo_id, receiver_id)
 
 print("Parameters are:")
 print(f"Proposal: {prop}")
 print(f"Memory cells: {mem_cells}/{max_cells}")
 print("Runs: {}".format([ v for v in offset_runs.values()]))
 print(f"Sequences: {sequences}")
 print(f"Interlaced mode: {IL_MODE}")
 print(f"Using DB: {db_output}")
 print(f"Input: {in_folder}")
 print(f"Output: {out_folder}")
 print(f"Bias voltage: {bias_voltage}V")
 print(f"Gain setting: {gain_setting}")
 ```
 
 %% Cell type:markdown id: tags:
 
 The following lines will create a queue of files which will the be executed module-parallel. Distiguishing between different gains.
 
 %% Cell type:code id: tags:
 
 ``` python
 # set everything up filewise
 os.makedirs(out_folder, exist_ok=True)
 gmf = map_gain_stages(in_folder, offset_runs, path_template, karabo_da, sequences)
 gain_mapped_files, total_sequences, total_file_size = gmf
 print(f"Will process a total of {total_sequences} files.")
 ```
 
 %% Cell type:markdown id: tags:
 
 ## Calculate Offsets, Noise and Thresholds ##
 
 The calculation is performed per-pixel and per-memory-cell. Offsets are simply the median value for a set of dark data taken at a given gain, noise the standard deviation, and gain-bit values the medians of the gain array.
 
 %% Cell type:code id: tags:
 
 ``` python
 import copy
 from functools import partial
 def characterize_module(il_mode, cells, bp_thresh, rawversion, loc, acq_rate,
                         h5path, h5path_idx, inp):
     import numpy as np
     import copy
     import h5py
     from cal_tools.enums import BadPixels
     from cal_tools.agipdlib import get_num_cells, get_acq_rate
 
     filename, channel, gg = inp
 
     if cells == 0:
         cells = get_num_cells(filename, loc, channel)
 
     print(f"Using {cells} memory cells")
 
     if acq_rate == 0.:
         acq_rate = get_acq_rate(filename, loc, channel)
 
     thresholds_offset, thresholds_offset_sigma, thresholds_noise, thresholds_noise_sigma = bp_thresh
     thresholds_offset_hard = thresholds_offset[gg]
     thresholds_noise_hard = thresholds_noise[gg]
     infile = h5py.File(filename, "r", driver="core")
 
     h5path = h5path.format(channel)
     h5path_idx = h5path_idx.format(channel)
 
     if rawversion == 2:
         count = np.squeeze(infile[f"{h5path_idx}/count"])
         first = np.squeeze(infile[f"{h5path_idx}/first"])
         last_index = int(first[count != 0][-1]+count[count != 0][-1])
         first_index = int(first[count != 0][0])
     else:
         status = np.squeeze(infile[f"{h5path_idx}/status"])
         if np.count_nonzero(status != 0) == 0:
             return
         last = np.squeeze(infile[f"{h5path_idx}/last"])
         first = np.squeeze(infile[f"{h5path_idx}/first"])
         last_index = int(last[status != 0][-1]) + 1
         first_index = int(first[status != 0][0])
     im = np.array(infile[f"{h5path}/data"][first_index:last_index,...])
     cellIds = np.squeeze(infile[f"{h5path}/cellId"][first_index:last_index,...])
 
     infile.close()
 
     if il_mode:
         ga = im[1::2, 0, ...]
         im = im[0::2, 0, ...].astype(np.float32)
         cellIds = cellIds[::2]
     else:
         ga = im[:, 1, ...]
         im = im[:, 0, ...].astype(np.float32)
 
     im = np.rollaxis(im, 2)
     im = np.rollaxis(im, 2, 1)
 
     ga = np.rollaxis(ga, 2)
     ga = np.rollaxis(ga, 2, 1)
 
     mcells = cells #max(cells, np.max(cellIds)+1)
     offset = np.zeros((im.shape[0], im.shape[1], mcells))
     gains = np.zeros((im.shape[0], im.shape[1], mcells))
     noise = np.zeros((im.shape[0], im.shape[1], mcells))
     gains_std = np.zeros((im.shape[0], im.shape[1], mcells))
 
     for cc in np.unique(cellIds[cellIds < mcells]):
         cellidx = cellIds == cc
         offset[...,cc] = np.median(im[..., cellidx], axis=2)
         noise[...,cc] = np.std(im[..., cellidx], axis=2)
         gains[...,cc] = np.median(ga[..., cellidx], axis=2)
         gains_std[...,cc] = np.std(ga[..., cellidx], axis=2)
 
     # bad pixels
     bp = np.zeros(offset.shape, np.uint32)
     # offset related bad pixels
     offset_mn = np.nanmedian(offset, axis=(0,1))
     offset_std = np.nanstd(offset, axis=(0,1))
 
     bp[(offset < offset_mn-thresholds_offset_sigma*offset_std) |
        (offset > offset_mn+thresholds_offset_sigma*offset_std)] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value
     bp[(offset < thresholds_offset_hard[0]) | (
         offset > thresholds_offset_hard[1])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value
     bp[~np.isfinite(offset)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value
 
     # noise related bad pixels
     noise_mn = np.nanmedian(noise, axis=(0,1))
     noise_std = np.nanstd(noise, axis=(0,1))
     bp[(noise < noise_mn-thresholds_noise_sigma*noise_std) |
        (noise > noise_mn+thresholds_noise_sigma*noise_std)] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value
     bp[(noise < thresholds_noise_hard[0]) | (noise > thresholds_noise_hard[1])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value
     bp[~np.isfinite(noise)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value
 
     return offset, noise, gains, gains_std, gg, bp, cells, acq_rate
 
 offset_g = OrderedDict()
 noise_g = OrderedDict()
 gain_g = OrderedDict()
 gainstd_g = OrderedDict()
 badpix_g = OrderedDict()
 gg = 0
 
 start = datetime.now()
 all_cells = []
 all_acq_rate = []
 
 if thresholds_offset_hard == [0, 0]:
     thresholds_offset_hard = [thresholds_offset_hard_hg, thresholds_offset_hard_mg, thresholds_offset_hard_lg]
 else:
     thresholds_offset_hard = [thresholds_offset_hard] * 3
 
 if thresholds_noise_hard == [0, 0]:
     thresholds_noise_hard = [thresholds_noise_hard_hg, thresholds_noise_hard_mg, thresholds_noise_hard_lg]
 else:
     thresholds_noise_hard = [thresholds_noise_hard] * 3
 
 
 inp = []
 for gain, mapped_files in gain_mapped_files.items():
     dones = []
     for i in modules:
         qm = f"Q{i//4+1}M{i%4+1}"
         if qm in mapped_files and not mapped_files[qm].empty():
             fname_in = mapped_files[qm].get()
             print("Process file: ", fname_in)
             dones.append(mapped_files[qm].empty())
         else:
             continue
         inp.append((fname_in, i, gg))
 
     gg += 1
 
 p = partial(characterize_module, IL_MODE, max_cells,
            (thresholds_offset_hard, thresholds_offset_sigma,
             thresholds_noise_hard, thresholds_noise_sigma),
             rawversion, karabo_id, acq_rate, h5path, h5path_idx)
 
 # Don't remove. Used for Debugging.
 #results = list(map(p, inp))
 results = view.map_sync(p, inp)
 
 for ii, r in enumerate(results):
     offset, noise, gains, gains_std, gg, bp, thiscell, thisacq = r
     all_cells.append(thiscell)
     all_acq_rate.append(thisacq)
     for i in modules:
         qm = f"Q{i//4+1}M{i%4+1}"
         if qm not in offset_g:
             offset_g[qm] = np.zeros((offset.shape[0], offset.shape[1], offset.shape[2], 3))
             noise_g[qm] = np.zeros_like(offset_g[qm])
             gain_g[qm] = np.zeros_like(offset_g[qm])
             gainstd_g[qm] = np.zeros_like(offset_g[qm])
             badpix_g[qm] = np.zeros_like(offset_g[qm], np.uint32)
 
         offset_g[qm][...,gg] = offset
         noise_g[qm][...,gg] = noise
         gain_g[qm][...,gg] = gains
         gainstd_g[qm][..., gg] = gains_std
         badpix_g[qm][...,gg] = bp
 
 
 duration = (datetime.now() - start).total_seconds()
 logger.runtime_summary_entry(success=True, runtime=duration,
                              total_sequences=total_sequences,
                              filesize=total_file_size)
 logger.send()
 max_cells = np.max(all_cells)
 print(f"Using {max_cells} memory cells")
 acq_rate = np.max(all_acq_rate)
 print(f"Using {acq_rate} MHz acquisition rate")
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 # Add a badpixel due to bad gain separation
 for g in range(2):
     # Bad pixels during bad gain separation.
     # Fraction of pixels in the module with separation lower than "thresholds_gain_sigma".
     bad_sep = (gain_g[qm][..., g+1] - gain_g[qm][..., g]) / np.sqrt(gainstd_g[qm][..., g+1]**2 + gainstd_g[qm][..., g]**2)
     badpix_g[qm][...,g+1][(bad_sep)<thresholds_gain_sigma]|= BadPixels.GAIN_THRESHOLDING_ERROR.value
 ```
 
 %% Cell type:markdown id: tags:
 
 The thresholds for gain switching are then defined as the mean value between in individual gain bit levels. Note that these thresholds need to be refined with charge induced thresholds, as the two are not the same.
 
 %% Cell type:code id: tags:
 
 ``` python
 thresholds_g = {}
 for qm in gain_g.keys():
     thresholds_g[qm] = np.zeros((gain_g[qm].shape[0], gain_g[qm].shape[1], gain_g[qm].shape[2], 5))
     thresholds_g[qm][...,0] = (gain_g[qm][...,1]+gain_g[qm][...,0])/2
     thresholds_g[qm][...,1] = (gain_g[qm][...,2]+gain_g[qm][...,1])/2
     for i in range(3):
         thresholds_g[qm][...,2+i] = gain_g[qm][...,i]
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 res = OrderedDict()
 for i in modules:
     qm = "Q{}M{}".format(i//4+1, i%4+1)
     res[qm] = {'Offset': offset_g[qm],
                'Noise': noise_g[qm],
                'ThresholdsDark': thresholds_g[qm],
                'BadPixelsDark': badpix_g[qm]
                }
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]
 file_loc = 'proposal:{} runs:{} {} {}'.format(proposal, run_low, run_med, run_high)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 # Retrieve existing constants for comparison
 clist = ["Offset", "Noise", "ThresholdsDark", "BadPixelsDark"]
 old_const = {}
 old_mdata = {}
 detinst = getattr(Detectors, dinstance)
 
 print('Retrieve pre-existing constants for comparison.')
 
 for qm in res:
     for const in res[qm]:
         dconst = getattr(Constants.AGIPD, const)()
         dconst.data = res[qm][const]
 
         # Setting conditions
         condition = Conditions.Dark.AGIPD(memory_cells=max_cells,
                                           bias_voltage=bias_voltage,
                                           acquisition_rate=acq_rate,
                                           gain_setting=gain_setting)
         device = getattr(detinst, qm)
         data, mdata = get_from_db(device,
                                   getattr(Constants.AGIPD, const)(),
                                   condition,
                                   None,
                                   cal_db_interface, creation_time=creation_time,
                                   verbosity=2, timeout=cal_db_timeout)
 
         old_const[const] = data
         if mdata is not None and data is not None:
             time = mdata.calibration_constant_version.begin_at
             old_mdata[const] = time.isoformat()
             os.makedirs('{}/old/'.format(out_folder), exist_ok=True)
             save_const_to_h5(device,
                              getattr(Constants.AGIPD, const)(),
                              condition, data, file_loc, creation_time,
                              f'{out_folder}/old/')
         else:
             old_mdata[const] = "Not found"
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 md = None
 for qm in res:
 
     for const in res[qm]:
         dconst = getattr(Constants.AGIPD, const)()
         dconst.data = res[qm][const]
 
         # set the operating condition
         condition = Conditions.Dark.AGIPD(memory_cells=max_cells,
                                           bias_voltage=bias_voltage,
                                           acquisition_rate=acq_rate,
                                           gain_setting=gain_setting)
         detinst = getattr(Detectors, dinstance)
         device = getattr(detinst, qm)
 
         if db_output:
             md = send_to_db(device, dconst, condition, file_loc,
                             cal_db_interface, creation_time=creation_time, timeout=cal_db_timeout)
 
         if local_output:
             md = save_const_to_h5(device, dconst, condition, dconst.data, file_loc, creation_time, out_folder)
             print(f"Calibration constant {const} is stored locally.\n")
 
     print("Constants parameter conditions are:\n")
     print(f"• memory_cells: {max_cells}\n• bias_voltage: {bias_voltage}\n"
           f"• acquisition_rate: {acq_rate}\n• gain_setting: {gain_setting}\n"
           f"• creation_time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\n")
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
-mnames=[]
-for i in modules:
-    qm = f"Q{i//4+1}M{i % 4+1}"
-    mnames.append(qm)
-    display(Markdown(f'## Position of the module {qm} and it\'s ASICs##'))
-show_processed_modules(dinstance, constants=None, mnames=mnames, mode="position")
+# TODO: add show_processed_modules diagram for Mini-Half AGIPD
+if dinstance != "AGIPD500K":
+    mnames=[]
+    for i in modules:
+        qm = f"Q{i//4+1}M{i % 4+1}"
+        mnames.append(qm)
+        display(Markdown(f'## Position of the module {qm} and it\'s ASICs##'))
+    show_processed_modules(dinstance, constants=None, mnames=mnames, mode="position")
 ```
 
 %% Cell type:markdown id: tags:
 
 ## Single-Cell Overviews ##
 
 Single cell overviews allow to identify potential effects on all memory cells, e.g. on sensor level. Additionally, they should serve as a first sanity check on expected behaviour, e.g. if structuring on the ASIC level is visible in the offsets, but otherwise no immediate artifacts are visible.
 
 %% Cell type:markdown id: tags:
 
 ### High Gain ###
 
 %% Cell type:code id: tags:
 
 ``` python
 cell = 3
 gain = 0
 show_overview(res, cell, gain, infix="{}-{}-{}".format(*offset_runs.values()))
 ```
 
 %% Cell type:markdown id: tags:
 
 ### Medium Gain ###
 
 %% Cell type:code id: tags:
 
 ``` python
 cell = 3
 gain = 1
 show_overview(res, cell, gain, infix="{}-{}-{}".format(*offset_runs.values()))
 ```
 
 %% Cell type:markdown id: tags:
 
 ### Low Gain ###
 
 %% Cell type:code id: tags:
 
 ``` python
 cell = 3
 gain = 2
 show_overview(res, cell, gain, infix="{}-{}-{}".format(*offset_runs.values()))
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 cols = {BadPixels.NOISE_OUT_OF_THRESHOLD.value: (BadPixels.NOISE_OUT_OF_THRESHOLD.name, '#FF000080'),
         BadPixels.OFFSET_NOISE_EVAL_ERROR.value: (BadPixels.OFFSET_NOISE_EVAL_ERROR.name, '#0000FF80'),
         BadPixels.OFFSET_OUT_OF_THRESHOLD.value: (BadPixels.OFFSET_OUT_OF_THRESHOLD.name, '#00FF0080'),
         BadPixels.GAIN_THRESHOLDING_ERROR.value: (BadPixels.GAIN_THRESHOLDING_ERROR.name, '#FF40FF40'),
         BadPixels.OFFSET_OUT_OF_THRESHOLD.value | BadPixels.NOISE_OUT_OF_THRESHOLD.value: ('OFFSET_OUT_OF_THRESHOLD + NOISE_OUT_OF_THRESHOLD', '#DD00DD80'),
         BadPixels.OFFSET_OUT_OF_THRESHOLD.value | BadPixels.NOISE_OUT_OF_THRESHOLD.value |
         BadPixels.GAIN_THRESHOLDING_ERROR.value: ('MIXED', '#BFDF009F')}
 
 if high_res_badpix_3d:
     display(Markdown("""
 
     ## Global Bad Pixel Behaviour ##
 
     The following plots show the results of bad pixel evaluation for all evaluated memory cells.
     Cells are stacked in the Z-dimension, while pixels values in x/y are rebinned with a factor of 2.
     This excludes single bad pixels present only in disconnected pixels.
     Hence, any bad pixels spanning at least 4 pixels in the x/y-plane, or across at least two memory cells are indicated.
     Colors encode the bad pixel type, or mixed type.
 
     """))
 
     gnames = ['High Gain', 'Medium Gain', 'Low Gain']
     for gain in range(3):
         display(Markdown(f'### {gnames[gain]} ###'))
         for mod, data in badpix_g.items():
             plot_badpix_3d(data[...,gain], cols, title=mod, rebin_fac=1)
             plt.show()
 ```
 
 %% Cell type:markdown id: tags:
 
 ## Aggregate values, and per Cell behaviour ##
 
 The following tables and plots give an overview of statistical aggregates for each constant, as well as per cell behavior.
 
 %% Cell type:code id: tags:
 
 ``` python
 create_constant_overview(offset_g, "Offset (ADU)", max_cells, 4000, 8000,
                          badpixels=[badpix_g, np.nan])
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 create_constant_overview(noise_g, "Noise (ADU)", max_cells, 0, 100,
                          badpixels=[badpix_g, np.nan])
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 # Plot only three gain threshold maps.
 bp_thresh = OrderedDict()
 for mod, con in badpix_g.items():
     bp_thresh[mod] = np.zeros((con.shape[0], con.shape[1], con.shape[2], 5), dtype=con.dtype)
     bp_thresh[mod][...,:2] = con[...,:2]
     bp_thresh[mod][...,2:] = con
 
 
 create_constant_overview(thresholds_g, "Threshold (ADU)", max_cells, 4000, 10000, 5,
                          badpixels=[bp_thresh, np.nan],
                          gmap=['HG-MG Threshold', 'MG-LG Threshold', 'High gain', 'Medium gain', 'low gain'],
                          marker=['d','d','','','']
                          )
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 bad_pixel_aggregate_g = OrderedDict()
 for m, d in badpix_g.items():
     bad_pixel_aggregate_g[m] = d.astype(np.bool).astype(np.float)
 create_constant_overview(bad_pixel_aggregate_g, "Bad pixel fraction", max_cells, 0, 0.10, 3)
 ```
 
 %% Cell type:markdown id: tags:
 
 ## Summary tables ##
 
 The following tables show summary information for the evaluated module. Values for currently evaluated constants are compared with values for pre-existing constants retrieved from the calibration database.
 
 %% Cell type:code id: tags:
 
 ``` python
 table = []
 gain_names = ['High', 'Medium', 'Low']
 bits = [BadPixels.NOISE_OUT_OF_THRESHOLD, BadPixels.OFFSET_OUT_OF_THRESHOLD, BadPixels.OFFSET_NOISE_EVAL_ERROR, BadPixels.GAIN_THRESHOLDING_ERROR]
 for qm in badpix_g.keys():
     for gain in range(3):
 
         l_data = []
         l_data_old = []
 
         data = np.copy(badpix_g[qm][:,:,:,gain])
         datau32 = data.astype(np.uint32)
         l_data.append(len(datau32[datau32>0].flatten()))
         for bit in bits:
             l_data.append(np.count_nonzero(badpix_g[qm][:,:,:,gain] & bit.value))
 
         if old_const['BadPixelsDark'] is not None:
             dataold = np.copy(old_const['BadPixelsDark'][:, :, :, gain])
             datau32old = dataold.astype(np.uint32)
             l_data_old.append(len(datau32old[datau32old>0].flatten()))
             for bit in bits:
                 l_data_old.append(np.count_nonzero(old_const['BadPixelsDark'][:, :, :, gain] & bit.value))
 
         l_data_name = ['All bad pixels', 'NOISE_OUT_OF_THRESHOLD',
                        'OFFSET_OUT_OF_THRESHOLD', 'OFFSET_NOISE_EVAL_ERROR', 'GAIN_THRESHOLDING_ERROR']
 
         l_threshold = ['', f'{thresholds_noise_sigma}' f'{thresholds_noise_hard[gain]}',
                        f'{thresholds_offset_sigma}' f'{thresholds_offset_hard[gain]}',
                        '', f'{thresholds_gain_sigma}']
 
         for i in range(len(l_data)):
             line = [f'{l_data_name[i]}, {gain_names[gain]} gain', l_threshold[i], l_data[i]]
 
             if old_const['BadPixelsDark'] is not None:
                 line += [l_data_old[i]]
             else:
                 line += ['-']
 
             table.append(line)
         table.append(['', '', '', ''])
 
 display(Markdown('''
 
 ### Number of bad pixels ###
 
 One pixel can be bad for different reasons, therefore, the sum of all types of bad pixels can be more than the number of all bad pixels.
 
 '''))
 if len(table)>0:
     md = display(Latex(tabulate.tabulate(table, tablefmt='latex',
                                          headers=["Pixel type", "Threshold",
                                                   "New constant", "Old constant"])))
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 header = ['Parameter',
           "New constant", "Old constant ",
           "New constant", "Old constant ",
           "New constant", "Old constant ",
           "New constant", "Old constant "]
 
 for const in ['Offset', 'Noise', 'ThresholdsDark']:
     if const != 'ThresholdsDark':
         table = [['','High gain', 'High gain', 'Medium gain', 'Medium gain', 'Low gain', 'Low gain']]
     else:
         table = [['','HG-MG threshold', 'HG-MG threshold', 'MG-LG threshold', 'MG-LG threshold']]
     for qm in res.keys():
 
         data = np.copy(res[qm][const])
         if const == 'ThresholdsDark':
             data[...,0][res[qm]['BadPixelsDark'][...,0]>0] = np.nan
             data[...,1][res[qm]['BadPixelsDark'][...,1]>0] = np.nan
         else:
             data[res[qm]['BadPixelsDark']>0] = np.nan
 
         if old_const[const] is not None and old_const['BadPixelsDark'] is not None:
             dataold = np.copy(old_const[const])
             if const == 'ThresholdsDark':
                 dataold[...,0][old_const['BadPixelsDark'][...,0]>0] = np.nan
                 dataold[...,1][old_const['BadPixelsDark'][...,1]>0] = np.nan
             else:
                 dataold[old_const['BadPixelsDark']>0] = np.nan
 
         f_list = [np.nanmedian, np.nanmean, np.nanstd, np.nanmin, np.nanmax]
         n_list = ['Median', 'Mean', 'Std', 'Min', 'Max']
 
         for i, f in enumerate(f_list):
             line = [n_list[i]]
             for gain in range(3):
                 # Compare only 3 threshold gain-maps
                 if gain == 2 and const == 'ThresholdsDark':
                     continue
                 line.append('{:6.1f}'.format(f(data[...,gain])))
                 if old_const[const] is not None and old_const['BadPixelsDark'] is not None:
                     line.append('{:6.1f}'.format(f(dataold[...,gain])))
                 else:
                     line.append('-')
 
             table.append(line)
 
     display(Markdown('### {} [ADU], good pixels only ###'.format(const)))
     md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=header)))
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 ```

notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb

 %% Cell type:code id: tags:
 
 ``` python
 #Author: K. Ahmed, M. Karnevsky, Version: 0.1
 #The following is a summary for the processing of dark images and calibration constants production.
 
-cluster_profile = "noDB" # The ipcluster profile to use
-out_folder = "/gpfs/exfel/data/scratch/ahmedk/test/agipd-test-report" # path to output to, required
-karabo_id = "SPB_DET_AGIPDM-1" # detector instance
+out_folder = "/gpfs/exfel/data/scratch/ahmedk/test/miniHalfAGIPD" # path to output to, required
+karabo_id = "DETLAB_DET_AGIPD500K2G" # detector instance
 gain_names = ['High gain', 'Medium gain', 'Low gain'] # a list of gain names to be used in plotting
 threshold_names = ['HG-MG threshold', 'MG_LG threshold'] # a list of gain names to be used in plotting
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 from collections import OrderedDict
 import copy
 from datetime import datetime
 import os
 import warnings
 warnings.filterwarnings('ignore')
 
 import glob
 import h5py
 from IPython.display import display, Markdown, Latex
 import numpy as np
 import matplotlib
 matplotlib.use("agg")
 import matplotlib.gridspec as gridspec
 import matplotlib.patches as patches
 import matplotlib.pyplot as plt
 %matplotlib inline
 import tabulate
 from cal_tools.ana_tools import get_range
 from cal_tools.plotting import show_processed_modules
 import extra_geom
 from iCalibrationDB import Detectors
 from XFELDetAna.plotting.heatmap import heatmapPlot
 from XFELDetAna.plotting.simpleplot import simplePlot
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
-SHOW=True
 if "AGIPD" in karabo_id:
     if "SPB" in karabo_id:
         dinstance = "AGIPD1M1"
+        nmods = 16
+        qm = f"Q{i//4+1}M{i%4+1}"
     elif "MID" in karabo_id:
         dinstance = "AGIPD1M2"
+        nmods = 16
+    elif "HED" in karabo_id or "DETLAB" in karabo_id :
+        dinstance = "AGIPD500K"
+        nmods = 8
     display(Markdown("""
 
 # Summary of AGIPD dark characterization #
 
 The following report shows a set of dark images taken with the AGIPD detector to deduce detector offsets, noise, bad-pixel maps and thresholding. All four types of constants are evaluated per-pixel and per-memory cell.
 
 
 **The offset** ($O$) is defined as the median ($M$) of the dark signal ($Ds$) over trains ($t$) for a given pixel ($x,y$) and memory cell ($c$).
 
 **The noise** $N$ is the standard deviation $\sigma$ of the dark signal.
 
 $$ O_{x,y,c} = M(Ds)_{t} ,\,\,\,\,\,\, N_{x,y,c} = \sigma(Ds)_{t}$$
 
 **The bad pixel** mask is encoded as a bit mask.
 
 **"OFFSET_OUT_OF_THRESHOLD":**
 
 Offset outside of bounds:
 
 $$M(O)_{x,y} - \sigma(O)_{x,y} * \mathrm{thresholds\_offset\_sigma} < O < M(O)_{x,y} + \sigma(O)_{x,y} * \mathrm{thresholds\_offset\_sigma} $$
 
 or offset outside of hard limits
 
 $$ \mathrm{thresholds\_offset\_hard}_\mathrm{low} < O < \mathrm{thresholds\_offset\_hard}_\mathrm{high} $$
 
 **"NOISE_OUT_OF_THRESHOLD":**
 
 Noise outside of bounds:
 
 $$M(N)_{x,y} - \sigma(N)_{x,y} * \mathrm{thresholds\_noise\_sigma} < N < M(N)_{x,y} + \sigma(N)_{x,y} * \mathrm{thresholds\_noise\_sigma} $$
 
 or noise outside of hard limits
 
 $$\mathrm{thresholds\_noise\_hard}_\mathrm{low} < N < \mathrm{thresholds\_noise\_hard}_\mathrm{high} $$
 
 **"OFFSET_NOISE_EVAL_ERROR":**
 
 Offset and Noise both not $nan$ values
 
 Values: $\mathrm{thresholds\_offset\_sigma}$, $\mathrm{thresholds\_offset\_hard}$, $\mathrm{thresholds\_noise\_sigma}$, $\mathrm{thresholds\_noise\_hard}$ are given as parameters.
 
 "**\"GAIN_THRESHOLDING_ERROR\":**
 
 Bad gain separated pixels with sigma separation less than gain_separation_sigma_threshold
 
 $$ sigma\_separation = \\frac{\mathrm{gain\_offset} - \mathrm{previous\_gain\_offset}}{\sqrt{\mathrm{gain\_offset_{std}}^\mathrm{2} + \mathrm{previuos\_gain\_offset_{std}}^\mathrm{2}}}$$
 $$ Bad\_separation = sigma\_separation < \mathrm{gain\_separation\_sigma\_threshold} $$
 
 """))
 
 elif "LPD" in karabo_id:
     dinstance = "LPD1M1"
+    nmods = 16
     display(Markdown("""
 
 # Summary of LPD dark characterization #
 
 The following report shows a set of dark images taken with the LPD detector to deduce detector offsets, noise, bad-pixel maps. All three types of constants are evaluated per-pixel and per-memory cell.
 
 **The offset** ($O$) is defined as the median ($M$) of the dark signal ($Ds$) over trains ($t$) for a given pixel ($x,y$) and memory cell ($c$).
 
 **The noise** $N$ is the standard deviation $\sigma$ of the dark signal.
 
 $$ O_{x,y,c} = M(Ds)_{t} ,\,\,\,\,\,\, N_{x,y,c} = \sigma(Ds)_{t}$$
 
 **The bad pixel** mask is encoded as a bit mask.
 
 **"OFFSET_OUT_OF_THRESHOLD":**
 
 Offset outside of bounds:
 
 $$M(O)_{x,y} - \sigma(O)_{x,y} * \mathrm{thresholds\_offset\_sigma} < O < M(O)_{x,y} + \sigma(O)_{x,y} * \mathrm{thresholds\_offset\_sigma} $$
 
 or offset outside of hard limits
 
 $$ \mathrm{thresholds\_offset\_hard}_\mathrm{low} < O < \mathrm{thresholds\_offset\_hard}_\mathrm{high} $$
 
 **"NOISE_OUT_OF_THRESHOLD":**
 
 Noise outside of bounds:
 
 $$M(N)_{x,y} - \sigma(N)_{x,y} * \mathrm{thresholds\_noise\_sigma} < N < M(N)_{x,y} + \sigma(N)_{x,y} * \mathrm{thresholds\_noise\_sigma} $$
 
 or noise outside of hard limits
 
 $$\mathrm{thresholds\_noise\_hard}_\mathrm{low} < N < \mathrm{thresholds\_noise\_hard}_\mathrm{high} $$
 
 **"OFFSET_NOISE_EVAL_ERROR":**
 
 Offset and Noise both not $nan$ values
 
 "Values: $\\mathrm{thresholds\\_offset\\_sigma}$, $\\mathrm{thresholds\\_offset\\_hard}$, $\\mathrm{thresholds\\_noise\\_sigma}$, $\\mathrm{thresholds\\_noise\\_hard}$ are given as parameters.\n",
 """))
 elif "DSSC" in karabo_id:
     dinstance = "DSSC1M1"
+    nmods = 16
     display(Markdown("""
 
 # Summary of DSSC dark characterization #
 
     """))
 ```
 
 %% Cell type:markdown id: tags:
 
 Preparing newly injected and previous constants from produced local folder in out_folder.
 
 %% Cell type:code id: tags:
 
 ``` python
 # TODO: After changes in the Cal DB interface read files from cal repository
 # Load constants from local files
 data = OrderedDict()
 old_cons = OrderedDict()
 mod_names = []
 # Loop over modules
-for i in range(16):
+for i in range(nmods):
     qm = f"Q{i//4+1}M{i%4+1}"
     # loop over constants
     detinst = getattr(Detectors, dinstance)
     for const in ['Offset', 'Noise', 'ThresholdsDark', 'BadPixelsDark']:
         det = getattr(detinst, qm)
         if det is None:
             continue
         det_name = det.device_name
         fpath = '{}/const_{}_{}.h5'.format(out_folder, const, det_name)
         oldfpath = '{}/old/const_{}_{}.h5'.format(out_folder, const, det_name)
         if not os.path.isfile(fpath):
             continue
         with h5py.File(fpath, 'r') as f:
             if qm not in data:
                 mod_names.append(qm)
                 data[qm] = OrderedDict()
 
             data[qm][const] = f["data"][()]
 
         if not os.path.isfile(oldfpath):
             continue
 
         with h5py.File(oldfpath, 'r') as oldf:
             if qm not in old_cons:
                 old_cons[qm] = OrderedDict()
             old_cons[qm][const] = oldf["data"][()]
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 cons_shape = {}
 # extracting constant shape.
 for qm, constant in data.items():
     for cname, cons in constant.items():
         shape = data[qm][cname].shape
         if cname not in cons_shape:
             cons_shape[cname] = shape
     break
 
 constants = {}
 prev_const = {}
 
 for cname, sh in cons_shape.items():
     constants[cname]= np.zeros((len(mod_names),) + sh[:])
     prev_const[cname]= np.zeros((len(mod_names),) + sh[:])
 for i in range(len(mod_names)):
     for cname, cval in constants.items():
         cval[i] = data[mod_names[i]][cname]
         if mod_names[i] in old_cons.keys():
             prev_const[cname][i] = old_cons[mod_names[i]][cname]
         else:
             print(f"No previous {cname} found for {mod_names[i]}")
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
-display(Markdown('## Processed modules ##'))
-show_processed_modules(dinstance, constants, mod_names, mode="processed")
+# TODO: add show_processed_modules diagram for Mini-Half AGIPD
+if dinstance != "AGIPD500K":
+    display(Markdown('## Processed modules ##'))
+    show_processed_modules(dinstance, constants, mod_names, mode="processed")
 ```
 
 %% Cell type:markdown id: tags:
 
 ## Summary figures across Modules ##
 
 The following plots give an overview of calibration constants averaged across pixels and memory cells. A bad pixel mask is applied.
 
 %% Cell type:code id: tags:
 
 ``` python
 if "LPD" in dinstance:
     geom = extra_geom.LPD_1MGeometry.from_quad_positions(quad_pos=[(11.4, 299),
                                                                    (-11.5, 8),
                                                                    (254.5, -16),
                                                                    (278.5, 275)])
     pixels_y = 256
     pixels_x = 256
 
 elif "AGIPD" in dinstance:
     geom = extra_geom.AGIPD_1MGeometry.from_quad_positions(quad_pos=[(-525, 625),
                                                                      (-550, -10),
                                                                      (520, -160),
                                                                      (542.5, 475)])
     pixels_y = 128
     pixels_x = 512
 
 elif "DSSC" in dinstance:
     pixels_y = 512
     pixels_x = 128
     quadpos = [(-130, 5), (-130, -125), (5, -125), (5, 5)]
 
     extrageom_pth = os.path.dirname(extra_geom.__file__)
     geom = extra_geom.DSSC_1MGeometry.from_h5_file_and_quad_positions(
                 f"{extrageom_pth}/tests/dssc_geo_june19.h5", positions=quadpos)
 
 Mod_data=OrderedDict()
 gainstages = 1
 
 for const_name, const in constants.items():
+
+
+    #TODO: add a Summary figure across modules
+    if dinstance == "AGIPD500K":
+        display(Markdown(f'WARNING: No summary figures are available for AGIPD mini-half at the moment.'))
+        break
+
+
     if const_name == 'BadPixelsDark':
         continue
     # Check if constant gain available in constant e.g. AGIPD, LPD
     if len(const.shape) == 5:
         gainstages = 3
     else:
         gainstages = 1
 
     for dname in ['{}', 'd-{}', 'dpct-{}']:
         Mod_data[dname.format(const_name)] = OrderedDict()
 
     display(Markdown(f'##### {const_name}'))
     print_once = True
     for gain in range(gainstages):
         if const_name == 'ThresholdsDark':
             if gain > 1:
                 continue
             glabel = threshold_names
         else:
             glabel = gain_names
-        for i in range(16):
+        for i in range(nmods):
             qm = f"Q{i//4+1}M{i%4+1}"
             if qm in mod_names:
                 m_idx = mod_names.index(qm)
                 # Check if constant shape of 5 indices e.g. AGIPD, LPD
                 if len(const.shape) == 5:
                     values = np.nanmean(const[m_idx, :, :, :, gain], axis=2)
                     dval = np.nanmean(prev_const[const_name][m_idx, :, :, :, gain], axis=2)
                 else:
                     values = np.nanmean(const[m_idx, :, :, :], axis=2)
                     dval = np.nanmean(prev_const[const_name][m_idx, :, :, :], axis=2)
                 values[values == 0] = np.nan
                 dval[dval == 0] = np.nan
                 dval = values - dval
                 dval_pct = dval/values * 100
 
                 values = np.moveaxis(values, 0, -1).reshape(1, values.shape[1], values.shape[0])
                 dval = np.moveaxis(dval, 0, -1).reshape(1, dval.shape[1], dval.shape[0])
                 dval_pct = np.moveaxis(dval_pct, 0, -1).reshape(1, dval_pct.shape[1], dval_pct.shape[0])
             else:
                 # if module not available fill arrays with nan
                 values = np.zeros((1, pixels_x, pixels_y),dtype=np.float64)
                 values[values == 0] = np.nan
                 dval = values
                 dval_pct = dval
             for k, v in {'{}': values, 'd-{}': dval , 'dpct-{}': dval_pct}.items():
                 try:
                     Mod_data[k.format(const_name)][gain_names[gain]] = \
                             np.concatenate((Mod_data[k.format(const_name)][gain_names[gain]],
                                                                          v), axis=0)
                 except:
                     Mod_data[k.format(const_name)][gain_names[gain]] = v
         if np.count_nonzero(dval) == 0 and print_once:
             display(Markdown(f'New and previous {const_name} are the same, hence there is no difference.'))
             print_once = False
         # Plotting constant overall modules.
         display(Markdown(f'###### {glabel[gain]} ######'))
 
         gs = gridspec.GridSpec(2, 2)
         fig = plt.figure(figsize=(24, 32))
 
         axis = OrderedDict()
         axis = {"ax0": {"cname": "{}" ,"gs": gs[0, :], "shrink": 0.9, "pad": 0.05, "label": "ADUs", "title": '{}'},
                 "ax1": {"cname": "d-{}","gs": gs[1, 0], "shrink": 0.6, "pad": 0.1, "label": "ADUs", "title": 'Difference with previous {}'},
                 "ax2": {"cname": "dpct-{}", "gs": gs[1, 1], "shrink": 0.6, "pad": 0.1, "label": "%", "title": 'Difference with previous {} %'}}
 
         for ax, axv in axis.items():
             # Add the min and max plot values for each axis.
             vmin, vmax =  get_range(Mod_data[axv["cname"].format(const_name)][gain_names[gain]], 2)
             ax = fig.add_subplot(axv["gs"])
             geom.plot_data_fast(Mod_data[axv["cname"].format(const_name)][gain_names[gain]],
                                 vmin=vmin, vmax=vmax, ax=ax,
                                 colorbar={'shrink': axv["shrink"],
                                           'pad': axv["pad"]
                                          }
                                )
 
             colorbar = ax.images[0].colorbar
             colorbar.set_label(axv["label"])
 
             ax.set_title(axv["title"].format(f"{const_name} {glabel[gain]}"), fontsize=15)
             ax.set_xlabel('Columns', fontsize=15)
             ax.set_ylabel('Rows', fontsize=15)
 
         plt.show()
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 # Loop over modules and constants
 for const_name, const in constants.items():
     display(Markdown(f'### Summary across Modules - {const_name}'))
 
     for gain in range(gainstages):
         if const_name == 'ThresholdsDark':
             if gain == 2:
                 continue
             glabel = threshold_names
         else:
             glabel = gain_names
 
         if len(const.shape) == 5:
             data = np.copy(const[:, :, :, :, gain])
         else:
             data = np.copy(const[:, :, :, :])
         if const_name != 'BadPixelsDark':
             if "BadPixelsDark" in constants.keys():
                 label = f'{const_name} value [ADU], good pixels only'
                 if len(const.shape) == 5:
                     data[constants['BadPixelsDark'][:, :, :, :, gain] > 0] = np.nan
                 else:
                     data[constants['BadPixelsDark'][:, :, :, :] > 0] = np.nan
             else:
                 label = f'{const_name} value [ADU], good and bad pixels'
             datamean = np.nanmean(data, axis=(1, 2))
 
             fig = plt.figure(figsize=(15, 6), tight_layout={
                              'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})
             ax = fig.add_subplot(121)
         else:
             label = 'Fraction of bad pixels'
             data[data > 0] = 1.0
             datamean = np.nanmean(data, axis=(1, 2))
             datamean[datamean == 1.0] = np.nan
 
             fig = plt.figure(figsize=(15, 6), tight_layout={
                              'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})
             ax = fig.add_subplot(111)
 
         d = []
         for im, mod in enumerate(datamean):
             d.append({'x': np.arange(mod.shape[0]),
                       'y': mod,
                       'drawstyle': 'steps-pre',
                       'label': mod_names[im],
                       })
 
         _ = simplePlot(d, figsize=(10, 10), xrange=(-12, 510),
                             x_label='Memory Cell ID',
                             y_label=label,
                             use_axis=ax,
                             title='{}'.format(glabel[gain]),
                             title_position=[0.5, 1.18],
                             legend='outside-top-ncol6-frame', legend_size='18%',
                             legend_pad=0.00)
         if const_name != 'BadPixelsDark':
             ax = fig.add_subplot(122)
             if "BadPixelsDark" in constants.keys():
                 label = f'$\sigma$ {const_name} [ADU], good pixels only'
             else:
                 label = f'$\sigma$ {const_name} [ADU], good and bad pixels'
             d = []
             for im, mod in enumerate(np.nanstd(data, axis=(1, 2))):
                 d.append({'x': np.arange(mod.shape[0]),
                           'y': mod,
                           'drawstyle': 'steps-pre',
                           'label': mod_names[im],
                           })
 
             _ = simplePlot(d, figsize=(10, 10), xrange=(-12, 510),
                                 x_label='Memory Cell ID',
                                 y_label=label,
                                 use_axis=ax,
                                 title='{} $\sigma$'.format(glabel[gain]),
                                 title_position=[0.5, 1.18],
                                 legend='outside-top-ncol6-frame', legend_size='18%',
                                 legend_pad=0.00)
 
         plt.show()
 ```
 
 %% Cell type:markdown id: tags:
 
 ## Summary tables across Modules ##
 
 Tables show values averaged across all pixels and memory cells of a given detector module.
 
 %% Cell type:code id: tags:
 
 ``` python
 if u'$' in tabulate.LATEX_ESCAPE_RULES:
     del(tabulate.LATEX_ESCAPE_RULES[u'$'])
 
 if u'\\' in tabulate.LATEX_ESCAPE_RULES:
     del(tabulate.LATEX_ESCAPE_RULES[u'\\'])
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 head = ['Module', 'High gain', 'Medium gain', 'Low gain']
 head_th = ['Module', 'HG_MG threshold', 'MG_LG threshold']
 for const_name, const in constants.items():
     table = []
 
     for i_mod, mod in enumerate(mod_names):
 
         t_line = [mod]
         for gain in range(gainstages):
 
             if const_name == 'ThresholdsDark':
                 if gain == 2:
                     continue
                 header = head_th
             else:
                 header = head
             if len(const.shape) == 5:
                 data = np.copy(const[i_mod, :, :, :, gain])
             else:
                 data = np.copy(const[i_mod, :, :, :])
             if const_name == 'BadPixelsDark':
                 data[data > 0] = 1.0
 
                 datasum = np.nansum(data)
                 datamean = np.nanmean(data)
                 if datamean == 1.0:
                     datamean = np.nan
                     datasum = np.nan
 
                 t_line.append(f'{datasum:6.0f} ({datamean:6.3f}) ')
                 label = '## Number(fraction) of bad pixels'
             else:
                 if "BadPixelsDark" in constants.keys():
                     data[constants['BadPixelsDark']
                          [i_mod, :, :, :, gain] > 0] = np.nan
                     label = f'### Average {const_name} [ADU], good pixels only'
                 else:
                     label = f'### Average {const_name} [ADU], good and bad pixels'
 
                 t_line.append(f'{np.nanmean(data):6.1f} $\\pm$ {np.nanstd(data):6.1f}')
                 label = f'## Average {const_name} [ADU], good pixels only'
 
         table.append(t_line)
 
     display(Markdown(label))
     md = display(Latex(tabulate.tabulate(
         table, tablefmt='latex', headers=header)))
 ```
+
+%% Cell type:code id: tags:
+
+``` python
+```
+
+%% Cell type:code id: tags:
+
+``` python
+```