[Gotthard-II] Dark Processing.

Introducing Gotthard2 dark notebook for processing Offset, Noise and BadPixels Dark constants.

Description

Gotthard-II Dark processing for Offset, Noise, and BadPixelsDark (Shape (1280, 2, 3)) out of 3 dark runs.

• RAW data is first converted into 10bit using the LUT. Then Offset(mean) and noise(std) and BadpixelsDark(wrong gain) are computed for each cell in parallel for a single run.

• BadPixelsDark consists of OFFSET_NOISE_EVAL_ERROR, NOISE_OUT_OF_THRESHOLD, and WRONG_GAIN_VALUE.

How Has This Been Tested?

Injected constants to test CALCAT

Relevant Documents (optional)

Types of changes

• New feature (non-breaking change which adds functionality)

Checklist:

• My code follows the code style of this project.
• Update cal_db_interactive https://git.xfel.eu/detectors/cal_db_interactive/-/merge_requests/81
• Add CALCAT mapping for Gotthard2 (Done for test_calcat only for now.)

Reviewers

@mramilli @schmidtp @hammerd

notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb

 ``` python
-in_folder = "/gpfs/exfel/exp/DETLAB/202230/p900276/raw"  # the folder to read data from, required
+in_folder = "/gpfs/exfel/exp/FXE/202221/p003225/raw"  # the folder to read data from, required
 out_folder =  "/gpfs/exfel/data/scratch/ahmedk/test/gotthard2/darks"  # the folder to output to, required
-run_high = 10  # run number for G0 dark run, required
-run_med = 10  # run number for G1 dark run, required
-run_low = 10  # run number for G2 dark run, required
+run_high = 50  # run number for G0 dark run, required
+run_med = 51  # run number for G1 dark run, required
+run_low = 52  # run number for G2 dark run, required
 sequences = [-1]  # sequences to correct, set to [-1] for all, range allowed
 sequences_per_node = 1  # number of sequence files per node if notebook executed through xfel-calibrate, set to 0 to not run SLURM parallel
 
 # Parameters used to access raw data.
-karabo_id = "DET_LAB_G2"  # karabo prefix of Jungfrau devices
-karabo_da = ["DA01"]  # data aggregators
-receiver_template = "GOT{:02d}"  # receiver template used to read INSTRUMENT keys.
-control_template = "CTRL{:02d}"  # control template used to read CONTROL keys.
+karabo_id = "FXE_XAD_G2XES"  # karabo prefix of Jungfrau devices
+karabo_da = ["GH201"]  # data aggregators
+receiver_template = "RECEIVER"  # receiver template used to read INSTRUMENT keys.
+control_template = "CONTROL"  # control template used to read CONTROL keys.
 instrument_source_template = '{}/DET/{}:daqOutput'  # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput'
 ctrl_source_template = '{}/DET/{}'  # template for control source name (filled with karabo_id_control)
 karabo_id_control = ""  # if control is on a different ID, set to empty string if it is the same a karabo-id
 cal_db_timeout = 300000 # timeout on caldb requests
 db_output = False # Output constants to the calibration database
 local_output = True # Output constants locally
+constants_file = "/gpfs/exfel/data/scratch/ahmedk/dont_remove/gotthard2/constants/calibration_constants_GH2.h5"
 
 # Conditions used for injected calibration constants.
+bias_voltage = -1  # Detector bias voltage, set to -1 to use value in raw file.
+pulses = -1  # Detector number of pulses, set to -1 to use value in raw file.
+acquisition_rate = -1  # Detector acquisition rate, set to -1 to use value in raw file.
+integration_time = -1  # Detector integration time, set to -1 to use value in raw file.
+gain_setting = -1  # Detector gain setting, set to -1 to use value in raw file.
+gain_mode = -1  # Detector gain mode, set to -1 to use value in raw file.
 
 # Parameters used during selecting raw data trains.
 min_trains = 1 # Minimum number of trains that should be available to process dark constants. Default 1.
 max_trains = 1000  # Maximum number of trains to use for processing dark constants. Set to 0 to use all available trains.
+badpixel_threshold_sigma = 5.  # bad pixels defined by values outside n times this std from median
 
 # Don't delete! myMDC sends this by default.
 operation_mode = ''  # Detector operation mode, optional
 import matplotlib.pyplot as plt
 import multiprocessing
 import pasha as psh
+from IPython.display import Markdown, display
 from extra_data import RunDirectory
 from pathlib import Path
 
     save_const_to_h5,
     send_to_db,
 )
 from iCalibrationDB import Conditions, Constants
+from XFELDetAna.plotting.heatmap import heatmapPlot
 %% Cell type:code id:18fe4379 tags:
 ``` python
-# load constants temporarely using defined local paths.
-
-constants_file = "/gpfs/exfel/data/user/mramilli/gotthard2/constants/GH2-0124/calibration_constants_GH2-0124.h5"
+# load constants temporarily using defined local paths.
 
 with h5py.File(constants_file, 'r') as cfile:
     lut = cfile["LUT"][()]
 %% Cell type:code id:ac9c5dc3-bc66-4e7e-b6a1-360259be535c tags:
 ``` python
 def convert_train(wid, index, tid, d):
     """Convert a Gotthard2 train from 12bit to 10bit."""
-    d_10bit = np.zeros_like(d[instr_mod_src]["data.adc"], dtype=np.float32)
-    convert_to_10bit(d[instr_mod_src]["data.adc"], lut, d_10bit)
-    data_10bit[index, ...] = d_10bit
+    convert_to_10bit(d[instr_mod_src]["data.adc"], lut, data_10bit[index, ...])
 %% Cell type:code id:4e8ffeae tags:
 ``` python
 # Calculate noise and offset per pixel and global average, std and median
 noise_map = dict()
     data_path = "INSTRUMENT/"+instr_mod_src+"/data"
 
     # TODO: Validate the final shape to store constants.
-    cshape = (3, 2, 1280)
-    offset_map[mod] = np.zeros(cshape, dtype=np.float32)
-    noise_map[mod] = np.zeros_like(offset_map[mod])
-    badpixels_map[mod] = np.zeros_like(offset_map[mod], dtype=np.uint32)
+    cshape = (1280, 2, 3)
+
+    offset_map[mod] = context.alloc(shape=cshape, dtype=np.float32)
+    noise_map[mod] = context.alloc(like=offset_map[mod])
+    badpixels_map[mod] = context.alloc(like=offset_map[mod], dtype=np.uint32)
 
     for run_num, [gain, run_dc] in run_dcs_dict.items():
         step_timer.start()
         step_timer.start()
 
         # Split even and odd data to calculate the two storage cell constants.
+        # Detector always operates in burst mode.
         even_data = data_10bit[:, ::2, :]
         odd_data = data_10bit[:, 1::2, :]
         data_gain = dc[instr_mod_src, "data.gain"].ndarray()
         even_gain = data_gain[:, ::2, :]
         odd_gain = data_gain[:, 1::2, :]
-
         def offset_noise_cell(wid, index, d):
-            offset_map[mod][gain, index, ...] = np.mean(d, axis=(0, 1))
-            noise_map[mod][gain, index, ...] = np.std(d, axis=(0, 1))
+            offset_map[mod][:, index, gain] = np.mean(d, axis=(0, 1))
+            noise_map[mod][:, index, gain] = np.std(d, axis=(0, 1))
 
-        offset_map[mod] = context.alloc(shape=cshape, dtype=np.float32)
-        noise_map[mod] = context.alloc(like=offset_map[mod])
-        badpixels_map[mod] = context.alloc(like=offset_map[mod], dtype=np.int32)
         context.map(offset_noise_cell, (even_data, odd_data))
 
         raw_g = 3 if gain == 2 else gain
-
         def badpixels_cell(wid, index, g):
             """Check if there are wrong bad gain values.
             Indicate pixels with wrong gain value across all trains for each cell."""
-            badpixels_map[mod][gain, index,
-                np.mean(g, axis=(0, 1)) != raw_g] |= BadPixels.WRONG_GAIN_VALUE.value
+            badpixels_map[mod][np.mean(g, axis=(0, 1)) != raw_g, index,
+                gain] |= BadPixels.WRONG_GAIN_VALUE.value
         context.map(badpixels_cell, (even_gain, odd_gain))
 
         step_timer.done_step("Processing darks")
-%% Output
-    DA01 has 1042 trains with empty frames out of 2041 trains
-    Processing 999 trains.
-    preparing raw data: 0.0 s
-    convert to 10bit: 4.3 s
-    Processing darks: 12.1 s
-%% Cell type:code id:4b81a3e7 tags:
-``` python
-badpixels_map[mod].shape
-%% Output
-    (3, 2, 1280)
-%% Cell type:code id:7ec0e569 tags:
-``` python
-np.where(badpixels_map[mod] != 0])
-%% Output
-    array([], dtype=uint8)
 %% Cell type:code id:3fc17e05-17ab-4ac4-9e79-c95399278bb9 tags:
 ``` python
 def print_bp_entry(bp):
-    print("{:<30s} {:032b} -> {}".format(bp.name, bp.value, int(bp.value)))
+    print(f"{bp.name:<30s} {bp.value:032b} -> {int(bp.value)}")
 
-print_bp_entry(BadPixels.OFFSET_OUT_OF_THRESHOLD)
 print_bp_entry(BadPixels.NOISE_OUT_OF_THRESHOLD)
 print_bp_entry(BadPixels.OFFSET_NOISE_EVAL_ERROR)
 print_bp_entry(BadPixels.WRONG_GAIN_VALUE)
 
 def eval_bpidx(d):
-
-    mdn = np.nanmedian(d, axis=(0, 1))[None, None, :, :]
-    std = np.nanstd(d, axis=(0, 1))[None, None, :, :]
+    mdn = np.nanmedian(d, axis=(0))[None, :, :]
+    std = np.nanstd(d, axis=(0))[None, :, :]
     idx = (d > badpixel_threshold_sigma*std+mdn) | (d < (-badpixel_threshold_sigma)*std+mdn)
 
     return idx
+%% Cell type:code id:9409d10f tags:
+``` python
+badpixels_map[mod][~np.isfinite(noise_map[mod])]
 %% Cell type:code id:40c34cc5-fe93-4b83-bf39-f465f37c40b4 tags:
 ``` python
 step_timer.start()
+g_name = ['G0', 'G1', 'G2']
 
 for mod in karabo_da:
     display(Markdown(f"### Badpixels for module {mod}:"))
-    offset_abs_threshold = np.array(offset_abs_threshold)
 
-    bad_pixels_map[mod][eval_bpidx(offset_map[mod])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value
+    badpixels_map[mod][~np.isfinite(offset_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value
+    badpixels_map[mod][eval_bpidx(noise_map[mod])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value
+
+    badpixels_map[mod][~np.isfinite(noise_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value
 
-    bad_pixels_map[mod][~np.isfinite(offset_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value
 
-    bad_pixels_map[mod][eval_bpidx(noise_map[mod])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value
+    for cell in [0, 1]:
+        fig, ax = plt.subplots(figsize=(10, 5))
+        for g_idx in [0, 1, 2]:
+            ax.plot(badpixels_map[mod][:, cell, g_idx], label=f"G{g_idx} Bad pixel map")
+        ax.set_xticks(np.arange(0, 1281, 80))
+        ax.set_xlabel("Stripes #")
+        ax.set_xlabel("BadPixels")
+        ax.set_title(f'Cell {cell} - Module {mod}')
+        ax.set_ylim([0, BadPixels.WRONG_GAIN_VALUE.value])
+        ax.set_yticks([0,
+            BadPixels.NOISE_OUT_OF_THRESHOLD.value,
+            BadPixels.OFFSET_NOISE_EVAL_ERROR.value,
+            BadPixels.WRONG_GAIN_VALUE.value])
+        ax.legend()
+        pass
+step_timer.done_step(f'Creating bad pixels constant and plotting it.')
+%% Cell type:code id:e5131112 tags:
+``` python
+step_timer.start()
+g_name = ['G0', 'G1', 'G2']
+
+for mod in karabo_da:
+    display(Markdown(f"### Badpixels for module {mod}:"))
 
-    bad_pixels_map[mod][~np.isfinite(noise_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value
+    badpixels_map[mod][~np.isfinite(offset_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value
+    badpixels_map[mod][eval_bpidx(noise_map[mod])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value
+    badpixels_map[mod][~np.isfinite(noise_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value
 
-    bad_pixels_map[mod][(offset_map[mod] < offset_abs_threshold[0][None, None, None, :]) | (offset_map[mod] > offset_abs_threshold[1][None, None, None, :])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value  # noqa
+    for g_idx in [0, 1, 2]:
+       # for cell in [0, 1]:
+        heatmapPlot(
+                badpixels_map[mod][:, :, g_idx],
+                y_label="Stripes",
+                x_label="Cells",
+#                lut_label=unit,
 
-    for g_idx in gains:
-        for cell in range(memory_cells):
-            bad_pixels = bad_pixels_map[mod][:, :, cell, g_idx]
-            fn_0 = heatmapPlot(
-                np.swapaxes(bad_pixels, 0, 1),
-                y_label="Row",
-                x_label="Column",
-                lut_label=f"Badpixels {g_name[g_idx]} [ADCu]",
-                aspect=1.,
                 vmin=0, vmax=5,
-                title=f'G{g_idx} Bad pixel map - Cell {cell:02d} - Module {mod}')
+                title=f"Even / Odd Offset map G{g_idx} - Module {mod}", # TODO: add PDU name({pdu})',
+        )
+        pass
 step_timer.done_step(f'Creating bad pixels constant and plotting it.')
 ``` python
-# # set the operating condition
-# # TODO: add the final conditions for constants.
-# condition = Conditions.Dark.Gotthard2(
-#     bias_voltage=bias_voltage,
-# )
-
-# db_modules = get_pdu_from_db(
-#     karabo_id=karabo_id,
-#     karabo_da=karabo_da,
-#     constant=Constants.Gotthard2.Offset(),
-#     condition=condition,
-#     cal_db_interface=cal_db_interface,
-#     snapshot_at=creation_time)
+# set the operating condition
+# TODO: add the final conditions for constants.
+condition = Conditions.Dark.Gotthard2(
+    bias_voltage=bias_voltage,
+    pulses=pulses,
+    integration_time=integration_time,
+    acquisition_rate=acquisition_rate,
+    gain_setting=gain_setting,
+    gain_mode=gain_mode,
+)
+
+db_modules = get_pdu_from_db(
+    karabo_id=karabo_id,
+    karabo_da=karabo_da,
+    constant=Constants.Gotthard2.Offset(),
+    condition=condition,
+    cal_db_interface=cal_db_interface,
+    snapshot_at=creation_time)
 %% Cell type:code id:fde8e1cf-bc74-462f-b6e5-cfee8279090d tags:
 for mod in karabo_da:
     for _, [gain, _] in run_dcs_dict.items():
         heatmapPlot(
-            offset_map[mod][gain],
-            y_label="Row",
-            x_label="Column",
+            offset_map[mod][:, :, gain],
+            y_label="Stripes",
+            x_label="Cells",
             lut_label=unit,
+            x_ticks=[0.5, 1.5],
+            x_ticklabels=["EVEN", "ODD"],
+            x_tick_rotation=0,
             title=f"Even / Odd Offset map G{gain} - Module {mod}", # TODO: add PDU name({pdu})',
         )
-        plt.show()
         heatmapPlot(
-            noise_map[mod][gain],
-            y_label="Row",
-            x_label="Column",
+            noise_map[mod][:, :, gain],
+            y_label="Stripes",
+            x_label="Cells",
             lut_label=unit,
+            x_ticks=[0.5, 1.5],
+            x_ticklabels=["EVEN", "ODD"],
+            x_tick_rotation=0,
             title=f"Even / Odd noise map G{gain} - Module {mod}", # TODO: add PDU name({pdu})',
         )
-        plt.show()
+        pass
+%% Cell type:code id:c8777cfe tags:
+``` python
+for mod in karabo_da:
+    for cons, cname in zip([offset_map, noise_map], ["Offset", "Noise"]):
+        for cell in [0, 1]:
+            fig, ax = plt.subplots(figsize=(10, 5))
+            for g_idx in [0, 1, 2]:
+                ax.plot(cons[mod][:, cell, g_idx], label=f"G{g_idx} {cname} map")
+
+            ax.set_xlabel("Stripes #")
+            ax.set_xlabel(cname)
+            ax.set_title(f"{cname} map - Cell {cell} - Module {mod}"), # TODO: add PDU name({pdu})'
+            ax.legend()
+            pass
 %% Cell type:code id:1c4eddf7-7d6e-49f4-8cbb-12d2bc496a8f tags:
 # TODO: add the final conditions for constants.
 print(
     f"• Bias voltage: {bias_voltage}\n"
+    f"• Pulses: {pulses}\n"
+    f"• Integration time: {integration_time}\n"
+    f"• Acquisition rate: {acquisition_rate}\n"
+    f"• Gain setting: {gain_setting}\n"
+    f"• Gain mode: {gain_mode}\n"
     f"• Creation time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\n")  # noqa
 step_timer.done_step("Injecting constants.")