[Gotthard-II] Dark Processing.

Introducing Gotthard2 dark notebook for processing Offset, Noise and BadPixels Dark constants.

Description

Gotthard-II Dark processing for Offset, Noise, and BadPixelsDark (Shape (1280, 2, 3)) out of 3 dark runs.

• RAW data is first converted into 10bit using the LUT. Then Offset(mean) and noise(std) and BadpixelsDark(wrong gain) are computed for each cell in parallel for a single run.

• BadPixelsDark consists of OFFSET_NOISE_EVAL_ERROR, NOISE_OUT_OF_THRESHOLD, and WRONG_GAIN_VALUE.

How Has This Been Tested?

Injected constants to test CALCAT

Relevant Documents (optional)

Types of changes

• New feature (non-breaking change which adds functionality)

Checklist:

• My code follows the code style of this project.
• Update cal_db_interactive https://git.xfel.eu/detectors/cal_db_interactive/-/merge_requests/81
• Add CALCAT mapping for Gotthard2 (Done for test_calcat only for now.)

Reviewers

@mramilli @schmidtp @hammerd

notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb

 
 # Parameters for the calibration database.
 use_dir_creation_date = True
-cal_db_interface = "tcp://max-exfl016:8020" # calibration DB interface to use
+cal_db_interface = "tcp://max-exfl017:8020" # calibration DB interface to use
 cal_db_timeout = 300000 # timeout on caldb requests
 db_output = False # Output constants to the calibration database
 local_output = True # Output constants locally
-constants_file = "/gpfs/exfel/data/scratch/ahmedk/dont_remove/gotthard2/constants/calibration_constants_GH2.h5"
 
 # Conditions used for injected calibration constants.
 bias_voltage = -1  # Detector bias voltage, set to -1 to use value in raw file.
 badpixel_threshold_sigma = 5.  # bad pixels defined by values outside n times this std from median
 
 # Don't delete! myMDC sends this by default.
-operation_mode = ''  # Detector operation mode, optional
+operation_mode = ''  # Detector dark run acquiring operation mode, optional
 %% Cell type:code id:8085f9aa tags:
 ``` python
-import h5py
 import numpy as np
 import matplotlib.pyplot as plt
 import multiprocessing
 from cal_tools.tools import (
     get_dir_creation_date,
     get_from_db,
+    get_constant_from_db_and_time,
     get_pdu_from_db,
     get_report,
     save_const_to_h5,
 if use_dir_creation_date:
     creation_time = get_dir_creation_date(in_folder, run_high)
     print(f"Using {creation_time.isoformat()} as creation time")
-
+# TODO: Remove later
+import datetime
+creation_time=datetime.datetime.strptime(
+    "2022-06-28 13:00:00.00",
+    "%Y-%m-%d %H:%M:%S.%f")
 if not karabo_id_control:
     karabo_id_control = karabo_id
 step_timer = StepTimer()
-%% Cell type:code id:fb80b98e tags:
-``` python
-# load constants temporarily using defined local paths.
-
-with h5py.File(constants_file, 'r') as cfile:
-    lut = cfile["LUT"][()]
 %% Cell type:code id:ff9149fc tags:
 ``` python
-# Read parameter conditions
+# Read parameter conditions and validate the values for the three runs.
 
 step_timer.start()
 run_dcs_dict = dict()
 
 ctrl_src = ctrl_source_template.format(karabo_id_control, control_template)
 conditions = {
-    "bias_voltage": [],
-    "exposure_time": [],
-    "exposure_period": [],
-    "operation_mode": [],
-    "single_photon": [],
+    "bias_voltage": set(),
+    "exposure_time": set(),
+    "exposure_period": set(),
+    #"operation_mode": set(),
+    #"single_photon": set(),
 }
 for gain, run in enumerate(run_nums):
     run_dc = RunDirectory(in_folder / f"r{run:04d}/")
     run_dcs_dict[run] = [gain, run_dc]
 
-#     # TODO: Read every condition from slow data.
-#     # Read slow data.
-#     g2ctrl = gotthard2lib.Gotthard2Ctrl(
-#         run_dc=run_dc, ctrl_src=ctrl_src)
-
-#     conditions["bias_voltage"].append(g2ctrl.get_bias_voltage())
-#     conditions["exposure_time"].append(g2ctrl.get_exposure_time())
-#     conditions["exposure_period"].append(g2ctrl.get_exposure_period())
-#     conditions["single_photon"].append(g2ctrl.get_single_photon())
-#     conditions["operation_mode"].append(g2ctrl.get_operation_mode())
-
-# # TODO: Validate that the conditions are the same and as expected for all runs.
-# for c, v in conditions.keys():
-#     assert len(set(v)) == 1, f"{c} value is not the same for the three runs."
+    # Read slow data.
+    g2ctrl = gotthard2lib.Gotthard2Ctrl(
+        run_dc=run_dc, ctrl_src=ctrl_src)
+    conditions["bias_voltage"].add(g2ctrl.get_bias_voltage() if bias_voltage == -1 else bias_voltage)  # noqa
+    conditions["exposure_time"].add(g2ctrl.get_exposure_time() if exposure_time == -1 else exposure_time)  # noqa
+    conditions["exposure_period"].add(g2ctrl.get_exposure_period() if exposure_period == -1 else exposure_period)  # noqa
+#   conditions["single_photon"].add(g2ctrl.get_single_photon() if single_photon == -1 else single_photon)  # noqa
+#   conditions["operation_mode"].add(g2ctrl.get_operation_mode() if operation_mode == -1 else operation_mode)  # noqa
+
+for c, v in conditions.items():
+    assert len(v) == 1, f"{c} value is not the same for the three runs."
+
+bias_voltage = conditions["bias_voltage"].pop()
+print("Bias voltage: ", bias_voltage)
+exposure_time = conditions["exposure_time"].pop()
+print("Exposure time: ", exposure_time)
+exposure_period = conditions["exposure_period"].pop()
+print("Exposure period: ", exposure_period)
+# single_photon = conditions["single_photon"].pop()
+# operation_mode = conditions["operation_mode"].pop()
 %% Cell type:code id:ac9c5dc3-bc66-4e7e-b6a1-360259be535c tags:
     return n_trains
+%% Cell type:code id:3c59c11d tags:
+``` python
+# set the operating condition
+# TODO: add the final conditions for constants.
+condition = Conditions.Dark.Gotthard2(
+    bias_voltage=bias_voltage,
+    exposure_time=exposure_time,
+    exposure_period=exposure_period,
+    #operation_mode=operation_mode,
+    #single_photon=single_photon,
+)
+
+db_modules = get_pdu_from_db(
+    karabo_id=karabo_id,
+    karabo_da=karabo_da,
+    constant=Constants.Gotthard2.LUT(),
+    condition=condition,
+    cal_db_interface=cal_db_interface,
+    snapshot_at=creation_time)
 %% Cell type:code id:e2eb2fc0-df9c-4887-9691-f81474f8c131 tags:
 context = psh.context.ProcessContext(num_workers=multiprocessing.cpu_count())
 
 for mod in karabo_da:
+
+    # Retrieve LUT constant
+    lut, time = get_constant_from_db_and_time(
+        constant=Constants.Gotthard2.LUT(),
+        condition=condition,
+        empty_constant=None,
+        karabo_id=karabo_id,
+        karabo_da=mod,
+        cal_db_interface=cal_db_interface,
+        creation_time=creation_time,
+        timeout=cal_db_timeout,
+        print_once=False,
+    )
+
+
     # Path to pixels ADC values
     instr_mod_src = instrument_src.format(int(mod[-2:]))
-    data_path = "INSTRUMENT/"+instr_mod_src+"/data"
 
     # TODO: Validate the final shape to store constants.
     cshape = (1280, 2, 3)
 step_timer.done_step(f'Creating bad pixels constant and plotting it.')
-%% Cell type:code id:6e10ed93-66de-4fb1-bf97-f8d25af22edb tags:
-``` python
-# set the operating condition
-# TODO: add the final conditions for constants.
-condition = Conditions.Dark.Gotthard2(
-    bias_voltage=bias_voltage,
-    exposure_time=exposure_time,
-    exposure_period=exposure_period,
-    operation_mode=operation_mode,
-    single_photon=single_photon,
-)
-
-db_modules = get_pdu_from_db(
-    karabo_id=karabo_id,
-    karabo_da=karabo_da,
-    constant=Constants.Gotthard2.Offset(),
-    condition=condition,
-    cal_db_interface=cal_db_interface,
-    snapshot_at=creation_time)
 %% Cell type:code id:fde8e1cf-bc74-462f-b6e5-cfee8279090d tags: