[GH2][CORRECT][DARK] Feat/add support for gh2 25um

Gotthard-II 25um Correction and Dark

Description

This Merge request is related to : https://git.xfel.eu/calibration/planning/-/issues/164

How Has This Been Tested?

• Tested against reference runs for GH2 50um.
• Tested against the production database for GH2 25um.

Relevant Documents (optional)

correct_gh2_25um.pdf

gh2_dark_25um.pdf

Correcting:

Into:

Types of changes

• New feature (non-breaking change which adds functionality)
• Bug fix (non-breaking change which fixes an issue)

Checklist:

• Follow up on learned lessons from LPDMini and propagate it here.
• Update the MR's history.
• Get Feedback on results from @mramilli

Reviewers

@schmidtp @hammerd

notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb

 
 Author: European XFEL Detector Group, Version: 1.0
 
-The following is a processing for offset, noise, and Badpixels maps using dark images taken with Gotthard2 detector.
+The following is a processing for the dark constants (`Offset`, `Noise`, and `BadPixelsDark`) maps using dark images taken with Gotthard2 detector (GH2 50um or 25um).
 All constants are evaluated per strip, per pulse, and per memory cell. The maps are calculated for each gain stage that is acquired in 3 separate runs.
 
-The three maps (calibration constants) can be injected to the database and stored locally.
+The three maps are of shape (stripes, cells, gains): (1280, 2, 3). They can be injected to the database (`db_output`) and/or stored locally (`local_output`).
 %% Cell type:code id:818e24e8 tags:
 karabo_da = ["GH201"]  # data aggregators
 receiver_template = "RECEIVER"  # receiver template used to read INSTRUMENT keys.
 control_template = "CONTROL"  # control template used to read CONTROL keys.
-instrument_source_template = '{}/DET/{}:daqOutput'  # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput'  # noqa
 ctrl_source_template = '{}/DET/{}'  # template for control source name (filled with karabo_id_control)
 karabo_id_control = ""  # Control karabo ID. Set to empty string to use the karabo-id
 
 # Parameters for the calibration database.
-use_dir_creation_date = True
 cal_db_interface = "tcp://max-exfl-cal001:8020"  # calibration DB interface to use
 cal_db_timeout = 300000  # timeout on caldb requests
-overwrite_creation_time = ""  # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. "2022-06-28 13:00:00.00"
+creation_time = ""  # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. "2022-06-28 13:00:00"
 db_output = False  # Output constants to the calibration database
 local_output = True  # Output constants locally
 
 ``` python
-import datetime
 import numpy as np
 import matplotlib.pyplot as plt
 import pasha as psh
 from extra_data import RunDirectory
 from pathlib import Path
 
+from cal_tools.calcat_interface import CalCatApi
 from cal_tools.enums import BadPixels
 from cal_tools.gotthard2 import gotthard2algs, gotthard2lib
 from cal_tools.step_timing import StepTimer
+from cal_tools.restful_config import calibration_client
 from cal_tools.tools import (
-    get_dir_creation_date,
+    calcat_creation_time,
     get_constant_from_db_and_time,
     get_pdu_from_db,
     get_report,
     save_const_to_h5,
     send_to_db,
 )
+
 from iCalibrationDB import Conditions, Constants
 
 %matplotlib inline
 run_nums = [run_high, run_med, run_low]
 in_folder = Path(in_folder)
 out_folder = Path(out_folder)
-out_folder.mkdir(exist_ok=True)
-
-print(f"Process modules: {karabo_da}")
-
-run_dc = RunDirectory(in_folder / f"r{run_high:04d}")
-file_loc = f"proposal:{run_dc.run_metadata()['proposalNumber']} runs:{run_high} {run_med} {run_low}"  # noqa
+out_folder.mkdir(parents=True, exist_ok=True)
 
-instrument_src = instrument_source_template.format(karabo_id, receiver_template)
 ctrl_src = ctrl_source_template.format(karabo_id_control, control_template)
 
 # Read report path to associate it later with injected constants.
 report = get_report(out_folder)
 
-
-if overwrite_creation_time:
-    creation_time = datetime.datetime.strptime(
-        overwrite_creation_time, "%Y-%m-%d %H:%M:%S.%f"
-    )
-elif use_dir_creation_date:
-    creation_time = get_dir_creation_date(in_folder, run_high)
-    print(f"Using {creation_time.isoformat()} as creation time")
-
+# Run's creation time:
+creation_time = calcat_creation_time(in_folder, run_high, creation_time)
+print(f"Creation time: {creation_time}")
 
 if not karabo_id_control:
     karabo_id_control = karabo_id
+%% Cell type:code id:c176a86f tags:
+``` python
+run_dc = RunDirectory(in_folder / f"r{run_high:04d}")
+file_loc = f"proposal:{run_dc.run_metadata()['proposalNumber']} runs:{run_high} {run_med} {run_low}"  # noqa
+
+receivers = list(run_dc.select(f'{karabo_id}/DET/{receiver_template}*').all_sources)
 %% Cell type:code id:108be688 tags:
 print("Acquisition rate: ", acquisition_rate)
+%% Cell type:code id:f64bc150-cfcd-4f98-83f9-a982fdacedd7 tags:
+``` python
+# Decide if GH2 is 25um or 50um
+gh2_hostname = run_dc.get_run_value(ctrl_src, "rxHostname")
+# gh2_hostname is a vector of bytes objects.
+# GH2 25um has two host-names unlike 50um which has one.
+
+calcat = CalCatApi(client=calibration_client())
+detector_id = calcat.detector(karabo_id)['id']
+pdus_by_da = calcat.physical_detector_units(detector_id, pdu_snapshot_at=creation_time)
+da_to_pdu = {da: p['physical_name'] for (da, p) in pdus_by_da.items()}
+
+if karabo_da != [""]:
+    # Filter DA connected to detector in CALCAT
+    karabo_da = [da for da in karabo_da if da in da_to_pdu]
+    # Exclude non selected DA from processing.
+    da_to_pdu = {da: da_to_pdu[da] for da in karabo_da}
+else:
+    karabo_da = sorted(da_to_pdu.keys())
+
+if gh2_hostname[1].decode("utf-8"):
+    print("Data is for 25um Gotthard2.")
+
+print(f"Processing {karabo_da}")
 %% Cell type:code id:ac9c5dc3-bc66-4e7e-b6a1-360259be535c tags:
 ``` python
 def specify_trains_to_process(
-    img_key_data: "extra_data.KeyData",  # noqa
-    max_trains: int = 0,
-    min_trains: int = 0,
+    img_key_data: "extra_data.KeyData",
 ):
     """Specify total number of trains to process.
     Based on given min_trains and max_trains, if given.
     Print number of trains to process and number of empty trains.
     Raise ValueError if specified trains are less than min_trains.
     """
-    # Specifies total number of trains to proccess.
+    # Specifies total number of trains to process.
     n_trains = img_key_data.shape[0]
     all_trains = len(img_key_data.train_ids)
     print(
-        f"{mod} has {all_trains - n_trains} "
+        f"{receiver} has {all_trains - n_trains} "
         f"trains with empty frames out of {all_trains} trains"
     )
 
     exposure_period=exposure_period,
     acquisition_rate=acquisition_rate,
     single_photon=single_photon,
 )
-
-db_modules = get_pdu_from_db(
-    karabo_id=karabo_id,
-    karabo_da=karabo_da,
-    constant=Constants.Gotthard2.LUT(),
-    condition=condition,
-    cal_db_interface=cal_db_interface,
-    snapshot_at=creation_time,
-)
 def convert_train(wid, index, tid, d):
     """Convert a Gotthard2 train from 12bit to 10bit."""
     gotthard2algs.convert_to_10bit(
-        d[instr_mod_src]["data.adc"], lut, data_10bit[index, ...]
+        d[receiver]["data.adc"], lut, data_10bit[index, ...]
     )
     np.uint16
 )
 empty_lut = np.stack(1280 * [np.stack([empty_lut] * 2)], axis=0)
-for mod in karabo_da:
+for mod, receiver in zip(karabo_da, receivers):
 
     # Retrieve LUT constant
     lut, time = get_constant_from_db_and_time(
         print_once=False,
     )
     print(f"Retrieved LUT constant with creation-time {time}")
-    # Path to pixels ADC values
-    instr_mod_src = instrument_src.format(int(mod[-2:]))
 
-    # TODO: Validate the final shape to store constants.
     cshape = (1280, 2, 3)
 
     offset_map[mod] = context.alloc(shape=cshape, dtype=np.float32)
 
     for run_num, [gain, run_dc] in run_dcs_dict.items():
         step_timer.start()
-        n_trains = specify_trains_to_process(run_dc[instr_mod_src, "data.adc"])
+        n_trains = specify_trains_to_process(run_dc[receiver, "data.adc"])
 
         # Select requested number of trains to process.
-        dc = run_dc.select(instr_mod_src, require_all=True).select_trains(
+        dc = run_dc.select(receiver, require_all=True).select_trains(
             np.s_[:n_trains]
         )
 
         step_timer.start()
         # Convert 12bit data to 10bit
         data_10bit = context.alloc(
-            shape=dc[instr_mod_src, "data.adc"].shape, dtype=np.float32
+            shape=dc[receiver, "data.adc"].shape, dtype=np.float32
         )
         context.map(convert_train, dc)
         step_timer.done_step("convert to 10bit")
         context.map(offset_noise_cell, (even_data, odd_data))
 
         # Split even and odd gain data.
-        data_gain = dc[instr_mod_src, "data.gain"].ndarray()
+        data_gain = dc[receiver, "data.gain"].ndarray()
         even_gain = data_gain[:, 20::2, :]
         odd_gain = data_gain[:, 21::2, :]
         raw_g = 3 if gain == 2 else gain
 step_timer.start()
 g_name = ["G0", "G1", "G2"]
 
-for mod, pdu in zip(karabo_da, db_modules):
+for mod, pdu in da_to_pdu.items():
     display(Markdown(f"### Badpixels for module {mod}:"))
 
     badpixels_map[mod][
         ax.set_xticks(np.arange(0, 1281, 80))
         ax.set_xlabel("Stripes #")
         ax.set_xlabel("BadPixels")
-        ax.set_title(f"Cell {cell} - Module {mod} ({pdu})")
+        ax.set_title(f"BadPixels map - Cell {cell} - Module {mod} ({pdu})")
         ax.set_ylim([0, 5])
         ax.legend()
-        pass
+        plt.show()
 step_timer.done_step(f"Creating bad pixels constant and plotting it.")
 ``` python
-for mod, pdu in zip(karabo_da, db_modules):
-    for cons, cname in zip([offset_map, noise_map], ["Offset", "Noise"]):
+for cons, cname in zip([offset_map, noise_map], ["Offset", "Noise"]):
+    for mod, pdu in da_to_pdu.items():
+        display(Markdown(f"### {cname} for module {mod}:"))
+
         for cell in [0, 1]:
             fig, ax = plt.subplots(figsize=(10, 5))
             for g_idx in [0, 1, 2]:
             ax.set_xlabel(cname)
             ax.set_title(f"{cname} map - Cell {cell} - Module {mod} ({pdu})")
             ax.legend()
-            pass
+            plt.show()
 %% Cell type:code id:1c4eddf7-7d6e-49f4-8cbb-12d2bc496a8f tags:
 ``` python
 step_timer.start()
-for mod, db_mod in zip(karabo_da, db_modules):
+for mod, db_mod in da_to_pdu.items():
     constants = {
         "Offset": offset_map[mod],
         "Noise": noise_map[mod],