Karim Ahmed
--- a/notebooks/LPD/inject_calibration_constants_from_h5files.ipynb 0 → 100644

+ 170

− 0
+++ b/notebooks/LPD/inject_calibration_constants_from_h5files.ipynb 0 → 100644

+ 170

− 0
+%% Cell type:markdown id: tags:
+
+# Injecting calibration constant data to the database #
+
+Author: European XFEL Detector Group, Version: 1.0
+
+Reading h5files of calibration constants to inject them to the database. Used for LPD
+
+%% Cell type:code id: tags:
+
+``` python
+# calibration constant parameters:
+constant_names = ["GainMap", "GainAmpMap", "FFMap", "BadPixelsFF"]  # calibration constant name, required.
+in_folder = "/gpfs/exfel/data/scratch/yousefh/LPD_CalinConst_Provisional/"  # calibration constant file, required.
+proposal = "" # Add proposal number to be sent to the database as a part of Raw data location.
+runs = [] # Add list of runs to be sent to the database as a part of Raw data location.
+
+# detector parameters:
+karabo_id = "FXE_DET_LPD1M-1"  # detector identifier, required.
+karabo_da = ["LPD00"]  # karabo data aggregators, required.
+
+# calibration database parameters:
+cal_db_interface = "tcp://max-exfl017:8020"
+
+# calibration constant conditions:
+memory_cells = 512
+bias_voltage = 250
+capacitor = 5
+creation_time = '2020-01-20T14:12:06'  # creation time for the injected constants. required format '2019-01-20T14:12:06'
+```
+
+%% Cell type:code id: tags:
+
+``` python
+import datetime
+import h5py
+from pathlib import Path
+
+from iCalibrationDB import (
+    Constants,
+    Conditions,
+)
+from cal_tools.tools import (
+    get_from_db,
+    get_pdu_from_db,
+    send_to_db,
+)
+```
+
+%% Cell type:code id: tags:
+
+``` python
+pixels_x = pixels_y = 256
+
+# if proposal or runs are given assign file_loc
+# for calibration constant versions metadata.
+file_loc = ""
+if proposal:
+    file_loc += f"proposal:{proposal}"
+if len(runs) > 0:
+    file_loc += f"runs: {runs}"
+
+if file_loc == ""
+    print(
+        "No proposal or runs were given for constant source."
+        " No \"Raw data location\" will be injected with the constants"
+    )
+```
+
+%% Cell type:code id: tags:
+
+``` python
+def validate_input_paths(in_folder):
+
+    # Validate input parameters:
+    if not (in_folder):
+        raise ValueError(
+            "ERROR: \"in_folder\" is not given."
+            " Please provide the constants input folder."
+        )
+
+    c_folder = Path(in_folder)
+
+    if not c_folder.isdir():
+        raise ValueError(
+            f"ERROR: in_folder {in_folder} directory doesn't exist."
+        )
+
+    try:
+        creation_time = datetime.datetime.strptime(creation_time, '%Y-%m-%dT%H:%M:%S')
+    except ValueError:
+        raise ValueError("Incorrect data format, should be YYYY-MM-DDTHH:MM:SS i.e. 2019-01-20T14:12:06")
+
+    return c_folder, creation_time
+```
+
+%% Cell type:code id: tags:
+
+``` python
+c_folder, creation_time = validate_input_paths(in_folder)
+
+# create a report path for calibration constant versions metadata.
+report_path = get_report(
+    out_folder=in_folder,
+    default_path=f"No_report/LPD_{datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S')}"
+)
+
+# Calibration constants condition object.
+condition = Conditions.Dark.LPD(
+    memory_cells=memory_cells,
+    bias_voltage=bias_voltage,
+    pixels_x=pixels_x,
+    pixels_y=pixels_y,
+    capacitor=capacitor,
+)
+
+mod_mapping = {}
+for const in constant_names:
+
+    # Calibration constant object.
+    constant = getattr(Constants.LPD, const)()
+
+    if len(mod_mapping.keys()) == 0
+        # Retrieve the detector modules if it is not known.
+        physical_units = get_pdu_from_db(
+            karabo_id=karabo_id,
+            karabo_da=karabo_da,
+            constant=constant,
+            condition=condition,
+            cal_db_interface=cal_db_interface,
+            snapshot_at=creation_time
+        )
+    mod_mapping = dict(zip(karabo_da, physical_units))
+    print("Physical detector units retrieved are: ", mod_mapping)
+
+    for pdu, k_da in zip(physical_units, karabo_da):
+
+        cfile = c_folder / f"{const}_{k_da}"
+
+        # load constant data.
+        with h5py.File(cfile, "r") as f:
+            cdata = f[const][()]
+
+        # Validate for only LPD at the moment.
+        if not cdata.shape == (pixels_x, pixels_y, memory_cells, 3):
+            raise ValueError(
+                f"ERROR: {const} constant data shape is not as expected."
+                f" {cdata.shape} != ({pixels_x}, {pixels_y}, {memory_cells}, 3)."
+            )
+
+        constant.data = cdata
+
+        md = send_to_db(
+            db_module=pdu,
+            karabo_id=karabo_id,
+            constant=constant,
+            condition=condition,
+            file_loc=file_loc,
+            report_path=report_path,
+            cal_db_interface=cal_db_interface,
+            creation_time=creation_time,
+        )
+```
+
+%% Output
+
+    No proposal or runs were given for constant source. No "Raw data location" will be injected with the constants
+    Physical detector units retrieved are:  {'LPD00': 'LPD_SIV1_LPDV2_SM015'}
+    BadPixelsFF is injected with creation-time: 2020-01-20 14:12:06
+