add conditions and psw for reading slow data

ba66cd98 · Karim Ahmed · c99d5321 · ba66cd98
Commit ba66cd98 authored 2 years ago by Karim Ahmed
--- a/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
+++ b/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
@@ -133,6 +133,23 @@
    "step_timer = StepTimer()"
   ]
  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1128e743",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Read slow data\n",
+    "# run_dc = RunDirectory(run_folder)\n",
+    "# g2ctrl = gotthard2lib.Gotthard2Ctrl(run_dc=run_dc, ctrl_src=ctrl_src)\n",
+    "# pulses = g2ctrl.pulses()\n",
+    "# bias_voltage = g2ctrl.get_bias_voltage()\n",
+    "# exposure_time = g2ctrl.exposure_time()\n",
+    "# acquisition_time = g2ctrl.get_acquisition_time()\n",
+    "# gain_setting = g2ctrl.get_gain_setting()"
+   ]
+  },
  {
   "cell_type": "code",
   "execution_count": null,
@@ -402,10 +419,9 @@
    "condition = Conditions.Dark.Gotthard2(\n",
    "    bias_voltage=bias_voltage,\n",
    "    pulses=pulses,\n",
-    "    integration_time=integration_time,\n",
+    "    integration_time=exposure_time,\n",
-    "    acquisition_rate=acquisition_rate,\n",
+    "    acquisition_rate=acquisition_time,\n",
    "    gain_setting=gain_setting,\n",
-    "    gain_mode=gain_mode,\n",
    ")\n",
    "\n",
    "db_modules = get_pdu_from_db(\n",

 %% Cell type:markdown id:49b6577f-96a5-4dd2-bdd9-da661b2c4619 tags:
 # Gotthard2 Dark Image Characterization
 Author: European XFEL Detector Group, Version: 1.0
 %% Cell type:code id:818e24e8 tags:
 ``` python
 in_folder = "/gpfs/exfel/exp/FXE/202221/p003225/raw"  # the folder to read data from, required
 out_folder =  "/gpfs/exfel/data/scratch/ahmedk/test/gotthard2/darks"  # the folder to output to, required
 run_high = 50  # run number for G0 dark run, required
 run_med = 51  # run number for G1 dark run, required
 run_low = 52  # run number for G2 dark run, required
 sequences = [-1]  # sequences to correct, set to [-1] for all, range allowed
 sequences_per_node = 1  # number of sequence files per node if notebook executed through xfel-calibrate, set to 0 to not run SLURM parallel
 # Parameters used to access raw data.
 karabo_id = "FXE_XAD_G2XES"  # karabo prefix of Jungfrau devices
 karabo_da = ["GH201"]  # data aggregators
 receiver_template = "RECEIVER"  # receiver template used to read INSTRUMENT keys.
 control_template = "CONTROL"  # control template used to read CONTROL keys.
 instrument_source_template = '{}/DET/{}:daqOutput'  # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput'
 ctrl_source_template = '{}/DET/{}'  # template for control source name (filled with karabo_id_control)
 karabo_id_control = ""  # if control is on a different ID, set to empty string if it is the same a karabo-id
 # Parameters for the calibration database.
 use_dir_creation_date = True
 cal_db_interface = "tcp://max-exfl016:8020" # calibration DB interface to use
 cal_db_timeout = 300000 # timeout on caldb requests
 db_output = False # Output constants to the calibration database
 local_output = True # Output constants locally
 constants_file = "/gpfs/exfel/data/scratch/ahmedk/dont_remove/gotthard2/constants/calibration_constants_GH2.h5"
 # Conditions used for injected calibration constants.
 bias_voltage = -1  # Detector bias voltage, set to -1 to use value in raw file.
 pulses = -1  # Detector number of pulses, set to -1 to use value in raw file.
 acquisition_rate = -1  # Detector acquisition rate, set to -1 to use value in raw file.
 integration_time = -1  # Detector integration time, set to -1 to use value in raw file.
 gain_setting = -1  # Detector gain setting, set to -1 to use value in raw file.
 gain_mode = -1  # Detector gain mode, set to -1 to use value in raw file.
 # Parameters used during selecting raw data trains.
 min_trains = 1 # Minimum number of trains that should be available to process dark constants. Default 1.
 max_trains = 1000  # Maximum number of trains to use for processing dark constants. Set to 0 to use all available trains.
 badpixel_threshold_sigma = 5.  # bad pixels defined by values outside n times this std from median
 # Don't delete! myMDC sends this by default.
 operation_mode = ''  # Detector operation mode, optional
 ```
 %% Cell type:code id:8085f9aa tags:
 ``` python
 import h5py
 import numpy as np
 import matplotlib.pyplot as plt
 import multiprocessing
 import pasha as psh
 from IPython.display import Markdown, display
 from extra_data import RunDirectory
 from pathlib import Path
 from cal_tools.enums import BadPixels
 from cal_tools.gotthard2algs import convert_to_10bit
 from cal_tools.step_timing import StepTimer
 from cal_tools.tools import (
    get_dir_creation_date,
    get_from_db,
    get_pdu_from_db,
    get_report,
    save_const_to_h5,
    send_to_db,
 )
 from iCalibrationDB import Conditions, Constants
 from XFELDetAna.plotting.heatmap import heatmapPlot
 ```
 %% Cell type:code id:18fe4379 tags:
 ``` python
 run_nums = [run_high, run_med, run_low]
 in_folder = Path(in_folder)
 out_folder = Path(out_folder)
 out_folder.mkdir(exist_ok=True)
 print(f"Process modules: {karabo_da}")
 run_dc = RunDirectory(in_folder / f"r{run_high:04d}")
 file_loc = f"proposal:{run_dc.run_metadata()['proposalNumber']} runs:{run_high} {run_med} {run_low}"  # noqa
 instrument_src = instrument_source_template.format(
    karabo_id, receiver_template)
 ctrl_src = ctrl_source_template.format(
    karabo_id_control, control_template)
 # Read report path to associate it later with injected constants.
 report = get_report(out_folder)
 if use_dir_creation_date:
    creation_time = get_dir_creation_date(in_folder, run_high)
    print(f"Using {creation_time.isoformat()} as creation time")
 if not karabo_id_control:
    karabo_id_control = karabo_id
 ```
 %% Cell type:code id:108be688 tags:
 ``` python
 step_timer = StepTimer()
 ```
+%% Cell type:code id:1128e743 tags:
+``` python
+# Read slow data
+# run_dc = RunDirectory(run_folder)
+# g2ctrl = gotthard2lib.Gotthard2Ctrl(run_dc=run_dc, ctrl_src=ctrl_src)
+# pulses = g2ctrl.pulses()
+# bias_voltage = g2ctrl.get_bias_voltage()
+# exposure_time = g2ctrl.exposure_time()
+# acquisition_time = g2ctrl.get_acquisition_time()
+# gain_setting = g2ctrl.get_gain_setting()
+```
 %% Cell type:code id:fb80b98e tags:
 ``` python
 # load constants temporarily using defined local paths.
 with h5py.File(constants_file, 'r') as cfile:
    lut = cfile["LUT"][()]
 ```
 %% Cell type:code id:ff9149fc tags:
 ``` python
 # Read parameter conditions
 step_timer.start()
 run_dcs_dict = dict()
 ctrl_src = ctrl_source_template.format(karabo_id_control, control_template)
 for gain, run in enumerate(run_nums):
    run_dc = RunDirectory(in_folder / f"r{run:04d}/")
    run_dcs_dict[run] = [gain, run_dc]
    # TODO: Read every condition from slow data.
    # TODO: Validate that the conditions are the same and as expected for all runs.
 ```
 %% Cell type:code id:ac9c5dc3-bc66-4e7e-b6a1-360259be535c tags:
 ``` python
 def specify_trains_to_process(
    img_key_data: "extra_data.KeyData",  # noqa
    max_trains: int = 0,
    min_trains: int = 0,
 ):
    """Specify total number of trains to process.
    Based on given min_trains and max_trains, if given.
    Print number of trains to process and number of empty trains.
    Raise ValueError if specified trains are less than min_trains.
    """
    # Specifies total number of trains to proccess.
    n_trains = img_key_data.shape[0]
    all_trains = len(img_key_data.train_ids)
    print(f"{mod} has {all_trains - n_trains} "
    f"trains with empty frames out of {all_trains} trains")
    if n_trains < min_trains:
        raise ValueError(
            f"Less than {min_trains} trains are available in RAW data."
            " Not enough data to process darks.")
    if max_trains > 0:
        n_trains = min(n_trains, max_trains)
    print(f"Processing {n_trains} trains.")
    return n_trains
 ```
 %% Cell type:code id:e2eb2fc0-df9c-4887-9691-f81474f8c131 tags:
 ``` python
 def convert_train(wid, index, tid, d):
    """Convert a Gotthard2 train from 12bit to 10bit."""
    convert_to_10bit(d[instr_mod_src]["data.adc"], lut, data_10bit[index, ...])
 ```
 %% Cell type:code id:4e8ffeae tags:
 ``` python
 # Calculate noise and offset per pixel and global average, std and median
 noise_map = dict()
 offset_map = dict()
 badpixels_map = dict()
 context = psh.context.ProcessContext(num_workers=multiprocessing.cpu_count())
 for mod in karabo_da:
    # Path to pixels ADC values
    instr_mod_src = instrument_src.format(int(mod[-2:]))
    data_path = "INSTRUMENT/"+instr_mod_src+"/data"
    # TODO: Validate the final shape to store constants.
    cshape = (1280, 2, 3)
    offset_map[mod] = context.alloc(shape=cshape, dtype=np.float32)
    noise_map[mod] = context.alloc(like=offset_map[mod])
    badpixels_map[mod] = context.alloc(like=offset_map[mod], dtype=np.uint32)
    for run_num, [gain, run_dc] in run_dcs_dict.items():
        step_timer.start()
        n_trains = specify_trains_to_process(run_dc[instr_mod_src, "data.adc"])
        # Select requested number of trains to process.
        dc = run_dc.select(
            instr_mod_src, require_all=True).select_trains(np.s_[:n_trains])  # noqa
        step_timer.done_step("preparing raw data")
        step_timer.start()
        # Convert 12bit data to 10bit
        data_10bit = context.alloc(
            shape=dc[instr_mod_src, "data.adc"].shape,
            dtype=np.float32
        )
        context.map(convert_train, dc)
        step_timer.done_step("convert to 10bit")
        step_timer.start()
        # Split even and odd data to calculate the two storage cell constants.
        # Detector always operates in burst mode.
        even_data = data_10bit[:, ::2, :]
        odd_data = data_10bit[:, 1::2, :]
        data_gain = dc[instr_mod_src, "data.gain"].ndarray()
        even_gain = data_gain[:, ::2, :]
        odd_gain = data_gain[:, 1::2, :]
        def offset_noise_cell(wid, index, d):
            offset_map[mod][:, index, gain] = np.mean(d, axis=(0, 1))
            noise_map[mod][:, index, gain] = np.std(d, axis=(0, 1))
        context.map(offset_noise_cell, (even_data, odd_data))
        raw_g = 3 if gain == 2 else gain
        def badpixels_cell(wid, index, g):
            """Check if there are wrong bad gain values.
            Indicate pixels with wrong gain value across all trains for each cell."""
            badpixels_map[mod][np.mean(g, axis=(0, 1)) != raw_g, index,
                gain] |= BadPixels.WRONG_GAIN_VALUE.value
        context.map(badpixels_cell, (even_gain, odd_gain))
        step_timer.done_step("Processing darks")
 ```
 %% Cell type:code id:3fc17e05-17ab-4ac4-9e79-c95399278bb9 tags:
 ``` python
 def print_bp_entry(bp):
    print(f"{bp.name:<30s} {bp.value:032b} -> {int(bp.value)}")
 print_bp_entry(BadPixels.NOISE_OUT_OF_THRESHOLD)
 print_bp_entry(BadPixels.OFFSET_NOISE_EVAL_ERROR)
 print_bp_entry(BadPixels.WRONG_GAIN_VALUE)
 def eval_bpidx(d):
    mdn = np.nanmedian(d, axis=(0))[None, :, :]
    std = np.nanstd(d, axis=(0))[None, :, :]
    idx = (d > badpixel_threshold_sigma*std+mdn) | (d < (-badpixel_threshold_sigma)*std+mdn)
    return idx
 ```
 %% Cell type:code id:9409d10f tags:
 ``` python
 badpixels_map[mod][~np.isfinite(noise_map[mod])]
 ```
 %% Cell type:code id:40c34cc5-fe93-4b83-bf39-f465f37c40b4 tags:
 ``` python
 step_timer.start()
 g_name = ['G0', 'G1', 'G2']
 for mod in karabo_da:
    display(Markdown(f"### Badpixels for module {mod}:"))
    badpixels_map[mod][~np.isfinite(offset_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value
    badpixels_map[mod][eval_bpidx(noise_map[mod])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value
    badpixels_map[mod][~np.isfinite(noise_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value
    for cell in [0, 1]:
        fig, ax = plt.subplots(figsize=(10, 5))
        for g_idx in [0, 1, 2]:
            ax.plot(badpixels_map[mod][:, cell, g_idx], label=f"G{g_idx} Bad pixel map")
        ax.set_xticks(np.arange(0, 1281, 80))
        ax.set_xlabel("Stripes #")
        ax.set_xlabel("BadPixels")
        ax.set_title(f'Cell {cell} - Module {mod}')
        ax.set_ylim([0, BadPixels.WRONG_GAIN_VALUE.value])
        ax.set_yticks([0,
            BadPixels.NOISE_OUT_OF_THRESHOLD.value,
            BadPixels.OFFSET_NOISE_EVAL_ERROR.value,
            BadPixels.WRONG_GAIN_VALUE.value])
        ax.legend()
        pass
 step_timer.done_step(f'Creating bad pixels constant and plotting it.')
 ```
 %% Cell type:code id:e5131112 tags:
 ``` python
 step_timer.start()
 g_name = ['G0', 'G1', 'G2']
 for mod in karabo_da:
    display(Markdown(f"### Badpixels for module {mod}:"))
    badpixels_map[mod][~np.isfinite(offset_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value
    badpixels_map[mod][eval_bpidx(noise_map[mod])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value
    badpixels_map[mod][~np.isfinite(noise_map[mod])] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value
    for g_idx in [0, 1, 2]:
       # for cell in [0, 1]:
        heatmapPlot(
                badpixels_map[mod][:, :, g_idx],
                y_label="Stripes",
                x_label="Cells",
 #                lut_label=unit,
                vmin=0, vmax=5,
                title=f"Even / Odd Offset map G{g_idx} - Module {mod}", # TODO: add PDU name({pdu})',
        )
        pass
 step_timer.done_step(f'Creating bad pixels constant and plotting it.')
 ```
 %% Cell type:code id:6e10ed93-66de-4fb1-bf97-f8d25af22edb tags:
 ``` python
 # set the operating condition
 # TODO: add the final conditions for constants.
 condition = Conditions.Dark.Gotthard2(
    bias_voltage=bias_voltage,
    pulses=pulses,
-    integration_time=integration_time,
+    integration_time=exposure_time,
-    acquisition_rate=acquisition_rate,
+    acquisition_rate=acquisition_time,
    gain_setting=gain_setting,
-    gain_mode=gain_mode,
 )
 db_modules = get_pdu_from_db(
    karabo_id=karabo_id,
    karabo_da=karabo_da,
    constant=Constants.Gotthard2.Offset(),
    condition=condition,
    cal_db_interface=cal_db_interface,
    snapshot_at=creation_time)
 ```
 %% Cell type:code id:fde8e1cf-bc74-462f-b6e5-cfee8279090d tags:
 ``` python
 from XFELDetAna.plotting.heatmap import heatmapPlot
 unit = '[ADCu]'
 for mod in karabo_da:
    for _, [gain, _] in run_dcs_dict.items():
        heatmapPlot(
            offset_map[mod][:, :, gain],
            y_label="Stripes",
            x_label="Cells",
            lut_label=unit,
            x_ticks=[0.5, 1.5],
            x_ticklabels=["EVEN", "ODD"],
            x_tick_rotation=0,
            title=f"Even / Odd Offset map G{gain} - Module {mod}", # TODO: add PDU name({pdu})',
        )
        heatmapPlot(
            noise_map[mod][:, :, gain],
            y_label="Stripes",
            x_label="Cells",
            lut_label=unit,
            x_ticks=[0.5, 1.5],
            x_ticklabels=["EVEN", "ODD"],
            x_tick_rotation=0,
            title=f"Even / Odd noise map G{gain} - Module {mod}", # TODO: add PDU name({pdu})',
        )
        pass
 ```
 %% Cell type:code id:c8777cfe tags:
 ``` python
 for mod in karabo_da:
    for cons, cname in zip([offset_map, noise_map], ["Offset", "Noise"]):
        for cell in [0, 1]:
            fig, ax = plt.subplots(figsize=(10, 5))
            for g_idx in [0, 1, 2]:
                ax.plot(cons[mod][:, cell, g_idx], label=f"G{g_idx} {cname} map")
            ax.set_xlabel("Stripes #")
            ax.set_xlabel(cname)
            ax.set_title(f"{cname} map - Cell {cell} - Module {mod}"), # TODO: add PDU name({pdu})'
            ax.legend()
            pass
 ```
 %% Cell type:code id:1c4eddf7-7d6e-49f4-8cbb-12d2bc496a8f tags:
 ``` python
 step_timer.start()
 for mod, db_mod in zip(karabo_da, db_modules):
    constants = {
        'Offset': offset_map[mod],
        'Noise': noise_map[mod],
    }
    md = None
    for key, const_data in constants.items():
        const =  getattr(Constants.Gotthard2, key)()
        const.data = const_data
        if db_output:
            md = send_to_db(
                db_module=db_mod,
                karabo_id=karabo_id,
                constant=const,
                condition=condition,
                file_loc=file_loc,
                report_path=report,
                cal_db_interface=cal_db_interface,
                creation_time=creation_time,
                timeout=cal_db_timeout,
            )
        if local_output:
            md = save_const_to_h5(
                db_module=db_mod,
                karabo_id=karabo_id,
                constant=const,
                condition=condition,
                data=const.data,
                file_loc=file_loc,
                report=report,
                creation_time=creation_time,
                out_folder=out_folder,
            )
            print(f"Calibration constant {key} is stored locally at {out_folder}.\n")
 print("Constants parameter conditions are:\n")
 # TODO: add the final conditions for constants.
 print(
    f"• Bias voltage: {bias_voltage}\n"
    f"• Pulses: {pulses}\n"
    f"• Integration time: {integration_time}\n"
    f"• Acquisition rate: {acquisition_rate}\n"
    f"• Gain setting: {gain_setting}\n"
    f"• Gain mode: {gain_mode}\n"
    f"• Creation time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\n")  # noqa
 step_timer.done_step("Injecting constants.")
 ```