Merge branch 'fix/Agipd_pre_notebook_import' into 'master'

[AGIPD][CORRECT] fix importing get_gain_mode See merge request detectors/pycalibration!626

Merge branch 'fix/Agipd_pre_notebook_import' into 'master'
[AGIPD][CORRECT] fix importing get_gain_mode See merge request detectors/pycalibration!626
6cf95a34 · Karim Ahmed · 955e86ac · cb9fee67 · 6cf95a34
Commit 6cf95a34 authored 3 years ago by Karim Ahmed
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -164,7 +164,7 @@
    "\n",
    "# Evaluate gain mode (operation mode)\n",
    "if gain_mode < 0:\n",
-    "    gain_mode = get_gain_mode(control_fn, h5path_ctrl)\n",
+    "    gain_mode = agipdlib.get_gain_mode(control_fn, h5path_ctrl)\n",
    "else:\n",
    "    gain_mode = AgipdGainMode(gain_mode)\n",
    "\n",

 %% Cell type:markdown id: tags:

 # AGIPD Retrieving Constants Pre-correction #

 Author: European XFEL Detector Group, Version: 1.0

 Retrieving Required Constants for Offline Calibration of the AGIPD Detector

 %% Cell type:code id: tags:

 ``` python
 in_folder = "/gpfs/exfel/exp/SPB/202030/p900119/raw" # the folder to read data from, required
 out_folder =  "/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_"  # the folder to output to, required
 sequences =  [-1] # sequences to correct, set to -1 for all, range allowed
 modules = [-1] # modules to correct, set to -1 for all, range allowed
 run = 80 # runs to process, required

 karabo_id = "SPB_DET_AGIPD1M-1" # karabo karabo_id
 karabo_da = ['-1']  # a list of data aggregators names, Default [-1] for selecting all data aggregators
 path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # the template to use to access data
 h5path_ctrl = '/CONTROL/{}/MDL/FPGA_COMP_TEST' # path to control information
 karabo_id_control = "SPB_IRU_AGIPD1M1" # karabo-id for control device
 karabo_da_control = 'AGIPD1MCTRL00' # karabo DA for control infromation

 use_dir_creation_date = True # use the creation data of the input dir for database queries
 cal_db_interface = "tcp://max-exfl016:8015#8045" # the database interface to use
 creation_date_offset = "00:00:00" # add an offset to creation date, e.g. to get different constants

 slopes_ff_from_files = "" # Path to locally stored SlopesFF and BadPixelsFF constants
 calfile =  "" # path to calibration file. Leave empty if all data should come from DB
 nodb = False # if set only file-based constants will be used
 mem_cells = 0 # number of memory cells used, set to 0 to automatically infer
 bias_voltage = 300
 acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine
 gain_setting = 0.1 # the gain setting, use 0.1 to try to auto-determine
 gain_mode = -1  # gain mode (0: adaptive, 1-3 fixed high/med/low, -1: read from CONTROL data)
 photon_energy = 9.2 # photon energy in keV
 max_cells_db_dark = 0  # set to a value different than 0 to use this value for dark data DB queries
 max_cells_db = 0 # set to a value different than 0 to use this value for DB queries
 integration_time = -1 # integration time, negative values for auto-detection.

 # Correction Booleans
 only_offset = False # Apply only Offset correction. if False, Offset is applied by Default. if True, Offset is only applied.
 rel_gain = False # do relative gain correction based on PC data
 xray_gain = True # do relative gain correction based on xray data
 blc_noise = False # if set, baseline correction via noise peak location is attempted
 blc_stripes = False # if set, baseline corrected via stripes
 blc_hmatch = False # if set, base line correction via histogram matching is attempted
 match_asics = False # if set, inner ASIC borders are matched to the same signal level
 adjust_mg_baseline = False # adjust medium gain baseline to match highest high gain value
 ```

 %% Cell type:code id: tags:

 ``` python
 # Fill dictionaries comprising bools and arguments for correction and data analysis
 # Here the hierarichy and dependencies for correction booleans are defined
 corr_bools = {}

 # offset is at the bottom of AGIPD correction pyramid.
 corr_bools["only_offset"] = only_offset

 # Dont apply any corrections if only_offset is requested
 if not only_offset:
    corr_bools["adjust_mg_baseline"] = adjust_mg_baseline
    corr_bools["rel_gain"] = rel_gain
    corr_bools["xray_corr"] = xray_gain
    corr_bools["blc_noise"] = blc_noise
    corr_bools["blc_hmatch"] = blc_hmatch
 ```

 %% Cell type:code id: tags:

 ``` python
 from typing import List, Tuple

 import matplotlib
 import numpy as np

 matplotlib.use("agg")
 import multiprocessing
 from datetime import timedelta
 from pathlib import Path

 import matplotlib.pyplot as plt
 from cal_tools import agipdlib, tools
 from cal_tools.enums import AgipdGainMode
 from dateutil import parser
 from iCalibrationDB import Conditions, Constants, Detectors
 ```

 %% Cell type:code id: tags:

 ``` python
 # slopes_ff_from_files left as str for now
 in_folder = Path(in_folder)
 out_folder = Path(out_folder)
 metadata = tools.CalibrationMetadata(out_folder)
 ```

 %% Cell type:code id: tags:

 ``` python
 max_cells = mem_cells

 creation_time = None
 if use_dir_creation_date:
    creation_time = tools.get_dir_creation_date(str(in_folder), run)
    offset = parser.parse(creation_date_offset)
    delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)
    creation_time += delta
    print(f"Using {creation_time} as creation time")

 if sequences[0] == -1:
    sequences = None

 print(f"Outputting to {out_folder}")
 out_folder.mkdir(parents=True, exist_ok=True)

 melt_snow = False if corr_bools["only_offset"] else agipdlib.SnowResolution.NONE
 ```

 %% Cell type:code id: tags:

 ``` python
 control_fn = in_folder / f'r{run:04d}' / f'RAW-R{run:04d}-{karabo_da_control}-S00000.h5'
 h5path_ctrl = h5path_ctrl.format(karabo_id_control)
 slow_paths = (control_fn, karabo_id_control)
 if gain_setting == 0.1:
    if creation_time.replace(tzinfo=None) < parser.parse('2020-01-31'):
        print("Set gain-setting to None for runs taken before 2020-01-31")
        gain_setting = None
    else:
        try:
            gain_setting = agipdlib.get_gain_setting(str(control_fn), h5path_ctrl)
        except Exception as e:
            print(f'ERROR: while reading gain setting from: \n{control_fn}')
            print(e)
            print("Set gain setting to 0")
            gain_setting = 0

 # Evaluate gain mode (operation mode)
 if gain_mode < 0:
-    gain_mode = get_gain_mode(control_fn, h5path_ctrl)
+    gain_mode = agipdlib.get_gain_mode(control_fn, h5path_ctrl)
 else:
    gain_mode = AgipdGainMode(gain_mode)

 # Evaluate integration time
 if integration_time < 0:
    integration_time = agipdlib.get_integration_time(control_fn, h5path_ctrl)

 print(f"Gain setting: {gain_setting}")
 print(f"Gain mode: {gain_mode.name}")
 print(f"Detector in use is {karabo_id}")


 # Extracting Instrument string
 instrument = karabo_id.split("_")[0]
 # Evaluate detector instance for mapping
 if instrument == "SPB":
    dinstance = "AGIPD1M1"
    nmods = 16
 elif instrument == "MID":
    dinstance = "AGIPD1M2"
    nmods = 16
 elif instrument == "HED":
    dinstance = "AGIPD500K"
    nmods = 8

 print(f"Instrument {instrument}")
 print(f"Detector instance {dinstance}")


 if karabo_da[0] == '-1':
    if modules[0] == -1:
        modules = list(range(nmods))
    karabo_da = ["AGIPD{:02d}".format(i) for i in modules]
 else:
    modules = [int(x[-2:]) for x in karabo_da]
 ```

 %% Cell type:markdown id: tags:

 ## Retrieve Constants ##

 %% Cell type:code id: tags:

 ``` python
 def retrieve_constants(
    qm_files: List[Path], qm: str, karabo_da: str, idx: int
 ) -> Tuple[str, str, float, float, str, dict]:
    """
    Retrieve constants for a module.

    :return:
            qm: module virtual name i.e. Q1M1.
            karabo_da: karabo data aggregator.
            acq_rate: acquisition rate parameter.
            max_cells: number of memory cells.
            mdata_dict: (DICT) dictionary with the metadata for the retrieved constants.
    """
    if max_cells != 0:
        # either use overriding notebook parameter
        local_max_cells = max_cells
    else:
        # or look around in sequence files
        for f in qm_files:
            local_max_cells = agipdlib.get_num_cells(f, karabo_id, idx)
            if local_max_cells is not None:
                break
    # maybe we never found this in a sequence file...
    if local_max_cells is None:
        raise ValueError(f"No raw images found for {qm} for all sequences")

    if acq_rate == 0:
        local_acq_rate = agipdlib.get_acq_rate(
            fast_paths=(f, karabo_id, idx), slow_paths=slow_paths)
    else:
        local_acq_rate = acq_rate

    # avoid retrieving constant, if requested.
    if nodb_with_dark:
        return

    const_dict = agipdlib.assemble_constant_dict(
        corr_bools,
        pc_bools,
        local_max_cells,
        bias_voltage,
        gain_setting,
        local_acq_rate,
        photon_energy,
        gain_mode=gain_mode,
        beam_energy=None,
        only_dark=only_dark,
        integration_time=integration_time
    )

    # Retrieve multiple constants through an input dictionary
    # to return a dict of useful metadata.
    mdata_dict = dict()
    mdata_dict["constants"] = dict()
    mdata_dict["physical-detector-unit"] = None  # initialization

    for const_name, (const_init_fun, const_shape, (cond_type, cond_param)) in const_dict.items():
        if gain_mode and const_name in ("ThresholdsDark",):
            continue

        # saving metadata in a dict
        const_mdata = dict()
        mdata_dict["constants"][const_name] = const_mdata

        if slopes_ff_from_files and const_name in ["SlopesFF", "BadPixelsFF"]:
            const_mdata["file-path"] = f"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5"
            const_mdata["creation-time"] = "00:00:00"
            continue

        if gain_mode and const_name in ("BadPixelsPC", "SlopesPC", "BadPixelsFF", "SlopesFF"):
            param_copy = cond_param.copy()
            del param_copy["gain_mode"]
            condition = getattr(Conditions, cond_type).AGIPD(**param_copy)
        else:
            condition = getattr(Conditions, cond_type).AGIPD(**cond_param)

        _, mdata = tools.get_from_db(
            karabo_id,
            karabo_da,
            getattr(Constants.AGIPD, const_name)(),
            condition,
            getattr(np, const_init_fun)(const_shape),
            cal_db_interface,
            creation_time,
            meta_only=True,
            verbosity=0,
        )
        mdata_const = mdata.calibration_constant_version
        # check if constant was sucessfully retrieved.
        if mdata.comm_db_success:
            const_mdata["file-path"] = (
                f"{mdata_const.hdf5path}" f"{mdata_const.filename}"
            )
            const_mdata["creation-time"] = f"{mdata_const.begin_at}"
            mdata_dict["physical-detector-unit"] = mdata_const.device_name
        else:
            const_mdata["file-path"] = const_dict[const_name][:2]
            const_mdata["creation-time"] = None

    return qm, mdata_dict, karabo_da, local_acq_rate, local_max_cells
 ```

 %% Cell type:code id: tags:

 ``` python
 # Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml
 retrieved_constants = metadata.setdefault("retrieved-constants", {})
 ```

 %% Cell type:code id: tags:

 ``` python
 # set everything up filewise
 mapped_files, _, _, _, _ = tools.map_modules_from_folder(
    str(in_folder), run, path_template, karabo_da, sequences
 )

 pc_bools = [corr_bools.get("rel_gain"),
            corr_bools.get("adjust_mg_baseline"),
            corr_bools.get('blc_noise'),
            corr_bools.get('blc_hmatch'),
            corr_bools.get('blc_stripes'),
            melt_snow]

 inp = []
 only_dark = False
 nodb_with_dark = False
 if not nodb:
    only_dark = (calfile != "")
 if calfile != "" and not corr_bools["only_offset"]:
    nodb_with_dark = nodb

 da_to_qm = dict()
 for module_index, k_da in zip(modules, karabo_da):
    qm = tools.module_index_to_qm(module_index)
    da_to_qm[k_da] = qm

    if k_da in retrieved_constants:
        print(f"Constant for {k_da} already in calibration_metadata.yml, won't query again.")
        continue

    if qm in mapped_files and not mapped_files[qm].empty():
        # TODO: make map_modules_from_folder just return list(s)
        qm_files = [Path(mapped_files[qm].get()) for _ in range(mapped_files[qm].qsize())]
    else:
        continue

    inp.append((qm_files, qm, k_da, module_index))
 ```

 %% Cell type:code id: tags:

 ``` python
 with multiprocessing.Pool(processes=nmods) as pool:
    results = pool.starmap(retrieve_constants, inp)
 ```

 %% Cell type:code id: tags:

 ``` python
 for qm, md_dict, karabo_da, acq_rate, max_cells in results:
    retrieved_constants[karabo_da] = md_dict

 # check if it is requested not to retrieve any constants from the database
 if nodb_with_dark:
    print("No constants were retrieved as calibrated files will be used.")
 else:

    print("\nRetrieved constants for modules:",
          ', '.join([tools.module_index_to_qm(x) for x in modules]))
    print(f"Operating conditions are:")
    print(f"• Bias voltage: {bias_voltage}")
    print(f"• Memory cells: {max_cells}")
    print(f"• Acquisition rate: {acq_rate}")
    print(f"• Gain mode: {gain_mode.name}")
    print(f"• Gain setting: {gain_setting}")
    print(f"• Integration time: {integration_time}")
    print(f"• Photon Energy: {photon_energy}")
    print("Constant metadata is saved under \"retrieved-constants\" in calibration_metadata.yml\n")
 ```

 %% Cell type:code id: tags:

 ``` python
 print("Using constants with creation times:")
 timestamps = {}

 for k_da, module_name in da_to_qm.items():
    module_timestamps = timestamps[module_name] = {}
    module_constants = retrieved_constants[k_da]

    print(f"{module_name}:")
    for cname, mdata in module_constants["constants"].items():
        if hasattr(mdata["creation-time"], 'strftime'):
            mdata["creation-time"] = mdata["creation-time"].strftime('%y-%m-%d %H:%M')
        print(f'{cname:.<12s}', mdata["creation-time"])

    for cname in ['Offset', 'SlopesPC', 'SlopesFF']:
        if cname in module_constants["constants"]:
            module_timestamps[cname] = module_constants["constants"][cname]["creation-time"]
        else:
            module_timestamps[cname] = "NA"

 time_summary = retrieved_constants.setdefault("time-summary", {})
 time_summary["SAll"] = timestamps

 metadata.save()
 ```