Merge branch 'fix/uncommented_line' into 'master'

Uncommented line causing syntax error See merge request detectors/pycalibration!728

Merge branch 'fix/uncommented_line' into 'master'
Uncommented line causing syntax error See merge request detectors/pycalibration!728
a6c9a7ce · Karim Ahmed · 49a1eee0 · 11c5e3da · a6c9a7ce
Commit a6c9a7ce authored 2 years ago by Karim Ahmed
--- a/notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb
@@ -45,7 +45,7 @@
    "integration_time = 4.96  # integration time in us, will be overwritten by value in file\n",
    "gain_setting = 0  # 0 for dynamic gain, 1 for dynamic HG0, will be overwritten by value in file\n",
    "gain_mode = 0  # 0 for runs with dynamic gain setting, 1 for fixgain. It will be overwritten by value in file, if manual_slow_data is set to True.\n",
-    "mem_cells = -1   Set mem_cells to -1 to automatically use the value stored in RAW data.#\n",
+    "mem_cells = -1  # Set mem_cells to -1 to automatically use the value stored in RAW data.#\n",
    "bias_voltage = 180  # will be overwritten by value in file"
   ]
  },

 %% Cell type:markdown id: tags:
 # JUNGFRAU Retrieving Constants Pre-correction #
 Author: European XFEL Detector Group, Version: 1.0
 Retrieving Required Constants for Offline Calibration of the JUNGFRAU Detector
 %% Cell type:code id: tags:
 ``` python
 in_folder = "/gpfs/exfel/exp/SPB/202130/p900204/raw"  # the folder to read data from, required
 out_folder =  "/gpfs/exfel/data/scratch/ahmedk/test/remove"  # the folder to output to, required
 metadata_folder = ""  # Directory containing calibration_metadata.yml when run by xfel-calibrate
 run = 112  # run to process, required
 sequences = [-1]  # sequences to correct, set to [-1] for all, range allowed
 sequences_per_node = 1  # number of sequence files per cluster node if run as slurm job, set to 0 to not run SLURM parallel
 # Parameters used to access raw data.
 karabo_id = "SPB_IRDA_JF4M"  # karabo prefix of Jungfrau devices
 karabo_da = ['JNGFR01', 'JNGFR02', 'JNGFR03', 'JNGFR04', 'JNGFR05', 'JNGFR06', 'JNGFR07', 'JNGFR08']  # data aggregators
 receiver_template = "JNGFR{:02d}"  # Detector receiver template for accessing raw data files. e.g. "JNGFR{:02d}"
 instrument_source_template = '{}/DET/{}:daqOutput'  # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput'
 ctrl_source_template = '{}/DET/CONTROL'  # template for control source name (filled with karabo_id_control)
 karabo_id_control = ""  # if control is on a different ID, set to empty string if it is the same a karabo-id
 # Parameters for calibration database.
 use_dir_creation_date = True  # use the creation data of the input dir for database queries
 cal_db_interface = "tcp://max-exfl016:8017#8025" # the database interface to use
 cal_db_timeout = 180000  # timeout on cal db requests
 # Parameters affecting corrected data.
 relative_gain = True  # do relative gain correction
 # Parameters for retrieving calibration constants
 manual_slow_data = False  # if true, use manually entered bias_voltage, integration_time, gain_setting, and gain_mode values
 integration_time = 4.96  # integration time in us, will be overwritten by value in file
 gain_setting = 0  # 0 for dynamic gain, 1 for dynamic HG0, will be overwritten by value in file
 gain_mode = 0  # 0 for runs with dynamic gain setting, 1 for fixgain. It will be overwritten by value in file, if manual_slow_data is set to True.
-mem_cells = -1   Set mem_cells to -1 to automatically use the value stored in RAW data.#
+mem_cells = -1  # Set mem_cells to -1 to automatically use the value stored in RAW data.#
 bias_voltage = 180  # will be overwritten by value in file
 ```
 %% Cell type:code id: tags:
 ``` python
 import datetime
 from functools import partial
 import multiprocessing
 from extra_data import RunDirectory
 from pathlib import Path
 from cal_tools.jungfraulib import JungfrauCtrl
 from cal_tools.tools import (
    get_dir_creation_date,
    get_from_db,
    CalibrationMetadata,
 )
 from iCalibrationDB import Conditions, Constants
 ```
 %% Cell type:code id: tags:
 ``` python
 in_folder = Path(in_folder)
 out_folder = Path(out_folder)
 metadata = CalibrationMetadata(metadata_folder or out_folder)
 run_folder = in_folder / f'r{run:04d}'
 run_dc = RunDirectory(run_folder)
 out_folder.mkdir(parents=True, exist_ok=True)
 creation_time = None
 if use_dir_creation_date:
    creation_time = get_dir_creation_date(in_folder, run)
    print(f"Using {creation_time} as creation time")
 if karabo_id_control == "":
    karabo_id_control = karabo_id
 ```
 %% Cell type:code id: tags:
 ``` python
 ctrl_src = ctrl_source_template.format(karabo_id_control)
 ctrl_data = JungfrauCtrl(run_dc, ctrl_src)
 if mem_cells < 0:
    memory_cells, sc_start = ctrl_data.get_memory_cells()
    mem_cells_name = "single cell" if memory_cells == 1 else "burst"
    print(f"Run is in {mem_cells_name} mode.\nStorage cell start: {sc_start:02d}")
 else:
    memory_cells = mem_cells
    mem_cells_name = "single cell" if memory_cells == 1 else "burst"
    print(f"Run is in manually set to {mem_cells_name} mode. With {memory_cells} memory cells")
 if not manual_slow_data:
    integration_time = ctrl_data.get_integration_time()
    bias_voltage = ctrl_data.get_bias_voltage()
    gain_setting = ctrl_data.get_gain_setting()
    gain_mode = ctrl_data.get_gain_mode()
 print(f"Integration time is {integration_time} us")
 print(f"Gain setting is {gain_setting} (run settings: "
      f"{ctrl_data.run_settings.value if ctrl_data.run_settings else ctrl_data.run_settings})")  # noqa
 print(f"Gain mode is {gain_mode}")
 print(f"Bias voltage is {bias_voltage} V")
 print(f"Number of memory cells are {memory_cells}")
 ```
 %% Cell type:code id: tags:
 ``` python
 condition = Conditions.Dark.jungfrau(
    memory_cells=memory_cells,
    bias_voltage=bias_voltage,
    integration_time=integration_time,
    gain_setting=gain_setting,
    gain_mode=gain_mode,
 )
 def get_constants_for_module(mod: str):
    """Get calibration constants for given module for Jungfrau."""
    retrieval_function = partial(
        get_from_db,
        karabo_id=karabo_id,
        karabo_da=mod,
        cal_db_interface=cal_db_interface,
        creation_time=creation_time,
        timeout=cal_db_timeout,
        verbosity=0,
        meta_only=True,
        load_data=False,
        empty_constant=None
    )
    mdata_dict = dict()
    mdata_dict["constants"] = dict()
    constants = [
        "Offset", "BadPixelsDark",
        "RelativeGain", "BadPixelsFF",
    ]
    for cname in constants:
        mdata_dict["constants"][cname] = dict()
        if not relative_gain and cname in ["BadPixelsFF", "RelativeGain"]:
            continue
        _, mdata = retrieval_function(
            condition=condition,
            constant=getattr(Constants.jungfrau, cname)(),
        )
        mdata_const = mdata.calibration_constant_version
        const_mdata = mdata_dict["constants"][cname]
        # check if constant was successfully retrieved.
        if mdata.comm_db_success:
            const_mdata["file-path"] = (
                f"{mdata_const.hdf5path}" f"{mdata_const.filename}"
            )
            const_mdata["dataset-name"] = mdata_const.h5path
            const_mdata["creation-time"] = f"{mdata_const.begin_at}"
            mdata_dict["physical-detector-unit"] = mdata_const.device_name
        else:
            const_mdata["file-path"] = None
            const_mdata["creation-time"] = None
    return mdata_dict, mod
 ```
 %% Cell type:code id: tags:
 ``` python
 # Constant paths are saved under retrieved-constants in calibration_metadata.yml
 retrieved_constants = metadata.setdefault("retrieved-constants", {})
 # Avoid retrieving constants for available modules in calibration_metadata.yml
 # This is used during reproducability.
 query_karabo_da = []
 for mod in karabo_da:
    if mod in retrieved_constants.keys():
        print(f"Constant for {mod} already in "
              "calibration_metadata.yml, won't query again.")
        continue
    query_karabo_da.append(mod)
 with multiprocessing.Pool() as pool:
    results = pool.map(get_constants_for_module, query_karabo_da)
 ```
 %% Cell type:code id: tags:
 ``` python
 timestamps = dict()
 for md_dict, mod in results:
    retrieved_constants[mod] = md_dict
    module_timestamps = timestamps[mod] = dict()
    module_constants = retrieved_constants[mod]
    print(f"Module: {mod}:")
    for cname, mdata in module_constants["constants"].items():
        if hasattr(mdata["creation-time"], 'strftime'):
            mdata["creation-time"] = mdata["creation-time"].strftime('%y-%m-%d %H:%M')
        print(f'{cname:.<12s}', mdata["creation-time"])
    for cname in ["Offset", "BadPixelsDark", "RelativeGain", "BadPixelsFF"]:
        if cname in module_constants["constants"]:
            module_timestamps[cname] = module_constants["constants"][cname]["creation-time"]
        else:
            module_timestamps[cname] = "NA"
 time_summary = retrieved_constants.setdefault("time-summary", {})
 time_summary = timestamps
 metadata.save()
 ```