Fix missed param for nomrmalization of SPI hitscores on pulse energy

295f706d · Egor Sobolev · c37e18ec · 295f706d
Commit 295f706d authored 3 months ago by Egor Sobolev
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
@@ -195,10 +195,11 @@
    "            min_scores=spi_hf_min_scores,\n",
    "            fixed_threshold=spi_hf_fixed_threshold,\n",
    "            hitrate_window_size=spi_hf_hitrate_window_size,\n",
+    "            xgm_norm=spi_hf_xgm_norm,\n",
    "            miss_fraction=spi_hf_miss_fraction,\n",
    "            miss_fraction_base=spi_hf_miss_fraction_base,\n",
    "        )\n",
-    "        hitfinder.find_hits(litpx_src)\n",
+    "        hitfinder.find_hits(litpx_src, litfrm_src)\n",
    "\n",
    "        # write hit-finder data in file\n",
    "        sources = {\n",

 %% Cell type:markdown id: tags:
 # Summary of the AGIPD offline correction #
 %% Cell type:code id: tags:
 ``` python
 run = 11 # runs to process, required
 in_folder = "/gpfs/exfel/exp/MID/202201/p002834/raw" # the folder to read data from, required
 out_folder = "/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_Corr" # path to output to, required
 metadata_folder = ""  # Directory containing calibration_metadata.yml when run by xfel-calibrate
 karabo_id = "SPB_DET_AGIPD1M-1" # karabo karabo_id
 modules = [-1]
 karabo_da = ['-1']  # a list of data aggregators names, Default [-1] for selecting all data aggregators
 rel_gain_mode = "off"  # Select relative gain correction. Choices [`PC`, `CS`, `off`]. (`PC`: Pulse Capacitor, `CS`: Current Source, `off`: Disable relative gain correction). Default: off.
 # Additional processing
 count_lit_pixels = False  # Count the number of pixels registering photons
 spi_hitfinding = False  # Find hits using lit-pixel counter
 # SPI hit-finder parameters
 spi_hf_modules = [3, 4, 8, 15]  # Use specified modules for SPI hitfinding
 spi_hf_mode = "adaptive"  # The method to compute threshold for hitscores in SPI hitfinding: `fixed` or `adaptive`
 spi_hf_snr = 4.0  # Siginal-to-noise ration for adaptive threshold in SPI hitfinding
 spi_hf_min_scores = 100  # The minimal size of events to compute adaptive threshold in SPI hitfinding
 spi_hf_fixed_threshold = 0  # The fixed threshold value
 spi_hf_hitrate_window_size = 200  # The window size for runnig average of hitrate in trains
 spi_hf_xgm_norm = False  # Use XGM pulse energy for hitscore normalization
 spi_hf_miss_fraction = 1  # The fraction of misses to select along with hits
 spi_hf_miss_fraction_base = "hit"  # The base to compute the number of misses to select: the number of hits (`hit`) or misses (`miss`)
 # Lit-frame finder
 use_litframe_finder = 'off' # Process only illuminated frames: 'off' - disable, 'device' - use online device data, 'offline' - use offline algorithm, 'auto' - choose online/offline source automatically (default)
 litframe_device_id = '' # Device ID for a lit frame finder device, empty string to auto detection
 ```
 %% Cell type:code id: tags:
 ``` python
 from pathlib import Path
 from logging import warning
 import yaml
 import tabulate
 from cal_tools.tools import CalibrationMetadata
 from IPython.display import Latex, display, Markdown
 import matplotlib
 import matplotlib.pyplot as plt
 matplotlib.use("agg")
 %matplotlib inline
 from extra_data import RunDirectory
 from extra_redu.fileutils import StackedPulseSource, exdf_save
 from extra_redu.spi import SpiHitfinder
 ```
 %% Cell type:code id: tags:
 ``` python
 out_folder = Path(out_folder)
 metadata = CalibrationMetadata(metadata_folder or out_folder)
 const_dict = metadata.setdefault("retrieved-constants", {})
 time_dict = const_dict.setdefault("time-summary", {})
 # Extracting Instrument string
 instrument = karabo_id.split("_")[0]
 # Evaluate detector instance for mapping
 if instrument == "SPB":
    dinstance = "AGIPD1M1"
    nmods = 16
 elif instrument == "MID":
    dinstance = "AGIPD1M2"
    nmods = 16
 elif instrument == "HED":
    dinstance = "AGIPD500K"
    nmods = 8
 if karabo_da[0] == '-1':
    if modules[0] == -1:
        modules = list(range(nmods))
    karabo_da = ["AGIPD{:02d}".format(i) for i in modules]
 else:
    modules = [int(x[-2:]) for x in karabo_da]
 # This is needed only if AGIPD Correction notebook had no precorrection notebooks for retrieving constants
 # gather all generated sequence yml files for time summary of retrieved constant under retrieved-constants in metadata.yml
 for fn in sorted(out_folder.glob("retrieved_constants_*.yml")):
    with fn.open("r") as fd:
        fdict = yaml.safe_load(fd)
    # append different sequences' time summary to the main yaml
    time_dict.update(fdict["time-summary"])
    fn.unlink()
 metadata.save()
 ```
 %% Cell type:code id: tags:
 ``` python
 def print_const_table(const):
    print(f"{const} constants were injected on:")
    table_data = {}
    for seq, mod_data in time_dict.items():
        for mod, const_data in mod_data.items():
            timestamp = const_data[const]
            table_data.setdefault(timestamp, []).append(f"{seq}:{mod}")
    table = []
    if not len(table_data):
        table.append(["No constants retrieved"])
    elif len(table_data) == 1:
        table.append([[*table_data][0], "All modules"])
    else:
        for timestamp, seqmod in table_data.items():
            table.append([timestamp, seqmod[0]])
            for further_module in seqmod[1:]:
                table.append(["", further_module])
    display(Latex(tabulate.tabulate(table,
                 tablefmt="latex",
                 headers=["Timestamps", "Modules and sequences"])))
 rel_gain_alias = "CS" if rel_gain_mode.lower() == "cs" else "PC"  # 'off' or 'pc'
 for const in ['Offset', f'Slopes{rel_gain_alias}', 'SlopesFF']:
    print_const_table(const)
 ```
 %% Cell type:code id: tags:
 ``` python
 if spi_hitfinding and not count_lit_pixels:
    # Operators are not expected to enable SPI hitfidnig without lit-pixels
    # counting. If this happens, warn on the mismatch in the configuration.
    warning("SPI hitfinding will be skipped because the required lit-pixel "
            "counting is disabled. To run hitfinding, enable also lit-pixel "
            "counting with the `--count-lit-pixels` option.")
 if spi_hitfinding and count_lit_pixels:
    display(Markdown("# SPI hit finding"))
    litfrm_src = None
    if use_litframe_finder != 'off':
        from extra_redu import make_litframe_finder, LitFrameFinderError
        from extra_redu.fileutils import PulseSource
        dc = RunDirectory(Path(in_folder) / f'r{run:04d}')
        litfrm = make_litframe_finder(karabo_id[:3], dc, litframe_device_id)
        try:
            get_data = {
                'auto': litfrm.read_or_process,
                'offline': litfrm.process,
                'online': litfrm.read
            }
            r = get_data[use_litframe_finder]()
            litfrm_src = PulseSource(
                r.meta.trainId, r.output.nFrame,
                r.output.detectorPulseId.ravel(),
                {
                    "energyPerFrame": r.output.energyPerFrame.ravel(),
                }
            )
        except LitFrameFinderError as err:
            pass
    try:
        dc = RunDirectory(out_folder)
    except FileNotFoundError:
        warning("No corrected files found to plot.")
    else:
        litpx_src = StackedPulseSource.from_datacollection(
            dc, f"{karabo_id}/CORR/(?P<key>\d+)CH0:output", "litpx")
        hitfinder = SpiHitfinder(
            modules=spi_hf_modules,
            mode=spi_hf_mode,
            snr=spi_hf_snr,
            min_scores=spi_hf_min_scores,
            fixed_threshold=spi_hf_fixed_threshold,
            hitrate_window_size=spi_hf_hitrate_window_size,
+            xgm_norm=spi_hf_xgm_norm,
            miss_fraction=spi_hf_miss_fraction,
            miss_fraction_base=spi_hf_miss_fraction_base,
        )
-        hitfinder.find_hits(litpx_src)
+        hitfinder.find_hits(litpx_src, litfrm_src)
        # write hit-finder data in file
        sources = {
            f"{karabo_id}/REDU/SPI_HITFINDER": hitfinder,
        }
        exdf_save(out_folder, "REDU00", run, sources, sequence_size=3500)
        # draw plots
        display(Markdown("## Hit-rate plot"))
        hitfinder.plot_hitrate()
        plt.show()
        display(Markdown("## Hitscore histogram"))
        hitfinder.plot_hitscore_hist()
        plt.show()
        display(Markdown("## Hitscore plots"))
        hitfinder.plot_hitscore()
 ```