Skip to content
Snippets Groups Projects
Commit 295f706d authored by Egor Sobolev's avatar Egor Sobolev
Browse files

Fix missed param for nomrmalization of SPI hitscores on pulse energy

parent c37e18ec
No related branches found
No related tags found
1 merge request!1125[AGIPD][CORRECT][SPI] Fix missed param
Hide whitespace changes
Inline Side-by-side
notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
+ 2
− 1
View file @ 295f706d
%% Cell type:markdown id: tags:
# Summary of the AGIPD offline correction #
%% Cell type:code id: tags:
``` python
run = 11 # runs to process, required
in_folder = "/gpfs/exfel/exp/MID/202201/p002834/raw" # the folder to read data from, required
out_folder = "/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_Corr" # path to output to, required
metadata_folder = "" # Directory containing calibration_metadata.yml when run by xfel-calibrate
karabo_id = "SPB_DET_AGIPD1M-1" # karabo karabo_id
modules = [-1]
karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators
rel_gain_mode = "off" # Select relative gain correction. Choices [`PC`, `CS`, `off`]. (`PC`: Pulse Capacitor, `CS`: Current Source, `off`: Disable relative gain correction). Default: off.
# Additional processing
count_lit_pixels = False # Count the number of pixels registering photons
spi_hitfinding = False # Find hits using lit-pixel counter
# SPI hit-finder parameters
spi_hf_modules = [3, 4, 8, 15] # Use specified modules for SPI hitfinding
spi_hf_mode = "adaptive" # The method to compute threshold for hitscores in SPI hitfinding: `fixed` or `adaptive`
spi_hf_snr = 4.0 # Siginal-to-noise ration for adaptive threshold in SPI hitfinding
spi_hf_min_scores = 100 # The minimal size of events to compute adaptive threshold in SPI hitfinding
spi_hf_fixed_threshold = 0 # The fixed threshold value
spi_hf_hitrate_window_size = 200 # The window size for runnig average of hitrate in trains
spi_hf_xgm_norm = False # Use XGM pulse energy for hitscore normalization
spi_hf_miss_fraction = 1 # The fraction of misses to select along with hits
spi_hf_miss_fraction_base = "hit" # The base to compute the number of misses to select: the number of hits (`hit`) or misses (`miss`)
# Lit-frame finder
use_litframe_finder = 'off' # Process only illuminated frames: 'off' - disable, 'device' - use online device data, 'offline' - use offline algorithm, 'auto' - choose online/offline source automatically (default)
litframe_device_id = '' # Device ID for a lit frame finder device, empty string to auto detection
```
%% Cell type:code id: tags:
``` python
from pathlib import Path
from logging import warning
import yaml
import tabulate
from cal_tools.tools import CalibrationMetadata
from IPython.display import Latex, display, Markdown
import matplotlib
import matplotlib.pyplot as plt
matplotlib.use("agg")
%matplotlib inline
from extra_data import RunDirectory
from extra_redu.fileutils import StackedPulseSource, exdf_save
from extra_redu.spi import SpiHitfinder
```
%% Cell type:code id: tags:
``` python
out_folder = Path(out_folder)
metadata = CalibrationMetadata(metadata_folder or out_folder)
const_dict = metadata.setdefault("retrieved-constants", {})
time_dict = const_dict.setdefault("time-summary", {})
# Extracting Instrument string
instrument = karabo_id.split("_")[0]
# Evaluate detector instance for mapping
if instrument == "SPB":
dinstance = "AGIPD1M1"
nmods = 16
elif instrument == "MID":
dinstance = "AGIPD1M2"
nmods = 16
elif instrument == "HED":
dinstance = "AGIPD500K"
nmods = 8
if karabo_da[0] == '-1':
if modules[0] == -1:
modules = list(range(nmods))
karabo_da = ["AGIPD{:02d}".format(i) for i in modules]
else:
modules = [int(x[-2:]) for x in karabo_da]
# This is needed only if AGIPD Correction notebook had no precorrection notebooks for retrieving constants
# gather all generated sequence yml files for time summary of retrieved constant under retrieved-constants in metadata.yml
for fn in sorted(out_folder.glob("retrieved_constants_*.yml")):
with fn.open("r") as fd:
fdict = yaml.safe_load(fd)
# append different sequences' time summary to the main yaml
time_dict.update(fdict["time-summary"])
fn.unlink()
metadata.save()
```
%% Cell type:code id: tags:
``` python
def print_const_table(const):
print(f"{const} constants were injected on:")
table_data = {}
for seq, mod_data in time_dict.items():
for mod, const_data in mod_data.items():
timestamp = const_data[const]
table_data.setdefault(timestamp, []).append(f"{seq}:{mod}")
table = []
if not len(table_data):
table.append(["No constants retrieved"])
elif len(table_data) == 1:
table.append([[*table_data][0], "All modules"])
else:
for timestamp, seqmod in table_data.items():
table.append([timestamp, seqmod[0]])
for further_module in seqmod[1:]:
table.append(["", further_module])
display(Latex(tabulate.tabulate(table,
tablefmt="latex",
headers=["Timestamps", "Modules and sequences"])))
rel_gain_alias = "CS" if rel_gain_mode.lower() == "cs" else "PC" # 'off' or 'pc'
for const in ['Offset', f'Slopes{rel_gain_alias}', 'SlopesFF']:
print_const_table(const)
```
%% Cell type:code id: tags:
``` python
if spi_hitfinding and not count_lit_pixels:
# Operators are not expected to enable SPI hitfidnig without lit-pixels
# counting. If this happens, warn on the mismatch in the configuration.
warning("SPI hitfinding will be skipped because the required lit-pixel "
"counting is disabled. To run hitfinding, enable also lit-pixel "
"counting with the `--count-lit-pixels` option.")
if spi_hitfinding and count_lit_pixels:
display(Markdown("# SPI hit finding"))
litfrm_src = None
if use_litframe_finder != 'off':
from extra_redu import make_litframe_finder, LitFrameFinderError
from extra_redu.fileutils import PulseSource
dc = RunDirectory(Path(in_folder) / f'r{run:04d}')
litfrm = make_litframe_finder(karabo_id[:3], dc, litframe_device_id)
try:
get_data = {
'auto': litfrm.read_or_process,
'offline': litfrm.process,
'online': litfrm.read
}
r = get_data[use_litframe_finder]()
litfrm_src = PulseSource(
r.meta.trainId, r.output.nFrame,
r.output.detectorPulseId.ravel(),
{
"energyPerFrame": r.output.energyPerFrame.ravel(),
}
)
except LitFrameFinderError as err:
pass
try:
dc = RunDirectory(out_folder)
except FileNotFoundError:
warning("No corrected files found to plot.")
else:
litpx_src = StackedPulseSource.from_datacollection(
dc, f"{karabo_id}/CORR/(?P<key>\d+)CH0:output", "litpx")
hitfinder = SpiHitfinder(
modules=spi_hf_modules,
mode=spi_hf_mode,
snr=spi_hf_snr,
min_scores=spi_hf_min_scores,
fixed_threshold=spi_hf_fixed_threshold,
hitrate_window_size=spi_hf_hitrate_window_size,
xgm_norm=spi_hf_xgm_norm,
miss_fraction=spi_hf_miss_fraction,
miss_fraction_base=spi_hf_miss_fraction_base,
)
hitfinder.find_hits(litpx_src)
hitfinder.find_hits(litpx_src, litfrm_src)
# write hit-finder data in file
sources = {
f"{karabo_id}/REDU/SPI_HITFINDER": hitfinder,
}
exdf_save(out_folder, "REDU00", run, sources, sequence_size=3500)
# draw plots
display(Markdown("## Hit-rate plot"))
hitfinder.plot_hitrate()
plt.show()
display(Markdown("## Hitscore histogram"))
hitfinder.plot_hitscore_hist()
plt.show()
display(Markdown("## Hitscore plots"))
hitfinder.plot_hitscore()
```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts