Skip to content
Snippets Groups Projects

WIP: Change BAM correction to absolute

Open Loïc Le Guyader requested to merge bozbam into master
1 unresolved thread
Files
3
%% Cell type:code id: tags:
``` python
import numpy as np
%matplotlib inline
import matplotlib.pyplot as plt
plt.rcParams['figure.constrained_layout.use'] = True
import dask
print(f'dask: {dask.__version__}')
import xarray as xr
from psutil import virtual_memory
mem = virtual_memory()
print(f'Physical memory: {mem.total/1024/1024/1024:.0f} Gb') # total physical memory available
import logging
logging.basicConfig(filename='example.log', level=logging.DEBUG)
import os
```
%% Output
dask: 2.11.0
Physical memory: 504 Gb
%% Cell type:code id: tags:
``` python
%load_ext autoreload
%autoreload 2
import toolbox_scs as tb
print(tb.__file__)
import toolbox_scs.routines.boz as boz
from extra_data.read_machinery import find_proposal
```
%% Output
/home/lleguy/notebooks/ToolBox/src/toolbox_scs/__init__.py
%% Cell type:markdown id: tags:
# Loading analysis parameters
%% Cell type:code id: tags:
``` python
proposal = 2937
runNB = 690
darkrunNB = 671
moduleNB = 15
gain = 3
```
%% Cell type:code id: tags:
``` python
root = find_proposal(f'p{proposal:06d}')
path = root + f'/usr/processed_runs/'
params = boz.parameters.load(path + 'r0614/parameters_p2937_d615_r614.json')
```
%% Cell type:code id: tags:
``` python
rpath = path + f'r{runNB:04d}/'
prefix = 'small-data'
os.makedirs(rpath, exist_ok=True)
```
%% Cell type:markdown id: tags:
# Dark run inspection
%% Cell type:markdown id: tags:
The aim is to check dark level and extract bad pixel map.
%% Cell type:code id: tags:
``` python
arr_dark, tid_dark = boz.load_dssc_module(proposal, darkrunNB, moduleNB)
```
%% Cell type:code id: tags:
``` python
f = boz.inspect_dark(arr_dark)
f.suptitle(f'p:{proposal} d:{darkrunNB}')
fname = rpath + prefix + '-inspect_dark.png'
if not os.path.isfile(fname):
f.savefig(fname, dpi=300)
```
%% Output
%% Cell type:markdown id: tags:
# Veto pattern check
%% Cell type:markdown id: tags:
Check potential veto pattern issue
%% Cell type:code id: tags:
``` python
dark = boz.average_module(arr_dark).compute()
arr, tid = boz.load_dssc_module(proposal, runNB, moduleNB)
data = boz.average_module(arr, dark=dark).compute()
pp = data.mean(axis=(1,2)) # pulseId resolved mean
dataM = data.mean(axis=0) # mean over pulseId
```
%% Cell type:code id: tags:
``` python
plt.figure()
plt.plot(pp)
plt.xlabel('pulseId')
plt.ylabel('dark corrected module mean')
plt.title(f'p:{params.proposal} r:{params.run} d:{params.darkrun}')
plt.savefig(rpath+prefix+'-inspect_veto.png', dpi=300)
```
%% Output
%% Cell type:code id: tags:
``` python
"""
# Thresholding out bad veto pulse
if False:
params.arr = params.arr[:, pp > 2, :, :]
params.arr_dark = params.arr_dark[:, pp > 2, :, :]
dark = boz.average_module(params.arr_dark).compute()
data = boz.average_module(params.arr, dark=dark).compute()
dataM = data.mean(axis=0) # mean over pulseId
"""
```
%% Output
'\n# Thresholding out bad veto pulse\nif False:\n params.arr = params.arr[:, pp > 2, :, :]\n params.arr_dark = params.arr_dark[:, pp > 2, :, :]\n dark = boz.average_module(params.arr_dark).compute()\n data = boz.average_module(params.arr, dark=dark).compute()\n dataM = data.mean(axis=0) # mean over pulseId\n'
%% Cell type:markdown id: tags:
# Check ROIs
%% Cell type:markdown id: tags:
Let's check the ROIs used in the part I on a run later
%% Cell type:markdown id: tags:
it's a bit off, also we can see from the blur that the photon energy was varied in this run.
%% Cell type:code id: tags:
``` python
rois_th = 1.2
rois = boz.find_rois(dataM, rois_th)
```
%% Cell type:code id: tags:
``` python
rois = params.rois
```
%% Cell type:code id: tags:
``` python
f = boz.inspect_rois(dataM, rois, rois_th)
f.suptitle(f'p:{proposal} r:{runNB} d:{darkrunNB}')
fname = rpath + prefix + '-inspect_rois.png'
f.savefig(fname, dpi=300)
```
%% Output
%% Cell type:markdown id: tags:
We got new rois.
%% Cell type:markdown id: tags:
# Compute flat field with new rois
%% Cell type:markdown id: tags:
We use the previously fitted plane on the new roi.
%% Cell type:code id: tags:
``` python
ff = boz.compute_flat_field_correction(rois, params.get_flat_field())
```
%% Cell type:code id: tags:
``` python
f = boz.inspect_plane_fitting(dataM/ff, rois)
f.suptitle(f'p:{proposal} r:{runNB} d:{darkrunNB}')
fname = rpath + prefix + '-inspect_plane_fitting.png'
f.savefig(fname, dpi=300)
```
%% Output
%% Cell type:markdown id: tags:
# Process a run
%% Cell type:code id: tags:
``` python
# load data
arr, tid = boz.load_dssc_module(proposal, runNB, moduleNB)
arr_dark, tid_dark = boz.load_dssc_module(proposal, darkrunNB, moduleNB)
# make sure to rechunk the arrays
arr = arr.rechunk((100,-1,-1,-1))
arr_dark = arr_dark.rechunk((100,-1,-1,-1))
#no correction
#data = boz.process(np.arange(2**9), arr_dark, arr, tid, rois, params.get_mask(),
# np.ones_like(ff), params.sat_level)
#with flat field correction
#data = boz.process(np.arange(2**9), arr_dark, arr, tid, rois, params.get_mask(),
# ff, params.sat_level)
# with flat field and non linear correction
data = boz.process(params.get_Fnl(), arr_dark, arr, tid, rois, params.get_mask(),
ff, params.sat_level)
```
%% Cell type:markdown id: tags:
# Saturation histogram
%% Cell type:code id: tags:
``` python
f, h = boz.inspect_saturation(data, gain)
f.suptitle(f'p:{proposal} r:{runNB} d:{darkrunNB}')
fname = rpath + prefix + '-inspect_saturation.png'
f.savefig(fname, dpi=300)
fname = rpath + prefix + f'-inspect_saturation-run{runNB}-darkrun{darkrunNB}.h5'
print(fname)
h.to_netcdf(fname, format='NETCDF4', engine='h5netcdf')
```
%% Output
/gpfs/exfel/exp/SCS/202122/p002937/usr/processed_runs/r0690/small-data-inspect_saturation-run690-darkrun671.h5
%% Cell type:markdown id: tags:
# Drop saturated shot
%% Cell type:code id: tags:
``` python
# filter saturated shots
d = data.where(data['sat_sat'] == False, drop=True)
d.attrs['saturation (%)'] = h.attrs['saturation (%)']
print(f"{d.attrs['saturation (%)']:.2f} % of shots were saturated")
```
%% Output
6.90 % of shots were saturated
%% Cell type:markdown id: tags:
# Load rest of data
%% Cell type:code id: tags:
``` python
run, data = tb.load(proposal, runNB, ['PP800_DelayLine', 'nrj', 'BAM1932M'])
run, data = tb.load(proposal, runNB, ['PP800_DelayLine', 'nrj', 'BAM1932S'])
```
%% Cell type:code id: tags:
``` python
# overwrite DSSC pulseId with BAM sa3_pId
d = d.rename(pulseId = 'sa3_pId')
d['sa3_pId'] = data['sa3_pId']
```
%% Cell type:code id: tags:
``` python
r = xr.merge([data, d], join='inner', combine_attrs='no_conflicts')
```
%% Cell type:code id: tags:
``` python
#BAM jitter correction
bam = r['BAM1932M'] - r['BAM1932M'].mean()
#BAM time correction
bam = r['BAM1932S']
c_ = 299792458 *1e-9 # speed of light in mm/ps
r['delay_ps'] = 2/c_*r['PP800_DelayLine'] - bam
# converting back to mm after BAM correction
r['delay_mm'] = r['delay_ps']*0.15
```
%% Cell type:markdown id: tags:
# Save the processed data
%% Cell type:code id: tags:
``` python
# save the result
fname = rpath + prefix + f'-run{runNB}-darkrun{darkrunNB}.h5'
print(fname)
r.to_netcdf(fname, format='NETCDF4', engine='h5netcdf')
```
%% Output
/gpfs/exfel/exp/SCS/202122/p002937/usr/processed_runs/r0690/small-data-run690-darkrun671.h5
%% Cell type:code id: tags:
``` python
```
Loading