Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • SCS/ToolBox
  • kluyvert/ToolBox
2 results
Show changes
Commits on Source (8)
%% Cell type:code id: tags:
 
``` python
import numpy as np
%matplotlib inline
import matplotlib.pyplot as plt
plt.rcParams['figure.constrained_layout.use'] = True
 
import dask
print(f'dask: {dask.__version__}')
import dask.array
dask.config.set({'array.chunk-size': '512MiB'})
 
import xarray as xr
 
from psutil import virtual_memory
 
mem = virtual_memory()
print(f'Physical memory: {mem.total/1024/1024/1024:.0f} Gb') # total physical memory available
 
import logging
logging.basicConfig(filename='example.log', level=logging.DEBUG)
 
import os
```
 
%% Output
 
dask: 2.11.0
Physical memory: 504 Gb
 
%% Cell type:code id: tags:
 
``` python
%load_ext autoreload
 
%autoreload 2
 
import toolbox_scs as tb
print(tb.__file__)
import toolbox_scs.routines.boz as boz
 
from extra_data.read_machinery import find_proposal
```
 
%% Output
 
/home/lleguy/notebooks/ToolBox/src/toolbox_scs/__init__.py
 
%% Cell type:markdown id: tags:
 
# Loading analysis parameters
 
%% Cell type:code id: tags:
 
``` python
proposal = 2937
runNB = 690
darkrunNB = 671
moduleNB = 15
gain = 3
```
 
%% Cell type:code id: tags:
 
``` python
root = find_proposal(f'p{proposal:06d}')
path = root + f'/usr/processed_runs/'
params = boz.parameters.load(path + 'r0614/parameters_p2937_d615_r614.json')
```
 
%% Cell type:code id: tags:
 
``` python
rpath = path + f'r{runNB:04d}/'
prefix = prefix = f'p{proposal}-r{runNB}-d{darkrunNB}-BOZ-II1'
os.makedirs(rpath, exist_ok=True)
```
 
%% Cell type:markdown id: tags:
 
# Dark run inspection
 
%% Cell type:markdown id: tags:
 
The aim is to check dark level and extract bad pixel map.
 
%% Cell type:code id: tags:
 
``` python
arr_dark, tid_dark = boz.load_dssc_module(proposal, darkrunNB, moduleNB)
```
 
%% Cell type:code id: tags:
 
``` python
f = boz.inspect_dark(arr_dark)
f.suptitle(f'p:{proposal} d:{darkrunNB}')
fname = rpath + prefix + '-inspect_dark.png'
f.savefig(fname, dpi=300)
```
 
%% Output
 
 
%% Cell type:markdown id: tags:
 
# Veto pattern check
 
%% Cell type:markdown id: tags:
 
Check potential veto pattern issue
 
%% Cell type:code id: tags:
 
``` python
dark = boz.average_module(arr_dark).compute()
arr, tid = boz.load_dssc_module(proposal, runNB, moduleNB)
data = boz.average_module(arr, dark=dark).compute()
pp = data.mean(axis=(1,2)) # pulseId resolved mean
dataM = data.mean(axis=0) # mean over pulseId
```
 
%% Cell type:code id: tags:
 
``` python
plt.figure()
plt.plot(pp)
plt.xlabel('pulseId')
plt.ylabel('dark corrected module mean')
plt.title(f'p:{proposal} r:{runNB} d:{darkrunNB}')
plt.savefig(rpath+prefix+'-inspect_veto.png', dpi=300)
```
 
%% Output
 
 
%% Cell type:code id: tags:
 
``` python
"""
# Thresholding out bad veto pulse
if False:
params.arr = params.arr[:, pp > 2, :, :]
params.arr_dark = params.arr_dark[:, pp > 2, :, :]
dark = boz.average_module(params.arr_dark).compute()
data = boz.average_module(params.arr, dark=dark).compute()
dataM = data.mean(axis=0) # mean over pulseId
"""
```
 
%% Output
 
'\n# Thresholding out bad veto pulse\nif False:\n params.arr = params.arr[:, pp > 2, :, :]\n params.arr_dark = params.arr_dark[:, pp > 2, :, :]\n dark = boz.average_module(params.arr_dark).compute()\n data = boz.average_module(params.arr, dark=dark).compute()\n dataM = data.mean(axis=0) # mean over pulseId\n'
 
%% Cell type:markdown id: tags:
 
# Check ROIs
 
%% Cell type:markdown id: tags:
 
Let's check the ROIs used in the part I on a run later
 
%% Cell type:markdown id: tags:
 
it's a bit off, also we can see from the blur that the photon energy was varied in this run.
 
%% Cell type:code id: tags:
 
``` python
rois_th = 1.2
rois = boz.find_rois(dataM, rois_th)
```
 
%% Cell type:code id: tags:
 
``` python
rois = params.rois
```
 
%% Cell type:code id: tags:
 
``` python
f = boz.inspect_rois(dataM, rois, rois_th)
 
f.suptitle(f'p:{proposal} r:{runNB} d:{darkrunNB}')
fname = rpath + prefix + '-inspect_rois.png'
f.savefig(fname, dpi=300)
```
 
%% Output
 
 
%% Cell type:markdown id: tags:
 
We got new rois.
 
%% Cell type:markdown id: tags:
 
# Compute flat field with new rois
 
%% Cell type:markdown id: tags:
 
We use the previously fitted plane on the new roi.
 
%% Cell type:code id: tags:
 
``` python
f, domains = boz.inspect_flat_field_domain(dataM, rois,
params.flat_field_prod_th, params.flat_field_ratio_th)
f.savefig(path+prefix+'-inspect-flat_field_domain.png', dpi=300)
```
 
%% Output
 
 
%% Cell type:code id: tags:
 
``` python
ff = boz.compute_flat_field_correction(rois, params)
```
 
%% Cell type:code id: tags:
 
``` python
f = boz.inspect_plane_fitting(dataM/ff, rois, domains)
 
f.suptitle(f'p:{proposal} r:{runNB} d:{darkrunNB}')
fname = rpath + prefix + '-inspect_plane_fitting.png'
f.savefig(fname, dpi=300)
```
 
%% Output
 
 
%% Cell type:markdown id: tags:
 
# Process a run
 
%% Cell type:code id: tags:
 
``` python
# load data
arr, tid = boz.load_dssc_module(proposal, runNB, moduleNB)
arr_dark, tid_dark = boz.load_dssc_module(proposal, darkrunNB, moduleNB)
 
# make sure to rechunk the arrays
arr = arr.rechunk(('auto',-1,-1,-1))
arr_dark = arr_dark.rechunk(('auto',-1,-1,-1))
 
#no correction
#data = boz.process(np.arange(2**9), arr_dark, arr, tid, rois, params.get_mask(),
# np.ones_like(ff), params.sat_level)
#with flat field correction
#data = boz.process(np.arange(2**9), arr_dark, arr, tid, rois, params.get_mask(),
# ff, params.sat_level)
# with flat field and non linear correction
data = boz.process(params.get_Fnl(), arr_dark, arr, tid, rois, params.get_mask(),
ff, params.sat_level)
```
 
%% Cell type:markdown id: tags:
 
# Saturation histogram
 
%% Cell type:code id: tags:
 
``` python
f, h = boz.inspect_saturation(data, gain)
 
f.suptitle(f'p:{proposal} r:{runNB} d:{darkrunNB}')
fname = rpath + prefix + '-inspect_saturation.png'
f.savefig(fname, dpi=300)
 
fname = rpath + prefix + f'-inspect_saturation.h5'
print(fname)
h.to_netcdf(fname, format='NETCDF4', engine='h5netcdf')
```
 
%% Output
 
/gpfs/exfel/exp/SCS/202122/p002937/usr/processed_runs/r0690/p2937-r690-d671-BOZ-II1-inspect_saturation.h5
 
 
%% Cell type:markdown id: tags:
 
# Drop saturated shot
 
%% Cell type:code id: tags:
 
``` python
# filter saturated shots
d = data.where(data['sat_sat'] == False, drop=True)
d.attrs['saturation (%)'] = h.attrs['saturation (%)']
print(f"{d.attrs['saturation (%)']:.2f} % of shots were saturated")
```
 
%% Output
 
6.90 % of shots were saturated
 
%% Cell type:markdown id: tags:
 
# Load rest of data
 
%% Cell type:code id: tags:
 
``` python
run, data = tb.load(proposal, runNB, ['PP800_DelayLine', 'nrj', 'BAM1932S'])
```
 
%% Cell type:code id: tags:
 
``` python
# overwrite DSSC pulseId with BAM sa3_pId
d = d.rename(pulseId = 'sa3_pId')
d['sa3_pId'] = data['sa3_pId']
```
 
%% Cell type:code id: tags:
 
``` python
r = xr.merge([data, d], join='inner', combine_attrs='no_conflicts')
```
 
%% Cell type:code id: tags:
 
``` python
#BAM jitter correction
bam = r['BAM1932S'] - r['BAM1932S'].mean()
c_ = 299792458 *1e-9 # speed of light in mm/ps
r['delay_ps'] = 0# 2/c_*r['PP800_DelayLine'] - bam
r['delay_ps'] = 2/c_*r['PP800_DelayLine'] - bam
# converting back to mm after BAM correction
r['delay_mm'] = r['delay_ps']*0.15
```
 
%% Cell type:markdown id: tags:
 
# Save the processed data
 
%% Cell type:code id: tags:
 
``` python
# save the result
fname = rpath + prefix + f'-small-data.h5'
print(fname)
r.to_netcdf(fname, format='NETCDF4', engine='h5netcdf')
```
 
%% Output
 
/gpfs/exfel/exp/SCS/202122/p002937/usr/processed_runs/r0690/p2937-r690-d671-BOZ-II1-small-data.h5
 
%% Cell type:code id: tags:
 
``` python
```
......
......@@ -15,6 +15,7 @@ unreleased
- fix :issue:`57` adds target mono energy mnemonic
- fix :issue:`55` implementingd dask auto rechunking in notebooks
- fix :issue:`53` wrong flat field correction sometimes being calculated
- fix :issue:`56` future warning on xarray.ufuncs :mr:`189`
- **Improvements**
......@@ -30,6 +31,7 @@ unreleased
- **New Features**
- add routine for fluence calibration :mr:`180`
- add Fast ADC 2 mnemonics :mr:`200`
1.5.0
-----
......
......@@ -660,6 +660,68 @@ mnemonics = {
'key': 'data.rawData',
'dim': ['fadc_samplesId']},),
# FastADC 2
"FastADC2_0peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_0.output',
'key': 'data.peaks',
'dim': ['peakId']},),
"FastADC2_0raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_0.output',
'key': 'data.rawData',
'dim': ['fadc2_samplesId']},),
"FastADC2_1peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_1.output',
'key': 'data.peaks',
'dim': ['peakId']},),
"FastADC2_1raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_1.output',
'key': 'data.rawData',
'dim': ['fadc2_samplesId']},),
"FastADC2_2peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_2.output',
'key': 'data.peaks',
'dim': ['peakId']},),
"FastADC2_2raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_2.output',
'key': 'data.rawData',
'dim': ['fadc2_samplesId']},),
"FastADC2_3peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_3.output',
'key': 'data.peaks',
'dim': ['peakId']},),
"FastADC2_3raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_3.output',
'key': 'data.rawData',
'dim': ['fadc2_samplesId']},),
"FastADC2_4peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_4.output',
'key': 'data.peaks',
'dim': ['peakId']},),
"FastADC2_4raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_4.output',
'key': 'data.rawData',
'dim': ['fadc2_samplesId']},),
"FastADC2_5peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_5.output',
'key': 'data.peaks',
'dim': ['peakId']},),
"FastADC2_5raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_5.output',
'key': 'data.rawData',
'dim': ['fadc2_samplesId']},),
"FastADC2_6peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_6.output',
'key': 'data.peaks',
'dim': ['peakId']},),
"FastADC2_6raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_6.output',
'key': 'data.rawData',
'dim': ['fadc2_samplesId']},),
"FastADC2_7peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_7.output',
'key': 'data.peaks',
'dim': ['peakId']},),
"FastADC2_7raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_7.output',
'key': 'data.rawData',
'dim': ['fadc2_samplesId']},),
"FastADC2_8peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_8.output',
'key': 'data.peaks',
'dim': ['peakId']},),
"FastADC2_8raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_8.output',
'key': 'data.rawData',
'dim': ['fadc2_samplesId']},),
"FastADC2_9peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_9.output',
'key': 'data.peaks',
'dim': ['peakId']},),
"FastADC2_9raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_9.output',
'key': 'data.rawData',
'dim': ['fadc2_samplesId']},),
# KARABACON
"KARABACON": ({'source': 'SCS_DAQ_SCAN/MDL/KARABACON',
'key': 'actualStep.value',
......
......@@ -146,7 +146,7 @@ def align_xgm_array(xgm_arr, bpt):
compute_sa3 = True
mask_sa1 = is_sase_1(bpt.sel(trainId=valid_tid))
mask_sa3 = is_sase_3(bpt.sel(trainId=valid_tid))
mask = xr.ufuncs.logical_or(mask_sa1, mask_sa3)
mask = np.logical_or(mask_sa1, mask_sa3)
tid = mask.where(mask.sum(dim='pulse_slot') > 0,
drop=True).trainId
mask_sa1 = mask_sa1.sel(trainId=tid).rename({'pulse_slot': 'sa1_pId'})
......