doc/BOZ analysis part II.1 Small data.ipynb

 %% Cell type:code id: tags:
 ``` python
 import numpy as np
 %matplotlib inline
 import matplotlib.pyplot as plt
 plt.rcParams['figure.constrained_layout.use'] = True
 import dask
 print(f'dask: {dask.__version__}')
 import dask.array
 dask.config.set({'array.chunk-size': '512MiB'})
 import xarray as xr
 from psutil import virtual_memory
 mem = virtual_memory()
 print(f'Physical memory: {mem.total/1024/1024/1024:.0f} Gb')  # total physical memory available
 import logging
 logging.basicConfig(filename='example.log', level=logging.DEBUG)
 import os
 ```
 %% Output
     dask: 2.11.0
     Physical memory: 504 Gb
 %% Cell type:code id: tags:
 ``` python
 %load_ext autoreload
 %autoreload 2
 import toolbox_scs as tb
 print(tb.__file__)
 import toolbox_scs.routines.boz as boz
 from extra_data.read_machinery import find_proposal
 ```
 %% Output
     /home/lleguy/notebooks/ToolBox/src/toolbox_scs/__init__.py
 %% Cell type:markdown id: tags:
 # Loading analysis parameters
 %% Cell type:code id: tags:
 ``` python
 proposal = 2937
 runNB = 690
 darkrunNB = 671
 moduleNB = 15
 gain = 3
 ```
 %% Cell type:code id: tags:
 ``` python
 root = find_proposal(f'p{proposal:06d}')
 path = root + f'/usr/processed_runs/'
 params = boz.parameters.load(path + 'r0614/parameters_p2937_d615_r614.json')
 ```
 %% Cell type:code id: tags:
 ``` python
 rpath = path + f'r{runNB:04d}/'
 prefix = prefix = f'p{proposal}-r{runNB}-d{darkrunNB}-BOZ-II1'
 os.makedirs(rpath, exist_ok=True)
 ```
 %% Cell type:markdown id: tags:
 # Dark run inspection
 %% Cell type:markdown id: tags:
 The aim is to check dark level and extract bad pixel map.
 %% Cell type:code id: tags:
 ``` python
 arr_dark, tid_dark = boz.load_dssc_module(proposal, darkrunNB, moduleNB)
 ```
 %% Cell type:code id: tags:
 ``` python
 f = boz.inspect_dark(arr_dark)
 f.suptitle(f'p:{proposal} d:{darkrunNB}')
 fname = rpath + prefix + '-inspect_dark.png'
 f.savefig(fname, dpi=300)
 ```
 %% Output
 %% Cell type:markdown id: tags:
 # Veto pattern check
 %% Cell type:markdown id: tags:
 Check potential veto pattern issue
 %% Cell type:code id: tags:
 ``` python
 dark = boz.average_module(arr_dark).compute()
 arr, tid = boz.load_dssc_module(proposal, runNB, moduleNB)
 data = boz.average_module(arr, dark=dark).compute()
 pp = data.mean(axis=(1,2)) # pulseId resolved mean
 dataM = data.mean(axis=0) # mean over pulseId
 ```
 %% Cell type:code id: tags:
 ``` python
 plt.figure()
 plt.plot(pp)
 plt.xlabel('pulseId')
 plt.ylabel('dark corrected module mean')
 plt.title(f'p:{proposal} r:{runNB} d:{darkrunNB}')
 plt.savefig(rpath+prefix+'-inspect_veto.png', dpi=300)
 ```
 %% Output
 %% Cell type:code id: tags:
 ``` python
 """
 # Thresholding out bad veto pulse
 if False:
     params.arr = params.arr[:, pp > 2, :, :]
     params.arr_dark = params.arr_dark[:, pp > 2, :, :]
     dark = boz.average_module(params.arr_dark).compute()
     data = boz.average_module(params.arr, dark=dark).compute()
     dataM = data.mean(axis=0) # mean over pulseId
 """
 ```
 %% Output
     '\n# Thresholding out bad veto pulse\nif False:\n    params.arr = params.arr[:, pp > 2, :, :]\n    params.arr_dark = params.arr_dark[:, pp > 2, :, :]\n    dark = boz.average_module(params.arr_dark).compute()\n    data = boz.average_module(params.arr, dark=dark).compute()\n    dataM = data.mean(axis=0) # mean over pulseId\n'
 %% Cell type:markdown id: tags:
 # Check ROIs
 %% Cell type:markdown id: tags:
 Let's check the ROIs used in the part I on a run later
 %% Cell type:markdown id: tags:
 it's a bit off, also we can see from the blur that the photon energy was varied in this run.
 %% Cell type:code id: tags:
 ``` python
 rois_th = 1.2
 rois = boz.find_rois(dataM, rois_th)
 ```
 %% Cell type:code id: tags:
 ``` python
 rois = params.rois
 ```
 %% Cell type:code id: tags:
 ``` python
 f = boz.inspect_rois(dataM, rois, rois_th)
 f.suptitle(f'p:{proposal} r:{runNB} d:{darkrunNB}')
 fname = rpath + prefix + '-inspect_rois.png'
 f.savefig(fname, dpi=300)
 ```
 %% Output
 %% Cell type:markdown id: tags:
 We got new rois.
 %% Cell type:markdown id: tags:
 # Compute flat field with new rois
 %% Cell type:markdown id: tags:
 We use the previously fitted plane on the new roi.
 %% Cell type:code id: tags:
 ``` python
 f, domains = boz.inspect_flat_field_domain(dataM, rois,
                                            params.flat_field_prod_th, params.flat_field_ratio_th)
 f.savefig(path+prefix+'-inspect-flat_field_domain.png', dpi=300)
 ```
 %% Output
 %% Cell type:code id: tags:
 ``` python
 ff = boz.compute_flat_field_correction(rois, params)
 ```
 %% Cell type:code id: tags:
 ``` python
 f = boz.inspect_plane_fitting(dataM/ff, rois, domains)
 f.suptitle(f'p:{proposal} r:{runNB} d:{darkrunNB}')
 fname = rpath + prefix + '-inspect_plane_fitting.png'
 f.savefig(fname, dpi=300)
 ```
 %% Output
 %% Cell type:markdown id: tags:
 # Process a run
 %% Cell type:code id: tags:
 ``` python
 # load data
 arr, tid = boz.load_dssc_module(proposal, runNB, moduleNB)
 arr_dark, tid_dark = boz.load_dssc_module(proposal, darkrunNB, moduleNB)
 # make sure to rechunk the arrays
 arr = arr.rechunk(('auto',-1,-1,-1))
 arr_dark = arr_dark.rechunk(('auto',-1,-1,-1))
 #no correction
 #data = boz.process(np.arange(2**9), arr_dark, arr, tid, rois, params.get_mask(),
 #                   np.ones_like(ff), params.sat_level)
 #with flat field correction
 #data = boz.process(np.arange(2**9), arr_dark, arr, tid, rois, params.get_mask(),
 #                      ff, params.sat_level)
 # with flat field and non linear correction
 data = boz.process(params.get_Fnl(), arr_dark, arr, tid, rois, params.get_mask(),
                          ff, params.sat_level)
 ```
 %% Cell type:markdown id: tags:
 # Saturation histogram
 %% Cell type:code id: tags:
 ``` python
 f, h = boz.inspect_saturation(data, gain)
 f.suptitle(f'p:{proposal} r:{runNB} d:{darkrunNB}')
 fname = rpath + prefix + '-inspect_saturation.png'
 f.savefig(fname, dpi=300)
 fname = rpath + prefix + f'-inspect_saturation.h5'
 print(fname)
 h.to_netcdf(fname, format='NETCDF4', engine='h5netcdf')
 ```
 %% Output
     /gpfs/exfel/exp/SCS/202122/p002937/usr/processed_runs/r0690/p2937-r690-d671-BOZ-II1-inspect_saturation.h5
 %% Cell type:markdown id: tags:
 # Drop saturated shot
 %% Cell type:code id: tags:
 ``` python
 # filter saturated shots
 d = data.where(data['sat_sat'] == False, drop=True)
 d.attrs['saturation (%)'] = h.attrs['saturation (%)']
 print(f"{d.attrs['saturation (%)']:.2f} % of shots were saturated")
 ```
 %% Output
     6.90 % of shots were saturated
 %% Cell type:markdown id: tags:
 # Load rest of data
 %% Cell type:code id: tags:
 ``` python
 run, data = tb.load(proposal, runNB, ['PP800_DelayLine', 'nrj', 'BAM1932S'])
 ```
 %% Cell type:code id: tags:
 ``` python
 # overwrite DSSC pulseId with BAM sa3_pId
 d = d.rename(pulseId = 'sa3_pId')
 d['sa3_pId'] = data['sa3_pId']
 ```
 %% Cell type:code id: tags:
 ``` python
 r = xr.merge([data, d], join='inner', combine_attrs='no_conflicts')
 ```
 %% Cell type:code id: tags:
 ``` python
 #BAM jitter correction
 bam = r['BAM1932S'] - r['BAM1932S'].mean()
 c_ = 299792458 *1e-9 # speed of light in mm/ps
-r['delay_ps'] = 0# 2/c_*r['PP800_DelayLine'] - bam
+r['delay_ps'] = 2/c_*r['PP800_DelayLine'] - bam
 # converting back to mm after BAM correction
 r['delay_mm'] = r['delay_ps']*0.15
 ```
 %% Cell type:markdown id: tags:
 # Save the processed data
 %% Cell type:code id: tags:
 ``` python
 # save the result
 fname = rpath + prefix + f'-small-data.h5'
 print(fname)
 r.to_netcdf(fname, format='NETCDF4', engine='h5netcdf')
 ```
 %% Output
     /gpfs/exfel/exp/SCS/202122/p002937/usr/processed_runs/r0690/p2937-r690-d671-BOZ-II1-small-data.h5
 %% Cell type:code id: tags:
 ``` python
 ```

doc/changelog.rst

@@ -15,6 +15,7 @@ unreleased
 		- fix :issue:`57` adds target mono energy mnemonic
     - fix :issue:`55` implementingd dask auto rechunking in notebooks
     - fix :issue:`53` wrong flat field correction sometimes being calculated 
+    - fix :issue:`56` future warning on xarray.ufuncs :mr:`189`
 
 - **Improvements**
 
@@ -30,6 +31,7 @@ unreleased
  
 - **New Features**
     - add routine for fluence calibration :mr:`180`
+    - add Fast ADC 2 mnemonics :mr:`200`
 
 1.5.0
 -----

src/toolbox_scs/constants.py

@@ -660,6 +660,68 @@ mnemonics = {
                      'key': 'data.rawData',
                      'dim': ['fadc_samplesId']},),
 
+    # FastADC 2
+    "FastADC2_0peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_0.output',
+                         'key': 'data.peaks',
+                         'dim': ['peakId']},),
+    "FastADC2_0raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_0.output',
+                       'key': 'data.rawData',
+                       'dim': ['fadc2_samplesId']},),
+    "FastADC2_1peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_1.output',
+                         'key': 'data.peaks',
+                         'dim': ['peakId']},),
+    "FastADC2_1raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_1.output',
+                       'key': 'data.rawData',
+                       'dim': ['fadc2_samplesId']},),
+    "FastADC2_2peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_2.output',
+                         'key': 'data.peaks',
+                         'dim': ['peakId']},),
+    "FastADC2_2raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_2.output',
+                       'key': 'data.rawData',
+                       'dim': ['fadc2_samplesId']},),
+    "FastADC2_3peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_3.output',
+                         'key': 'data.peaks',
+                         'dim': ['peakId']},),
+    "FastADC2_3raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_3.output',
+                       'key': 'data.rawData',
+                       'dim': ['fadc2_samplesId']},),
+    "FastADC2_4peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_4.output',
+                         'key': 'data.peaks',
+                         'dim': ['peakId']},),
+    "FastADC2_4raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_4.output',
+                       'key': 'data.rawData',
+                       'dim': ['fadc2_samplesId']},),
+    "FastADC2_5peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_5.output',
+                         'key': 'data.peaks',
+                         'dim': ['peakId']},),
+    "FastADC2_5raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_5.output',
+                       'key': 'data.rawData',
+                       'dim': ['fadc2_samplesId']},),
+    "FastADC2_6peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_6.output',
+                        'key': 'data.peaks',
+                        'dim': ['peakId']},),
+    "FastADC2_6raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_6.output',
+                       'key': 'data.rawData',
+                       'dim': ['fadc2_samplesId']},),
+    "FastADC2_7peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_7.output',
+                         'key': 'data.peaks',
+                         'dim': ['peakId']},),
+    "FastADC2_7raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_7.output',
+                       'key': 'data.rawData',
+                       'dim': ['fadc2_samplesId']},),
+    "FastADC2_8peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_8.output',
+                         'key': 'data.peaks',
+                        'dim': ['peakId']},),
+    "FastADC2_8raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_8.output',
+                       'key': 'data.rawData',
+                       'dim': ['fadc2_samplesId']},),
+    "FastADC2_9peaks": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_9.output',
+                         'key': 'data.peaks',
+                         'dim': ['peakId']},),
+    "FastADC2_9raw": ({'source': 'SCS_UTC2_FADC/ADC/1:channel_9.output',
+                       'key': 'data.rawData',
+                       'dim': ['fadc2_samplesId']},),
+
     # KARABACON
     "KARABACON": ({'source': 'SCS_DAQ_SCAN/MDL/KARABACON',
                    'key': 'actualStep.value',

src/toolbox_scs/detectors/xgm.py

@@ -146,7 +146,7 @@ def align_xgm_array(xgm_arr, bpt):
         compute_sa3 = True
         mask_sa1 = is_sase_1(bpt.sel(trainId=valid_tid))
         mask_sa3 = is_sase_3(bpt.sel(trainId=valid_tid))
-        mask = xr.ufuncs.logical_or(mask_sa1, mask_sa3)
+        mask = np.logical_or(mask_sa1, mask_sa3)
         tid = mask.where(mask.sum(dim='pulse_slot') > 0,
                          drop=True).trainId
         mask_sa1 = mask_sa1.sel(trainId=tid).rename({'pulse_slot': 'sa1_pId'})