Beam splitting off-axis zone plate analysis

This adds the beam splitting off-axis zone plate analysis code to the toolbox:

• determination of correction parameters are shown in a notebook documentation
• correction parameters and parameters used in their determination are all saved in one single JSON file for reproducibility
• analysis of actual data using these correction parameters is shown in a second notebook documentation

doc/BOZ analysis part II run processing.ipynb

 %% Cell type:code id: tags:
 
 ``` python
 import numpy as np
 %matplotlib notebook
 import matplotlib.pyplot as plt
 plt.rcParams['figure.constrained_layout.use'] = True
 
 import dask
 print(f'dask: {dask.__version__}')
 import dask.array as da
 from dask.distributed import Client, progress
 from dask_jobqueue import SLURMCluster
 
 import netCDF4
 import xarray as xr
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 from psutil import virtual_memory
 import gc
 # gc.collect() # run garbage collection to free possible memory
 
 mem = virtual_memory()
 print(f'Physical memory: {mem.total/1024/1024/1024:.0f} Gb')  # total physical memory available
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 import logging
 logging.basicConfig(filename='example.log', level=logging.DEBUG)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 %load_ext autoreload
 
 %autoreload 2
 
 import toolbox_scs as tb
 print(tb.__file__)
 import toolbox_scs.routines.boz as boz
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 %load_ext line_profiler
 %load_ext memory_profiler
 ```
 
 %% Cell type:markdown id: tags:
 
 # Slum cluster setup
 
 %% Cell type:markdown id: tags:
 
 Change to *True* to allocate a slurm cluster and run the calculation on it instead of the current machine.
 
 %% Cell type:code id: tags:
 
 ``` python
 if False:
     partition = 'exfel'   # For users
     # partition = 'upex'   # For users
 
     cluster = SLURMCluster(
         queue=partition,
         local_directory='/scratch',  # Local disk space for workers to use
 
         # Resources per SLURM job (per node, the way SLURM is configured on Maxwell)
         # processes=16 runs 16 Dask workers in a job, so each worker has 1 core & 32 GB RAM.
         processes=8, cores=16, memory='512GB',
         walltime='24:00:00',
         #interface='ib0',
         #job_extra=["--reservation=upex_002619"] # reserved partition
     )
 
     # dashboard link
     # print('https://max-jhub.desy.de/user/lleguy/proxy/8787/graph')
 
     # Submit 2 SLURM jobs, for 32 Dask workers
     cluster.scale(32)
     #cluster.adapt(minimum=0, maximum=32)
 
     client = Client(cluster)
     print("Created dask client:", client)
 
     # Get a notbook widget showing the cluster state
     cluster
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 try:
     client.close()
     cluster.close()
 except:
     print('No client defined')
 ```
 
 %% Cell type:markdown id: tags:
 
 # Loading analysis parameters
 
 %% Cell type:code id: tags:
 
 ``` python
 params = boz.parameters.load('./parameters_p2712_d5_r6.json')
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 print(params)
 ```
 
 %% Cell type:markdown id: tags:
 
 # Dark run inspection
 
 %% Cell type:markdown id: tags:
 
 The aim is to check dark level and extract bad pixel map.
 
 %% Cell type:code id: tags:
 
 ``` python
 proposalNB = 2712
 darkrunNB = 10
 moduleNB = 15
 
 arr_dark, tid_dark = boz.load_dssc_module(proposalNB, darkrunNB, moduleNB)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 f = boz.inspect_dark(arr_dark)
 ```
 
 %% Cell type:markdown id: tags:
 
 # Check ROIs
 
 %% Cell type:markdown id: tags:
 
 Let's check the ROIs used in the part I on a run later
 
 %% Cell type:code id: tags:
 
 ``` python
 runNB = 13
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 dark = boz.average_module(arr_dark).compute()
 arr, tid = boz.load_dssc_module(proposalNB, runNB, moduleNB)
 data = boz.average_module(arr, dark=dark).compute()
 dataM = data.mean(axis=0) # mean over pulseId
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 f = boz.inspect_rois(dataM, params.rois, params.rois_th)
 ```
 
 %% Cell type:markdown id: tags:
 
 it's a bit off, also we can see from the blur that the photon energy was varied in this run.
 
 %% Cell type:code id: tags:
 
 ``` python
 rois_th = 35
 rois = boz.find_rois(dataM, rois_th)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 f = boz.inspect_rois(dataM, rois, rois_th)
 ```
 
 %% Cell type:markdown id: tags:
 
 We got new rois.
 
 %% Cell type:markdown id: tags:
 
 # Compute flat field with new rois
 
 %% Cell type:markdown id: tags:
 
 We use the previously fitted plane on the new roi.
 
 %% Cell type:code id: tags:
 
 ``` python
 ff = boz.compute_flat_field_correction(rois, params.get_flat_field())
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 f = boz.inspect_plane_fitting(dataM/ff, rois)
 ```
 
 %% Cell type:markdown id: tags:
 
 some issue in the plotting since the rois have different size
 
 %% Cell type:markdown id: tags:
 
 # Process a run
 
 %% Cell type:code id: tags:
 
 ``` python
 # load data
 arr, tid = boz.load_dssc_module(proposalNB, runNB, moduleNB)
 arr_dark, tid_dark = boz.load_dssc_module(proposalNB, darkrunNB, moduleNB)
 
 # make sure to rechunk the arrays
 arr = arr.rechunk((100,-1,-1,-1))
 arr_dark = arr_dark.rechunk((100,-1,-1,-1))
 
-data = boz._process(params.get_Fnl(), arr_dark, arr, tid, rois, params.get_mask(), ff, params.sat_level)
+#no correction
+data = boz.process(np.arange(2**9), arr_dark, arr, tid, rois, params.get_mask(),
+                   np.ones_like(ff), params.sat_level)
+#with flat field correction
+data_ff = boz.process(np.arange(2**9), arr_dark, arr, tid, rois, params.get_mask(),
+                      ff, params.sat_level)
+# with flat field and non linear correction
+data_ff_nl = boz.process(params.get_Fnl(), arr_dark, arr, tid, rois, params.get_mask(),
+                         ff, params.sat_level)
 
 # drop saturated shots
 d = data.where(data['sat_sat'] == False, drop=True)
+d_ff = data_ff.where(data_ff['sat_sat'] == False, drop=True)
+d_ff_nl = data_ff_nl.where(data_ff_nl['sat_sat'] == False, drop=True)
 ```
 
+%% Cell type:markdown id: tags:
 
+# Load rest of data
 
 %% Cell type:code id: tags:
 
 ``` python
-d
+run, data = tb.load(proposalNB, runNB, ['nrj'])
 ```
 
-%% Cell type:markdown id: tags:
+%% Cell type:code id: tags:
 
-# Load rest of data
+``` python
+r = xr.merge([data, d], join='inner')
+r_ff = xr.merge([data, d_ff], join='inner')
+r_ff_nl = xr.merge([data, d_ff_nl], join='inner')
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
-run, data = tb.load(proposalNB, runNB, ['nrj'])
+res = tb.xas(r_ff_nl, 0.1, Iokey = '0', Itkey = 'n', nrjkey = 'nrj', plot=True)
 ```
 
+%% Cell type:markdown id: tags:
 
+# Comparison spectra between the different correction
 
 %% Cell type:code id: tags:
 
 ``` python
-data
+res = tb.xas(r, 0.1, Iokey = '0', Itkey = 'n', nrjkey = 'nrj', plot=False)
+res_ff = tb.xas(r_ff, 0.1, Iokey = '0', Itkey = 'n', nrjkey = 'nrj', plot=False)
+res_ff_nl = tb.xas(r_ff_nl, 0.1, Iokey = '0', Itkey = 'n', nrjkey = 'nrj', plot=False)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
-r = xr.merge([data, d], join='inner')
-r
+f = plt.figure()
+ax = f.gca()
+ax.errorbar(res['nrj'], res['muA'] - res['muA'][0], res['sterrA'], label='raw')
+ax.errorbar(res_ff['nrj'], res_ff['muA'] - res_ff['muA'][0], res_ff['sterrA'], label='ff')
+ax.errorbar(res_ff_nl['nrj'], res_ff_nl['muA'] - res_ff_nl['muA'][0], res_ff_nl['sterrA'], label='ff+nl')
+ax.set_xlabel('Energy (eV)')
+ax.set_ylabel('XAS')
+ax.set_xlim([695, 740])
+ax.legend(loc=2)
+
+from mpl_toolkits.axes_grid1.inset_locator import inset_axes, mark_inset
+
+axins = inset_axes(ax, width='40%', height='30%', loc=1)
+axins.errorbar(res['nrj'], res['muA'] - res['muA'][0], res['sterrA'], label='raw')
+axins.errorbar(res_ff['nrj'], res_ff['muA'] - res_ff['muA'][0], res_ff['sterrA'], label='ff')
+axins.errorbar(res_ff_nl['nrj'], res_ff_nl['muA'] - res_ff_nl['muA'][0], res_ff_nl['sterrA'], label='ff+nl')
+axins.set_xlim([695, 698])
+axins.set_ylim([-0.005, 0.005])
+mark_inset(ax, axins, loc1=2, loc2=4, fc="none", ec="0.5")
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
-res = tb.xas(r, 0.1, Iokey = '0', Itkey = 'n', nrjkey = 'nrj', plot=True)
 ```