Beam splitting off-axis zone plate analysis

This adds the beam splitting off-axis zone plate analysis code to the toolbox:

• determination of correction parameters are shown in a notebook documentation
• correction parameters and parameters used in their determination are all saved in one single JSON file for reproducibility
• analysis of actual data using these correction parameters is shown in a second notebook documentation

doc/BOZ analysis part I parameters determination.ipynb

 %% Cell type:code id: tags:
 
 ``` python
 import numpy as np
 %matplotlib notebook
 import matplotlib.pyplot as plt
 plt.rcParams['figure.constrained_layout.use'] = True
 
 import dask
 print(f'dask: {dask.__version__}')
 import dask.array as da
 from dask.distributed import Client, progress
 from dask_jobqueue import SLURMCluster
 
 import netCDF4
 import pickle
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 from psutil import virtual_memory
 import gc
 # gc.collect() # run garbage collection to free possible memory
 
 mem = virtual_memory()
 print(f'Physical memory: {mem.total/1024/1024/1024:.0f} Gb')  # total physical memory available
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 import logging
 logging.basicConfig(filename='example.log', level=logging.DEBUG)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 %load_ext autoreload
 
 %autoreload 2
 
 import toolbox_scs as tb
 print(tb.__file__)
 import toolbox_scs.routines.boz as boz
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 %load_ext line_profiler
 %load_ext memory_profiler
 ```
 
 %% Cell type:markdown id: tags:
 
 # Slum cluster setup
 
 %% Cell type:markdown id: tags:
 
 Change to *True* to allocate a slurm cluster and run the calculation on it instead of the current machine.
 
 %% Cell type:code id: tags:
 
 ``` python
 if False:
     partition = 'exfel'   # For users
     # partition = 'upex'   # For users
 
     cluster = SLURMCluster(
         queue=partition,
         local_directory='/scratch',  # Local disk space for workers to use
 
         # Resources per SLURM job (per node, the way SLURM is configured on Maxwell)
         # processes=16 runs 16 Dask workers in a job, so each worker has 1 core & 32 GB RAM.
         processes=8, cores=16, memory='512GB',
         walltime='9:00:00',
         #interface='ib0',
         #job_extra=["--reservation=upex_002619"] # reserved partition
     )
 
     # dashboard link
     # print('https://max-jhub.desy.de/user/lleguy/proxy/8787/graph')
 
     # Submit 2 SLURM jobs, for 32 Dask workers
     cluster.scale(32)
     #cluster.adapt(minimum=0, maximum=32)
 
     client = Client(cluster)
     print("Created dask client:", client)
 
     # Get a notbook widget showing the cluster state
     cluster
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 try:
     client.close()
     cluster.close()
 except:
     print('No client defined')
 ```
 
 %% Cell type:markdown id: tags:
 
 # Create parameters object
 
 %% Cell type:code id: tags:
 
 ``` python
 params = boz.parameters(proposal=2712, darkrun=5, run=6, module=15)
+#params = boz.parameters(proposal=2619, darkrun=33, run=38, module=15)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 print(params)
 ```
 
 %% Cell type:markdown id: tags:
 
-# Dark run inspection
 
-%% Cell type:markdown id: tags:
 
-The aim is to check dark level and extract bad pixel map.
+### load data persistently
 
 %% Cell type:code id: tags:
 
 ``` python
-arr_dark, tid_dark = boz.load_dssc_module(params.proposal, params.darkrun, params.module)
+params.dask_load()
 ```
 
+%% Cell type:markdown id: tags:
 
+# Dark run inspection
 
+%% Cell type:markdown id: tags:
 
+The aim is to check dark level and extract bad pixel map.
 
 %% Cell type:code id: tags:
 
 ``` python
 mean_th = (30, 56)
-f = boz.inspect_dark(arr_dark, mean_th=mean_th)
+f = boz.inspect_dark(params.arr_dark, mean_th=mean_th)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 params.mean_th = mean_th
 params.set_mask(boz.bad_pixel_map(params))
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 print(params)
 ```
 
 %% Cell type:markdown id: tags:
 
 # Histogram
 
 %% Cell type:code id: tags:
 
 ``` python
 h, f = boz.inspect_histogram(params.proposal, params.run, params.module,
                              mask=params.get_mask() #, extra_lines=True
                          )
 ```
 
 %% Cell type:markdown id: tags:
 
 adding guide to the eye
 
 %% Cell type:code id: tags:
 
 ``` python
 ax = f.gca()
 pf = np.polyfit([60, 220], [7, 4], 1)
 ax.plot([40, 400], 2*10**np.polyval(pf, [40, 400]))
 ax.plot([40, 400], 0.25*2*10**np.polyval(pf, [40, 400]))
 ```
 
 %% Cell type:markdown id: tags:
 
 # ROIs extraction
 
 %% Cell type:code id: tags:
 
 ``` python
-params.rois_th = 40
+params.rois_th = 50
 params.rois = boz.find_rois_from_params(params)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 print(params)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
-dark = boz.average_module(arr_dark).compute()
-arr, tid = boz.load_dssc_module(params.proposal, params.run, params.module)
-data = boz.average_module(arr, dark=dark).compute()
+dark = boz.average_module(params.arr_dark).compute()
+data = boz.average_module(params.arr, dark=dark).compute()
 dataM = data.mean(axis=0) # mean over pulseId
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 f = boz.inspect_rois(dataM, params.rois, params.rois_th)
 ```
 
 %% Cell type:markdown id: tags:
 
 # Flat field extraction
 
 %% Cell type:markdown id: tags:
 
 The first step is to compute a good average image, this mean remove saturated shots and ignoring bad pixels
 
 %% Cell type:code id: tags:
 
 ``` python
 params.sat_level = 511
-res = boz.average_module(arr, dark=dark,
+res = boz.average_module(params.arr, dark=dark,
                          ret='mean', mask=params.get_mask(), sat_roi=params.rois['sat'],
                          sat_level=params.sat_level)
 avg = res.compute().mean(axis=0)
 ```
 
 %% Cell type:markdown id: tags:
 
 The second step is from that good average image to fit the plane field on n/0 and p/0 rois. We have to make sure that the rois have same width.
 
 %% Cell type:code id: tags:
 
 ``` python
 f = boz.inspect_plane_fitting(avg, params.rois)
 ```
 
 %% Cell type:markdown id: tags:
 
 fit the plane field correction
 
 %% Cell type:code id: tags:
 
 ``` python
 plane = boz.plane_fitting(params)
 ```
 
+%% Cell type:code id: tags:
 
+``` python
+plane
 ```
 
 %% Cell type:markdown id: tags:
 
 compute the correction and inspect the result of its application
 
 %% Cell type:code id: tags:
 
 ``` python
-params.set_flat_field(plane)
+params.set_flat_field(plane.x)
 ff = boz.compute_flat_field_correction(params.rois, params.get_flat_field())
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 f = boz.inspect_plane_fitting(avg/ff, params.rois)
 ```
 
 %% Cell type:markdown id: tags:
 
 # Non-linearities correction extraction
 
 %% Cell type:code id: tags:
 
 ``` python
 N = 40
 domain = boz.nl_domain(N, 40, 280)
 params.alpha = 0.5
 params.max_iter = 25
 ```
 
 %% Cell type:markdown id: tags:
 
 ## minimizing
 
 %% Cell type:code id: tags:
 
 ``` python
 res, fit_res = boz.nl_fit(params, domain)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 params.set_Fnl(boz.nl_lut(domain, res.x))
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 f = boz.inspect_correction(params, gain=3)
 ```
 
+%% Cell type:code id: tags:
 
+``` python
+f.savefig(f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-correction.png', dpi=300)
 ```
 
 %% Cell type:markdown id: tags:
 
 ### plotting the fitted correction
 
 %% Cell type:code id: tags:
 
 ``` python
 f = boz.inspect_Fnl(params.get_Fnl())
 ```
 
+%% Cell type:code id: tags:
 
+``` python
+f.savefig(f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-Fnl.png', dpi=300)
 ```
 
 %% Cell type:markdown id: tags:
 
 ### plotting the fit progresion
 
 %% Cell type:code id: tags:
 
 ``` python
 f = boz.inspect_nl_fit(fit_res)
 ```
 
+%% Cell type:code id: tags:
 
+``` python
+f.savefig(f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-nl-fit.png', dpi=300)
 ```
 
 %% Cell type:markdown id: tags:
 
 # Save the analysis parameters
 
 %% Cell type:code id: tags:
 
 ``` python
 params.save()
 ```