Update BOZ I notebook for 2.25 MHz data and 0.33 ph/bin dark figure

• compute mean pedestal from the dark run and use it as center value to define threshold to find bad pixels
• update nl fitting to 80 parameters extending to 511

doc/BOZ analysis part I parameters determination.ipynb

 %% Cell type:code id: tags:
 
 ``` python
 import numpy as np
 %matplotlib notebook
 import matplotlib.pyplot as plt
 plt.rcParams['figure.constrained_layout.use'] = True
 
 import dask
 print(f'dask: {dask.__version__}')
 import dask.array as da
 from dask.distributed import Client, progress
 from dask_jobqueue import SLURMCluster
 
 import netCDF4
 import pickle
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 from psutil import virtual_memory
 import gc
 # gc.collect() # run garbage collection to free possible memory
 
 mem = virtual_memory()
 print(f'Physical memory: {mem.total/1024/1024/1024:.0f} Gb')  # total physical memory available
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 import logging
 logging.basicConfig(filename='example.log', level=logging.DEBUG)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 %load_ext autoreload
 
 %autoreload 2
 
 import toolbox_scs as tb
 print(tb.__file__)
 import toolbox_scs.routines.boz as boz
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 %load_ext line_profiler
 %load_ext memory_profiler
 ```
 
 %% Cell type:markdown id: tags:
 
 # Slum cluster setup
 
 %% Cell type:markdown id: tags:
 
 Change to *True* to allocate a slurm cluster and run the calculation on it instead of the current machine.
 
 %% Cell type:code id: tags:
 
 ``` python
 if False:
     partition = 'exfel'   # For users
     # partition = 'upex'   # For users
 
     cluster = SLURMCluster(
         queue=partition,
         local_directory='/scratch',  # Local disk space for workers to use
 
         # Resources per SLURM job (per node, the way SLURM is configured on Maxwell)
         # processes=16 runs 16 Dask workers in a job, so each worker has 1 core & 32 GB RAM.
         processes=8, cores=16, memory='512GB',
         walltime='9:00:00',
         #interface='ib0',
         #job_extra=["--reservation=upex_002619"] # reserved partition
     )
 
     # dashboard link
     # print('https://max-jhub.desy.de/user/lleguy/proxy/8787/graph')
 
     # Submit 2 SLURM jobs, for 32 Dask workers
     cluster.scale(32)
     #cluster.adapt(minimum=0, maximum=32)
 
     client = Client(cluster)
     print("Created dask client:", client)
 
     # Get a notbook widget showing the cluster state
     cluster
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 try:
     client.close()
     cluster.close()
 except:
     print('No client defined')
 ```
 
 %% Cell type:markdown id: tags:
 
 # Create parameters object
 
 %% Cell type:code id: tags:parameters
 
 ``` python
-proposal = 2619
-darkrun = 33
-run = 38
+proposal = 2937
+darkrun = 478
+run = 477
 module = 15
-gain = 4
+gain = 0.5
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 params = boz.parameters(proposal=proposal, darkrun=darkrun, run=run, module=module, gain=gain)
 path = f'r{params.run}/'
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 print(params)
 ```
 
 %% Cell type:markdown id: tags:
 
 ### load data persistently
 
 %% Cell type:code id: tags:
 
 ``` python
 params.dask_load()
 ```
 
 %% Cell type:markdown id: tags:
 
 # Dark run inspection
 
 %% Cell type:markdown id: tags:
 
 The aim is to check dark level and extract bad pixel map.
 
 %% Cell type:code id: tags:
 
 ``` python
-mean_th = (30, 56)
+dark = boz.average_module(params.arr_dark).compute()
 ```
 
+%% Cell type:code id: tags:
 
+``` python
+pedestal = np.mean(dark)
+pedestal
 ```
 
+%% Cell type:code id: tags:
 
+``` python
+mean_th = (pedestal-25, pedestal+30)
 f = boz.inspect_dark(params.arr_dark, mean_th=mean_th)
 f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-dark.png', dpi=300)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 params.mean_th = mean_th
 params.set_mask(boz.bad_pixel_map(params))
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 print(params)
 ```
 
 %% Cell type:markdown id: tags:
 
 # Histogram
 
 %% Cell type:code id: tags:
 
 ``` python
 h, f = boz.inspect_histogram(params.proposal, params.run, params.module,
                              mask=params.get_mask() #, extra_lines=True
                          )
 f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-histogram.png', dpi=300)
 ```
 
 %% Cell type:markdown id: tags:
 
 adding guide to the eye
 
 %% Cell type:code id: tags:
 
 ``` python
 ax = f.gca()
 pf = np.polyfit([60, 220], [7, 4], 1)
 ax.plot([40, 400], 2*10**np.polyval(pf, [40, 400]))
 ax.plot([40, 400], 0.25*2*10**np.polyval(pf, [40, 400]))
 ```
 
 %% Cell type:markdown id: tags:
 
 # ROIs extraction
 
 %% Cell type:code id: tags:
 
 ``` python
-params.rois_th = 130
+params.rois_th = 1
 params.rois = boz.find_rois_from_params(params)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 print(params)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
-dark = boz.average_module(params.arr_dark).compute()
 data = boz.average_module(params.arr, dark=dark).compute()
 dataM = data.mean(axis=0) # mean over pulseId
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 f = boz.inspect_rois(dataM, params.rois, params.rois_th)
 f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-rois.png', dpi=300)
 ```
 
 %% Cell type:markdown id: tags:
 
 # Flat field extraction
 
 %% Cell type:markdown id: tags:
 
 The first step is to compute a good average image, this mean remove saturated shots and ignoring bad pixels
 
 %% Cell type:code id: tags:
 
 ``` python
 params.sat_level = 511
 res = boz.average_module(params.arr, dark=dark,
                          ret='mean', mask=params.get_mask(), sat_roi=params.rois['sat'],
                          sat_level=params.sat_level)
 avg = res.compute().mean(axis=0)
 ```
 
 %% Cell type:markdown id: tags:
 
 The second step is from that good average image to fit the plane field on n/0 and p/0 rois. We have to make sure that the rois have same width.
 
 %% Cell type:code id: tags:
 
 ``` python
 f = boz.inspect_plane_fitting(avg, params.rois)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-noflatfield.png', dpi=300)
 ```
 
 %% Cell type:markdown id: tags:
 
 fit the plane field correction
 
 %% Cell type:code id: tags:
 
 ``` python
 plane = boz.plane_fitting(params)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 plane
 ```
 
 %% Cell type:markdown id: tags:
 
 compute the correction and inspect the result of its application
 
 %% Cell type:code id: tags:
 
 ``` python
 params.set_flat_field(plane.x)
 ff = boz.compute_flat_field_correction(params.rois, params.get_flat_field())
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 f = boz.inspect_plane_fitting(avg/ff, params.rois)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-withflatfield.png', dpi=300)
 ```
 
 %% Cell type:markdown id: tags:
 
 # Non-linearities correction extraction
 
 %% Cell type:code id: tags:
 
 ``` python
-N = 40
-domain = boz.nl_domain(N, 40, 280)
+N = 80
+domain = boz.nl_domain(N, 40, 511)
 params.alpha = 0.5
 params.max_iter = 25
 ```
 
 %% Cell type:markdown id: tags:
 
 ## minimizing
 
 %% Cell type:code id: tags:
 
 ``` python
 res, fit_res = boz.nl_fit(params, domain)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 params.set_Fnl(boz.nl_lut(domain, res.x))
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 print(params)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 f = boz.inspect_correction(params, gain=params.gain)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-correction.png', dpi=300)
 ```
 
 %% Cell type:markdown id: tags:
 
 ### plotting the fitted correction
 
 %% Cell type:code id: tags:
 
 ``` python
 f = boz.inspect_Fnl(params.get_Fnl())
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-Fnl.png', dpi=300)
 ```
 
 %% Cell type:markdown id: tags:
 
 ### plotting the fit progresion
 
 %% Cell type:code id: tags:
 
 ``` python
 f = boz.inspect_nl_fit(fit_res)
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-nl-fit.png', dpi=300)
 ```
 
 %% Cell type:markdown id: tags:
 
 # Save the analysis parameters
 
 %% Cell type:code id: tags:
 
 ``` python
 params.save(path=path)
 ```