Skip to content
Snippets Groups Projects
Commit 3ac0cdea authored by Loïc Le Guyader's avatar Loïc Le Guyader
Browse files

Merge branch 'boz-issue26' into 'master' 

Boz issue26

Closes #26

See merge request !135
parents 1fe93611 82d17134
No related branches found
No related tags found
1 merge request!135Boz issue26
Expand all Hide whitespace changes
Inline Side-by-side
doc/BOZ analysis part I parameters determination.ipynb
+ 35
− 112
View file @ 3ac0cdea
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import numpy as np import numpy as np
%matplotlib notebook %matplotlib notebook
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
plt.rcParams['figure.constrained_layout.use'] = True plt.rcParams['figure.constrained_layout.use'] = True
import dask import dask
print(f'dask: {dask.__version__}') print(f'dask: {dask.__version__}')
import dask.array as da
from dask.distributed import Client, progress
from dask_jobqueue import SLURMCluster
import netCDF4
import pickle
```
%% Cell type:code id: tags:
``` python
from psutil import virtual_memory from psutil import virtual_memory
import gc
# gc.collect() # run garbage collection to free possible memory
mem = virtual_memory() mem = virtual_memory()
print(f'Physical memory: {mem.total/1024/1024/1024:.0f} Gb') # total physical memory available print(f'Physical memory: {mem.total/1024/1024/1024:.0f} Gb') # total physical memory available
```
%% Cell type:code id: tags:
``` python
import logging import logging
logging.basicConfig(filename='example.log', level=logging.DEBUG) logging.basicConfig(filename='example.log', level=logging.DEBUG)
```
%% Cell type:code id: tags:
``` python
%load_ext autoreload
%autoreload 2
import toolbox_scs as tb import toolbox_scs as tb
print(tb.__file__) print(tb.__file__)
import toolbox_scs.routines.boz as boz import toolbox_scs.routines.boz as boz
``` ```
%% Cell type:code id: tags:
``` python
%load_ext line_profiler
%load_ext memory_profiler
```
%% Cell type:markdown id: tags:
# Slum cluster setup
%% Cell type:markdown id: tags:
Change to *True* to allocate a slurm cluster and run the calculation on it instead of the current machine.
%% Cell type:code id: tags:
``` python
if False:
partition = 'exfel' # For users
# partition = 'upex' # For users
cluster = SLURMCluster(
queue=partition,
local_directory='/scratch', # Local disk space for workers to use
# Resources per SLURM job (per node, the way SLURM is configured on Maxwell)
# processes=16 runs 16 Dask workers in a job, so each worker has 1 core & 32 GB RAM.
processes=8, cores=16, memory='512GB',
walltime='9:00:00',
#interface='ib0',
#job_extra=["--reservation=upex_002619"] # reserved partition
)
# dashboard link
# print('https://max-jhub.desy.de/user/lleguy/proxy/8787/graph')
# Submit 2 SLURM jobs, for 32 Dask workers
cluster.scale(32)
#cluster.adapt(minimum=0, maximum=32)
client = Client(cluster)
print("Created dask client:", client)
# Get a notbook widget showing the cluster state
cluster
```
%% Cell type:code id: tags:
``` python
try:
client.close()
cluster.close()
except:
print('No client defined')
```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Create parameters object # Create parameters object
%% Cell type:code id: tags:parameters %% Cell type:code id: tags:parameters
``` python ``` python
proposal = 2937 proposal = 2937
darkrun = 478 darkrun = 478
run = 477 run = 477
module = 15 module = 15
gain = 0.5 gain = 0.5
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
params = boz.parameters(proposal=proposal, darkrun=darkrun, run=run, module=module, gain=gain) params = boz.parameters(proposal=proposal, darkrun=darkrun, run=run, module=module, gain=gain)
path = f'r{params.run}/' ```
%% Cell type:code id: tags:
``` python
from extra_data.read_machinery import find_proposal
root = find_proposal(f'p{proposal:06d}')
path = root + '/usr/processed_runs/' + f'r{params.run}/'
print(path)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print(params) print(params)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### load data persistently ### load data persistently
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
params.dask_load() params.dask_load()
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Dark run inspection # Dark run inspection
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
The aim is to check dark level and extract bad pixel map. The aim is to check dark level and extract bad pixel map.
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
dark = boz.average_module(params.arr_dark).compute() dark = boz.average_module(params.arr_dark).compute()
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
pedestal = np.mean(dark) pedestal = np.mean(dark)
pedestal pedestal
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
mean_th = (pedestal-25, pedestal+30) mean_th = (pedestal-25, pedestal+30)
f = boz.inspect_dark(params.arr_dark, mean_th=mean_th) f = boz.inspect_dark(params.arr_dark, mean_th=mean_th)
f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-dark.png', dpi=300) f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-dark.png', dpi=300)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
params.mean_th = mean_th params.mean_th = mean_th
params.set_mask(boz.bad_pixel_map(params)) params.set_mask(boz.bad_pixel_map(params))
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print(params) print(params)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Histogram # Histogram
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
h, f = boz.inspect_histogram(params.proposal, params.run, params.module, h, f = boz.inspect_histogram(params.proposal, params.run, params.module,
mask=params.get_mask() #, extra_lines=True mask=params.get_mask() #, extra_lines=True
) )
f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-histogram.png', dpi=300) f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-histogram.png', dpi=300)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
adding guide to the eye adding guide to the eye
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
ax = f.gca() ax = f.gca()
pf = np.polyfit([60, 220], [7, 4], 1) pf = np.polyfit([60, 220], [7, 4], 1)
ax.plot([40, 400], 2*10**np.polyval(pf, [40, 400])) ax.plot([40, 400], 2*10**np.polyval(pf, [40, 400]))
ax.plot([40, 400], 0.25*2*10**np.polyval(pf, [40, 400])) ax.plot([40, 400], 0.25*2*10**np.polyval(pf, [40, 400]))
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# ROIs extraction # ROIs extraction
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
params.rois_th = 1 params.rois_th = 4
params.rois = boz.find_rois_from_params(params) params.rois = boz.find_rois_from_params(params)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print(params) print(params)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
data = boz.average_module(params.arr, dark=dark).compute() data = boz.average_module(params.arr, dark=dark).compute()
dataM = data.mean(axis=0) # mean over pulseId dataM = data.mean(axis=0) # mean over pulseId
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f = boz.inspect_rois(dataM, params.rois, params.rois_th) f = boz.inspect_rois(dataM, params.rois, params.rois_th)
f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-rois.png', dpi=300) f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-rois.png', dpi=300)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Flat field extraction # Flat field extraction
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
The first step is to compute a good average image, this mean remove saturated shots and ignoring bad pixels The first step is to compute a good average image, this mean remove saturated shots and ignoring bad pixels
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
params.sat_level = 511 params.sat_level = 511
res = boz.average_module(params.arr, dark=dark, res = boz.average_module(params.arr, dark=dark,
ret='mean', mask=params.get_mask(), sat_roi=params.rois['sat'], ret='mean', mask=params.get_mask(), sat_roi=params.rois['sat'],
sat_level=params.sat_level) sat_level=params.sat_level)
avg = res.compute().mean(axis=0) avg = res.compute().mean(axis=0)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
The second step is from that good average image to fit the plane field on n/0 and p/0 rois. We have to make sure that the rois have same width. The second step is from that good average image to fit the plane field on n/0 and p/0 rois. We have to make sure that the rois have same width.
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f = boz.inspect_plane_fitting(avg, params.rois) f = boz.inspect_plane_fitting(avg, params.rois)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-noflatfield.png', dpi=300) f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-noflatfield.png', dpi=300)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
fit the plane field correction fit the plane field correction
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
plane = boz.plane_fitting(params) plane = boz.plane_fitting(params)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
plane plane
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
compute the correction and inspect the result of its application compute the correction and inspect the result of its application
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
params.set_flat_field(plane.x) params.set_flat_field(plane.x)
ff = boz.compute_flat_field_correction(params.rois, params.get_flat_field()) ff = boz.compute_flat_field_correction(params.rois, params.get_flat_field())
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f = boz.inspect_plane_fitting(avg/ff, params.rois) f = boz.inspect_plane_fitting(avg/ff, params.rois)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-withflatfield.png', dpi=300) f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-withflatfield.png', dpi=300)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Non-linearities correction extraction # Non-linearities correction extraction
%% Cell type:markdown id: tags:
To speed up online analysis, we save the corrections with a dummy non-linearity correction. The saved file can be used for online analysis as soon as it appears.
%% Cell type:code id: tags:
``` python
params.set_Fnl(np.arange(2**9))
params.save(path=path)
```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
N = 80 N = 80
domain = boz.nl_domain(N, 40, 511) domain = boz.nl_domain(N, 40, 511)
params.alpha = 0.5 params.alpha = 0.5
params.max_iter = 25 params.max_iter = 25
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## minimizing ## minimizing
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
res, fit_res = boz.nl_fit(params, domain) res, fit_res = boz.nl_fit(params, domain)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
params.set_Fnl(boz.nl_lut(domain, res.x)) params.set_Fnl(boz.nl_lut(domain, res.x))
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print(params) print(params)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f = boz.inspect_correction(params, gain=params.gain) f = boz.inspect_correction(params, gain=params.gain)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-correction.png', dpi=300) f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-correction.png', dpi=300)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### plotting the fitted correction ### plotting the fitted correction
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f = boz.inspect_Fnl(params.get_Fnl()) f = boz.inspect_Fnl(params.get_Fnl())
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-Fnl.png', dpi=300) f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-Fnl.png', dpi=300)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### plotting the fit progresion ### plotting the fit progresion
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f = boz.inspect_nl_fit(fit_res) f = boz.inspect_nl_fit(fit_res)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-nl-fit.png', dpi=300) f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-nl-fit.png', dpi=300)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Save the analysis parameters # Save the analysis parameters
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
params.save(path=path) params.save(path=path)
``` ```
......
doc/BOZ analysis part II run processing.ipynb
+ 4220
− 166
View file @ 3ac0cdea
This diff is collapsed.
doc/howtos.rst
+ 30
− 11
View file @ 3ac0cdea
...@@ -41,14 +41,33 @@ routines ...@@ -41,14 +41,33 @@ routines
BOZ: Beam-Splitting Off-axis Zone plate analysis BOZ: Beam-Splitting Off-axis Zone plate analysis
++++++++++++++++++++++++++++++++++++++++++++++++ ++++++++++++++++++++++++++++++++++++++++++++++++
The BOZ analysis consists of 2 notebooks and a script. The first notebook is used to determine The BOZ analysis consists of 2 notebooks and a script. The first notebook
all the correction. The input are a dark run and a run with X-ray on empty sample or at the pre-edge. :doc:`BOZ analysis part I parameters determination <BOZ analysis part I parameters determination>`
The result is a JSON file that contains the flat field and non-linearity correction. The determination is used to determine all the necessary correction, that is the flat field
of the non-linearity correction can take some 2 to 8 hours depending on the number of pulses in the correction from the zone plate optics and the non-linearity correction from the
train. For this reason it is possible to use a script 'scripts/boz_parameters_job.sh' in the toolbox to DSSC gain. The inputs are a dark run and a run with X-rays on three broken or
launch the first notebook via slurm. It requires 'papermill' to be installed. empty membranes. For the latter, an alternative is to use pre-edge data on an
actual sample. The result is a JSON file that contains the flat field and
The second notebook then use the JSON file to load all needed correction and process an run with a non-linearity correction as well as the parameters used for their determination
corresponding dark run to bin data and compute a spectrum or a time resolved XAS scan. such that this can be reproduced and investigated in case of issues. The
* :doc:`BOZ analysis part I parameters determination <BOZ analysis part I parameters determination>`. determination of the flat field correction is rather quick, few minutes and is
* :doc:`BOZ analysis part II run processing <BOZ analysis part II run processing>`. the most important correction for the change in XAS computed from the 1st and
1st order. For quick correction of the online preview one can bypass the
non-linearity calculation by taking the JSON file as soon as it appears.
The determination of the non-linearity correction is a lot longer and can take
some 2 to 8 hours depending on the number of pulses in the
train. For this reason it is possible to use a script
``scripts/boz_parameters_job.sh`` in the toolbox to launch the first notebook
via slurm:
``sbatch ./boz_parameters_job.sh 615 614 3``
where 615 is the dark run number, 614 is the run on 3 broken membranes and 3 is
the DSSC gain in photon per bin. The proposal run number is defined inside the
script file.
The second notebook
:doc:`BOZ analysis part II run processing <BOZ analysis part II run processing>`
then use the JSON file to load all needed corrections and
process an run with a corresponding dark run to bin data and compute a
spectrum or a time resolved XAS scan
setup.py
+ 2
− 2
View file @ 3ac0cdea
...@@ -2,7 +2,7 @@ from setuptools import setup, find_packages ...@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
with open('README.rst') as f: with open('README.rst') as f:
readme = f.read() readme = f.read()
with open('VERSION') as f: with open('VERSION') as f:
_version = f.read() _version = f.read()
_version = _version.strip("\n") _version = _version.strip("\n")
...@@ -12,7 +12,7 @@ basic_analysis_reqs = ['numpy', 'scipy',] # and is readily available in Karabo ...@@ -12,7 +12,7 @@ basic_analysis_reqs = ['numpy', 'scipy',] # and is readily available in Karabo
advanced_analysis_reqs = [ advanced_analysis_reqs = [
'pandas', 'imageio', 'xarray>=0.13.0', 'h5py', 'h5netcdf',] 'pandas', 'imageio', 'xarray>=0.13.0', 'h5py', 'h5netcdf',]
interactive_reqs = ['ipykernel', 'matplotlib', 'tqdm',] interactive_reqs = ['ipykernel', 'matplotlib', 'tqdm',]
maxwell_reqs = ['joblib', 'extra_data', 'euxfel_bunch_pattern>=0.6'] maxwell_reqs = ['joblib', 'papermill', 'extra_data', 'euxfel_bunch_pattern>=0.6']
setup(name='toolbox_scs', setup(name='toolbox_scs',
......
src/toolbox_scs/routines/boz.py
+ 2
− 2
View file @ 3ac0cdea
...@@ -577,7 +577,7 @@ def inspect_histogram(proposalNB, runNB, moduleNB, ...@@ -577,7 +577,7 @@ def inspect_histogram(proposalNB, runNB, moduleNB,
def load_dssc_module(proposalNB, runNB, moduleNB=15, def load_dssc_module(proposalNB, runNB, moduleNB=15,
subset=slice(None), drop_intra_darks=True, persist=True): subset=slice(None), drop_intra_darks=True, persist=False):
"""Load single module dssc data as dask array. """Load single module dssc data as dask array.
Inputs Inputs
...@@ -587,7 +587,7 @@ def load_dssc_module(proposalNB, runNB, moduleNB=15, ...@@ -587,7 +587,7 @@ def load_dssc_module(proposalNB, runNB, moduleNB=15,
moduleNB: default 15, module number moduleNB: default 15, module number
subset: default slice(None), subset of trains to load subset: default slice(None), subset of trains to load
drop_intra_darks: boolean, default True, remove intra darks from the data drop_intra_darks: boolean, default True, remove intra darks from the data
persist: load all data in memory persist: default False, load all data persistently in memory
Returns Returns
------- -------
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts