Skip to content
Snippets Groups Projects
Commit f76d59cb authored by Loïc Le Guyader's avatar Loïc Le Guyader
Browse files

Merge branch 'boz-saturation' into 'master' 

Boz saturation and rois threshold parametrized

See merge request !136
parents 3ac0cdea 0126db83
No related branches found
No related tags found
1 merge request!136Boz saturation and rois threshold parametrized
Hide whitespace changes
Inline Side-by-side
doc/BOZ analysis part I parameters determination.ipynb
+ 8
− 16
View file @ f76d59cb
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import numpy as np import numpy as np
%matplotlib notebook %matplotlib notebook
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
plt.rcParams['figure.constrained_layout.use'] = True plt.rcParams['figure.constrained_layout.use'] = True
import dask import dask
print(f'dask: {dask.__version__}') print(f'dask: {dask.__version__}')
from psutil import virtual_memory from psutil import virtual_memory
mem = virtual_memory() mem = virtual_memory()
print(f'Physical memory: {mem.total/1024/1024/1024:.0f} Gb') # total physical memory available print(f'Physical memory: {mem.total/1024/1024/1024:.0f} Gb') # total physical memory available
import logging import logging
logging.basicConfig(filename='example.log', level=logging.DEBUG) logging.basicConfig(filename='example.log', level=logging.DEBUG)
import toolbox_scs as tb import toolbox_scs as tb
print(tb.__file__) print(tb.__file__)
import toolbox_scs.routines.boz as boz import toolbox_scs.routines.boz as boz
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Create parameters object # Create parameters object
%% Cell type:code id: tags:parameters %% Cell type:code id: tags:parameters
``` python ``` python
proposal = 2937 proposal = 2937
darkrun = 478 darkrun = 478
run = 477 run = 477
module = 15 module = 15
gain = 0.5 gain = 0.5
sat_level = 500
rois_th = 1
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
params = boz.parameters(proposal=proposal, darkrun=darkrun, run=run, module=module, gain=gain) params = boz.parameters(proposal=proposal, darkrun=darkrun, run=run, module=module, gain=gain)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
from extra_data.read_machinery import find_proposal from extra_data.read_machinery import find_proposal
root = find_proposal(f'p{proposal:06d}') root = find_proposal(f'p{proposal:06d}')
path = root + '/usr/processed_runs/' + f'r{params.run}/' path = root + '/usr/processed_runs/' + f'r{params.run}/'
print(path) print(path)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print(params) print(params)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### load data persistently ### load data persistently
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
params.dask_load() params.dask_load()
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Dark run inspection # Dark run inspection
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
The aim is to check dark level and extract bad pixel map. The aim is to check dark level and extract bad pixel map.
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
dark = boz.average_module(params.arr_dark).compute() dark = boz.average_module(params.arr_dark).compute()
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
pedestal = np.mean(dark) pedestal = np.mean(dark)
pedestal pedestal
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
mean_th = (pedestal-25, pedestal+30) mean_th = (pedestal-25, pedestal+30)
f = boz.inspect_dark(params.arr_dark, mean_th=mean_th) f = boz.inspect_dark(params.arr_dark, mean_th=mean_th)
f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-dark.png', dpi=300) f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-dark.png', dpi=300)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
params.mean_th = mean_th params.mean_th = mean_th
params.set_mask(boz.bad_pixel_map(params)) params.set_mask(boz.bad_pixel_map(params))
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print(params) print(params)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Histogram # Histogram
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
h, f = boz.inspect_histogram(params.proposal, params.run, params.module, h, f = boz.inspect_histogram(params.arr,
params.arr_dark,
mask=params.get_mask() #, extra_lines=True mask=params.get_mask() #, extra_lines=True
) )
f.suptitle(f'p:{params.proposal} r:{params.run} d:{params.darkrun}')
f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-histogram.png', dpi=300) f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-histogram.png', dpi=300)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
adding guide to the eye adding guide to the eye
%% Cell type:code id: tags:
``` python
ax = f.gca()
pf = np.polyfit([60, 220], [7, 4], 1)
ax.plot([40, 400], 2*10**np.polyval(pf, [40, 400]))
ax.plot([40, 400], 0.25*2*10**np.polyval(pf, [40, 400]))
```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# ROIs extraction # ROIs extraction
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
params.rois_th = 4 params.rois_th = rois_th
params.rois = boz.find_rois_from_params(params) params.rois = boz.find_rois_from_params(params)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print(params) print(params)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
data = boz.average_module(params.arr, dark=dark).compute() data = boz.average_module(params.arr, dark=dark).compute()
dataM = data.mean(axis=0) # mean over pulseId dataM = data.mean(axis=0) # mean over pulseId
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f = boz.inspect_rois(dataM, params.rois, params.rois_th) f = boz.inspect_rois(dataM, params.rois, params.rois_th)
f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-rois.png', dpi=300) f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-rois.png', dpi=300)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Flat field extraction # Flat field extraction
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
The first step is to compute a good average image, this mean remove saturated shots and ignoring bad pixels The first step is to compute a good average image, this mean remove saturated shots and ignoring bad pixels
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
params.sat_level = 511 params.sat_level = sat_level
res = boz.average_module(params.arr, dark=dark, res = boz.average_module(params.arr, dark=dark,
ret='mean', mask=params.get_mask(), sat_roi=params.rois['sat'], ret='mean', mask=params.get_mask(), sat_roi=params.rois['sat'],
sat_level=params.sat_level) sat_level=params.sat_level)
avg = res.compute().mean(axis=0) avg = res.compute().mean(axis=0)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
The second step is from that good average image to fit the plane field on n/0 and p/0 rois. We have to make sure that the rois have same width. The second step is from that good average image to fit the plane field on n/0 and p/0 rois. We have to make sure that the rois have same width.
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f = boz.inspect_plane_fitting(avg, params.rois) f = boz.inspect_plane_fitting(avg, params.rois)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-noflatfield.png', dpi=300) f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-noflatfield.png', dpi=300)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
fit the plane field correction fit the plane field correction
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
plane = boz.plane_fitting(params) plane = boz.plane_fitting(params)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
plane plane
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
compute the correction and inspect the result of its application compute the correction and inspect the result of its application
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
params.set_flat_field(plane.x) params.set_flat_field(plane.x)
ff = boz.compute_flat_field_correction(params.rois, params.get_flat_field()) ff = boz.compute_flat_field_correction(params.rois, params.get_flat_field())
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f = boz.inspect_plane_fitting(avg/ff, params.rois) f = boz.inspect_plane_fitting(avg/ff, params.rois)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-withflatfield.png', dpi=300) f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-withflatfield.png', dpi=300)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Non-linearities correction extraction # Non-linearities correction extraction
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
To speed up online analysis, we save the corrections with a dummy non-linearity correction. The saved file can be used for online analysis as soon as it appears. To speed up online analysis, we save the corrections with a dummy non-linearity correction. The saved file can be used for online analysis as soon as it appears.
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
params.set_Fnl(np.arange(2**9)) params.set_Fnl(np.arange(2**9))
params.save(path=path) params.save(path=path)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
N = 80 N = 80
domain = boz.nl_domain(N, 40, 511) domain = boz.nl_domain(N, 40, 511)
params.alpha = 0.5 params.alpha = 0.5
params.max_iter = 25 params.max_iter = 25
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
## minimizing ## minimizing
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
res, fit_res = boz.nl_fit(params, domain) res, fit_res = boz.nl_fit(params, domain)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
params.set_Fnl(boz.nl_lut(domain, res.x)) params.set_Fnl(boz.nl_lut(domain, res.x))
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print(params) print(params)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f = boz.inspect_correction(params, gain=params.gain) f = boz.inspect_correction(params, gain=params.gain)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-correction.png', dpi=300) f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-correction.png', dpi=300)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### plotting the fitted correction ### plotting the fitted correction
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f = boz.inspect_Fnl(params.get_Fnl()) f = boz.inspect_Fnl(params.get_Fnl())
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-Fnl.png', dpi=300) f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-Fnl.png', dpi=300)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
### plotting the fit progresion ### plotting the fit progresion
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f = boz.inspect_nl_fit(fit_res) f = boz.inspect_nl_fit(fit_res)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-nl-fit.png', dpi=300) f.savefig(path+f'p{params.proposal}-r{params.run}-d{params.darkrun}-inspect-nl-fit.png', dpi=300)
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Save the analysis parameters # Save the analysis parameters
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
params.save(path=path) params.save(path=path)
``` ```
......
scripts/boz_parameters_job.sh
+ 4
− 3
View file @ f76d59cb
...@@ -9,6 +9,7 @@ PROPOSAL=2619 ...@@ -9,6 +9,7 @@ PROPOSAL=2619
DARK=$1 DARK=$1
RUN=$2 RUN=$2
GAIN=$3 GAIN=$3
ROISTH=${4:-1}
MODULE=15 MODULE=15
source /etc/profile.d/modules.sh source /etc/profile.d/modules.sh
...@@ -16,7 +17,7 @@ module load exfel ...@@ -16,7 +17,7 @@ module load exfel
module load exfel_anaconda3/1.1 module load exfel_anaconda3/1.1
echo processing run $RUN echo processing run $RUN
mkdir r$RUN mkdir ../../processed_runs/r$RUN
papermill 'BOZ analysis part I parameters determination.ipynb' r$RUN/output.ipynb \ papermill 'BOZ analysis part I parameters determination.ipynb' ../../processed_runs/r$RUN/output.ipynb \
-p proposal $PROPOSAL -p darkrun $DARK -p run $RUN -p module $MODULE \ -p proposal $PROPOSAL -p darkrun $DARK -p run $RUN -p module $MODULE \
-p gain $GAIN -k xfel -p gain $GAIN -p rois_th $ROISTH -k xfel
src/toolbox_scs/routines/boz.py
+ 19
− 26
View file @ f76d59cb
...@@ -497,29 +497,18 @@ def inspect_rois(data_mean, rois, threshold=None, allrois=False): ...@@ -497,29 +497,18 @@ def inspect_rois(data_mean, rois, threshold=None, allrois=False):
return fig return fig
def histogram_module(proposalNB, runNB, moduleNB, mask=None): def histogram_module(arr, mask=None):
"""Compute a histogram of the 9 bits raw pixel values over a module. """Compute a histogram of the 9 bits raw pixel values over a module.
Inputs Inputs
------ ------
proposalNB: proposal number arr: dask array of reshaped dssc data (trainId, pulseId, x, y)
runNB: run number
moduleNB: module number
mask: optional bad pixel mask mask: optional bad pixel mask
Returns Returns
------- -------
histogram histogram
""" """
run = open_run(proposal=proposalNB, run=runNB)
# DSSC
source = f'SCS_DET_DSSC1M-1/DET/{moduleNB}CH0:xtdf'
key = 'image.data'
arr = run[source, key].dask_array()
arr = arr.rechunk((100, -1, -1, -1))
if mask is not None: if mask is not None:
w = da.repeat(da.repeat(da.array(mask[None, None, :, :]), w = da.repeat(da.repeat(da.array(mask[None, None, :, :]),
arr.shape[1], axis=1), arr.shape[0], axis=0) arr.shape[1], axis=1), arr.shape[0], axis=0)
...@@ -529,30 +518,37 @@ def histogram_module(proposalNB, runNB, moduleNB, mask=None): ...@@ -529,30 +518,37 @@ def histogram_module(proposalNB, runNB, moduleNB, mask=None):
return da.bincount(arr.ravel(), minlength=512).compute() return da.bincount(arr.ravel(), minlength=512).compute()
def inspect_histogram(proposalNB, runNB, moduleNB, def inspect_histogram(arr, arr_dark=None, mask=None, extra_lines=False):
mask=None, extra_lines=False):
"""Compute and plot a histogram of the 9 bits raw pixel values. """Compute and plot a histogram of the 9 bits raw pixel values.
Inputs Inputs
------ ------
proposalNB: proposal number arr: dask array of reshaped dssc data (trainId, pulseId, x, y)
runNB: run number arr: dask array of reshaped dssc dark data (trainId, pulseId, x, y)
moduleNB: module number
mask: optional bad pixel mask mask: optional bad pixel mask
extra_lines: boolean, default False, plot extra lines at period values extra_lines: boolean, default False, plot extra lines at period values
Returns Returns
------- -------
histogram (h, hd): histogram of arr, arr_dark
figure figure
""" """
h = histogram_module(proposalNB, runNB, moduleNB, mask=mask)
from matplotlib.ticker import MultipleLocator from matplotlib.ticker import MultipleLocator
f = plt.figure(figsize=(6, 3)) f = plt.figure(figsize=(6, 3))
plt.plot(np.arange(2**9), h, marker='o', h = histogram_module(arr, mask=mask)
Sum_h = np.sum(h)
plt.plot(np.arange(2**9), h/Sum_h, marker='o',
ms=3, markerfacecolor='none', lw=1) ms=3, markerfacecolor='none', lw=1)
if arr_dark is not None:
hd = histogram_module(arr_dark, mask=mask)
Sum_hd = np.sum(hd)
plt.plot(np.arange(2**9), hd/Sum_hd, marker='o',
ms=3, markerfacecolor='none', lw=1, c='k', alpha=.5)
else:
hd = None
if extra_lines: if extra_lines:
for k in range(50, 271): for k in range(50, 271):
if not (k - 2) % 8: if not (k - 2) % 8:
...@@ -563,17 +559,14 @@ def inspect_histogram(proposalNB, runNB, moduleNB, ...@@ -563,17 +559,14 @@ def inspect_histogram(proposalNB, runNB, moduleNB,
plt.axvline(k, c='r', alpha=0.3, ls='--') plt.axvline(k, c='r', alpha=0.3, ls='--')
plt.axvline(271, c='C1', alpha=0.5, ls='--') plt.axvline(271, c='C1', alpha=0.5, ls='--')
plt.fill_between(np.arange(2**9)[30:51], h[30:51], 1, alpha=0.5)
plt.ylim([1, None])
plt.xlim([0, 2**9-1]) plt.xlim([0, 2**9-1])
plt.yscale('log') plt.yscale('log')
plt.axes().xaxis.set_minor_locator(MultipleLocator(10)) plt.axes().xaxis.set_minor_locator(MultipleLocator(10))
plt.xlabel('DSSC pixel value') plt.xlabel('DSSC pixel value')
plt.ylabel('counts') plt.ylabel('count frequency')
plt.title(f'p{proposalNB} run {runNB}')
return h, f return (h, hd), f
def load_dssc_module(proposalNB, runNB, moduleNB=15, def load_dssc_module(proposalNB, runNB, moduleNB=15,
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts