Skip to content
Snippets Groups Projects
Commit c3234477 authored by Karim Ahmed's avatar Karim Ahmed
Browse files

Merge branch 'feat/read_with_EXtra_data' into 'master' 

[EPIX100] Reading ePix100 data with EXtra-data, Correct and Dark notebooks.

See merge request detectors/pycalibration!500
parents b468a401 183003a7
No related branches found
No related tags found
1 merge request!500[EPIX100] Reading ePix100 data with EXtra-data, Correct and Dark notebooks.
Expand all Hide whitespace changes
Inline Side-by-side
notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
+ 97
− 128
View file @ c3234477
%% Cell type:markdown id: tags:
# ePix100 Dark Characterization
Author: M. Karnevskiy, Version 1.0
Author: European XFEL Detector Group, Version: 2.0
The following notebook provides dark image analysis of the ePix100 detector.
The following notebook provides dark image analysis and calibration constants of the ePix100 detector.
Dark characterization evaluates offset and noise of the detector and gives information about bad pixels. Resulting maps are saved as .h5 files for a latter use and injected to calibration DB.
%% Cell type:code id: tags:
``` python
cluster_profile = "noDB" # ipcluster profile to use
in_folder = '/gpfs/exfel/exp/HED/202030/p900136/raw' # input folder, required
out_folder = '/gpfs/exfel/data/scratch/ahmedk/test/HED_dark/' # output folder, required
out_folder = '' # output folder, required
sequence = 0 # sequence file to use
run = 182 # which run to read data from, required
# Parameters for accessing the raw data.
karabo_id = "HED_IA1_EPX100-2" # karabo karabo_id
karabo_da = ["EPIX02"] # data aggregators
receiver_id = "RECEIVER" # inset for receiver devices
receiver_template = "RECEIVER" # detector receiver template for accessing raw data files
path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data
h5path = '/INSTRUMENT/{}/DET/{}:daqOutput/data/image/pixels' # path in the HDF5 file to images
h5path_t = '/INSTRUMENT/{}/DET/{}:daqOutput/data/backTemp' # path to find temperature at
h5path_cntrl = '/CONTROL/{}/DET' # path to control data
instrument_source_template = '{}/DET/{}:daqOutput' # instrument detector data source in h5files
# Parameters for the calibration database.
use_dir_creation_date = True
cal_db_interface = "tcp://max-exfl016:8020" # calibration DB interface to use
cal_db_timeout = 300000 # timeout on caldb requests
db_output = False # Output constants to the calibration database
local_output = True # output constants locally
number_dark_frames = 0 # number of images to be used, if set to 0 all available images are used
temp_limits = 5 # limit for parameter Operational temperature
db_module = 'ePix100_M17' # detector karabo_id
# Conditions used for injected calibration constants.
bias_voltage = 200 # bias voltage
in_vacuum = False # detector operated in vacuum
fix_temperature = 290. # fix temperature to this value
temp_limits = 5 # limit for parameter Operational temperature
# Parameters used during selecting raw data trains.
min_trains = 1 # Minimum number of trains that should be available to process dark constants. Default 1.
max_trains = 1000 # Maximum number of trains to use for processing dark constants. Set to 0 to use all available trains.
# Don't delete! myMDC sends this by default.
operation_mode = '' # Detector operation mode, optional
# TODO: delete after removing from calibration_configurations
db_module = '' # ID of module in calibration database, this parameter is ignore in the notebook. TODO: remove from calibration_configurations.
```
%% Cell type:code id: tags:
``` python
import os
import warnings
warnings.filterwarnings('ignore')
import h5py
import matplotlib.pyplot as plt
from IPython.display import Latex, Markdown, display
%matplotlib inline
import numpy as np
import pasha as psh
from extra_data import RunDirectory
import XFELDetAna.xfelprofiler as xprof
from XFELDetAna import xfelpyanatools as xana
from XFELDetAna.plotting.util import prettyPlotting
from cal_tools.tools import (
get_dir_creation_date,
get_pdu_from_db,
get_random_db_interface,
get_report,
save_const_to_h5,
send_to_db,
)
from iCalibrationDB import Conditions, Constants, Detectors, Versions
from iCalibrationDB.detectors import DetectorTypes
from XFELDetAna.util import env
from iCalibrationDB import Conditions, Constants
```
env.iprofile = cluster_profile
from XFELDetAna import xfelpyanatools as xana
from XFELDetAna import xfelpycaltools as xcal
from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5
from XFELDetAna.plotting.util import prettyPlotting
%% Cell type:code id: tags:
``` python
%matplotlib inline
warnings.filterwarnings('ignore')
prettyPlotting = True
import XFELDetAna.xfelprofiler as xprof
profiler = xprof.Profiler()
profiler.disable()
from XFELDetAna.xfelreaders import ChunkReader
h5path = h5path.format(karabo_id, receiver_id)
h5path_t = h5path_t.format(karabo_id, receiver_id)
h5path_cntrl = h5path_cntrl.format(karabo_id)
def nImagesOrLimit(nImages, limit):
if limit == 0:
return nImages
else:
return min(nImages, limit)
instrument_src = instrument_source_template.format(karabo_id, receiver_template)
```
%% Cell type:code id: tags:
``` python
# Read report path and create file location tuple to add with the injection
proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]
file_loc = f'proposal:{proposal} runs:{run}'
pixels_x = 708
pixels_y = 768
report = get_report(out_folder)
x = 708 # rows of the xPix100
y = 768 # columns of the xPix100
ped_dir = os.path.join(in_folder, f"r{run:04d}")
fp_name = path_template.format(run, karabo_da[0]).format(sequence)
filename = os.path.join(ped_dir, fp_name)
run_dir = RunDirectory(ped_dir)
print(f"Reading data from: {filename}\n")
print(f"Run number: {run}")
print(f"HDF5 path: {h5path}")
print(f"Instrument H5File source: {instrument_src}")
if use_dir_creation_date:
creation_time = get_dir_creation_date(in_folder, run)
print(f"Using {creation_time.isoformat()} as creation time")
os.makedirs(out_folder, exist_ok=True)
```
%% Cell type:code id: tags:
``` python
sensorSize = [x, y]
chunkSize = 100 #Number of images to read per chunk
pixel_data = (instrument_src, "data.image.pixels")
# Specifies total number of images to proceed
n_trains = run_dir.get_data_counts(*pixel_data).shape[0]
#Sensor area will be analysed according to blocksize
blockSize = [sensorSize[0] // 2, sensorSize[1] // 2]
xcal.defaultBlockSize = blockSize
cpuCores = 4 #Specifies the number of running cpu cores
memoryCells = 1 #No mamery cells
#Specifies total number of images to proceed
nImages = fastccdreaderh5.getDataSize(filename, h5path)[0]
nImages = nImagesOrLimit(nImages, number_dark_frames)
print("\nNumber of dark images to analyze: ", nImages)
run_parallel = False
with h5py.File(filename, 'r') as f:
integration_time = int(f[os.path.join(h5path_cntrl, "CONTROL","expTime", "value")][0])
temperature = np.mean(f[h5path_t])/100.
# Modify n_trains to process based on given maximum
# and minimun number of trains.
if max_trains:
n_trains = min(max_trains, n_trains)
if n_trains < min_trains:
raise ValueError(
f"Less than {min_trains} trains are available in RAW data."
" Not enough data to process darks.")
print(f"Number of dark images to analyze: {n_trains}.")
integration_time = int(run_dir.get_array(
f"{karabo_id}/DET/CONTROL",
"expTime.value")[0])
temperature = np.mean(run_dir.get_array(
instrument_src,
"data.backTemp").values) / 100.
if fix_temperature != 0:
temperature_k = fix_temperature
print("Temperature is fixed!")
else:
temperature_k = temperature + 273.15
if fix_temperature != 0:
temperature_k = fix_temperature
print("Temperature is fixed!")
print(f"Bias voltage is {bias_voltage} V")
print(f"Detector integration time is set to {integration_time}")
print(f"Mean temperature was {temperature:0.2f} °C / {temperature_k:0.2f} K")
print(f"Operated in vacuum: {in_vacuum} ")
print(f"Bias voltage is {bias_voltage} V")
print(f"Detector integration time is set to {integration_time}")
print(f"Mean temperature was {temperature:0.2f} °C / {temperature_k:0.2f} K")
print(f"Operated in vacuum: {in_vacuum} ")
```
%% Cell type:code id: tags:
``` python
reader = ChunkReader(filename, fastccdreaderh5.readData,
nImages, chunkSize,
path=h5path,
pixels_x=sensorSize[0],
pixels_y=sensorSize[1], )
```
data_dc = run_dir.select(
*pixel_data, require_all=True).select_trains(np.s_[:n_trains])
print(f"Reading data from: {data_dc.files}\n")
%% Cell type:code id: tags:
data = data_dc[pixel_data].ndarray()
``` python
noiseCal = xcal.NoiseCalculator(sensorSize, memoryCells,
cores=cpuCores, blockSize=blockSize,
parallel=run_parallel)
histCalRaw = xcal.HistogramCalculator(sensorSize, bins=1000,
range=[0, 10000], parallel=False,
memoryCells=memoryCells,
cores=cpuCores, blockSize=blockSize)
noise_data = np.std(data, axis=0)
offset_data = np.mean(data, axis=0)
```
%% Cell type:code id: tags:
``` python
for data in reader.readChunks():
dx = np.count_nonzero(data, axis=(0, 1))
data = data[:, :, dx != 0]
histCalRaw.fill(data)
noiseCal.fill(data) #Fill calculators with data
constant_maps = {}
constant_maps['Offset'] = noiseCal.getOffset() #Produce offset map
constant_maps['Noise'] = noiseCal.get() #Produce noise map
noiseCal.reset() #Reset noise calculator
print("Initial maps were created")
constant_maps['Offset'] = offset_data[..., np.newaxis]
constant_maps['Noise'] = noise_data[..., np.newaxis]
print("Initial constant maps are created")
```
%% Cell type:code id: tags:
``` python
#**************OFFSET MAP HISTOGRAM***********#
ho, co = np.histogram(constant_maps['Offset'].flatten(), bins=700)
do = {'x': co[:-1],
'y': ho,
'y_err': np.sqrt(ho[:]),
'drawstyle': 'bars',
'color': 'cornflowerblue',
}
fig = xana.simplePlot(do, figsize='1col', aspect=2,
x_label='Offset (ADU)',
y_label="Counts", y_log=True,
)
#*****NOISE MAP HISTOGRAM FROM THE OFFSET CORRECTED DATA*******#
hn, cn = np.histogram(constant_maps['Noise'].flatten(), bins=200)
dn = {'x': cn[:-1],
'y': hn,
'y_err': np.sqrt(hn[:]),
'drawstyle': 'bars',
'color': 'cornflowerblue',
}
fig = xana.simplePlot(dn, figsize='1col', aspect=2,
x_label='Noise (ADU)',
y_label="Counts",
y_log=True)
#**************HEAT MAPS*******************#
fig = xana.heatmapPlot(constant_maps['Offset'][:, :, 0],
x_label='Columns', y_label='Rows',
lut_label='Offset (ADU)',
x_range=(0, y),
y_range=(0, x), vmin=1000, vmax=4000)
x_range=(0, pixels_y),
y_range=(0, pixels_x), vmin=1000, vmax=4000)
fig = xana.heatmapPlot(constant_maps['Noise'][:, :, 0],
x_label='Columns', y_label='Rows',
lut_label='Noise (ADU)',
x_range=(0, y),
y_range=(0, x), vmax=2 * np.mean(constant_maps['Noise']))
x_range=(0, pixels_y),
y_range=(0, pixels_x), vmax=2 * np.mean(constant_maps['Noise']))
```
%% Cell type:code id: tags:
``` python
# Save constants to DB
dclass="ePix100"
md = None
# If PDU(db_module) is not given with input parameters
# retrieve the connected physical detector unit
for const_name in constant_maps.keys():
det = getattr(Constants, dclass)
const = getattr(det, const_name)()
const.data = constant_maps[const_name].data
# set the operating condition
condition = getattr(Conditions.Dark, dclass)(bias_voltage=bias_voltage,
integration_time=integration_time,
temperature=temperature_k,
in_vacuum=in_vacuum)
for parm in condition.parameters:
if parm.name == "Sensor Temperature":
parm.lower_deviation = temp_limits
parm.upper_deviation = temp_limits
# This should be used in case of running notebook
# by a different method other than myMDC which already
# sends CalCat info.
# TODO: Set db_module to "" by default in the first cell
if not db_module:
db_module = get_pdu_from_db(karabo_id, karabo_da, const,
condition, cal_db_interface,
snapshot_at=creation_time)[0]
db_module = get_pdu_from_db(karabo_id, karabo_da, const,
condition, cal_db_interface,
snapshot_at=creation_time)[0]
if db_output:
md = send_to_db(db_module, karabo_id, const, condition,
file_loc=file_loc, report_path=report,
cal_db_interface=cal_db_interface,
creation_time=creation_time,
timeout=cal_db_timeout)
if local_output:
md = save_const_to_h5(db_module, karabo_id, const, condition,
const.data, file_loc, report,
creation_time, out_folder)
print(f"Calibration constant {const_name} is stored locally at {out_folder} \n")
print("Constants parameter conditions are:\n")
print(f"• Bias voltage: {bias_voltage}\n• Integration time: {integration_time}\n"
f"• Temperature: {temperature_k}\n• In Vacuum: {in_vacuum}\n"
f"• Creation time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\n")
```
%% Cell type:code id: tags:
``` python
```
......
notebooks/ePix100/Correction_ePix100_NBC.ipynb
+ 298
− 222
View file @ c3234477
This diff is collapsed.
src/cal_tools/step_timing.py
+ 1
− 1
View file @ c3234477
......@@ -33,4 +33,4 @@ class StepTimer:
"""Show mean & std for each step"""
for step, data in self.steps.items():
data = np.asarray(data)
print(f'{step}: {data.mean():.01f} +- {data.std():.02f}s')
print(f'{step}: {data.mean():.01f} +- {data.std():.02f} s')
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts