From 61f9d09906b214d8d90c8206633c81df8cd2a1b0 Mon Sep 17 00:00:00 2001
From: Mikhail Karnevskiy <mikhail.karnevskiy@xfel.eu>
Date: Tue, 25 Feb 2020 10:57:11 +0100
Subject: [PATCH] Fix: 2.8.0 hp1
---
cal_tools/cal_tools/agipdlib.py | 546 ++++++++------
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 104 +--
...AGIPD_Correct_and_Verify_Summary_NBC.ipynb | 18 +-
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 48 +-
.../playground/AGIPD_SingleM_test_Dark.ipynb | 676 ++++++++++++++++++
webservice/serve_overview.py | 4 +-
webservice/templates/dark_overview.html | 2 +-
webservice/update_config.py | 4 +
xfel_calibrate/notebooks.py | 10 +
9 files changed, 1118 insertions(+), 294 deletions(-)
create mode 100644 notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 0877b8dde..691a3c418 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -36,6 +36,37 @@ def get_acq_rate(fname, loc, module):
return options.get(diff, None)
+def get_gain_setting(fname, h5path_ctrl):
+ """
+ Return gain setting base on setupr and patternTypeIndex
+
+ gain-setting 1: setupr@dark=8, setupr@slopespc=40
+ gain-setting 0: setupr@dark=0, setupr@slopespc=32
+
+ patternTypeIndex 1: High-gian
+ patternTypeIndex 2: Medium-gain
+ patternTypeIndex 3: Low-gain
+ patternTypeIndex 4: SlopesPC
+
+ :param fname: path to file with control information
+ :param h5path_ctrl: path to control information inside the file
+ :return: gain setting
+ """
+ with h5py.File(fname, "r") as f:
+ train_id = f["INDEX/trainId"][()]
+ idx = np.nonzero(train_id)[0][0]
+ setupr = f[f'{h5path_ctrl}/setupr/value'][idx]
+ pattern_type_idx = f[f'{h5path_ctrl}/patternTypeIndex/value'][idx]
+ if (setupr == 0 and pattern_type_idx < 4) or (
+ setupr == 32 and pattern_type_idx == 4):
+ return 0
+ elif(setupr == 8 and pattern_type_idx < 4) or (
+ setupr == 40 and pattern_type_idx == 4):
+ return 1
+ else:
+ raise ValueError('Could not derive gain setting from setupr and patternTypeIndex') # noqa
+
+
class SnowResolution(Enum):
""" An Enum specifying how to resolve snowy pixels
"""
@@ -80,7 +111,7 @@ class AgipdCorrections:
chunk_size_idim=512, il_mode=False,
cal_det_instance="AGIPD1M1", karabo_data_mode=False,
force_hg_if_below=None, force_mg_if_below=None,
- mask_noisy_adc=False, acquisition_rate=None,
+ mask_noisy_adc=False, acquisition_rate=None, gain_setting=None,
corr_bools=None):
"""
Initialize an AgipdCorrections Class
@@ -141,7 +172,6 @@ class AgipdCorrections:
self.sig_zero_mask = None
self.base_offset = None
self.baseline_corr_noise_threshold = 100
- self.baseline_corr_using_hmatch = True
self.melt_snow = SnowResolution.NONE
self.chunk_size_idim = chunk_size_idim
self.dohigh = 0
@@ -154,14 +184,18 @@ class AgipdCorrections:
self.mask_noisy_adc = mask_noisy_adc
self.adc_mask = None
self.gain_stats = [0, 0, 0]
- self.mg_bl_adjust = 0
self.acquisition_rate = acquisition_rate
+ self.gain_setting = gain_setting
self.valid_indices = None
+ self.frac_high_med = 0
+ self.md_additional_offset = 0
+ self.xray_cor = 0
# check if given corr_bools are correct
- tot_corr_bools = ['only_offset', 'pc_corr', 'ff_corr',
- 'adjust_mg_baseline', 'do_rel_gain', 'blc_noise',
- 'match_asics', 'corr_asic_diag', 'dont_zero_nans',
+ tot_corr_bools = ['only_offset', 'adjust_mg_baseline', 'rel_gain',
+ 'xray_corr', 'blc_noise', 'blc_hmatch',
+ 'blc_stripes', 'blc_set_min', 'match_asics',
+ 'corr_asic_diag', 'dont_zero_nans',
'dont_zero_orange']
if set(corr_bools).issubset(tot_corr_bools):
@@ -177,8 +211,9 @@ class AgipdCorrections:
splits = np.arange(0, self.firange.size, self.chunksize)
return [a for a in np.array_split(self.firange, splits) if a.size > 0]
- def initialize(self, offset, rel_gain, mask, noise, thresholds,
- base_offset, slopesPC=None, swap_axis=True):
+ def initialize(self, offset, rel_gain, xray_cor=None, mask=None,
+ noise=None, thresholds=None, swap_axis=True,
+ frac_high_med=None, md_additional_offset=None):
""" Initialize the calibration constants and the output data file.
Any data that is not touched by the corrections is copied into the
@@ -187,15 +222,14 @@ class AgipdCorrections:
:param offset: offset map to use for corrections
:param rel_gain: relative gain map to use for corrections
+ :param xray_cor: xray-driven correction map
:param mask: bad pixel mask to use for corrections
:param noise: noise map to use for corrections
:param thresholds: thresholds for gain determination
- :param base_offset: base offsets from PC and FF data: should be a
- tuple:
- PC offset for high gain, PC offset for medium gain, FF offset
- for high gain
:param swap_axis: set to True if using data from the calibration
database.
+ :param frac_high_med: ratio high to medium gain across cells
+ :param md_additional_offset: additional medium-gain offset
Note that this function may be called twice, when initializing
partially from DB and file.
@@ -211,34 +245,32 @@ class AgipdCorrections:
if rel_gain is not None:
mvd = np.moveaxis(np.moveaxis(rel_gain, 2, 0), 2, 1)
rel_gain = np.ascontiguousarray(mvd)
+ if xray_cor is not None:
+ mvd = np.moveaxis(xray_cor, 1, 0)
+ xray_cor = np.ascontiguousarray(mvd)
+ if md_additional_offset is not None:
+ mvd = np.moveaxis(np.moveaxis(md_additional_offset, 2, 0), 2, 1)
+ md_additional_offset = np.ascontiguousarray(mvd)
if mask is not None:
mvd = np.moveaxis(np.moveaxis(mask, 2, 0), 2, 1)
mask = np.ascontiguousarray(mvd)
if noise is not None:
mvd = np.moveaxis(np.moveaxis(noise, 2, 0), 2, 1)
noise = np.ascontiguousarray(mvd)
- if base_offset is not None:
- fallback_bo = np.nanmedian(np.array(
- [b for b in base_offset[:-1] if
- isinstance(b, np.ndarray)]), axis=0)
- for i in range(len(base_offset)):
- if base_offset[i] is not None:
- if not isinstance(base_offset[i], np.ndarray):
- base_offset[i] = fallback_bo
- mvd = np.moveaxis(base_offset[i], 2, 0)
- mvd = np.moveaxis(mvd, 2, 1)
- base_offset[i] = np.ascontiguousarray(mvd)
if thresholds is not None:
mvd = np.moveaxis(np.moveaxis(thresholds, 2, 0), 2, 1)
thresholds = np.ascontiguousarray(mvd)
- if base_offset is not None:
- self.base_offset = base_offset
if offset is not None:
self.offset = offset
-
if rel_gain is not None:
- self.rel_gain = 1. / rel_gain
+ self.rel_gain = rel_gain
+ if xray_cor is not None:
+ self.xray_cor = xray_cor
+ if frac_high_med is not None:
+ self.frac_high_med = frac_high_med
+ if md_additional_offset is not None:
+ self.md_additional_offset = md_additional_offset
# if a mask already exists we OR the mask to maintain what we
# already have
@@ -254,56 +286,6 @@ class AgipdCorrections:
if thresholds is not None:
self.thresholds = thresholds
- if self.base_offset is not None and self.offset is not None:
-
- # calculate a delta offset from base offsets and normal offsets
- dohigh = np.zeros(self.offset.shape[:-1], np.float32)
- for i in range(8):
- for j in range(2):
- asico = self.offset[:, i * 64:(i + 1) * 64,
- j * 64:(j + 1) * 64, 0]
- asic_m = np.nanmedian(asico, axis=(1, 2))
- global_m = np.nanmedian(self.offset[:, ..., 0],
- axis=(1, 2))
- doa = (asic_m - global_m)[:, None, None]
- dohigh[:, i * 64:(i + 1) * 64, j * 64:(j + 1) * 64] = doa
-
- self.dohigh = dohigh
-
- dopc = self.base_offset[1] - self.base_offset[0]
- dooff = self.offset[..., 1] - self.offset[..., 0]
- doff = dopc - dooff
- for i in range(8):
- for j in range(2):
- ado = doff[:, i * 64:(i + 1) * 64, j * 64:(j + 1) * 64]
- med = np.nanmedian(ado, axis=(1, 2))[:, None, None]
- doff[:, i * 64:(i + 1) * 64, j * 64:(j + 1) * 64] = med
-
- # This is deprecated part of code, which is not compatible with
- # existing PC constants.
- # self.offset[..., 1] += doff
-
- if self.corr_bools.get('adjust_mg_baseline') and slopesPC is not None:
- x = np.linspace(0, 1000, 1000)
- m_h = np.moveaxis(
- np.moveaxis(slopesPC[..., :self.max_cells, 0], 0, 2), 0, 1)
- b_h = np.moveaxis(
- np.moveaxis(slopesPC[..., :self.max_cells, 1], 0, 2), 0, 1)
- m_m = np.moveaxis(
- np.moveaxis(slopesPC[..., :self.max_cells, 3], 0, 2), 0, 1)
- b_m = np.moveaxis(
- np.moveaxis(slopesPC[..., :self.max_cells, 4], 0, 2), 0, 1)
- b_ho = self.offset[..., 0]
- b_mo = self.offset[..., 1]
- mz = -(b_m - b_mo) / m_m
- cx = np.zeros(mz.shape)
- for i in range(cx.shape[0]):
- for j in range(cx.shape[1]):
- for k in range(cx.shape[2]):
- cx[i, j, k] = x[np.argmin(np.abs(mz[i, j, k] - x))]
- doz = (m_h * cx + b_h - m_m * cx - b_m)
- self.mg_bl_adjust = doz
-
# this is the first call, usually from the database
# we also generate file structures here, as the second call is optional
if not self.initialized:
@@ -326,11 +308,84 @@ class AgipdCorrections:
print("After reading from file: ")
print("Offset medians are {}".format(
np.nanmedian(self.offset, axis=(0, 1, 2))))
- print("Gain medians are {}".format(
- np.nanmedian(self.rel_gain, axis=(0, 1, 2))))
+ if hasattr(self, "rel_gain"):
+ print("Gain medians are {}".format(
+ np.nanmedian(self.rel_gain, axis=(0, 1, 2))))
print("Threshold medians are {}".format(
np.nanmedian(self.thresholds, axis=(0, 1, 2))))
+ def get_shadowed_stripe(self, data, threshold, fraction):
+ """
+ Return list of shadowed regions.
+ Shadowed region is presented by the list of lines of pixels along
+ numpy axis 1. Regions are defined with respect of threshold
+ and fraction. Margin of one pixel is used.
+ Double-pixels are stored in separate regions
+
+ :param data: (numpy.ndarray, rank=2) offset corrected single image
+ :param threshold: (float) a threshold, below which
+ pixes is considered as dark
+ :param fraction: (float) a fraction of pixels in a line along axis 1
+ below which a full line is considered as dark
+ """
+
+ npx_all = np.count_nonzero(~np.isnan(data), axis=1)
+ npx_sh = np.count_nonzero(data < threshold, axis=1)
+
+ A = np.array(np.where(npx_sh / npx_all > fraction)[0])
+
+ dp_idx = np.arange(64, 512, 64)
+ dp_idx = np.sort(np.hstack((dp_idx, dp_idx - 1)))
+
+ # grouping indices
+ sh_idx = []
+ tmp_idx = []
+ for idx, i in enumerate(A[1:-1]):
+ if i - 1 not in A:
+ continue
+ if len(tmp_idx) == 0:
+ tmp_idx.append(i)
+ continue
+ if tmp_idx[-1] + 1 == i and (
+ (tmp_idx[-1] in dp_idx) == (i in dp_idx)) and (i + 1 in A):
+ tmp_idx.append(i)
+ if i != A[-2]:
+ continue
+ if len(tmp_idx) > 1:
+ sh_idx.append(list(tmp_idx))
+ tmp_idx = [i]
+
+ return sh_idx
+
+ def baseline_correct_via_stripe(self, d, g, m, frac_high_med):
+ """ Correct baseline shifts using shadowed stripes
+
+ :param d: the data to correct, should be offset corrected single image
+ :param frac_high_med: ratio of high to medium PC slopes
+ :param g: gain array matching `d` array
+ :param m: bad pixel mask
+ """
+
+ dd = copy.copy(d)
+ dd[g != 0] = np.nan # only high gain data
+ dd[m != 0] = np.nan # only good pixels
+
+ sh_idxs = self.get_shadowed_stripe(dd, 30, 0.95)
+
+ # collect all shadowed regions excluding double pixels
+ idx = []
+ for sh_idx in sh_idxs:
+ if len(sh_idx)>2:
+ idx += sh_idx
+
+ if len(idx)<3:
+ return d
+
+ shift = np.nanmean(dd[idx, :])
+ d[g==0] -= shift
+ d[g==1] -= shift/frac_high_med
+ return d
+
def split_gain(self, d):
""" Split gain information off AGIPD Data
"""
@@ -945,6 +1000,7 @@ class AgipdCorrections:
# choose constants according to gain setting
off = np.choose(gain,
(offsetb[..., 0], offsetb[..., 1], offsetb[..., 2]))
+
msk = np.choose(gain, (tmask[..., 0], tmask[..., 1], tmask[..., 2]))
# same for relative gain and then bad pixel mask
@@ -964,7 +1020,9 @@ class AgipdCorrections:
# before doing relative gain correction we need to evaluate any
# baseline shifts
# as they are effectively and additional offset in the data
- if self.corr_bools.get('blc_noise') or self.baseline_corr_using_hmatch:
+ if (self.corr_bools.get('blc_noise') or
+ self.corr_bools.get('blc_hmatch') or
+ self.corr_bools.get('blc_stripes') ):
# do this image wise, as the shift is per image
for i in range(im.shape[0]):
@@ -983,7 +1041,7 @@ class AgipdCorrections:
# if we have enough pixels in medium or low gain and
# correction via hist matching is requested to this now
gcrit = np.count_nonzero(gain[i, ...] > 0) > 1000
- if gcrit and self.baseline_corr_using_hmatch and hasattr(self, "rel_gain"):
+ if gcrit and self.corr_bools.get('blc_hmatch') and hasattr(self, "rel_gain"):
dd2 = self.correct_baseline_via_hist(im[i, ...],
rel_cor[i, ...],
gain[i, ...])
@@ -1000,13 +1058,29 @@ class AgipdCorrections:
else:
im[i, ...] = dd
+ if self.corr_bools.get('blc_stripes'):
+ fmh = self.frac_high_med[cellid[i]]
+ dd = self.baseline_correct_via_stripe(im[i, ...],
+ gain[i, ...],
+ msk[i, ...],
+ fmh)
+ im[i, ...] = dd
+
# now we can correct for relative gain if requested
- if self.corr_bools.get("do_rel_gain") and hasattr(self, "rel_gain"):
+ if self.corr_bools.get("rel_gain") and hasattr(self, "rel_gain"):
im *= rel_cor
if self.corr_bools.get("adjust_mg_baseline"):
- mgbc = self.mg_bl_adjust[cellid, ...]
- im[gain == 1] += 1.5 * mgbc[gain == 1]
+ mgbc = self.md_additional_offset[cellid, ...]
+ im[gain == 1] += mgbc[gain == 1]
+
+ # Set negative values for medium gain to 0
+ if self.corr_bools.get('blc_set_min'):
+ im[(im < 0) & (gain == 1)] = 0
+
+ # Do xray correction if requested
+ if self.corr_bools.get("xray_corr"):
+ im *= self.xray_cor
# try to identify snowy pixels at this point
if self.melt_snow is not False:
@@ -1121,10 +1195,12 @@ class AgipdCorrections:
highok = (idxtrains < medianTrain + 1e4)
valid &= lowok & highok
- uq, fidxv, cntsv = np.unique(idxtrains,
- return_index=True,
- return_counts=True)
- valid &= cntsv == 1 # filter out double trains
+ # Do not filter out double trains
+ #uq, fidxv, cntsv = np.unique(idxtrains,
+ # return_index=True,
+ # return_counts=True)
+ #valid &= cntsv == 1 # filter out double trains
+
self.valid = valid
last_index = int(first[valid][-1] + count[valid][-1])
first_index = int(first[valid][0])
@@ -1300,6 +1376,9 @@ class AgipdCorrections:
# fidxv and cntsv should have same length as
# raw INDEX/.../image/first and INDEX/.../image/count,
# respectively
+
+ # first_inc = first incrementation
+ first_inc = True
for i, diff in enumerate(train_diff):
if diff:
if i < len(cntsv):
@@ -1307,11 +1386,18 @@ class AgipdCorrections:
if i == 0:
fidxv = np.insert(fidxv, i, 0)
else:
- fidxv = np.insert(fidxv, i, fidxv[i-1])
+ fidxv = np.insert(fidxv, i, fidxv[i])
else:
# append if at the end of the array
cntsv = np.append(cntsv, 0)
- fidxv = np.append(fidxv, 0)
+ # increment fidxv once with the
+ # no. of processed mem-cells.
+ if first_inc:
+ fidxv = np.append(fidxv,
+ (2 * fidxv[i-1]) - fidxv[i-2])
+ first_inc = False
+ else:
+ fidxv = np.append(fidxv, fidxv[i-1])
# save INDEX contents (first, count) in CORR files
self.outfile.create_dataset(idx_base + "{}/first".format(do),
@@ -1372,7 +1458,8 @@ class AgipdCorrections:
""" Return preview histograms computed from the first chunk
"""
return ((self.hists_signal_low, self.hists_signal_high,
- self.hists_gain_vs_signal, self.hists_dig_gain_vs_signal, self.hist_pulses),
+ self.hists_gain_vs_signal, self.hists_dig_gain_vs_signal,
+ self.hist_pulses),
(self.low_edges, self.high_edges, self.signal_edges,
self.dig_signal_edges))
@@ -1518,6 +1605,7 @@ class AgipdCorrections:
Conditions.Dark.AGIPD(
memory_cells=max_cells_db_dark,
bias_voltage=bias_voltage,
+ gain_setting=self.gain_setting,
acquisition_rate=self.acquisition_rate), # noqa
np.zeros((128, 512,
max_cells_db,
@@ -1532,6 +1620,7 @@ class AgipdCorrections:
Conditions.Dark.AGIPD(
memory_cells=max_cells_db_dark,
bias_voltage=bias_voltage,
+ gain_setting=self.gain_setting,
acquisition_rate=self.acquisition_rate), # noqa
np.zeros(
(128, 512, max_cells_db, 3)),
@@ -1545,6 +1634,7 @@ class AgipdCorrections:
Conditions.Dark.AGIPD(
memory_cells=max_cells_db_dark,
bias_voltage=bias_voltage,
+ gain_setting=self.gain_setting,
acquisition_rate=self.acquisition_rate), # noqa
np.zeros(
(128, 512, max_cells_db, 3)),
@@ -1559,6 +1649,7 @@ class AgipdCorrections:
Conditions.Dark.AGIPD(
memory_cells=max_cells_db_dark,
bias_voltage=bias_voltage,
+ gain_setting=self.gain_setting,
acquisition_rate=self.acquisition_rate), # noqa
np.ones((128, 512, max_cells_db, 2)),
cal_db_interface,
@@ -1569,140 +1660,144 @@ class AgipdCorrections:
# initialize and return
if only_dark:
self.initialize(offset=offset, rel_gain=None, mask=bpixels,
- noise=noise, thresholds=thresholds,
- base_offset=None, slopesPC=None)
+ noise=noise, thresholds=thresholds)
return when
- # load also non-dark constants from the database
- slopesPC, when['slopesPC'] = \
- get_constant_from_db_and_time(getattr(dinstance, qm),
- Constants.AGIPD.SlopesPC(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db,
- bias_voltage=bias_voltage,
- acquisition_rate=self.acquisition_rate), # noqa
- np.ones(
- (128, 512, max_cells_db, 10)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
- slopesPC = slopesPC.astype(np.float32)
-
- # this will handle some historical data in a slighly different format
- # constant dimension injected first
- if slopesPC.shape[0] == 10 or slopesPC.shape[0] == 11:
- slopesPC = np.moveaxis(slopesPC, 0, 3)
- slopesPC = np.moveaxis(slopesPC, 0, 2)
-
- slopesFF, when['slopesFF'] = \
- get_constant_from_db_and_time(getattr(dinstance, qm),
- Constants.AGIPD.SlopesFF(),
- Conditions.Illuminated.AGIPD(
- max_cells_db,
- bias_voltage,
- photon_energy,
- pixels_x=512,
- pixels_y=128,
- beam_energy=None,
- acquisition_rate=self.acquisition_rate), # noqa
- np.ones((128, 512, max_cells_db, 2)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
- slopesFF = np.squeeze(slopesFF)
-
- # calculate relative gain from the constants
- # we default to a value of one, so basically a unity transform
- rel_gain = np.ones((128, 512, self.max_cells, 3), np.float32)
-
- # high gain slope from pulse capacitor data
- pc_high_m = slopesPC[..., :self.max_cells, 0]
-
- # factor of high vs. medium gain for each pixel as derived from PC data
- fac_high_med = slopesPC[..., :self.max_cells, 3] / pc_high_m
-
- # if the flat field data contains a field for X-ray offset we can
- # make used of it
- xrayOffset = 0
- if len(slopesFF.shape) == 4:
- xrayOffset = slopesFF[..., :self.max_cells, 1]
- slopesFF = slopesFF[..., 0]
-
- # first 32 cells are known to behave differently so if we can avoid
- # them
- # when calculating the mean X-ray derived gain slope for each pixel
- if slopesFF.shape[2] > 32:
- xraygain = np.nanmedian(slopesFF[..., 32:min(96, self.max_cells)],
- axis=2)
- elif slopesFF.shape[2] > 2:
- xraygain = np.nanmedian(slopesFF[..., :min(96, self.max_cells)],
- axis=2)
- else:
- xraygain = np.squeeze(slopesFF[..., 0])
-
- # relative X-ray gain is then each pixels gain, by the median gain
- # of all pixels
- xraygain /= np.nanmedian(xraygain)
+ if self.corr_bools.get("xray_corr"):
+ bpff, when['bpff'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.BadPixelsFF(),
+ Conditions.Illuminated.AGIPD(
+ max_cells_db,
+ bias_voltage,
+ photon_energy,
+ pixels_x=512,
+ pixels_y=128,
+ beam_energy=None,
+ gain_setting=self.gain_setting,
+ acquisition_rate=self.acquisition_rate),
+ # noqa
+ np.zeros(
+ (128, 512, max_cells_db)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
+
+ bpixels |= bpff.astype(np.uint32)[..., :bpixels.shape[2], None]
+
+ slopesFF, when['slopesFF'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.SlopesFF(),
+ Conditions.Illuminated.AGIPD(
+ max_cells_db,
+ bias_voltage,
+ photon_energy,
+ pixels_x=512,
+ pixels_y=128,
+ beam_energy=None,
+ gain_setting=self.gain_setting,
+ acquisition_rate=self.acquisition_rate), # noqa
+ np.ones((128, 512, max_cells_db, 2)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
+ slopesFF = np.squeeze(slopesFF)
- # scale the relative PC gain of each memory cell in a pixel to the
- # median value of all cells
- pcrel = pc_high_m / np.nanmedian(pc_high_m, axis=2)[..., None]
-
- # and get high gain relative gain by multiplying with x-ray derived
- # gain
- rel_gain[..., 0] = pcrel * xraygain[..., None]
-
- # same scaling for high-to medium gain factor
- medfhm = np.nanmedian(fac_high_med, axis=2)[..., None]
- mfac = fac_high_med / medfhm * np.nanmedian(fac_high_med)
-
- # and scale with X-ray derived gain
- rel_gain[..., 1] = xraygain[..., None] * mfac
+ if len(slopesFF.shape) == 4:
+ slopesFF = slopesFF[..., 0]
- # finally low gain with a constant factor
- rel_gain[..., 2] = rel_gain[..., 1] * 0.223
+ # first 32 cells are known to behave differently so if we can avoid
+ # them
+ # when calculating the mean X-ray derived gain slope for each pixel
+ if slopesFF.shape[2] > 32:
+ xray_cor = np.nanmedian(slopesFF[..., 32:min(96, self.max_cells)],
+ axis=2)
+ elif slopesFF.shape[2] > 2:
+ xray_cor = np.nanmedian(slopesFF[..., :min(96, self.max_cells)],
+ axis=2)
+ else:
+ xray_cor = np.squeeze(slopesFF[..., 0])
- # for base offsets we pass a tuple containing the relevant information
- base_offset = [slopesPC[..., :self.max_cells, 1],
- slopesPC[..., :self.max_cells, 4], xrayOffset]
+ # relative X-ray correction is normalized by the median of all pixels
+ xray_cor /= np.nanmedian(xray_cor)
+ else:
+ xray_cor = None
+
+ if (self.corr_bools.get("rel_gain") or
+ self.corr_bools.get("adjust_mg_baseline") or
+ self.corr_bools.get('blc_noise') or
+ self.corr_bools.get('blc_hmatch') or
+ self.corr_bools.get('blc_stripes') or
+ self.melt_snow is not False):
+
+ # add additonal bad pixel information
+ bppc, when['bppc'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.BadPixelsPC(),
+ Conditions.Dark.AGIPD(
+ memory_cells=max_cells_db,
+ bias_voltage=bias_voltage,
+ gain_setting=self.gain_setting,
+ acquisition_rate=self.acquisition_rate),
+ # noqa
+ np.zeros(
+ (max_cells_db, 128, 512)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
+ bppc = np.moveaxis(bppc.astype(np.uint32), 0, 2)
+ bpixels |= bppc[..., :bpixels.shape[2], None]
+
+ # load also non-dark constants from the database
+ slopesPC, when['slopesPC'] = \
+ get_constant_from_db_and_time(getattr(dinstance, qm),
+ Constants.AGIPD.SlopesPC(),
+ Conditions.Dark.AGIPD(
+ memory_cells=max_cells_db,
+ bias_voltage=bias_voltage,
+ acquisition_rate=self.acquisition_rate), # noqa
+ np.ones(
+ (128, 512, max_cells_db, 10)),
+ cal_db_interface,
+ creation_time=creation_time,
+ timeout=timeout_cal_db)
+ slopesPC = slopesPC.astype(np.float32)
+
+ # this will handle some historical data in a slighly different format
+ # constant dimension injected first
+ if slopesPC.shape[0] == 10 or slopesPC.shape[0] == 11:
+ slopesPC = np.moveaxis(slopesPC, 0, 3)
+ slopesPC = np.moveaxis(slopesPC, 0, 2)
- # add additonal bad pixel information
- bppc, when['bppc'] = \
- get_constant_from_db_and_time(getattr(dinstance, qm),
- Constants.AGIPD.BadPixelsPC(),
- Conditions.Dark.AGIPD(
- memory_cells=max_cells_db,
- bias_voltage=bias_voltage,
- acquisition_rate=self.acquisition_rate), # noqa
- np.zeros((max_cells_db, 128, 512)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
- bppc = np.moveaxis(bppc.astype(np.uint32), 0, 2)
- bpixels |= bppc[..., :bpixels.shape[2], None]
+ # calculate relative gain from the constants
+ rel_gain = np.ones((128, 512, self.max_cells, 3), np.float32)
+ # high gain slope from pulse capacitor data
+ pc_high_m = slopesPC[..., :self.max_cells, 0]
+ pc_high_l = slopesPC[..., :self.max_cells, 1]
+ # medium gain slope from pulse capacitor data
+ pc_med_m = slopesPC[..., :self.max_cells, 3]
+ pc_med_l = slopesPC[..., :self.max_cells, 4]
- bpff, when['bpff'] = \
- get_constant_from_db_and_time(getattr(dinstance, qm),
- Constants.AGIPD.BadPixelsFF(),
- Conditions.Illuminated.AGIPD(
- max_cells_db,
- bias_voltage,
- photon_energy,
- pixels_x=512,
- pixels_y=128,
- beam_energy=None,
- acquisition_rate=self.acquisition_rate), # noqa
- np.zeros((128, 512, max_cells_db)),
- cal_db_interface,
- creation_time=creation_time,
- timeout=timeout_cal_db)
+ # calculate ratio high to medium gain over memory cells
+ pc_high_ave = np.nanmean(pc_high_m, axis=(0,1))
+ pc_med_ave = np.nanmean(pc_med_m, axis=(0,1))
+ frac_high_med = pc_high_ave / pc_med_ave
- bpixels |= bpff.astype(np.uint32)[..., :bpixels.shape[2], None]
+ # calculate additional medium-gain offset
+ md_additional_offset = pc_high_l - pc_med_l * pc_high_m / pc_med_m
- self.initialize(offset=offset, rel_gain=rel_gain, mask=bpixels,
- noise=noise, thresholds=thresholds,
- base_offset=base_offset, slopesPC=slopesPC)
+ # Calculate relative gain
+ rel_gain[..., 0] = pc_high_m / pc_high_ave
+ rel_gain[..., 1] = pc_med_m / pc_med_ave * frac_high_med
+ rel_gain[..., 2] = rel_gain[..., 1] * 0.233
+ else:
+ md_additional_offset = None
+ rel_gain = None
+ frac_high_med = None
+ self.initialize(offset, rel_gain, xray_cor, bpixels, noise, thresholds,
+ frac_high_med=frac_high_med,
+ md_additional_offset=md_additional_offset)
return when
def initialize_from_file(self, filename, qm, with_dark=True):
@@ -1823,30 +1918,25 @@ class AgipdCorrections:
pc_med_m = slopesPC[..., :self.max_cells, 3]
fac_high_med = pc_med_m / pc_high_m
- xrayOffset = 0
if len(slopesFF.shape) == 4:
- xrayOffset = slopesFF[..., :self.max_cells, 1]
slopesFF = slopesFF[..., 0]
if slopesFF.shape[2] > 32:
- xraygain = np.nanmedian(slopesFF[..., 32:], axis=2)
+ xray_cor = np.nanmedian(slopesFF[..., 32:], axis=2)
elif slopesFF.shape[2] > 2:
- xraygain = np.nanmedian(
+ xray_cor = np.nanmedian(
slopesFF[..., :min(96, self.max_cells)], axis=2)
else:
- xraygain = np.squeeze(slopesFF[..., 0])
+ xray_cor = np.squeeze(slopesFF[..., 0])
- xraygain /= np.nanmedian(xraygain)
+ xray_cor /= np.nanmedian(xray_cor)
pcrel = pc_high_m / np.nanmedian(pc_high_m, axis=2)[..., None]
- rel_gain[..., 0] = pcrel * xraygain[..., None]
+ rel_gain[..., 0] = pcrel
mfac = fac_high_med / np.nanmedian(fac_high_med, axis=2)[
..., None] * np.nanmedian(fac_high_med)
- rel_gain[..., 1] = xraygain[..., None] * mfac
+ rel_gain[..., 1] = pcrel * mfac
rel_gain[..., 2] = rel_gain[..., 1] * 0.223
- base_offset = [slopesPC[..., :self.max_cells, 1],
- slopesPC[..., :self.max_cells, 4], xrayOffset]
+ self.initialize(offsets, rel_gain, xray_cor, bpixels, noises,
+ thresholds, frac_high_med=fac_high_med)
- self.initialize(offset=offsets, rel_gain=rel_gain, mask=bpixels,
- noise=noises, thresholds=thresholds,
- base_offset=base_offset, slopesPC=slopesPC)
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 2286d4640..63a57ef73 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -24,13 +24,12 @@
"source": [
"in_folder = \"/gpfs/exfel/exp/MID/201931/p900107/raw\" # the folder to read data from, required\n",
"run = 11 # runs to process, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_Corr\" # the folder to output to, required\n",
- "calfile = \"/gpfs/exfel/data/scratch/haufs/agipd_on_demand/agipd_store_mid.h5\" # path to calibration file. Leave empty if all data should come from DB\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/karnem/test/AGIPD_Corr\" # the folder to output to, required\n",
+ "calfile = \"\" # path to calibration file. Leave empty if all data should come from DB\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
"interlaced = False # whether data is in interlaced layout\n",
"overwrite = True # set to True if existing data should be overwritten\n",
- "relative_gain = False # do relative gain correction\n",
"cluster_profile = \"noDB\"\n",
"max_pulses = [0, 500, 1] # range list [st, end, step] of maximum pulse indices. 3 allowed maximum list input elements. \n",
"local_input = False\n",
@@ -42,7 +41,6 @@
"index_v = 2 # version of RAW index type\n",
"nodb = False # if set only file-based constants will be used\n",
"blc_noise_threshold = 5000 # above this mean signal intensity now baseline correction via noise is attempted\n",
- "blc_hist = False # if set, base line correction via histogram matching is attempted \n",
"corr_asic_diag = False # if set, diagonal drop offs on ASICs are correted \n",
"melt_snow = \"\" # if set to \"none\" snowy pixels are identified and resolved to NaN, if set to \"interpolate\", the value is interpolated from neighbouring pixels\n",
"cal_db_timeout = 30000 # in milli seconds\n",
@@ -55,37 +53,27 @@
"force_mg_if_below = 1000 # set to a value other than 0 to force a pixel into medium gain if it's medium gain offset subtracted value is below this threshold\n",
"mask_noisy_adc = 0.25 # set to a value other than 0 and below 1 to mask entire ADC if fraction of noisy pixels is above\n",
"acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
+ "gain_setting = 0.1 # the gain setting, use 0.1 to try to auto-determine\n",
+ "\n",
+ "h5path_ctrl = '/CONTROL/SPB_IRU_AGIPD1M1/MDL/FPGA_COMP_TEST' # path to control information\n",
+ "karabo_da_control = 'DA02' # karabo DA for control infromation\n",
"\n",
"# Correction Booleans\n",
- "only_offset = False # Apply only Offset correction.\n",
- "pc_corr = False # Apply only Pulse Capictor correction.\n",
- "ff_corr = False # Apply only Flat Field correction.\n",
+ "only_offset = False # Apply only Offset correction. if False, Offset is applied by Default. if True, Offset is only applied.\n",
+ "rel_gain = False # do relative gain correction based on PC data\n",
+ "xray_gain = False # do relative gain correction based on xray data\n",
"blc_noise = False # if set, baseline correction via noise peak location is attempted\n",
+ "blc_stripes = False # if set, baseline corrected via stripes\n",
+ "blc_hmatch = False # if set, base line correction via histogram matching is attempted \n",
"match_asics = False # if set, inner ASIC borders are matched to the same signal level\n",
"adjust_mg_baseline = False # adjust medium gain baseline to match highest high gain value\n",
"dont_zero_nans = False # do not zero NaN values in corrected data\n",
"dont_zero_orange = False # do not zero very negative and very large values\n",
+ "blc_set_min = False # Shift to 0 negative medium gain pixels after offset corr\n",
"\n",
"def balance_sequences(in_folder, run, sequences, sequences_per_node):\n",
- " import glob\n",
- " import re\n",
- " import numpy as np\n",
- " if sequences[0] == -1:\n",
- " sequence_files = glob.glob(\"{}/r{:04d}/*-S*.h5\".format(in_folder, run))\n",
- " seq_nums = set()\n",
- " for sf in sequence_files:\n",
- " seqnum = re.findall(r\".*-S([0-9]*).h5\", sf)[0]\n",
- " seq_nums.add(int(seqnum))\n",
- " seq_nums -= set(sequences)\n",
- " else:\n",
- " seq_nums = set(sequences)\n",
- " nsplits = len(seq_nums)//sequences_per_node+1\n",
- " while nsplits > 32:\n",
- " sequences_per_node += 1\n",
- " nsplits = len(seq_nums)//sequences_per_node+1\n",
- " print(\"Changed to {} sequences per node to have a maximum of 8 concurrent jobs\".format(sequences_per_node))\n",
- " return [l.tolist() for l in np.array_split(list(seq_nums), nsplits) if l.size > 0]\n",
- " "
+ " from xfel_calibrate.calibrate import balance_sequences as bs\n",
+ " return bs(in_folder, run, sequences, sequences_per_node, \"AGIPD*\")"
]
},
{
@@ -104,18 +92,13 @@
"\n",
"# Dont apply any corrections if only_offset is requested \n",
"if not only_offset:\n",
- " \n",
- " # Apply PC correction only if requested\n",
- " if pc_corr:\n",
- " corr_bools[\"pc_corr\"] = pc_corr\n",
- " \n",
- " # Apply FF correction only if requested\n",
- " if ff_corr:\n",
- " corr_bools[\"ff_corr\"] = ff_corr\n",
- " \n",
" corr_bools[\"adjust_mg_baseline\"] = adjust_mg_baseline\n",
- " corr_bools[\"do_rel_gain\"] = relative_gain\n",
+ " corr_bools[\"rel_gain\"] = rel_gain\n",
+ " corr_bools[\"xray_corr\"] = xray_gain\n",
" corr_bools[\"blc_noise\"] = blc_noise\n",
+ " corr_bools[\"blc_stripes\"] = blc_stripes\n",
+ " corr_bools[\"blc_hmatch\"] = blc_hmatch\n",
+ " corr_bools[\"blc_set_min\"] = blc_set_min\n",
" corr_bools[\"match_asics\"] = match_asics\n",
" corr_bools[\"corr_asic_diag\"] = corr_asic_diag\n",
" corr_bools[\"dont_zero_nans\"] = dont_zero_nans\n",
@@ -154,7 +137,7 @@
"from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
"from cal_tools.tools import (gain_map_files, parse_runs, run_prop_seq_from_path, get_notebook_name,\n",
" get_dir_creation_date, get_constant_from_db)\n",
- "\n",
+ "from cal_tools.agipdlib import get_gain_setting\n",
"from dateutil import parser\n",
"from datetime import timedelta\n",
"\n",
@@ -177,7 +160,7 @@
"\n",
"if sequences[0] == -1:\n",
" sequences = None\n",
- "\n",
+ " \n",
"QUADRANTS = 4\n",
"MODULES_PER_QUAD = 4\n",
"DET_FILE_INSET = \"AGIPD\"\n",
@@ -199,7 +182,7 @@
"from cal_tools.agipdlib import SnowResolution\n",
"melt_snow = False if melt_snow == \"\" else SnowResolution(melt_snow)\n",
"\n",
- "special_opts = blc_noise_threshold, blc_hist, melt_snow\n",
+ "special_opts = blc_noise_threshold, blc_hmatch, melt_snow\n",
"\n",
"loc = None\n",
"if instrument == \"SPB\":\n",
@@ -208,7 +191,31 @@
"else:\n",
" loc = \"MID_DET_AGIPD1M-1\"\n",
" dinstance = \"AGIPD1M2\"\n",
- "print(\"Detector in use is {}\".format(loc)) "
+ "print(\"Detector in use is {}\".format(loc))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "control_fname = '{}/RAW-R{:04d}-{}-S00000.h5'.format(in_folder, run, karabo_da_control)\n",
+ "\n",
+ "if gain_setting == 0.1:\n",
+ " if creation_time.replace(tzinfo=None) < parser.parse('2020-01-31'):\n",
+ " print(\"Set gain-setting to None for runs taken before 2020-01-31\")\n",
+ " gain_setting = None\n",
+ " else:\n",
+ " try:\n",
+ " gain_setting = get_gain_setting(control_fname, h5path_ctrl)\n",
+ " except Exception as e:\n",
+ " print(f'Error while reading gain setting from: \\n{control_fname}')\n",
+ " print(e)\n",
+ " print(\"Set gain settion to 0\")\n",
+ " gain_setting = 0\n",
+ " \n",
+ "print(\"Gain setting: {}\".format(gain_setting))"
]
},
{
@@ -386,7 +393,7 @@
" bins_gain_vs_signal, bins_signal_low_range, bins_signal_high_range,\n",
" bins_dig_gain_vs_signal, max_pulses, dbparms, fileparms, nodb, chunk_size_idim,\n",
" special_opts, il_mode, loc, dinstance, force_hg_if_below, force_mg_if_below,\n",
- " mask_noisy_adc, acq_rate, corr_bools, inp):\n",
+ " mask_noisy_adc, acq_rate, gain_setting, corr_bools, inp):\n",
" print(\"foo\")\n",
" import numpy as np\n",
" import copy\n",
@@ -396,6 +403,7 @@
" import re\n",
" import os\n",
" from influxdb import InfluxDBClient\n",
+ " import subprocess\n",
" from cal_tools.enums import BadPixels\n",
" from cal_tools.agipdlib import AgipdCorrections, SnowResolution\n",
" from cal_tools.agipdlib import get_num_cells, get_acq_rate\n",
@@ -443,6 +451,7 @@
" when = None\n",
" err = None\n",
" \n",
+ " \n",
" try:\n",
" start = datetime.now()\n",
" success = True\n",
@@ -483,14 +492,14 @@
" h5_index_path=\"INDEX/{}/DET/{{}}CH0:xtdf/\".format(loc),\n",
" cal_det_instance=dinstance, force_hg_if_below=force_hg_if_below,\n",
" force_mg_if_below=force_mg_if_below, mask_noisy_adc=mask_noisy_adc,\n",
- " acquisition_rate=acq_rate, corr_bools=corr_bools)\n",
+ " acquisition_rate=acq_rate, gain_setting=gain_setting,\n",
+ " corr_bools=corr_bools)\n",
"\n",
- " blc_noise_threshold, blc_hist, melt_snow = special_opts\n",
+ " blc_noise_threshold, blc_hmatch, melt_snow = special_opts\n",
" if not corr_bools[\"only_offset\"]:\n",
- " blc_hist = False\n",
+ " blc_hmatch = False\n",
" melt_snow = False\n",
" agipd_corr.baseline_corr_noise_threshold = blc_noise_threshold\n",
- " agipd_corr.baseline_corr_using_hmatch = blc_hist\n",
" agipd_corr.melt_snow = melt_snow\n",
" try:\n",
" agipd_corr.get_valid_image_idx()\n",
@@ -511,6 +520,7 @@
" hists_signal_low, hists_signal_high, hists_gain_vs_signal, hists_dig_gain_vs_signal, hist_pulses = hists\n",
" low_edges, high_edges, signal_edges, dig_signal_edges = edges\n",
" gain_stats = np.array(agipd_corr.gain_stats)\n",
+ " \n",
" finally:\n",
" outfile.close()\n",
" infile.close()\n",
@@ -593,7 +603,7 @@
" sequences_qm, bins_gain_vs_signal, bins_signal_low_range, bins_signal_high_range,\n",
" bins_dig_gain_vs_signal, max_pulses, dbparms, fileparms, nodb, chunk_size_idim,\n",
" special_opts, il_mode, loc, dinstance, force_hg_if_below, force_mg_if_below,\n",
- " mask_noisy_adc, acq_rate, corr_bools)\n",
+ " mask_noisy_adc, acq_rate, gain_setting, corr_bools)\n",
"\n",
" r = view.map_sync(p, inp)\n",
"\n",
@@ -783,7 +793,7 @@
"outputs": [],
"source": [
"fig, ax = plt.subplots()\n",
- "if gain_stats != 0:\n",
+ "if isinstance(gain_stats, np.ndarray):\n",
" ax.pie(gain_stats, labels=[\"high\", \"medium\", \"low\"], autopct='%1.2f%%',\n",
" shadow=True, startangle=27)\n",
" a = ax.axis('equal') # Equal aspect ratio ensures that pie is drawn as a circle."
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
index eed58a388..7fb5ed474 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
@@ -14,8 +14,8 @@
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
- "run = 555 # runs to process, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/karnem/AGIPD_Bla3\" # path to output to, required"
+ "run = 119 # runs to process, required\n",
+ "out_folder = \"/gpfs/exfel/exp/SPB/201802/p002157/usr/test/test5/r0119\" # path to output to, required"
]
},
{
@@ -45,7 +45,7 @@
"outputs": [],
"source": [
"# load temporary data from csv files\n",
- "fnames = sorted(glob.glob('{}/r{:04d}/tmp_const_beginat_*'.format(out_folder,run)))\n",
+ "fnames = sorted(glob.glob('{}/tmp_const_beginat_*'.format(out_folder)))\n",
"\n",
"const_times = []\n",
"seq = []\n",
@@ -81,19 +81,23 @@
"\n",
" unique, idx, counts = np.unique(const_times[:,:,pos], return_inverse=True, return_counts=True)\n",
" idx = idx.reshape((const_times.shape[0],16))\n",
+ " \n",
+ " table = []\n",
"\n",
- " table = [ [const_times[:,:,pos][idx==0][0] , 'All modules'] ]\n",
- "\n",
- " for i in range(1, counts.shape[0]):\n",
+ " for i in range(0, counts.shape[0]):\n",
" line = [ const_times[:,:,pos][idx==i][0] ]\n",
" mods = ''\n",
" for i_s, s in enumerate(seq):\n",
" if( const_times[i_s,:,0][idx[i_s]==i].shape[0]>0 ):\n",
- " table[0][1] = 'Most of the modules'\n",
" mods = mods+ 'S{}: {}, '.format(seq[i_s], const_times[i_s,:,0][idx[i_s]==i])\n",
" line.append(mods)\n",
" table.append(line)\n",
"\n",
+ " if counts.shape[0] == 1:\n",
+ " table[0][1] = 'All modules'\n",
+ " else:\n",
+ " table[np.argmax(counts)][1] = 'Rest of the modules'\n",
+ "\n",
" md = display(Latex(tabulate.tabulate(table, tablefmt='latex', \n",
" headers=[\"Time stamps\", \"Modules and sequences\"]))) \n",
" \n",
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 5b9a3a171..41116eb52 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -25,13 +25,13 @@
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
- "in_folder = \"/gpfs/exfel/exp/SPB/201930/p900062/raw\" # path to input data, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/haufs/test/\" # path to output to, required\n",
+ "in_folder = \"/gpfs/exfel/d/raw/SPB/202030/p900138/\" # path to input data, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/karnem/test/\" # path to output to, required\n",
"sequences = [0] # sequence files to evaluate.\n",
"\n",
- "run_high = 264 # run number in which high gain data was recorded, required\n",
- "run_med = 265 # run number in which medium gain data was recorded, required\n",
- "run_low = 266 # run number in which low gain data was recorded, required\n",
+ "run_high = 64 # run number in which high gain data was recorded, required\n",
+ "run_med = 61 # run number in which medium gain data was recorded, required\n",
+ "run_low = 62 # run number in which low gain data was recorded, required\n",
"\n",
"mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
"local_output = True # output constants locally\n",
@@ -52,7 +52,11 @@
"instrument = \"SPB\"\n",
"high_res_badpix_3d = False # set this to True if you need high-resolution 3d bad pixel plots. Runtime: ~ 1h\n",
"modules = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15] # module to run for\n",
- "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine"
+ "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
+ "gain_setting = 0.1 # the gain setting, use 0.1 to try to auto-determine\n",
+ "\n",
+ "h5path_ctrl = '/CONTROL/SPB_IRU_AGIPD1M1/MDL/FPGA_COMP_TEST' # path to control information\n",
+ "karabo_da_control = 'DA02' # karabo DA for control infromation"
]
},
{
@@ -68,6 +72,7 @@
"source": [
"# imports and things that do not usually need to be changed\n",
"from datetime import datetime\n",
+ "import dateutil.parser\n",
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"from collections import OrderedDict\n",
@@ -86,6 +91,7 @@
"from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import show_overview, plot_badpix_3d, create_constant_overview\n",
+ "from cal_tools.agipdlib import get_gain_setting\n",
"\n",
"# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
"from ipyparallel import Client\n",
@@ -135,6 +141,30 @@
"print(\"Detector in use is {}\".format(loc)) "
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "control_fname = '{}/r{:04d}/RAW-R{:04d}-{}-S00000.h5'.format(in_folder, run_high, \n",
+ " run_high, karabo_da_control)\n",
+ "\n",
+ "if gain_setting == 0.1:\n",
+ " if creation_time.replace(tzinfo=None) < dateutil.parser.parse('2020-01-31'):\n",
+ " print(\"Set gain-setting to None for runs taken before 2020-01-31\")\n",
+ " gain_setting = None\n",
+ " else:\n",
+ " try:\n",
+ " gain_setting = get_gain_setting(control_fname, h5path_ctrl)\n",
+ " except Exception as e:\n",
+ " print(f'Error while reading gain setting from: \\n{control_fname}')\n",
+ " print(e)\n",
+ " print(\"Gain setting is not found in the control information\")\n",
+ " print(\"Data will not be processed\")\n",
+ " sequences = []"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -155,7 +185,8 @@
"print(\"Using DB: {}\".format(db_output))\n",
"print(\"Input: {}\".format(in_folder))\n",
"print(\"Output: {}\".format(out_folder))\n",
- "print(\"Bias voltage: {}V\".format(bias_voltage))"
+ "print(\"Bias voltage: {}V\".format(bias_voltage))\n",
+ "print(\"Gain setting: {}\".format(gain_setting))"
]
},
{
@@ -442,7 +473,8 @@
" # set the operating condition\n",
" condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
" bias_voltage=bias_voltage,\n",
- " acquisition_rate=acq_rate)\n",
+ " acquisition_rate=acq_rate,\n",
+ " gain_setting=gain_setting)\n",
" detinst = getattr(Detectors, dinstance)\n",
" device = getattr(detinst, qm)\n",
" \n",
diff --git a/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb b/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb
new file mode 100644
index 000000000..eeffc84fe
--- /dev/null
+++ b/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb
@@ -0,0 +1,676 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Characterize Dark Images for AGIPD64K #\n",
+ "\n",
+ "Author: K. Ahmed, Version: 0.1\n",
+ "\n",
+ "The following code analyzes a set of dark images taken with the Single Module AGIPD (AGIPD64K) detector to deduce detector offsets and noise. Data for the detector's three gain stages needs to be present, separated into separate runs."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Inputs exposed to xfel-calibrate package should be in this first cell.\n",
+ "\n",
+ "# Parameters for accessing files.\n",
+ "in_folder = \"/gpfs/exfel/exp/SPB/202030/p900138/raw\" # path to input data, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/SPB/AGIPD/DARK/202030/p900138/\" # path to output to, required\n",
+ "sequences = [0] # sequence files to evaluate.\n",
+ "\n",
+ "run_high = 33 # run number in which high gain data was recorded, required\n",
+ "run_med = 34 # run number in which medium gain data was recorded, required\n",
+ "run_low = 35 # run number in which low gain data was recorded, required\n",
+ "\n",
+ "local_output = True # output constants locally\n",
+ "db_output = False # output constants to database\n",
+ "\n",
+ "cal_db_interface = \"tcp://max-exfl016:8020\" # the database interface to use\n",
+ "cal_db_timeout = 3000000 # timeout on caldb requests\"\n",
+ "\n",
+ "instrument = \"SPB\"\n",
+ "module_name = 'AGIPD64K'\n",
+ "\n",
+ "channel = 16\n",
+ "\n",
+ "path_template = \"RAW-R{:04d}-{}-S{:05d}\"\n",
+ "\n",
+ "# Parameters for taking dark.\n",
+ "mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
+ "bias_voltage = 300 # detector bias voltage\n",
+ "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
+ "gain_setting = 0.1 # gain setting can have value 0 or 1, Default=0.1 for no (None) gain-setting\n",
+ "\n",
+ "dont_use_dir_date = False # don't use the dir creation date for determining the creation time\n",
+ "\n",
+ "thresholds_offset_sigma = 3. # thresholds in terms of n sigma noise for offset deduced bad pixels\n",
+ "thresholds_offset_hard = [4000, 8500] # thresholds in absolute ADU terms for offset deduced bad pixels\n",
+ "\n",
+ "thresholds_noise_sigma = 5. # thresholds in terms of n sigma noise for offset deduced bad pixels\n",
+ "thresholds_noise_hard = [4, 20] # thresholds in absolute ADU terms for offset deduced bad pixels\n",
+ "\n",
+ "# Plotting parameters\n",
+ "high_res_badpix_3d = False # set this to True if you need high-resolution 3d bad pixel plots. Runtime: ~ 1h\n",
+ "\n",
+ "# Parameters for ipcluster\n",
+ "cluster_profile = \"noDB\" # The ipcluster profile to use\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# imports and things that do not usually need to be changed\n",
+ "from datetime import datetime\n",
+ "import warnings\n",
+ "warnings.filterwarnings('ignore')\n",
+ "from collections import OrderedDict\n",
+ "import os\n",
+ "import h5py\n",
+ "import numpy as np\n",
+ "import traceback\n",
+ "import matplotlib\n",
+ "matplotlib.use('agg')\n",
+ "import matplotlib.pyplot as plt\n",
+ "%matplotlib inline\n",
+ "\n",
+ "from cal_tools.tools import (gain_map_files, parse_runs, \n",
+ " run_prop_seq_from_path, get_notebook_name, \n",
+ " get_dir_creation_date, save_const_to_h5,\n",
+ " get_random_db_interface)\n",
+ "from cal_tools.influx import InfluxLogger\n",
+ "from cal_tools.enums import BadPixels\n",
+ "from cal_tools.plotting import show_overview, plot_badpix_3d, create_constant_overview\n",
+ "\n",
+ "# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
+ "from ipyparallel import Client\n",
+ "\n",
+ "view = Client(profile=cluster_profile)[:]\n",
+ "view.use_dill()\n",
+ "\n",
+ "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
+ "\n",
+ "gains = np.arange(3)\n",
+ "\n",
+ "max_cells = mem_cells\n",
+ " \n",
+ "offset_runs = OrderedDict()\n",
+ "offset_runs[\"high\"] = parse_runs(run_high)[0]\n",
+ "offset_runs[\"med\"] = parse_runs(run_med)[0]\n",
+ "offset_runs[\"low\"] = parse_runs(run_low)[0]\n",
+ "\n",
+ "creation_time=None\n",
+ "if not dont_use_dir_date:\n",
+ " creation_time = get_dir_creation_date(in_folder, run_high)\n",
+ "\n",
+ "\n",
+ "run, prop, seq = run_prop_seq_from_path(in_folder)\n",
+ "logger = InfluxLogger(detector=\"AGIPD\", instrument=instrument, mem_cells=mem_cells,\n",
+ " notebook=get_notebook_name(), proposal=prop)\n",
+ "\n",
+ "print(\"Using {} as creation time of constant.\".format(creation_time))\n",
+ "\n",
+ "cal_db_interface = get_random_db_interface(cal_db_interface)\n",
+ "print('Calibration database interface: {}'.format(cal_db_interface))\n",
+ "\n",
+ "# Same used for testing(Temporary) the Single Module\n",
+ "loc = \"SPB_DET_AGIPD1M-1\"\n",
+ "\n",
+ "# Same used for testing(Temporary) the Single Module\n",
+ "dinstance = \"AGIPD1M1\"\n",
+ "print(\"Detector in use is {}\".format(loc))\n",
+ "\n",
+ "# Convert gain-setting in case of still being 0.1\n",
+ "if gain_setting == 0.1:\n",
+ " gain_setting = None"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(\"Parameters are:\")\n",
+ "print(\"Proposal: {}\".format(prop))\n",
+ "print(\"Memory cells: {}/{}\".format(mem_cells, max_cells))\n",
+ "print(\"Runs: {}\".format([ v for v in offset_runs.values()]))\n",
+ "print(\"Sequences: {}\".format(sequences))\n",
+ "print(\"Using DB: {}\".format(db_output))\n",
+ "print(\"Input: {}\".format(in_folder))\n",
+ "print(\"Output: {}\".format(out_folder))\n",
+ "print(\"Bias voltage: {}V\".format(bias_voltage))\n",
+ "print(\"Gain setting: {}\".format(gain_setting))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Dark raw files:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# set everything up filewise\n",
+ "if not os.path.exists(out_folder):\n",
+ " os.makedirs(out_folder)\n",
+ "\n",
+ "path_inset = \"AGIPD{}\".format(channel)\n",
+ "raw_files = []\n",
+ "total_file_size = 0\n",
+ "\n",
+ "for sequence in sequences:\n",
+ " for run in [run_high, run_med, run_low]:\n",
+ " h5file = path_template.format(run, path_inset, sequence)\n",
+ " print(h5file)\n",
+ " rfile = \"{}/r{:04d}/{}.h5\".format(in_folder, run, h5file)\n",
+ " raw_files.append(rfile)\n",
+ " total_file_size += os.path.getsize(rfile) \n",
+ "\n",
+ "\n",
+ "\n",
+ "if len(raw_files) < 1:\n",
+ " print(\"WARNING: NO FILES TO CREATE THE DARK!\")\n",
+ "else:\n",
+ " total_file_size = total_file_size / 1e9\n",
+ " total_sequences = len(raw_files)\n",
+ " print(\"The total size of the processed data: {}GB\".format(total_file_size))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Calculate Offsets, Noise and Thresholds ##\n",
+ "\n",
+ "The calculation is performed per-pixel and per-memory-cell. Offsets are simply the median value for a set of dark data taken at a given gain, noise the standard deviation, and gain-bit values the medians of the gain array."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import copy\n",
+ "from functools import partial\n",
+ "def characterize_module(cells, bp_thresh, loc, acq_rate, inp):\n",
+ " import numpy as np\n",
+ " import copy\n",
+ " import h5py\n",
+ " import traceback\n",
+ " from cal_tools.enums import BadPixels\n",
+ " from cal_tools.agipdlib import get_num_cells, get_acq_rate\n",
+ " \n",
+ " filename, filename_out, channel = inp\n",
+ " \n",
+ " if cells == 0:\n",
+ " cells = get_num_cells(filename, loc, channel)\n",
+ " \n",
+ " if acq_rate == 0.:\n",
+ " acq_rate = get_acq_rate(filename, loc, channel)\n",
+ " \n",
+ " thresholds_offset_hard, thresholds_offset_sigma, thresholds_noise_hard, thresholds_noise_sigma = bp_thresh \n",
+ " infile = h5py.File(filename, \"r\", driver=\"core\")\n",
+ " count = np.squeeze(infile[\"/INDEX/{}/DET/{}CH0:xtdf/image/count\".format(loc, channel)])\n",
+ " first = np.squeeze(infile[\"/INDEX/{}/DET/{}CH0:xtdf/image/first\".format(loc, channel)])\n",
+ " last_index = int(first[count != 0][-1]+count[count != 0][-1])\n",
+ " first_index = int(first[count != 0][0])\n",
+ " im = np.array(infile[\"/INSTRUMENT/{}/DET/{}CH0:xtdf/image/data\".format(loc, channel)][first_index:last_index,...]) \n",
+ " cellIds = np.squeeze(infile[\"/INSTRUMENT/{}/DET/{}CH0:xtdf/image/cellId\".format(loc, channel)][first_index:last_index,...]) \n",
+ " \n",
+ " infile.close()\n",
+ "\n",
+ " ga = im[:, 1, ...]\n",
+ " im = im[:, 0, ...].astype(np.float32)\n",
+ " \n",
+ " im = np.rollaxis(im, 2)\n",
+ " im = np.rollaxis(im, 2, 1)\n",
+ "\n",
+ " ga = np.rollaxis(ga, 2)\n",
+ " ga = np.rollaxis(ga, 2, 1)\n",
+ "\n",
+ " mcells = cells\n",
+ " offset = np.zeros((im.shape[0], im.shape[1], mcells))\n",
+ " gains = np.zeros((im.shape[0], im.shape[1], mcells))\n",
+ " noise = np.zeros((im.shape[0], im.shape[1], mcells))\n",
+ " \n",
+ " for cc in np.unique(cellIds[cellIds < mcells]):\n",
+ " cellidx = cellIds == cc\n",
+ " offset[...,cc] = np.median(im[..., cellidx], axis=2)\n",
+ " noise[...,cc] = np.std(im[..., cellidx], axis=2)\n",
+ " gains[...,cc] = np.median(ga[..., cellidx], axis=2)\n",
+ " \n",
+ " # bad pixels\n",
+ " bp = np.zeros(offset.shape, np.uint32)\n",
+ " \n",
+ " # offset related bad pixels\n",
+ " offset_mn = np.nanmedian(offset, axis=(0,1))\n",
+ " offset_std = np.nanstd(offset, axis=(0,1)) \n",
+ " \n",
+ " bp[(offset < offset_mn-thresholds_offset_sigma*offset_std) |\n",
+ " (offset > offset_mn+thresholds_offset_sigma*offset_std)] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
+ " bp[(offset < thresholds_offset_hard[0]) | (offset > thresholds_offset_hard[1])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
+ " bp[~np.isfinite(offset)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
+ " \n",
+ " # noise related bad pixels\n",
+ " noise_mn = np.nanmedian(noise, axis=(0,1))\n",
+ " noise_std = np.nanstd(noise, axis=(0,1)) \n",
+ " \n",
+ " bp[(noise < noise_mn-thresholds_noise_sigma*noise_std) |\n",
+ " (noise > noise_mn+thresholds_noise_sigma*noise_std)] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
+ " bp[(noise < thresholds_noise_hard[0]) | (noise > thresholds_noise_hard[1])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
+ " bp[~np.isfinite(noise)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
+ "\n",
+ "\n",
+ " return offset, noise, gains, bp, cells, acq_rate\n",
+ "\n",
+ "gg = 0\n",
+ "\n",
+ "start = datetime.now()\n",
+ "all_cells = []\n",
+ "all_acq_rate = []\n",
+ "\n",
+ "for gain, fname_in in enumerate(raw_files):\n",
+ "\n",
+ " inp = []\n",
+ "\n",
+ " #replace RAW with CORR in .hf5 file name.\n",
+ " fout = os.path.abspath(\"{}/{}\".format(out_folder, (os.path.split(fname_in)[-1]).replace(\"RAW\", \"CORR\")))\n",
+ " inp.append((fname_in, fout, channel))\n",
+ "\n",
+ " p = partial(characterize_module, max_cells,\n",
+ " (thresholds_offset_hard, thresholds_offset_sigma,\n",
+ " thresholds_noise_hard, thresholds_noise_sigma), loc, acq_rate)\n",
+ "\n",
+ " results = list(map(p, inp))\n",
+ " #results = view.map_sync(p, inp)\n",
+ "\n",
+ " for ii, r in enumerate(results):\n",
+ " i = 0\n",
+ " offset, noise, gain, bp, thiscell, thisacq = r\n",
+ " all_cells.append(thiscell)\n",
+ " all_acq_rate.append(thisacq)\n",
+ " \n",
+ " # only at the first gain.\n",
+ " if gg == 0:\n",
+ " offset_g = np.zeros((offset.shape[0], offset.shape[1], offset.shape[2], 3))\n",
+ " noise_g = np.zeros_like(offset_g)\n",
+ " gain_g = np.zeros_like(offset_g)\n",
+ " badpix_g = np.zeros_like(offset_g, np.uint32)\n",
+ " first = False\n",
+ "\n",
+ " offset_g[...,gg] = offset\n",
+ " noise_g[...,gg] = noise\n",
+ " gain_g[...,gg] = gain\n",
+ " badpix_g[...,gg] = bp\n",
+ " gg +=1\n",
+ "\n",
+ "duration = (datetime.now()-start).total_seconds()\n",
+ "logger.runtime_summary_entry(success=True, runtime=duration,\n",
+ " total_sequences=total_sequences,\n",
+ " filesize=total_file_size)\n",
+ "logger.send()\n",
+ "\n",
+ "max_cells = np.max(all_cells)\n",
+ "print(\"Using {} memory cells\".format(max_cells))\n",
+ "\n",
+ "acq_rate = np.max(all_acq_rate)\n",
+ "print(\"Using {} MHz acquisition rate\".format(acq_rate))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The thresholds for gain switching are then defined as the mean value between in individual gain bit levels. Note that these thresholds need to be refined with charge induced thresholds, as the two are not the same."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "thresholds_g = {}\n",
+ "thresholds_g = np.zeros((gain_g.shape[0], gain_g.shape[1], gain_g.shape[2], 5))\n",
+ "thresholds_g[...,0] = (gain_g[...,1]+gain_g[...,0])/2\n",
+ "thresholds_g[...,1] = (gain_g[...,2]+gain_g[...,1])/2\n",
+ "\n",
+ "# loop over the 3 files of each gain\n",
+ "for i in range(3):\n",
+ " thresholds_g[...,2+i] = gain_g[...,i]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "res = OrderedDict()\n",
+ "\n",
+ "res = {'Offset': offset_g,\n",
+ " 'Noise': noise_g,\n",
+ " 'ThresholdsDark': thresholds_g,\n",
+ " 'BadPixelsDark': badpix_g \n",
+ " }\n",
+ "\n",
+ "if local_output:\n",
+ " ofile = \"{}/agipd_offset_store_{}_{}.h5\".format(out_folder, \"_\".join(offset_runs.values()), module_name)\n",
+ " store_file = h5py.File(ofile, \"w\")\n",
+ " store_file[\"{}/Offset/0/data\".format(module_name)] = offset_g\n",
+ " store_file[\"{}/Noise/0/data\".format(module_name)] = noise_g\n",
+ " store_file[\"{}/Threshold/0/data\".format(module_name)] = thresholds_g\n",
+ " store_file[\"{}/BadPixels/0/data\".format(module_name)] = badpix_g\n",
+ " store_file.close()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
+ "file_loc = 'proposal:{} runs:{} {} {}'.format(proposal, run_low, run_med, run_high)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for const in res:\n",
+ " metadata = ConstantMetaData()\n",
+ " dconst = getattr(Constants.AGIPD, const)()\n",
+ " dconst.data = res[const]\n",
+ " metadata.calibration_constant = dconst\n",
+ "\n",
+ " # set the operating condition\n",
+ " condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
+ " bias_voltage=bias_voltage,\n",
+ " acquisition_rate=acq_rate,\n",
+ " gain_setting=gain_setting)\n",
+ " detinst = getattr(Detectors, dinstance)\n",
+ "\n",
+ " # AGIPD_SIV1_AGIPDV11_M001Test\n",
+ " device = Detectors.AGIPD.AGIPD_SIV1_AGIPDV11_M001Test\n",
+ "\n",
+ " metadata.detector_condition = condition\n",
+ "\n",
+ " # specify the a version for this constant\n",
+ " if creation_time is None:\n",
+ " metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ " else:\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
+ " start=creation_time)\n",
+ "\n",
+ " metadata.calibration_constant_version.raw_data_location = file_loc\n",
+ " if db_output:\n",
+ " try:\n",
+ " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
+ " msg = 'Const {} was injected to the calibration DB. Begin at: {}'\n",
+ " print(msg.format(const,\n",
+ " metadata.calibration_constant_version.begin_at))\n",
+ " except Exception as e:\n",
+ " print(\"Error sending constant to the DataBase:\", e)\n",
+ "\n",
+ " if local_output:\n",
+ " save_const_to_h5(metadata, out_folder)\n",
+ " print(\"Calibration constant {} is stored locally.\".format(const))\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Single-Cell Overviews ##\n",
+ "\n",
+ "Single cell overviews allow to identify potential effects on all memory cells, e.g. on sensor level. Additionally, they should serve as a first sanity check on expected behaviour, e.g. if structuring on the ASIC level is visible in the offsets, but otherwise no immediate artifacts are visible."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### High Gain ###"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cell = 3\n",
+ "gain = 0\n",
+ "out_folder = None\n",
+ "# attach module name for plotting.\n",
+ "res_da = {}\n",
+ "res_da[module_name] = res\n",
+ "\n",
+ "show_overview(res_da, cell, gain, out_folder=out_folder, infix=\"_\".join(offset_runs.values()))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Medium Gain ###"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cell = 3\n",
+ "gain = 1\n",
+ "show_overview(res_da, cell, gain, out_folder=out_folder, infix=\"_\".join(offset_runs.values()))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Low Gain ###"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cell = 3\n",
+ "gain = 2\n",
+ "show_overview(res_da, cell, gain, out_folder=out_folder, infix=\"_\".join(offset_runs.values()))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Global Bad Pixel Behaviour ##\n",
+ "\n",
+ "The following plots show the results of bad pixel evaluation for all evaluated memory cells. Cells are stacked in the Z-dimension, while pixels values in x/y are rebinned with a factor of 2. This excludes single bad pixels present only in disconnected pixels. Hence, any bad pixels spanning at least 4 pixels in the x/y-plane, or across at least two memory cells are indicated. Colors encode the bad pixel type, or mixed type."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### High Gain ###"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cols = {BadPixels.NOISE_OUT_OF_THRESHOLD.value: (BadPixels.NOISE_OUT_OF_THRESHOLD.name, '#FF000080'),\n",
+ " BadPixels.OFFSET_NOISE_EVAL_ERROR.value: (BadPixels.OFFSET_NOISE_EVAL_ERROR.name, '#0000FF80'),\n",
+ " BadPixels.OFFSET_OUT_OF_THRESHOLD.value: (BadPixels.OFFSET_OUT_OF_THRESHOLD.name, '#00FF0080'),\n",
+ " BadPixels.OFFSET_OUT_OF_THRESHOLD.value | BadPixels.NOISE_OUT_OF_THRESHOLD.value: ('MIXED', '#DD00DD80')}\n",
+ "\n",
+ "rebin = 8 if not high_res_badpix_3d else 2\n",
+ "\n",
+ "gain = 0\n",
+ "badpix_g_da = {}\n",
+ "badpix_g_da[module_name] = badpix_g\n",
+ "for mod, data in badpix_g_da.items():\n",
+ " plot_badpix_3d(data[...,gain], cols, title=mod, rebin_fac=rebin)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Medium Gain ###"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "gain = 1\n",
+ "for mod, data in badpix_g_da.items():\n",
+ " plot_badpix_3d(data[...,gain], cols, title=mod, rebin_fac=rebin)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Low Gain ###"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "gain = 2\n",
+ "for mod, data in badpix_g_da.items():\n",
+ " plot_badpix_3d(data[...,gain], cols, title=mod, rebin_fac=rebin)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Aggregate values, and per Cell behaviour ##\n",
+ "\n",
+ "The following tables and plots give an overview of statistical aggregates for each constant, as well as per cell behavior."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "offset_g_da = {}\n",
+ "offset_g_da[module_name] = offset_g\n",
+ "\n",
+ "create_constant_overview(offset_g_da, \"Offset (ADU)\", max_cells, 4000, 8000,\n",
+ " out_folder=out_folder, infix=\"_\".join(offset_runs.values()))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "noise_g_da = {}\n",
+ "noise_g_da[module_name] = noise_g\n",
+ "create_constant_overview(noise_g_da, \"Noise (ADU)\", max_cells, 0, 100,\n",
+ " out_folder=out_folder, infix=\"_\".join(offset_runs.values()))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "thresholds_g_da = {}\n",
+ "thresholds_g_da[module_name] = thresholds_g\n",
+ "create_constant_overview(thresholds_g_da, \"Threshold (ADU)\", max_cells, 3000, 8000, 2,\n",
+ " out_folder=out_folder, infix=\"_\".join(offset_runs.values()))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "bad_pixel_aggregate_g = OrderedDict()\n",
+ "\n",
+ "for m, d in badpix_g_da.items():\n",
+ " bad_pixel_aggregate_g[m] = d.astype(np.bool).astype(np.float)\n",
+ "create_constant_overview(bad_pixel_aggregate_g, \"Bad pixel fraction\", max_cells, 0, 0.10, 3,\n",
+ " out_folder=out_folder, infix=\"_\".join(offset_runs.values()))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Calibration_VENV",
+ "language": "python",
+ "name": "calibration_venv"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/webservice/serve_overview.py b/webservice/serve_overview.py
index edcf71510..ab34a06f6 100644
--- a/webservice/serve_overview.py
+++ b/webservice/serve_overview.py
@@ -343,10 +343,8 @@ class RequestHandler(BaseHTTPRequestHandler):
if len(pdfs):
# pdfs = ", ".join(pdfs)
for pdf, p in pdfs.items():
- puuid = uuid4().hex
- self.pdf_queue[puuid] = p
fpdfs.append(
- (pdf, f"http://{host}:{port}/pdf?{puuid}"))
+ (pdf, f"http://{host}:{port}/{p}"))
pdfs = fpdfs
tsize = 0
for run in runs:
diff --git a/webservice/templates/dark_overview.html b/webservice/templates/dark_overview.html
index 200c63b3b..9d965ea11 100644
--- a/webservice/templates/dark_overview.html
+++ b/webservice/templates/dark_overview.html
@@ -17,7 +17,7 @@
{% for file in files %}
<tr bgcolor="{{ file[2] }}">
- <td> <a href=http://{{host}}:{{port}}/file?{{ file[0] }} target="_blank">{{ file[0] }}</a> </td>
+ <td> <a href=http://{{host}}:{{port}}/{{ file[0] }} target="_blank">{{ file[0] }}</a> </td>
<td> {{ file[1] }} </td>
</tr>
diff --git a/webservice/update_config.py b/webservice/update_config.py
index d1972e6e6..917f252f5 100644
--- a/webservice/update_config.py
+++ b/webservice/update_config.py
@@ -7,7 +7,11 @@ import zmq
available_options = {
"AGIPD": {"force-hg-if-below": float,
+ "rel_gain": bool,
+ "xray_gain": bool,
"blc-noise": bool,
+ "blc-stripes": bool,
+ "blc-set-min": bool,
"dont-zero-nans": bool,
"dont-zero-orange": bool,
"max-pulses": list,
diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py
index ff69d5435..9de92ccc2 100644
--- a/xfel_calibrate/notebooks.py
+++ b/xfel_calibrate/notebooks.py
@@ -35,6 +35,16 @@ notebooks = {
"cluster cores": 8},
},
},
+
+ "AGIPD64K": {
+ "DARK": {
+ "notebook": "notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb",
+ "concurrency": {"parameter": None,
+ "default concurrency": None,
+ "cluster cores": 4},
+ },
+ },
+
"LPD": {
"DARK": {
"notebook": "notebooks/LPD/LPDChar_Darks_NBC.ipynb",
--
GitLab