diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 971689cf5985c54fc73da4a43ef48fc5cee2b02f..84eaa0090c2ad0fe3363cca944260dcc0f701b6d 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -63,10 +63,13 @@
"ff_gain = 7.2 # conversion gain for absolute FlatField constants, while applying xray_gain\n",
"photon_energy = -1.0 # photon energy in keV, non-positive value for XGM autodetection\n",
"rounding_threshold = 0.5 # the fraction to round to down, 0.5 for standard rounding rule\n",
+ "cs_mg_adjust = 7e3 # Value to adjust medium gain when correcting with current source. This is used when `adjust_mg_baseline` is True.\n",
"\n",
"# Correction Booleans\n",
"only_offset = False # Apply only Offset correction. if False, Offset is applied by Default. if True, Offset is only applied.\n",
- "rel_gain = False # do relative gain correction based on PC data\n",
+ "# TODO: Remove this boolean parameter an replace rel_gain_mode with it.\n",
+ "rel_gain = False \n",
+ "rel_gain_mode = \"off\" # Select relative gain correction. Choices [`PC`, `CS`, `off`]. (`PC`: Pulse Capacitor, `CS`: Current Source, `off`: Disable relative gain correction). Default: off.\n",
"xray_gain = False # do relative gain correction based on xray data\n",
"blc_noise = False # if set, baseline correction via noise peak location is attempted\n",
"blc_stripes = False # if set, baseline corrected via stripes\n",
@@ -213,13 +216,32 @@
"# Here the hierarchy and dependability for correction booleans are defined\n",
"corr_bools = {}\n",
"\n",
+ "cs_corr = False\n",
+ "pc_corr = False\n",
+ "if rel_gain_mode.lower() == \"off\":\n",
+ " # TODO: Remove this part after replacing rel_gain with rel_gain_mode\n",
+ " if rel_gain:\n",
+ " pc_corr = True\n",
+ "\n",
+ "elif rel_gain_mode.lower() == \"cs\":\n",
+ " cs_corr = True\n",
+ "\n",
+ "elif rel_gain_mode.lower() == \"pc\":\n",
+ " pc_corr = True\n",
+ "\n",
+ "else:\n",
+ " raise ValueError(\n",
+ " \"Selected `rel_gain_mode` is unexpected. \"\n",
+ " \"Please select between CS or PC.\")\n",
+ "\n",
"# offset is at the bottom of AGIPD correction pyramid.\n",
"corr_bools[\"only_offset\"] = only_offset\n",
"\n",
"# Dont apply any corrections if only_offset is requested\n",
"if not only_offset:\n",
+ " corr_bools[\"cs_corr\"] = cs_corr\n",
+ " corr_bools[\"pc_corr\"] = pc_corr\n",
" corr_bools[\"adjust_mg_baseline\"] = adjust_mg_baseline\n",
- " corr_bools[\"rel_gain\"] = rel_gain\n",
" corr_bools[\"xray_corr\"] = xray_gain\n",
" corr_bools[\"blc_noise\"] = blc_noise\n",
" corr_bools[\"blc_stripes\"] = blc_stripes\n",
@@ -246,7 +268,8 @@
" \"force_mg_if_below\",\n",
" \"low_medium_gap\",\n",
" \"melt_snow\",\n",
- " \"rel_gain\"\n",
+ " \"pc_corr\",\n",
+ " \"cs_corr\",\n",
"]"
]
},
@@ -564,6 +587,7 @@
"agipd_corr.ff_gain = ff_gain\n",
"agipd_corr.photon_energy = photon_energy\n",
"agipd_corr.rounding_threshold = rounding_threshold\n",
+ "agipd_corr.cs_mg_adjust = cs_mg_adjust\n",
"\n",
"agipd_corr.compress_fields = compress_fields\n",
"if recast_image_data:\n",
@@ -623,22 +647,34 @@
"agipd_metadata = agipd_cal.metadata(dark_constants)\n",
"\n",
"agipd_cal.gain_mode = None # gain_mode is not used for gain constants\n",
- "pc_constants, ff_constants = [], []\n",
- "if any(agipd_corr.pc_bools):\n",
- " pc_constants = [\"SlopesPC\", \"BadPixelsPC\"]\n",
- " get_constants_and_update_metadata(\n",
- " agipd_cal, agipd_metadata, pc_constants)\n",
+ "pc_constants, ff_constants, cs_constants = [], [], []\n",
"\n",
"if agipd_corr.corr_bools.get('xray_corr'):\n",
" ff_constants = list(agipd_cal.illuminated_calibrations)\n",
" get_constants_and_update_metadata(\n",
" agipd_cal, agipd_metadata, ff_constants)\n",
"\n",
+ "if any(agipd_corr.relgain_bools):\n",
+ "\n",
+ " if cs_corr:\n",
+ " # Integration time is not used with CS\n",
+ " agipd_cal.integration_time = None\n",
+ " cs_constants = [\"SlopesCS\", \"BadPixelsCS\"]\n",
+ " get_constants_and_update_metadata(\n",
+ " agipd_cal, agipd_metadata, cs_constants)\n",
+ " \n",
+ "\n",
+ " else: # rel_gain_mode == \"pc\" or \"off\"\n",
+ " pc_constants = [\"SlopesPC\", \"BadPixelsPC\"]\n",
+ " get_constants_and_update_metadata(\n",
+ " agipd_cal, agipd_metadata, pc_constants)\n",
+ "\n",
"step_timer.done_step(\"Constants were retrieved in\")\n",
"\n",
+ "relgain_alias = \"CS\" if cs_corr else \"PC\"\n",
"print(\"Preparing constants (\"\n",
" f\"FF: {agipd_corr.corr_bools.get('xray_corr', False)}, \"\n",
- " f\"PC: {any(agipd_corr.pc_bools)}, \"\n",
+ " f\"{relgain_alias}: {any(agipd_corr.relgain_bools)}, \"\n",
" f\"BLC: {any(agipd_corr.blc_bools)})\")\n",
"# Display retrieved calibration constants timestamps\n",
"agipd_cal.display_markdown_retrieved_constants(metadata=agipd_metadata)"
@@ -666,7 +702,7 @@
" karabo_da.remove(da)\n",
" modules.remove(mod)\n",
"\n",
- " warn_missing_constants = set(dark_constants + pc_constants + ff_constants)\n",
+ " warn_missing_constants = set(dark_constants + pc_constants + ff_constants + cs_constants)\n",
" warn_missing_constants -= error_missing_constants\n",
" warn_missing_constants -= set(calibrations)\n",
" if warn_missing_constants:\n",
@@ -731,7 +767,7 @@
"metadata": {},
"outputs": [],
"source": [
- "# Store timestamps for Offset, SlopesPC, and SlopesFF\n",
+ "# Store timestamps for Offset, SlopesPC/SlopesCS, and SlopesFF\n",
"# in YAML file for time-summary table.\n",
"timestamps = {}\n",
"\n",
@@ -742,7 +778,7 @@
"\n",
" # Store few time stamps if exists\n",
" # Add NA to keep array structure\n",
- " for key in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
+ " for key in ['Offset', f'Slopes{relgain_alias}', 'SlopesFF']:\n",
" if key in mod_mdata:\n",
" module_timestamps[key] = mod_mdata[key][\"begin_validity_at\"]\n",
" else:\n",
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
index 36e21e9220cc0fcda5f1f4f4a235fcdef1208034..17a162ea0017533e62fa34002537908032c46654 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
@@ -18,7 +18,8 @@
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"karabo_id = \"SPB_DET_AGIPD1M-1\" # karabo karabo_id\n",
"modules = [-1]\n",
- "karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators"
+ "karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
+ "rel_gain_mode = \"off\" # Select relative gain correction. Choices [`PC`, `CS`, `off`]. (`PC`: Pulse Capacitor, `CS`: Current Source, `off`: Disable relative gain correction). Default: off."
]
},
{
@@ -105,8 +106,8 @@
" display(Latex(tabulate.tabulate(table,\n",
" tablefmt=\"latex\",\n",
" headers=[\"Timestamps\", \"Modules and sequences\"])))\n",
- "\n",
- "for const in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
+ "rel_gain_alias = \"CS\" if rel_gain_mode.lower() == \"cs\" else \"PC\" # 'off' or 'pc'\n",
+ "for const in ['Offset', f'Slopes{rel_gain_alias}', 'SlopesFF']:\n",
" print_const_table(const)"
]
}
diff --git a/src/cal_tools/agipdlib.py b/src/cal_tools/agipdlib.py
index f22d739c9c30e5bc5cf61de5497614a7dc779f0c..348d621c9f253b28380739069b4286bff85180f7 100644
--- a/src/cal_tools/agipdlib.py
+++ b/src/cal_tools/agipdlib.py
@@ -21,13 +21,14 @@ from cal_tools.agipdutils import (
cast_array_inplace,
correct_baseline_via_hist,
correct_baseline_via_hist_asic,
+ get_gain_pc_slopes,
make_noisy_adc_mask,
match_asic_borders,
melt_snowy_pixels,
)
from cal_tools.enums import AgipdGainMode, BadPixels, SnowResolution
from cal_tools.h5_copy_except import h5_copy_except_paths
-
+from logging import warning
@dataclass
class AgipdCtrl:
@@ -599,6 +600,7 @@ class AgipdCorrections:
self.ff_gain = 1
self.photon_energy = 9.2
self.rounding_threshold = 0.5
+ self.cs_mg_adjust = 7e3
# Output parameters
self.compress_fields = ['gain', 'mask']
@@ -624,8 +626,8 @@ class AgipdCorrections:
self.hist_lock = Manager().Lock()
# check if given corr_bools are correct
- tot_corr_bools = ['only_offset', 'adjust_mg_baseline', 'rel_gain',
- 'xray_corr', 'blc_noise', 'blc_hmatch',
+ tot_corr_bools = ['only_offset', 'adjust_mg_baseline', 'pc_corr',
+ 'cs_corr', 'xray_corr', 'blc_noise', 'blc_hmatch',
'blc_stripes', 'blc_set_min', 'match_asics',
'corr_asic_diag', 'zero_nans',
'zero_orange', 'force_hg_if_below',
@@ -640,13 +642,17 @@ class AgipdCorrections:
raise Exception('Correction Booleans: '
f'{bools} are not available!')
- # Flags allowing for pulse capacitor constant retrieval.
- self.pc_bools = [self.corr_bools.get("rel_gain"),
- self.corr_bools.get("adjust_mg_baseline"),
- self.corr_bools.get('blc_noise'),
- self.corr_bools.get('blc_hmatch'),
- self.corr_bools.get('blc_stripes'),
- self.corr_bools.get('melt_snow')]
+ # Flags allowing for pulse capacitor / current source constant retrieval.
+ self.relgain_bools = [
+ self.corr_bools.get("pc_corr"),
+ self.corr_bools.get("cs_corr"),
+ # This correction is disabled if cs_corr == True
+ self.corr_bools.get("adjust_mg_baseline"),
+ self.corr_bools.get('blc_noise'),
+ self.corr_bools.get('blc_hmatch'),
+ self.corr_bools.get('blc_stripes'),
+ self.corr_bools.get('melt_snow')
+ ]
self.blc_bools = [self.corr_bools.get('blc_noise'),
self.corr_bools.get('blc_hmatch'),
@@ -906,6 +912,8 @@ class AgipdCorrections:
t0 = self.thresholds[module_idx][0]
t1 = self.thresholds[module_idx][1]
+ # This correction has not been used for years.
+ # TODO: decide about removing it.
if self.corr_bools.get("melt_snow"):
# load raw_data and rgain to be used during gain_correction
self.shared_dict[i_proc]["t0_rgain"][first:last] = rawgain / t0[cellid, ...] # noqa
@@ -989,6 +997,8 @@ class AgipdCorrections:
# if we have enough pixels in medium or low gain and
# correction via hist matching is requested to this now
gcrit = np.count_nonzero(gain[i] > 0) > 1000
+ # blc_hmatch correction has not been used for years.
+ # TODO: decide about removing it.
if (gcrit and self.corr_bools.get('blc_hmatch') and
hasattr(self, "rel_gain")):
dd2, sh2 = correct_baseline_via_hist(data[i],
@@ -997,6 +1007,8 @@ class AgipdCorrections:
data[i] = np.maximum(dd, dd2)
sh = np.minimum(sh, sh2)
# finally correct diagonal effects if requested
+ # This correction has not been used for years.
+ # TODO: decide about removing it.
if self.corr_bools.get('corr_asic_diag'):
ii = data[i, ...]
gg = gain[i, ...]
@@ -1044,15 +1056,17 @@ class AgipdCorrections:
rel_corr = calgs.gain_choose(gain, self.rel_gain[module_idx][:, cellid]) # noqa
# Correct for relative gain
- if self.corr_bools.get("rel_gain") and hasattr(self, "rel_gain"):
+ if (
+ self.corr_bools.get("pc_corr") or self.corr_bools.get("cs_corr")
+ ) and hasattr(self, "rel_gain"):
data *= rel_corr
del rel_corr
# Adjust medium gain baseline to match highest high gain value
if self.corr_bools.get("adjust_mg_baseline"):
- mgbc = self.md_additional_offset[module_idx][cellid, ...]
- data[gain == 1] += mgbc[gain == 1]
- del mgbc
+ mgbc = self.md_additional_offset[module_idx][cellid, ...]
+ data[gain == 1] += mgbc[gain == 1]
+ del mgbc
# Set negative values for medium gain to 0
# TODO: Probably it would be better to add it to badpixel maps,
@@ -1413,16 +1427,29 @@ class AgipdCorrections:
self.xray_cor[module_idx][...] = xray_cor.transpose()[...]
# add additional bad pixel information
- if any(self.pc_bools):
- if "BadPixelsPC" in cons_data:
- bppc = np.moveaxis(cons_data["BadPixelsPC"].astype(np.uint32),
- 0, 2)
- bpixels |= bppc[..., :bpixels.shape[2], None]
+ if any(self.relgain_bools):
+ for rg in ["CS", "PC"]:
+ if f"BadPixels{rg}" in cons_data:
+ bp_relgain = np.moveaxis(
+ cons_data[f"BadPixels{rg}"].astype(np.uint32), 0, 2)
+ bpixels |= bp_relgain[..., :bpixels.shape[2], None]
# calculate relative gain from the constants
rel_gain = np.ones((128, 512, self.max_cells, 3), np.float32)
- if "SlopesPC" in cons_data:
+ # Either SlopesCS or SlopesPC is applied.
+ if "SlopesCS" in cons_data:
+ rel_gain[..., 1] = rel_gain[..., 0] * cons_data["SlopesCS"][..., :self.max_cells, 6] # noqa
+ rel_gain[..., 2] = rel_gain[..., 1] * cons_data["SlopesCS"][..., :self.max_cells, 7] # noqa
+ frac_high_med = np.median(
+ cons_data["SlopesCS"][..., :self.max_cells, 6])
+
+ md_additional_offset = np.full(
+ (128, 512, self.max_cells),
+ fill_value=self.cs_mg_adjust,
+ dtype=np.float32)
+
+ elif "SlopesPC" in cons_data:
slopesPC = cons_data["SlopesPC"].astype(np.float32, copy=False)
# This will handle some historical data in a different format
@@ -1431,55 +1458,14 @@ class AgipdCorrections:
slopesPC = np.moveaxis(slopesPC, 0, 3)
slopesPC = np.moveaxis(slopesPC, 0, 2)
- # high gain slope from pulse capacitor data
- pc_high_m = slopesPC[..., :self.max_cells, 0]
- pc_high_l = slopesPC[..., :self.max_cells, 1]
- # medium gain slope from pulse capacitor data
- pc_med_m = slopesPC[..., :self.max_cells, 3]
- pc_med_l = slopesPC[..., :self.max_cells, 4]
-
- # calculate median for slopes
- pc_high_med = np.nanmedian(pc_high_m, axis=(0, 1))
- pc_med_med = np.nanmedian(pc_med_m, axis=(0, 1))
-
- if variant.get("SlopesPC", 0) == 0:
- # calculate median for intercepts:
- pc_high_l_med = np.nanmedian(pc_high_l, axis=(0, 1))
- pc_med_l_med = np.nanmedian(pc_med_l, axis=(0, 1))
-
- # sanitize PC data with CCV variant = 0.
- # Sanitization is already done for constants
- # with CCV variant = 1
- # In the following loop,
- # replace `nan`s across memory cells with
- # the median value calculated previously.
- # Then, values outside of the valid range (0.8 and 1.2)
- # are fixed to the median value.
- # This is applied for high and medium gain stages
- for i in range(self.max_cells):
- pc_high_m[
- np.isnan(pc_high_m[..., i]), i] = pc_high_med[i]
- pc_med_m[
- np.isnan(pc_med_m[..., i]), i] = pc_med_med[i]
-
- pc_high_l[
- np.isnan(pc_high_l[..., i]), i] = pc_high_l_med[i]
- pc_med_l[
- np.isnan(pc_med_l[..., i]), i] = pc_med_l_med[i]
-
- pc_high_m[
- (pc_high_m[..., i] < 0.8 * pc_high_med[i]) |
- (pc_high_m[..., i] > 1.2 * pc_high_med[i]), i] = pc_high_med[i] # noqa
- pc_med_m[
- (pc_med_m[..., i] < 0.8 * pc_med_med[i]) |
- (pc_med_m[..., i] > 1.2 * pc_med_med[i]), i] = pc_med_med[i] # noqa
-
- pc_high_l[
- (pc_high_l[..., i] < 0.8 * pc_high_l_med[i]) |
- (pc_high_l[..., i] > 1.2 * pc_high_l_med[i]), i] = pc_high_l_med[i] # noqa
- pc_med_l[
- (pc_med_l[..., i] < 0.8 * pc_med_l_med[i]) |
- (pc_med_l[..., i] > 1.2 * pc_med_l_med[i]), i] = pc_med_l_med[i] # noqa
+ (
+ pc_high_m, pc_med_m, pc_high_l,
+ pc_med_l, pc_high_med, pc_med_med
+ ) = get_gain_pc_slopes(
+ slopes_pc=slopesPC,
+ mem_cells=self.max_cells,
+ variant=variant
+ )
# ration between HG and MG per pixel per mem cell used
# for rel gain calculation
diff --git a/src/cal_tools/agipdutils.py b/src/cal_tools/agipdutils.py
index a7ee52de1c110921dbc652325c6441afed3cc008..e34f5846e827edd1e82dbc2566d788921e527ced 100644
--- a/src/cal_tools/agipdutils.py
+++ b/src/cal_tools/agipdutils.py
@@ -650,3 +650,79 @@ def cast_array_inplace(inp, dtype):
outp.shape = orig_shape
return outp
+
+def get_gain_pc_slopes(
+ slopes_pc: np.ndarray,
+ mem_cells: int,
+ variant: int = 0,
+ ) -> (
+ np.ndarray, np.ndarray, np.ndarray,
+ np.ndarray, np.ndarray, np.ndarray
+ ):
+ """Calculate gain PC slopes and the high and medium gain medians.
+
+ Args:
+ slopes_pc (np.ndarray): SlopesPC gain constant array.
+ mem_cells (int): Number of operating memory cells.
+ variant (int): Calibration Constant Version variant.
+
+ Returns:
+ np.ndarray: _description_
+ np.ndarray: _description_
+ np.ndarray: _description_
+ np.ndarray: _description_
+ np.ndarray: _description_
+ np.ndarray: _description_
+ """
+ # high gain slope from pulse capacitor data
+ pc_high_m = slopes_pc[..., :mem_cells, 0]
+ pc_high_l = slopes_pc[..., :mem_cells, 1]
+ # medium gain slope from pulse capacitor data
+ pc_med_m = slopes_pc[..., :mem_cells, 3]
+ pc_med_l = slopes_pc[..., :mem_cells, 4]
+
+ # calculate median for slopes
+ pc_high_med = np.nanmedian(pc_high_m, axis=(0, 1))
+ pc_med_med = np.nanmedian(pc_med_m, axis=(0, 1))
+
+ if variant.get("SlopesPC", 0) == 0:
+ # calculate median for intercepts:
+ pc_high_l_med = np.nanmedian(pc_high_l, axis=(0, 1))
+ pc_med_l_med = np.nanmedian(pc_med_l, axis=(0, 1))
+
+ # sanitize PC data with CCV variant = 0.
+ # Sanitization is already done for constants
+ # with CCV variant = 1
+ # In the following loop,
+ # replace `nan`s across memory cells with
+ # the median value calculated previously.
+ # Then, values outside of the valid range (0.8 and 1.2)
+ # are fixed to the median value.
+ # This is applied for high and medium gain stages
+ for i in range(mem_cells):
+
+ pc_high_m[
+ np.isnan(pc_high_m[..., i]), i] = pc_high_med[i]
+ pc_med_m[
+ np.isnan(pc_med_m[..., i]), i] = pc_med_med[i]
+
+ pc_high_l[
+ np.isnan(pc_high_l[..., i]), i] = pc_high_l_med[i]
+ pc_med_l[
+ np.isnan(pc_med_l[..., i]), i] = pc_med_l_med[i]
+
+ pc_high_m[
+ (pc_high_m[..., i] < 0.8 * pc_high_med[i]) |
+ (pc_high_m[..., i] > 1.2 * pc_high_med[i]), i] = pc_high_med[i] # noqa
+ pc_med_m[
+ (pc_med_m[..., i] < 0.8 * pc_med_med[i]) |
+ (pc_med_m[..., i] > 1.2 * pc_med_med[i]), i] = pc_med_med[i] # noqa
+
+ pc_high_l[
+ (pc_high_l[..., i] < 0.8 * pc_high_l_med[i]) |
+ (pc_high_l[..., i] > 1.2 * pc_high_l_med[i]), i] = pc_high_l_med[i] # noqa
+ pc_med_l[
+ (pc_med_l[..., i] < 0.8 * pc_med_l_med[i]) |
+ (pc_med_l[..., i] > 1.2 * pc_med_l_med[i]), i] = pc_med_l_med[i] # noqa
+
+ return pc_high_m, pc_med_m, pc_high_l, pc_med_l, pc_high_med, pc_med_med
\ No newline at end of file
diff --git a/src/cal_tools/calcat_interface.py b/src/cal_tools/calcat_interface.py
index 164ce0ee0f52c08a7e1f48f81a2a505f6ab454aa..aabee45033593c22797bc97362ed06f6517108a6 100644
--- a/src/cal_tools/calcat_interface.py
+++ b/src/cal_tools/calcat_interface.py
@@ -1014,6 +1014,8 @@ class AGIPD_CalibrationData(SplitConditionCalibrationData):
"BadPixelsDark",
"BadPixelsPC",
"SlopesPC",
+ "SlopesCS",
+ "BadPixelsCS",
}
illuminated_calibrations = {
"BadPixelsFF",