From bd9ec4965b4057fea80ef21b298280660a099a6b Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Fri, 12 Mar 2021 17:27:13 +0100
Subject: [PATCH] Additional cleanup, starting to add gain_mode in agipdutils
---
cal_tools/cal_tools/agipdutils.py | 53 ++++---
...IPD_Retrieve_Constants_Precorrection.ipynb | 143 ++++++++++--------
2 files changed, 106 insertions(+), 90 deletions(-)
diff --git a/cal_tools/cal_tools/agipdutils.py b/cal_tools/cal_tools/agipdutils.py
index 9d58ccf83..aec289dd6 100644
--- a/cal_tools/cal_tools/agipdutils.py
+++ b/cal_tools/cal_tools/agipdutils.py
@@ -2,18 +2,27 @@ import copy
from typing import Tuple
import numpy as np
-from cal_tools.enums import BadPixels, SnowResolution
-from scipy.signal import cwt, ricker
+from cal_tools.enums import AgipdGainMode, BadPixels, SnowResolution
+from scipy.signal import cwt, find_peaks_cwt, ricker
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import StandardScaler
-def assemble_constant_dict(corr_bools, pc_bools, memory_cells, bias_voltage,
- gain_setting, acquisition_rate,
- photon_energy, beam_energy=None, only_dark=False):
+def assemble_constant_dict(
+ corr_bools,
+ pc_bools,
+ memory_cells,
+ bias_voltage,
+ gain_setting,
+ acquisition_rate,
+ photon_energy,
+ beam_energy=None,
+ only_dark=False,
+ gain_mode=AgipdGainMode.ADAPTIVE_GAIN,
+):
"""
Assemble a dictionary with the iCalibrationDB constant names and
- the operating conditions for retrieveing the required constants
+ the operating conditions for retrieving the required constants
for correction.
:param corr_bools: (Dict) A dict of booleans for applying
@@ -25,20 +34,23 @@ def assemble_constant_dict(corr_bools, pc_bools, memory_cells, bias_voltage,
:param acquisition_rate: (Float) Acquisition rate
:param photon_energy: (Float) Photong energy
:param beam_energy: (Float) Beam Energy
- :param only_dark: (Bool) Indicating a retrieval for dark
- constants only from db
+ :param only_dark: (Bool) Indicating a retrieval for dark constants only from db
+ :param gain_mode: Operation mode of the detector (default to adaptive gain)
:return: const_dict: (Dict) An assembeld dictionary that can be used
to retrieve the required constants
"""
darkcond = [
- "Dark", {
+ "Dark",
+ {
"memory_cells": memory_cells,
"bias_voltage": bias_voltage,
"acquisition_rate": acquisition_rate,
"gain_setting": gain_setting,
+ "gain_mode": gain_mode,
"pixels_x": 512,
- "pixels_y": 128, }
+ "pixels_y": 128,
+ },
]
const_dict = {
"Offset": ["zeros", (128, 512, memory_cells, 3), darkcond],
@@ -47,28 +59,21 @@ def assemble_constant_dict(corr_bools, pc_bools, memory_cells, bias_voltage,
"BadPixelsDark": ["zeros", (128, 512, memory_cells, 3), darkcond],
}
- if not (corr_bools.get('only_offset') or only_dark):
-
+ if not (corr_bools.get("only_offset") or only_dark):
if any(pc_bools):
- const_dict["BadPixelsPC"] = \
- ["zeros", (memory_cells, 128, 512), darkcond]
- const_dict["SlopesPC"] = \
- ["ones", (128, 512, memory_cells, 10), darkcond]
+ const_dict["BadPixelsPC"] = ["zeros", (memory_cells, 128, 512), darkcond]
+ const_dict["SlopesPC"] = ["ones", (128, 512, memory_cells, 10), darkcond]
if corr_bools.get("xray_corr"):
# Add illuminated conditions
illumcond = [
- "Illuminated", {
- "beam_energy": beam_energy,
- "photon_energy": photon_energy
- }
+ "Illuminated",
+ {"beam_energy": beam_energy, "photon_energy": photon_energy},
]
illumcond[1].update(darkcond[1])
- const_dict["BadPixelsFF"] = ["zeros", (128, 512, memory_cells),
- illumcond]
- const_dict["SlopesFF"] = ["ones", (128, 512, memory_cells, 2),
- illumcond]
+ const_dict["BadPixelsFF"] = ["zeros", (128, 512, memory_cells), illumcond]
+ const_dict["SlopesFF"] = ["ones", (128, 512, memory_cells, 2), illumcond]
return const_dict
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 78386a95e..ed1574138 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -63,8 +63,7 @@
"outputs": [],
"source": [
"# Fill dictionaries comprising bools and arguments for correction and data analysis\n",
- "\n",
- "# Here the herarichy and dependability for correction booleans are defined \n",
+ "# Here the hierarichy and dependencies for correction booleans are defined \n",
"corr_bools = {}\n",
"\n",
"# offset is at the bottom of AGIPD correction pyramid.\n",
@@ -91,7 +90,7 @@
"import numpy as np\n",
"\n",
"matplotlib.use(\"agg\")\n",
- "import multiprocessing as mp\n",
+ "import multiprocessing\n",
"from datetime import timedelta\n",
"from pathlib import Path\n",
"\n",
@@ -139,9 +138,7 @@
"\n",
"warnings.filterwarnings('ignore')\n",
"\n",
- "from cal_tools.agipdlib import SnowResolution\n",
- "\n",
- "melt_snow = False if corr_bools[\"only_offset\"] else SnowResolution.NONE"
+ "melt_snow = False if corr_bools[\"only_offset\"] else agipdlib.SnowResolution.NONE"
]
},
{
@@ -166,7 +163,11 @@
" print(\"Set gain setting to 0\")\n",
" gain_setting = 0\n",
"\n",
+ "# Evaluate gain mode (operation mode)\n",
+ "gain_mode = agipdlib.get_gain_mode(control_fn, h5path_ctrl)\n",
+ " \n",
"print(f\"Gain setting: {gain_setting}\")\n",
+ "print(f\"Gain mode: {gain_mode.name}\")\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"\n",
"\n",
@@ -215,7 +216,7 @@
" idx: int\n",
") -> Tuple[str, str, float, float, str, dict]:\n",
" \"\"\"\n",
- " Retreive constants for a module.\n",
+ " Retrieve constants for a module.\n",
" \n",
" :return:\n",
" qm: module virtual name i.e. Q1M1.\n",
@@ -245,56 +246,61 @@
" else:\n",
" local_acq_rate = acq_rate\n",
"\n",
- " # avoid creating retireving constant, if requested.\n",
- " if not nodb_with_dark:\n",
- " const_dict = agipdlib.assemble_constant_dict(\n",
- " corr_bools,\n",
- " pc_bools,\n",
- " local_max_cells,\n",
- " bias_voltage,\n",
- " gain_setting,\n",
- " local_acq_rate,\n",
- " photon_energy,\n",
- " beam_energy=None,\n",
- " only_dark=only_dark,\n",
- " )\n",
- "\n",
- " # Retrieve multiple constants through an input dictionary\n",
- " # to return a dict of useful metadata.\n",
- " mdata_dict = dict()\n",
- " mdata_dict['constants'] = dict()\n",
- " mdata_dict['physical-detector-unit'] = None # initialization\n",
- "\n",
- " for cname, cval in const_dict.items():\n",
- " # saving metadata in a dict\n",
- " mdata_dict['constants'][cname] = dict()\n",
- "\n",
- " if slopes_ff_from_files and cname in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = \"00:00:00\"\n",
+ " # avoid retrieving constant, if requested.\n",
+ " if nodb_with_dark:\n",
+ " return\n",
+ " \n",
+ " const_dict = agipdlib.assemble_constant_dict(\n",
+ " corr_bools,\n",
+ " pc_bools,\n",
+ " local_max_cells,\n",
+ " bias_voltage,\n",
+ " gain_setting,\n",
+ " local_acq_rate,\n",
+ " photon_energy,\n",
+ " gain_mode=gain_mode,\n",
+ " beam_energy=None,\n",
+ " only_dark=only_dark,\n",
+ " )\n",
+ "\n",
+ " # Retrieve multiple constants through an input dictionary\n",
+ " # to return a dict of useful metadata.\n",
+ " mdata_dict = dict()\n",
+ " mdata_dict['constants'] = dict()\n",
+ " mdata_dict['physical-detector-unit'] = None # initialization\n",
+ "\n",
+ " for cname, cval in const_dict.items():\n",
+ " print(cname)\n",
+ " print(cval)\n",
+ " # saving metadata in a dict\n",
+ " mdata_dict['constants'][cname] = dict()\n",
+ "\n",
+ " if slopes_ff_from_files and cname in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = \"00:00:00\"\n",
+ " else:\n",
+ " condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
+ " co, mdata = tools.get_from_db(\n",
+ " karabo_id,\n",
+ " karabo_da,\n",
+ " getattr(Constants.AGIPD, cname)(),\n",
+ " condition,\n",
+ " getattr(np, cval[0])(cval[1]),\n",
+ " cal_db_interface,\n",
+ " creation_time,\n",
+ " meta_only=True,\n",
+ " verbosity=1,\n",
+ " )\n",
+ " mdata_const = mdata.calibration_constant_version\n",
+ " # check if constant was sucessfully retrieved.\n",
+ " if mdata.comm_db_success:\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
+ " f\"{mdata_const.filename}\"\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
+ " mdata_dict['physical-detector-unit'] = mdata_const.device_name\n",
" else:\n",
- " condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
- " co, mdata = tools.get_from_db(\n",
- " karabo_id,\n",
- " karabo_da,\n",
- " getattr(Constants.AGIPD, cname)(),\n",
- " condition,\n",
- " getattr(np, cval[0])(cval[1]),\n",
- " cal_db_interface,\n",
- " creation_time,\n",
- " meta_only=True,\n",
- " verbosity=1,\n",
- " )\n",
- " mdata_const = mdata.calibration_constant_version\n",
- " # check if constant was sucessfully retrieved.\n",
- " if mdata.comm_db_success:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
- " f\"{mdata_const.filename}\"\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
- " mdata_dict['physical-detector-unit'] = mdata_const.device_name\n",
- " else:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = const_dict[cname][:2]\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = None\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = const_dict[cname][:2]\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = None\n",
"\n",
" return qm, mdata_dict, karabo_da, acq_rate, local_max_cells, err"
]
@@ -321,7 +327,7 @@
"only_dark = False\n",
"nodb_with_dark = False\n",
"if not nodb:\n",
- " only_dark=(calfile != \"\")\n",
+ " only_dark = (calfile != \"\")\n",
"if calfile != \"\" and not corr_bools[\"only_offset\"]:\n",
" nodb_with_dark = nodb\n",
"\n",
@@ -347,7 +353,7 @@
"outputs": [],
"source": [
"with multiprocessing.Pool(processes=nmods) as pool:\n",
- " results = pool.starmap(p, inp)"
+ " results = pool.starmap(retrieve_constants, inp)"
]
},
{
@@ -366,16 +372,21 @@
" print(f\"Error for module {qm}: {err}\")\n",
" mdata_dict[karabo_da] = md_dict\n",
"# check if it is requested not to retrieve any constants from the database\n",
- "if not nodb_with_dark:\n",
+ "if nodb_with_dark:\n",
+ " print(\"No constants were retrieved as calibrated files will be used.\")\n",
+ "else:\n",
" metadata.update({\"retrieved-constants\": mdata_dict})\n",
"\n",
- " print(\"\\nRetrieved constants for modules: \",\n",
- " f\"{[', '.join([f'Q{x//4+1}M{x%4+1}' for x in modules])]}\")\n",
- " print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Memory cells: {max_cells}\\n\"\n",
- " f\"• Acquisition rate: {acq_rate}\\n• Gain setting: {gain_setting}\\n• Photon Energy: {photon_energy}\\n\")\n",
- " print(\"Constant metadata is saved under \\\"retrieved-constants\\\" in calibration_metadata.yml\\n\")\n",
- "else:\n",
- " print(\"No constants were retrieved as calibrated files will be used.\")"
+ " print(\"\\nRetrieved constants for modules:\",\n",
+ " ', '.join([tools.module_index_to_qm(x) for x in modules]))\n",
+ " print(f\"Operating conditions are:\")\n",
+ " print(f\"• Bias voltage: {bias_voltage}\")\n",
+ " print(f\"• Memory cells: {max_cells}\\n\")\n",
+ " print(f\"• Acquisition rate: {acq_rate}\")\n",
+ " print(f\"• Gain mode: {gain_mode.name}\")\n",
+ " print(f\"• Gain setting: {gain_setting}\")\n",
+ " print(f\"• Photon Energy: {photon_energy}\")\n",
+ " print(\"Constant metadata is saved under \\\"retrieved-constants\\\" in calibration_metadata.yml\\n\")"
]
},
{
--
GitLab