From 4be9018a2049700d6c9dd7b9232e7dfa2026cdce Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Wed, 24 Feb 2021 12:19:21 +0100
Subject: [PATCH 01/72] Start refactoring of darks notebook
Applying isort and nbstripout. Changing from ipyparallel to multiprocessing - and while doing so,
simplifying the call signature of characterize_module.
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 271 +++++++-----------
1 file changed, 96 insertions(+), 175 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 18d532b16..8e76bafce 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -16,22 +16,16 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-20T12:42:51.255184Z",
- "start_time": "2019-02-20T12:42:51.225500Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
- "cluster_profile = \"noDB\" # The ipcluster profile to use\n",
- "in_folder = \"/gpfs/exfel/d/raw/DETLAB/202031/p900172/\" # path to input data, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/miniHalfAGIPD\" # path to output to, required\n",
+ "in_folder = \"/gpfs/exfel/d/raw/CALLAB/202031/p900113\" # path to input data, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/hammerd/agipd-fixed-gain\" # path to output to, required\n",
"sequences = [0] # sequence files to evaluate.\n",
"modules = [-1] # list of modules to evaluate, RANGE ALLOWED\n",
- "run_high = 84 # run number in which high gain data was recorded, required\n",
- "run_med = 87 # run number in which medium gain data was recorded, required\n",
- "run_low = 88 # run number in which low gain data was recorded, required\n",
+ "run_high = 9985 # run number in which high gain data was recorded, required\n",
+ "run_med = 9984 # run number in which medium gain data was recorded, required\n",
+ "run_low = 9983 # run number in which low gain data was recorded, required\n",
"operation_mode = 'ADAPTIVE_GAIN' # Detector operation mode, optional\n",
"\n",
"karabo_id = \"HED_DET_AGIPD500K2G\" # karabo karabo_id\n",
@@ -40,9 +34,9 @@
"path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # the template to use to access data\n",
"h5path = '/INSTRUMENT/{}/DET/{}:xtdf/image' # path in the HDF5 file to images\n",
"h5path_idx = '/INDEX/{}/DET/{}:xtdf/image' # path in the HDF5 file to images\n",
- "h5path_ctrl = '/CONTROL/{}/MDL/FPGA_COMP_TEST' # path to control information\n",
- "karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for control device '\n",
- "karabo_da_control = \"AGIPD1MCTRL00\" # karabo DA for control infromation\n",
+ "h5path_ctrl = '/CONTROL/{}/MDL/FPGA_COMP' # path to control information\n",
+ "karabo_id_control = \"HED_EXP_AGIPD500K2G\" # karabo-id for control device '\n",
+ "karabo_da_control = \"AGIPD500K2G00\" # karabo DA for control infromation\n",
"\n",
"use_dir_creation_date = True # use dir creation date as data production reference date\n",
"cal_db_interface = \"tcp://max-exfl016:8020\" # the database interface to use\n",
@@ -77,52 +71,55 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-20T12:42:52.599660Z",
- "start_time": "2019-02-20T12:42:51.472138Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
- "# imports and things that do not usually need to be changed\n",
+ "import warnings\n",
"from datetime import datetime\n",
+ "\n",
"import dateutil.parser\n",
- "import warnings\n",
+ "\n",
"warnings.filterwarnings('ignore')\n",
- "from collections import OrderedDict\n",
+ "import copy\n",
"import os\n",
- "from typing import Tuple, List\n",
+ "from collections import OrderedDict\n",
+ "from functools import partial\n",
+ "from typing import List, Tuple\n",
+ "\n",
"import h5py\n",
- "import numpy as np\n",
"import matplotlib\n",
+ "import numpy as np\n",
"import tabulate\n",
+ "from cal_tools.agipdlib import get_acq_rate, get_num_cells\n",
+ "from cal_tools.enums import BadPixels\n",
"\n",
"matplotlib.use('agg')\n",
"import matplotlib.pyplot as plt\n",
- "from IPython.display import display, Markdown, Latex\n",
+ "from IPython.display import Latex, Markdown, display\n",
+ "\n",
"%matplotlib inline\n",
"\n",
+ "import multiprocessing\n",
+ "\n",
+ "from cal_tools.agipdlib import get_bias_voltage, get_gain_setting\n",
+ "from cal_tools.enums import BadPixels\n",
+ "from cal_tools.plotting import (create_constant_overview, plot_badpix_3d,\n",
+ " show_overview, show_processed_modules)\n",
"from cal_tools.tools import (get_dir_creation_date, get_from_db,\n",
" get_notebook_name, get_pdu_from_db,\n",
" get_random_db_interface, get_report,\n",
" map_gain_stages, parse_runs,\n",
" run_prop_seq_from_path, save_const_to_h5,\n",
" send_to_db)\n",
- "from cal_tools.enums import BadPixels\n",
- "from cal_tools.plotting import (create_constant_overview,\n",
- " plot_badpix_3d, show_processed_modules,\n",
- " show_overview)\n",
- "from cal_tools.agipdlib import get_bias_voltage, get_gain_setting\n",
- "\n",
- "# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
- "from ipyparallel import Client\n",
- "\n",
- "view = Client(profile=cluster_profile)[:]\n",
- "view.use_dill()\n",
- "\n",
- "from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
- "\n",
+ "from iCalibrationDB import Conditions, Constants, Detectors, Versions"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
"gains = np.arange(3)\n",
"\n",
"IL_MODE = interlaced\n",
@@ -203,12 +200,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-20T12:42:52.608214Z",
- "start_time": "2019-02-20T12:42:52.601257Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"if karabo_da[0] == '-1':\n",
@@ -249,12 +241,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-20T12:42:54.024731Z",
- "start_time": "2019-02-20T12:42:53.901555Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"# set everything up filewise\n",
@@ -276,71 +263,50 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-20T10:50:55.839958Z",
- "start_time": "2019-02-20T10:50:55.468134Z"
- },
- "scrolled": true
- },
+ "metadata": {},
"outputs": [],
"source": [
- "import copy\n",
- "from functools import partial\n",
- "def characterize_module(il_mode: bool,\n",
- " cells: int,\n",
- " bp_thresh: Tuple[List[int], float, List[int], float], \n",
- " rawversion: int,\n",
- " loc: str, \n",
- " acq_rate: float,\n",
- " h5path: str,\n",
- " h5path_idx: str,\n",
- " control_names: List[str],\n",
- " karabo_id_control: str,\n",
- " inp: Tuple[str, int, int]) -> Tuple[np.array, np.array, np.array, np.array, int, np.array, int, float]:\n",
- " import numpy as np\n",
- " import copy\n",
- " import h5py\n",
- " from cal_tools.enums import BadPixels\n",
- " from cal_tools.agipdlib import get_num_cells, get_acq_rate\n",
- "\n",
+ "def characterize_module(inp: Tuple[str, int, int]) -> Tuple[np.array, np.array, np.array, np.array, int, np.array, int, float]:\n",
" fast_data_filename, channel, gg = inp\n",
- " \n",
- " if cells == 0:\n",
- " cells = get_num_cells(fast_data_filename, loc, channel)\n",
"\n",
- " print(f\"Using {cells} memory cells\")\n",
- " \n",
+ " if max_cells == 0:\n",
+ " num_cells = get_num_cells(fast_data_filename, karabo_id, channel)\n",
+ " else:\n",
+ " num_cells = max_cells\n",
+ "\n",
+ " print(f\"Using {num_cells} memory cells\")\n",
+ "\n",
" if acq_rate == 0.:\n",
" slow_paths = control_names[gg], karabo_id_control\n",
- " fast_paths = fast_data_filename, loc, channel\n",
- " acq_rate = get_acq_rate(fast_paths, slow_paths)\n",
+ " fast_paths = fast_data_filename, karabo_id, channel\n",
+ " local_acq_rate = get_acq_rate(fast_paths, slow_paths)\n",
+ " else:\n",
+ " local_acq_rate = acq_rate\n",
+ "\n",
+ " local_thresholds_offset_hard = thresholds_offset_hard[gg]\n",
+ " local_thresholds_noise_hard = thresholds_noise_hard[gg]\n",
+ "\n",
+ " h5path_f = h5path.format(channel)\n",
+ " h5path_idx_f = h5path_idx.format(channel)\n",
"\n",
- " thresholds_offset, thresholds_offset_sigma, thresholds_noise, thresholds_noise_sigma = bp_thresh \n",
- " thresholds_offset_hard = thresholds_offset[gg]\n",
- " thresholds_noise_hard = thresholds_noise[gg]\n",
- " \n",
- " h5path = h5path.format(channel)\n",
- " h5path_idx = h5path_idx.format(channel)\n",
- " \n",
" with h5py.File(fast_data_filename, \"r\", driver=\"core\") as infile:\n",
" if rawversion == 2:\n",
- " count = np.squeeze(infile[f\"{h5path_idx}/count\"])\n",
- " first = np.squeeze(infile[f\"{h5path_idx}/first\"])\n",
+ " count = np.squeeze(infile[f\"{h5path_idx_f}/count\"])\n",
+ " first = np.squeeze(infile[f\"{h5path_idx_f}/first\"])\n",
" last_index = int(first[count != 0][-1]+count[count != 0][-1])\n",
" first_index = int(first[count != 0][0])\n",
" else:\n",
- " status = np.squeeze(infile[f\"{h5path_idx}/status\"])\n",
+ " status = np.squeeze(infile[f\"{h5path_idx_f}/status\"])\n",
" if np.count_nonzero(status != 0) == 0:\n",
" return\n",
- " last = np.squeeze(infile[f\"{h5path_idx}/last\"])\n",
- " first = np.squeeze(infile[f\"{h5path_idx}/first\"])\n",
+ " last = np.squeeze(infile[f\"{h5path_idx_f}/last\"])\n",
+ " first = np.squeeze(infile[f\"{h5path_idx_f}/first\"])\n",
" last_index = int(last[status != 0][-1]) + 1\n",
" first_index = int(first[status != 0][0])\n",
- " im = np.array(infile[f\"{h5path}/data\"][first_index:last_index,...]) \n",
- " cellIds = np.squeeze(infile[f\"{h5path}/cellId\"][first_index:last_index,...]) \n",
+ " im = np.array(infile[f\"{h5path_f}/data\"][first_index:last_index,...])\n",
+ " cellIds = np.squeeze(infile[f\"{h5path_f}/cellId\"][first_index:last_index,...])\n",
"\n",
- " if il_mode:\n",
+ " if IL_MODE:\n",
" ga = im[1::2, 0, ...]\n",
" im = im[0::2, 0, ...].astype(np.float32)\n",
" cellIds = cellIds[::2]\n",
@@ -354,13 +320,12 @@
" ga = np.rollaxis(ga, 2)\n",
" ga = np.rollaxis(ga, 2, 1)\n",
"\n",
- " mcells = cells #max(cells, np.max(cellIds)+1)\n",
- " offset = np.zeros((im.shape[0], im.shape[1], mcells))\n",
- " gains = np.zeros((im.shape[0], im.shape[1], mcells))\n",
- " noise = np.zeros((im.shape[0], im.shape[1], mcells))\n",
- " gains_std = np.zeros((im.shape[0], im.shape[1], mcells))\n",
- " \n",
- " for cc in np.unique(cellIds[cellIds < mcells]):\n",
+ " offset = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
+ " gains = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
+ " noise = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
+ " gains_std = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
+ "\n",
+ " for cc in np.unique(cellIds[cellIds < num_cells]):\n",
" cellidx = cellIds == cc\n",
" offset[...,cc] = np.median(im[..., cellidx], axis=2)\n",
" noise[...,cc] = np.std(im[..., cellidx], axis=2)\n",
@@ -375,19 +340,19 @@
"\n",
" bp[(offset < offset_mn-thresholds_offset_sigma*offset_std) |\n",
" (offset > offset_mn+thresholds_offset_sigma*offset_std)] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
- " bp[(offset < thresholds_offset_hard[0]) | (\n",
- " offset > thresholds_offset_hard[1])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
+ " bp[(offset < local_thresholds_offset_hard[0]) | (\n",
+ " offset > local_thresholds_offset_hard[1])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
" bp[~np.isfinite(offset)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
"\n",
" # noise related bad pixels\n",
" noise_mn = np.nanmedian(noise, axis=(0,1))\n",
- " noise_std = np.nanstd(noise, axis=(0,1)) \n",
+ " noise_std = np.nanstd(noise, axis=(0,1))\n",
" bp[(noise < noise_mn-thresholds_noise_sigma*noise_std) |\n",
" (noise > noise_mn+thresholds_noise_sigma*noise_std)] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
- " bp[(noise < thresholds_noise_hard[0]) | (noise > thresholds_noise_hard[1])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
+ " bp[(noise < local_thresholds_noise_hard[0]) | (noise > local_thresholds_noise_hard[1])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
" bp[~np.isfinite(noise)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
"\n",
- " return offset, noise, gains, gains_std, gg, bp, cells, acq_rate\n",
+ " return offset, noise, gains, gains_std, gg, bp, num_cells, local_acq_rate\n",
"\n",
"offset_g = OrderedDict()\n",
"noise_g = OrderedDict()\n",
@@ -410,7 +375,7 @@
"else:\n",
" thresholds_noise_hard = [thresholds_noise_hard] * 3\n",
"\n",
- " \n",
+ "\n",
"inp = []\n",
"for gain, mapped_files in gain_mapped_files.items():\n",
" dones = []\n",
@@ -423,18 +388,11 @@
" else:\n",
" continue\n",
" inp.append((fname_in, i, gg))\n",
- " \n",
- " gg += 1\n",
"\n",
- "p = partial(characterize_module, IL_MODE, max_cells,\n",
- " (thresholds_offset_hard, thresholds_offset_sigma,\n",
- " thresholds_noise_hard, thresholds_noise_sigma),\n",
- " rawversion, karabo_id, acq_rate, h5path, h5path_idx,\n",
- " control_names, karabo_id_control)\n",
+ " gg += 1\n",
"\n",
- "# Don't remove. Used for Debugging.\n",
- "#results = list(map(p, inp))\n",
- "results = view.map_sync(p, inp)\n",
+ "with multiprocessing.Pool() as pool:\n",
+ " results = pool.map(p, inp)\n",
"\n",
"for ii, r in enumerate(results):\n",
" offset, noise, gains, gains_std, gg, bp, thiscell, thisacq = r\n",
@@ -454,7 +412,7 @@
" gain_g[qm][...,gg] = gains\n",
" gainstd_g[qm][..., gg] = gains_std\n",
" badpix_g[qm][...,gg] = bp\n",
- " \n",
+ "\n",
"\n",
"duration = (datetime.now() - start).total_seconds()\n",
"\n",
@@ -488,12 +446,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2018-12-06T09:38:18.220833Z",
- "start_time": "2018-12-06T09:38:17.926616Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"thresholds_g = {}\n",
@@ -508,12 +461,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2018-12-06T09:38:18.234582Z",
- "start_time": "2018-12-06T09:38:18.222838Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"res = OrderedDict()\n",
@@ -557,9 +505,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Retrieve existing constants for comparison\n",
@@ -614,12 +560,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2018-12-06T09:49:32.449330Z",
- "start_time": "2018-12-06T09:49:20.231607Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"md = None\n",
@@ -655,9 +596,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"mnames=[]\n",
@@ -687,13 +626,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2018-12-06T09:49:14.540552Z",
- "start_time": "2018-12-06T09:49:13.009674Z"
- },
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"cell = 3\n",
@@ -711,9 +644,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"cell = 3\n",
@@ -731,9 +662,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"cell = 3\n",
@@ -744,9 +673,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"cols = {BadPixels.NOISE_OUT_OF_THRESHOLD.value: (BadPixels.NOISE_OUT_OF_THRESHOLD.name, '#FF000080'),\n",
@@ -790,9 +717,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"create_constant_overview(offset_g, \"Offset (ADU)\", max_cells, 4000, 8000,\n",
@@ -802,9 +727,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"create_constant_overview(noise_g, \"Noise (ADU)\", max_cells, 0, 100,\n",
@@ -814,9 +737,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Plot only three gain threshold maps.\n",
@@ -997,5 +918,5 @@
}
},
"nbformat": 4,
- "nbformat_minor": 1
+ "nbformat_minor": 4
}
--
GitLab
From 128ac0a9fec2654216211be42cb0bf5e6bd8a153 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Wed, 24 Feb 2021 14:48:30 +0100
Subject: [PATCH 02/72] Cleaning up imports
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 25 ++++++-------------
1 file changed, 7 insertions(+), 18 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 8e76bafce..f1b0759d2 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -80,17 +80,14 @@
"import dateutil.parser\n",
"\n",
"warnings.filterwarnings('ignore')\n",
- "import copy\n",
"import os\n",
"from collections import OrderedDict\n",
- "from functools import partial\n",
"from typing import List, Tuple\n",
"\n",
"import h5py\n",
"import matplotlib\n",
"import numpy as np\n",
"import tabulate\n",
- "from cal_tools.agipdlib import get_acq_rate, get_num_cells\n",
"from cal_tools.enums import BadPixels\n",
"\n",
"matplotlib.use('agg')\n",
@@ -101,17 +98,16 @@
"\n",
"import multiprocessing\n",
"\n",
- "from cal_tools.agipdlib import get_bias_voltage, get_gain_setting\n",
- "from cal_tools.enums import BadPixels\n",
+ "from cal_tools.agipdlib import (get_acq_rate, get_bias_voltage,\n",
+ " get_gain_setting, get_num_cells)\n",
"from cal_tools.plotting import (create_constant_overview, plot_badpix_3d,\n",
" show_overview, show_processed_modules)\n",
"from cal_tools.tools import (get_dir_creation_date, get_from_db,\n",
- " get_notebook_name, get_pdu_from_db,\n",
- " get_random_db_interface, get_report,\n",
- " map_gain_stages, parse_runs,\n",
+ " get_pdu_from_db, get_random_db_interface,\n",
+ " get_report, map_gain_stages,\n",
" run_prop_seq_from_path, save_const_to_h5,\n",
" send_to_db)\n",
- "from iCalibrationDB import Conditions, Constants, Detectors, Versions"
+ "from iCalibrationDB import Conditions, Constants, Detectors"
]
},
{
@@ -132,7 +128,7 @@
"\n",
"creation_time=None\n",
"if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run_high)\n",
+ " creation_time = get_dir_creation_date(in_folder, run_high, verbosity=2)\n",
"\n",
"print(f\"Using {creation_time} as creation time of constant.\")\n",
"\n",
@@ -392,7 +388,7 @@
" gg += 1\n",
"\n",
"with multiprocessing.Pool() as pool:\n",
- " results = pool.map(p, inp)\n",
+ " results = pool.map(characterize_module, inp)\n",
"\n",
"for ii, r in enumerate(results):\n",
" offset, noise, gains, gains_std, gg, bp, thiscell, thisacq = r\n",
@@ -889,13 +885,6 @@
" display(Markdown('### {} [ADU], good pixels only ###'.format(const)))\n",
" md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=header))) "
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
--
GitLab
From cf2ff561dff3dcc73d2d4eb4e2ad54dba1ff0ebf Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 25 Feb 2021 13:57:22 +0100
Subject: [PATCH 03/72] Additional refactoring
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 72 +++++++++++--------
1 file changed, 41 insertions(+), 31 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index f1b0759d2..c9e207778 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -118,7 +118,6 @@
"source": [
"gains = np.arange(3)\n",
"\n",
- "IL_MODE = interlaced\n",
"max_cells = mem_cells\n",
" \n",
"offset_runs = OrderedDict()\n",
@@ -128,7 +127,7 @@
"\n",
"creation_time=None\n",
"if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run_high, verbosity=2)\n",
+ " creation_time = get_dir_creation_date(in_folder, run_high)\n",
"\n",
"print(f\"Using {creation_time} as creation time of constant.\")\n",
"\n",
@@ -166,8 +165,8 @@
"source": [
"runs = [run_high, run_med, run_low]\n",
"\n",
- "if \"{\" in h5path_ctrl:\n",
- " h5path_ctrl = h5path_ctrl.format(karabo_id_control)\n",
+ "# insert control device if format string (does nothing otherwise)\n",
+ "h5path_ctrl = h5path_ctrl.format(karabo_id_control)\n",
"\n",
"if gain_setting == 0.1:\n",
" if creation_time.replace(tzinfo=None) < dateutil.parser.parse('2020-01-31'):\n",
@@ -219,7 +218,7 @@
"print(f\"Memory cells: {mem_cells}/{max_cells}\")\n",
"print(\"Runs: {}\".format([ v for v in offset_runs.values()]))\n",
"print(f\"Sequences: {sequences}\")\n",
- "print(f\"Interlaced mode: {IL_MODE}\")\n",
+ "print(f\"Interlaced mode: {interlaced}\")\n",
"print(f\"Using DB: {db_output}\")\n",
"print(f\"Input: {in_folder}\")\n",
"print(f\"Output: {out_folder}\")\n",
@@ -247,6 +246,16 @@
"print(f\"Will process a total of {total_sequences} files.\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def index_to_qm(i: int):\n",
+ " return f\"Q{i//4+1}M{i%4+1}\""
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -262,9 +271,24 @@
"metadata": {},
"outputs": [],
"source": [
- "def characterize_module(inp: Tuple[str, int, int]) -> Tuple[np.array, np.array, np.array, np.array, int, np.array, int, float]:\n",
- " fast_data_filename, channel, gg = inp\n",
+ "if thresholds_offset_hard == [0, 0]:\n",
+ " thresholds_offset_hard = [thresholds_offset_hard_hg, thresholds_offset_hard_mg, thresholds_offset_hard_lg]\n",
+ "else:\n",
+ " thresholds_offset_hard = [thresholds_offset_hard] * 3\n",
"\n",
+ "if thresholds_noise_hard == [0, 0]:\n",
+ " thresholds_noise_hard = [thresholds_noise_hard_hg, thresholds_noise_hard_mg, thresholds_noise_hard_lg]\n",
+ "else:\n",
+ " thresholds_noise_hard = [thresholds_noise_hard] * 3"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def characterize_module(fast_data_filename: str, channel: int, gg: int) -> Tuple[np.array, np.array, np.array, np.array, int, np.array, int, float]:\n",
" if max_cells == 0:\n",
" num_cells = get_num_cells(fast_data_filename, karabo_id, channel)\n",
" else:\n",
@@ -302,7 +326,7 @@
" im = np.array(infile[f\"{h5path_f}/data\"][first_index:last_index,...])\n",
" cellIds = np.squeeze(infile[f\"{h5path_f}/cellId\"][first_index:last_index,...])\n",
"\n",
- " if IL_MODE:\n",
+ " if interlaced:\n",
" ga = im[1::2, 0, ...]\n",
" im = im[0::2, 0, ...].astype(np.float32)\n",
" cellIds = cellIds[::2]\n",
@@ -350,33 +374,22 @@
"\n",
" return offset, noise, gains, gains_std, gg, bp, num_cells, local_acq_rate\n",
"\n",
+ "\n",
"offset_g = OrderedDict()\n",
"noise_g = OrderedDict()\n",
"gain_g = OrderedDict()\n",
"gainstd_g = OrderedDict()\n",
"badpix_g = OrderedDict()\n",
- "gg = 0\n",
"\n",
"start = datetime.now()\n",
"all_cells = []\n",
"all_acq_rate = []\n",
"\n",
- "if thresholds_offset_hard == [0, 0]:\n",
- " thresholds_offset_hard = [thresholds_offset_hard_hg, thresholds_offset_hard_mg, thresholds_offset_hard_lg]\n",
- "else:\n",
- " thresholds_offset_hard = [thresholds_offset_hard] * 3\n",
- "\n",
- "if thresholds_noise_hard == [0, 0]:\n",
- " thresholds_noise_hard = [thresholds_noise_hard_hg, thresholds_noise_hard_mg, thresholds_noise_hard_lg]\n",
- "else:\n",
- " thresholds_noise_hard = [thresholds_noise_hard] * 3\n",
- "\n",
- "\n",
"inp = []\n",
- "for gain, mapped_files in gain_mapped_files.items():\n",
+ "for gg, (gain, mapped_files) in enumerate(gain_mapped_files.items()):\n",
" dones = []\n",
" for i in modules:\n",
- " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ " qm = index_to_qm(i)\n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
" fname_in = mapped_files[qm].get()\n",
" print(\"Process file: \", fname_in)\n",
@@ -385,17 +398,14 @@
" continue\n",
" inp.append((fname_in, i, gg))\n",
"\n",
- " gg += 1\n",
- "\n",
"with multiprocessing.Pool() as pool:\n",
- " results = pool.map(characterize_module, inp)\n",
+ " results = pool.starmap(characterize_module, inp)\n",
"\n",
- "for ii, r in enumerate(results):\n",
- " offset, noise, gains, gains_std, gg, bp, thiscell, thisacq = r\n",
+ "for offset, noise, gains, gains_std, gg, bp, thiscell, thisacq in results:\n",
" all_cells.append(thiscell)\n",
" all_acq_rate.append(thisacq)\n",
" for i in modules:\n",
- " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ " qm = index_to_qm(i)\n",
" if qm not in offset_g:\n",
" offset_g[qm] = np.zeros((offset.shape[0], offset.shape[1], offset.shape[2], 3))\n",
" noise_g[qm] = np.zeros_like(offset_g[qm])\n",
@@ -462,7 +472,7 @@
"source": [
"res = OrderedDict()\n",
"for i in modules:\n",
- " qm = \"Q{}M{}\".format(i//4+1, i%4+1)\n",
+ " qm = index_to_qm(i)\n",
" res[qm] = {'Offset': offset_g[qm],\n",
" 'Noise': noise_g[qm],\n",
" 'ThresholdsDark': thresholds_g[qm],\n",
@@ -493,7 +503,7 @@
"# Create the modules dict of karabo_das and PDUs\n",
"qm_dict = OrderedDict()\n",
"for i, k_da in zip(modules, karabo_da):\n",
- " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ " qm = index_to_qm(i)\n",
" qm_dict[qm] = {\"karabo_da\": k_da,\n",
" \"db_module\": \"\"}"
]
@@ -597,7 +607,7 @@
"source": [
"mnames=[]\n",
"for i in modules:\n",
- " qm = f\"Q{i//4+1}M{i % 4+1}\"\n",
+ " qm = index_to_qm(i)\n",
" mnames.append(qm)\n",
" display(Markdown(f'## Position of the module {qm} and its ASICs##'))\n",
"show_processed_modules(dinstance, constants=None, mnames=mnames, mode=\"position\")"
--
GitLab
From 4b7035338f9c5e990f1c96c6432f344b32d4a9ac Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 25 Feb 2021 16:18:28 +0100
Subject: [PATCH 04/72] Adding enum encoding AGIPD gain mode
---
cal_tools/cal_tools/enums.py | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/cal_tools/cal_tools/enums.py b/cal_tools/cal_tools/enums.py
index a516dee2f..27ce86511 100644
--- a/cal_tools/cal_tools/enums.py
+++ b/cal_tools/cal_tools/enums.py
@@ -49,3 +49,11 @@ class SnowResolution(Enum):
NONE = "none"
INTERPOLATE = "interpolate"
+
+class AgipdGainMode(Enum):
+ """Encoding added to distinguish between adaptive and fixed gain"""
+
+ ADAPTIVE_GAIN = 0 # adaptive is default (if gain mode missing in slow data)
+ FIXED_HIGH_GAIN = 1 # non-zero means fixed gain
+ FIXED_MEDIUM_GAIN = 2
+ FIXED_LOW_GAIN = 3
--
GitLab
From 59db0511ddfa898da06747f9d8c2df1f2268b076 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Fri, 26 Feb 2021 17:19:59 +0100
Subject: [PATCH 05/72] Adding prototype gain mode detection from slow data
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 34 +++++++++++++++----
1 file changed, 28 insertions(+), 6 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index c9e207778..d4b913f10 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -26,7 +26,6 @@
"run_high = 9985 # run number in which high gain data was recorded, required\n",
"run_med = 9984 # run number in which medium gain data was recorded, required\n",
"run_low = 9983 # run number in which low gain data was recorded, required\n",
- "operation_mode = 'ADAPTIVE_GAIN' # Detector operation mode, optional\n",
"\n",
"karabo_id = \"HED_DET_AGIPD500K2G\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
@@ -100,6 +99,7 @@
"\n",
"from cal_tools.agipdlib import (get_acq_rate, get_bias_voltage,\n",
" get_gain_setting, get_num_cells)\n",
+ "from cal_tools.enums import AgipdGainMode\n",
"from cal_tools.plotting import (create_constant_overview, plot_badpix_3d,\n",
" show_overview, show_processed_modules)\n",
"from cal_tools.tools import (get_dir_creation_date, get_from_db,\n",
@@ -116,6 +116,23 @@
"metadata": {},
"outputs": [],
"source": [
+ "def get_gain_mode(h5fn):\n",
+ " h5path_gainmode = f'{h5path_ctrl.replace(\"/CONTROL/\", \"/RUN/\", 1)}/gainModeIndex/value'\n",
+ " with h5py.File(h5fn, 'r') as fd:\n",
+ " if h5path_gainmode in fd:\n",
+ " return AgipdGainMode(fd[h5path_gainmode][0])\n",
+ " return AgipdGainMode.ADAPTIVE_GAIN"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# insert control device if format string (does nothing otherwise)\n",
+ "h5path_ctrl = h5path_ctrl.format(karabo_id_control)\n",
+ "\n",
"gains = np.arange(3)\n",
"\n",
"max_cells = mem_cells\n",
@@ -149,12 +166,20 @@
" nmods = 8\n",
"\n",
"control_names = [f'{in_folder}/r{r:04d}/RAW-R{r:04d}-{karabo_da_control}-S00000.h5'\n",
- " for r in (run_high, run_med, run_low)] \n",
+ " for r in (run_high, run_med, run_low)]\n",
+ "\n",
+ "run_gain_modes = [get_gain_mode(fn) for fn in control_names]\n",
+ "if all(gm == AgipdGainMode.ADAPTIVE_GAIN for gm in run_gain_modes):\n",
+ " fixed_gain_mode = False\n",
+ "elif run_gain_modes == [AgipdGainMode.FIXED_HIGH_GAIN, AgipdGainMode.FIXED_MEDIUM_GAIN, AgipdGainMode.FIXED_LOW_GAIN]:\n",
+ " fixed_gain_mode = True\n",
+ "else:\n",
+ " print(f'Something is clearly wrong; slow data indicates gain modes {run_gain_modes}')\n",
"\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"print(f\"Instrument {instrument}\")\n",
"print(f\"Detector instance {dinstance}\")\n",
- "print(f\"Operation mode is {operation_mode}\")"
+ "print(f\"Operation mode is {'fixed' if fixed_gain_mode else 'adaptive'} gain mode\")"
]
},
{
@@ -165,9 +190,6 @@
"source": [
"runs = [run_high, run_med, run_low]\n",
"\n",
- "# insert control device if format string (does nothing otherwise)\n",
- "h5path_ctrl = h5path_ctrl.format(karabo_id_control)\n",
- "\n",
"if gain_setting == 0.1:\n",
" if creation_time.replace(tzinfo=None) < dateutil.parser.parse('2020-01-31'):\n",
" print(\"Set gain-setting to None for runs taken before 2020-01-31\")\n",
--
GitLab
From a74fad831d5bc1f81efd57a97e09e06a91c400fa Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Tue, 2 Mar 2021 09:20:45 +0100
Subject: [PATCH 06/72] Formatting, splitting a cell
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 29 ++++++++++++-------
1 file changed, 19 insertions(+), 10 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index d4b913f10..4da456954 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -394,9 +394,15 @@
" bp[(noise < local_thresholds_noise_hard[0]) | (noise > local_thresholds_noise_hard[1])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
" bp[~np.isfinite(noise)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
"\n",
- " return offset, noise, gains, gains_std, gg, bp, num_cells, local_acq_rate\n",
- "\n",
- "\n",
+ " return offset, noise, gains, gains_std, gg, bp, num_cells, local_acq_rate"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
"offset_g = OrderedDict()\n",
"noise_g = OrderedDict()\n",
"gain_g = OrderedDict()\n",
@@ -495,11 +501,12 @@
"res = OrderedDict()\n",
"for i in modules:\n",
" qm = index_to_qm(i)\n",
- " res[qm] = {'Offset': offset_g[qm],\n",
- " 'Noise': noise_g[qm],\n",
- " 'ThresholdsDark': thresholds_g[qm],\n",
- " 'BadPixelsDark': badpix_g[qm] \n",
- " }"
+ " res[qm] = {\n",
+ " 'Offset': offset_g[qm],\n",
+ " 'Noise': noise_g[qm],\n",
+ " 'ThresholdsDark': thresholds_g[qm],\n",
+ " 'BadPixelsDark': badpix_g[qm]\n",
+ " }"
]
},
{
@@ -526,8 +533,10 @@
"qm_dict = OrderedDict()\n",
"for i, k_da in zip(modules, karabo_da):\n",
" qm = index_to_qm(i)\n",
- " qm_dict[qm] = {\"karabo_da\": k_da,\n",
- " \"db_module\": \"\"}"
+ " qm_dict[qm] = {\n",
+ " \"karabo_da\": k_da,\n",
+ " \"db_module\": \"\"\n",
+ " }"
]
},
{
--
GitLab
From fdd736f4a743de35f33b6196df07819adf18618f Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Tue, 2 Mar 2021 10:02:12 +0100
Subject: [PATCH 07/72] Skipping thresholds, gain separation bp for fixed gain
mode
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 76 +++++++++++--------
1 file changed, 45 insertions(+), 31 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 4da456954..f4f269bc9 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -463,11 +463,12 @@
"outputs": [],
"source": [
"# Add a badpixel due to bad gain separation\n",
- "for g in range(2):\n",
- " # Bad pixels during bad gain separation.\n",
- " # Fraction of pixels in the module with separation lower than \"thresholds_gain_sigma\".\n",
- " bad_sep = (gain_g[qm][..., g+1] - gain_g[qm][..., g]) / np.sqrt(gainstd_g[qm][..., g+1]**2 + gainstd_g[qm][..., g]**2)\n",
- " badpix_g[qm][...,g+1][(bad_sep)<thresholds_gain_sigma]|= BadPixels.GAIN_THRESHOLDING_ERROR.value"
+ "if not fixed_gain_mode:\n",
+ " for g in range(2):\n",
+ " # Bad pixels during bad gain separation.\n",
+ " # Fraction of pixels in the module with separation lower than \"thresholds_gain_sigma\".\n",
+ " bad_sep = (gain_g[qm][..., g+1] - gain_g[qm][..., g]) / np.sqrt(gainstd_g[qm][..., g+1]**2 + gainstd_g[qm][..., g]**2)\n",
+ " badpix_g[qm][...,g+1][(bad_sep)<thresholds_gain_sigma]|= BadPixels.GAIN_THRESHOLDING_ERROR.value"
]
},
{
@@ -483,13 +484,14 @@
"metadata": {},
"outputs": [],
"source": [
- "thresholds_g = {}\n",
- "for qm in gain_g.keys():\n",
- " thresholds_g[qm] = np.zeros((gain_g[qm].shape[0], gain_g[qm].shape[1], gain_g[qm].shape[2], 5))\n",
- " thresholds_g[qm][...,0] = (gain_g[qm][...,1]+gain_g[qm][...,0])/2\n",
- " thresholds_g[qm][...,1] = (gain_g[qm][...,2]+gain_g[qm][...,1])/2\n",
- " for i in range(3):\n",
- " thresholds_g[qm][...,2+i] = gain_g[qm][...,i]"
+ "if not fixed_gain_mode:\n",
+ " thresholds_g = {}\n",
+ " for qm in gain_g.keys():\n",
+ " thresholds_g[qm] = np.zeros((gain_g[qm].shape[0], gain_g[qm].shape[1], gain_g[qm].shape[2], 5))\n",
+ " thresholds_g[qm][...,0] = (gain_g[qm][...,1]+gain_g[qm][...,0])/2\n",
+ " thresholds_g[qm][...,1] = (gain_g[qm][...,2]+gain_g[qm][...,1])/2\n",
+ " for i in range(3):\n",
+ " thresholds_g[qm][...,2+i] = gain_g[qm][...,i]"
]
},
{
@@ -504,9 +506,10 @@
" res[qm] = {\n",
" 'Offset': offset_g[qm],\n",
" 'Noise': noise_g[qm],\n",
- " 'ThresholdsDark': thresholds_g[qm],\n",
" 'BadPixelsDark': badpix_g[qm]\n",
- " }"
+ " }\n",
+ " if not fixed_gain_mode:\n",
+ " res[qm]['ThresholdsDark'] = thresholds_g[qm]"
]
},
{
@@ -546,7 +549,10 @@
"outputs": [],
"source": [
"# Retrieve existing constants for comparison\n",
- "clist = [\"Offset\", \"Noise\", \"ThresholdsDark\", \"BadPixelsDark\"]\n",
+ "if fixed_gain_mode:\n",
+ " clist = [\"Offset\", \"Noise\", \"BadPixelsDark\"]\n",
+ "else:\n",
+ " clist = [\"Offset\", \"Noise\", \"ThresholdsDark\", \"BadPixelsDark\"]\n",
"old_const = {}\n",
"old_mdata = {}\n",
"detinst = getattr(Detectors, dinstance)\n",
@@ -777,19 +783,20 @@
"metadata": {},
"outputs": [],
"source": [
- "# Plot only three gain threshold maps.\n",
- "bp_thresh = OrderedDict()\n",
- "for mod, con in badpix_g.items():\n",
- " bp_thresh[mod] = np.zeros((con.shape[0], con.shape[1], con.shape[2], 5), dtype=con.dtype)\n",
- " bp_thresh[mod][...,:2] = con[...,:2]\n",
- " bp_thresh[mod][...,2:] = con\n",
+ "if not fixed_gain_mode:\n",
+ " # Plot only three gain threshold maps.\n",
+ " bp_thresh = OrderedDict()\n",
+ " for mod, con in badpix_g.items():\n",
+ " bp_thresh[mod] = np.zeros((con.shape[0], con.shape[1], con.shape[2], 5), dtype=con.dtype)\n",
+ " bp_thresh[mod][...,:2] = con[...,:2]\n",
+ " bp_thresh[mod][...,2:] = con\n",
"\n",
"\n",
- "create_constant_overview(thresholds_g, \"Threshold (ADU)\", max_cells, 4000, 10000, 5,\n",
- " badpixels=[bp_thresh, np.nan],\n",
- " gmap=['HG-MG Threshold', 'MG-LG Threshold', 'High gain', 'Medium gain', 'low gain'],\n",
- " marker=['d','d','','','']\n",
- " )"
+ " create_constant_overview(thresholds_g, \"Threshold (ADU)\", max_cells, 4000, 10000, 5,\n",
+ " badpixels=[bp_thresh, np.nan],\n",
+ " gmap=['HG-MG Threshold', 'MG-LG Threshold', 'High gain', 'Medium gain', 'low gain'],\n",
+ " marker=['d','d','','','']\n",
+ " )"
]
},
{
@@ -884,14 +891,21 @@
" \"New constant\", \"Old constant \",\n",
" \"New constant\", \"Old constant \"]\n",
"\n",
- "for const in ['Offset', 'Noise', 'ThresholdsDark']:\n",
- " if const != 'ThresholdsDark':\n",
- " table = [['','High gain', 'High gain', 'Medium gain', 'Medium gain', 'Low gain', 'Low gain']]\n",
- " else:\n",
+ "if fixed_gain_mode:\n",
+ " constants = ['Offset', 'Noise']\n",
+ "else:\n",
+ " ['Offset', 'Noise', 'ThresholdsDark']\n",
+ "\n",
+ "for const in constants:\n",
+ " \n",
+ " if const == 'ThresholdsDark':\n",
" table = [['','HG-MG threshold', 'HG-MG threshold', 'MG-LG threshold', 'MG-LG threshold']]\n",
- " for qm in res.keys():\n",
+ " else: \n",
+ " table = [['','High gain', 'High gain', 'Medium gain', 'Medium gain', 'Low gain', 'Low gain']]\n",
"\n",
+ " for qm in res.keys():\n",
" data = np.copy(res[qm][const])\n",
+ " \n",
" if const == 'ThresholdsDark':\n",
" data[...,0][res[qm]['BadPixelsDark'][...,0]>0] = np.nan\n",
" data[...,1][res[qm]['BadPixelsDark'][...,1]>0] = np.nan\n",
--
GitLab
From 49d3185137ca7aa19c06a5c628e21496ea90685d Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Tue, 2 Mar 2021 10:09:06 +0100
Subject: [PATCH 08/72] Stripping some trailing spaces
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 48 +++++++++----------
1 file changed, 24 insertions(+), 24 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index f4f269bc9..ca2c3ba7b 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -136,7 +136,7 @@
"gains = np.arange(3)\n",
"\n",
"max_cells = mem_cells\n",
- " \n",
+ "\n",
"offset_runs = OrderedDict()\n",
"offset_runs[\"high\"] = run_high\n",
"offset_runs[\"med\"] = run_med\n",
@@ -204,7 +204,7 @@
" gsettings.append(get_gain_setting(control_fname, h5path_ctrl))\n",
" if not all(g == gsettings[0] for g in gsettings):\n",
" raise ValueError(f\"Different gain settings for the 3 input runs {gsettings}\")\n",
- " gain_setting = gsettings[0] \n",
+ " gain_setting = gsettings[0]\n",
" except Exception as e:\n",
" print(f'Error while reading gain setting from: \\n{control_fname}')\n",
" print(f'Error: {e}')\n",
@@ -229,7 +229,7 @@
"h5path = h5path.format(karabo_id, receiver_id)\n",
"h5path_idx = h5path_idx.format(karabo_id, receiver_id)\n",
"\n",
- "if bias_voltage == 0: \n",
+ "if bias_voltage == 0:\n",
" # Read the bias voltage from files, if recorded.\n",
" # If not available, make use of the historical voltage the detector is running at\n",
" bias_voltage = get_bias_voltage(control_names[0], karabo_id_control)\n",
@@ -621,15 +621,15 @@
" acquisition_rate=acq_rate,\n",
" gain_setting=gain_setting)\n",
" if db_output:\n",
- " md = send_to_db(db_module, karabo_id, dconst, condition, file_loc, \n",
+ " md = send_to_db(db_module, karabo_id, dconst, condition, file_loc,\n",
" report, cal_db_interface, creation_time=creation_time,\n",
" timeout=cal_db_timeout)\n",
"\n",
" if local_output:\n",
- " md = save_const_to_h5(db_module, karabo_id, dconst, condition, dconst.data, \n",
+ " md = save_const_to_h5(db_module, karabo_id, dconst, condition, dconst.data,\n",
" file_loc, report, creation_time, out_folder)\n",
" print(f\"Calibration constant {const} is stored locally.\\n\")\n",
- " \n",
+ "\n",
" print(\"Constants parameter conditions are:\\n\")\n",
" print(f\"• memory_cells: {max_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
" f\"• acquisition_rate: {acq_rate}\\n• gain_setting: {gain_setting}\\n\"\n",
@@ -724,18 +724,18 @@
" BadPixels.OFFSET_OUT_OF_THRESHOLD.value: (BadPixels.OFFSET_OUT_OF_THRESHOLD.name, '#00FF0080'),\n",
" BadPixels.GAIN_THRESHOLDING_ERROR.value: (BadPixels.GAIN_THRESHOLDING_ERROR.name, '#FF40FF40'),\n",
" BadPixels.OFFSET_OUT_OF_THRESHOLD.value | BadPixels.NOISE_OUT_OF_THRESHOLD.value: ('OFFSET_OUT_OF_THRESHOLD + NOISE_OUT_OF_THRESHOLD', '#DD00DD80'),\n",
- " BadPixels.OFFSET_OUT_OF_THRESHOLD.value | BadPixels.NOISE_OUT_OF_THRESHOLD.value | \n",
+ " BadPixels.OFFSET_OUT_OF_THRESHOLD.value | BadPixels.NOISE_OUT_OF_THRESHOLD.value |\n",
" BadPixels.GAIN_THRESHOLDING_ERROR.value: ('MIXED', '#BFDF009F')}\n",
"\n",
"if high_res_badpix_3d:\n",
" display(Markdown(\"\"\"\n",
- " \n",
+ "\n",
" ## Global Bad Pixel Behaviour ##\n",
"\n",
- " The following plots show the results of bad pixel evaluation for all evaluated memory cells. \n",
- " Cells are stacked in the Z-dimension, while pixels values in x/y are rebinned with a factor of 2. \n",
- " This excludes single bad pixels present only in disconnected pixels. \n",
- " Hence, any bad pixels spanning at least 4 pixels in the x/y-plane, or across at least two memory cells are indicated. \n",
+ " The following plots show the results of bad pixel evaluation for all evaluated memory cells.\n",
+ " Cells are stacked in the Z-dimension, while pixels values in x/y are rebinned with a factor of 2.\n",
+ " This excludes single bad pixels present only in disconnected pixels.\n",
+ " Hence, any bad pixels spanning at least 4 pixels in the x/y-plane, or across at least two memory cells are indicated.\n",
" Colors encode the bad pixel type, or mixed type.\n",
"\n",
" \"\"\"))\n",
@@ -848,10 +848,10 @@
" for bit in bits:\n",
" l_data_old.append(np.count_nonzero(old_const['BadPixelsDark'][:, :, :, gain] & bit.value))\n",
"\n",
- " l_data_name = ['All bad pixels', 'NOISE_OUT_OF_THRESHOLD', \n",
+ " l_data_name = ['All bad pixels', 'NOISE_OUT_OF_THRESHOLD',\n",
" 'OFFSET_OUT_OF_THRESHOLD', 'OFFSET_NOISE_EVAL_ERROR', 'GAIN_THRESHOLDING_ERROR']\n",
"\n",
- " l_threshold = ['', f'{thresholds_noise_sigma}' f'{thresholds_noise_hard[gain]}', \n",
+ " l_threshold = ['', f'{thresholds_noise_sigma}' f'{thresholds_noise_hard[gain]}',\n",
" f'{thresholds_offset_sigma}' f'{thresholds_offset_hard[gain]}',\n",
" '', f'{thresholds_gain_sigma}']\n",
"\n",
@@ -874,9 +874,9 @@
"\n",
"'''))\n",
"if len(table)>0:\n",
- " md = display(Latex(tabulate.tabulate(table, tablefmt='latex', \n",
- " headers=[\"Pixel type\", \"Threshold\", \n",
- " \"New constant\", \"Old constant\"]))) "
+ " md = display(Latex(tabulate.tabulate(table, tablefmt='latex',\n",
+ " headers=[\"Pixel type\", \"Threshold\",\n",
+ " \"New constant\", \"Old constant\"])))"
]
},
{
@@ -885,9 +885,9 @@
"metadata": {},
"outputs": [],
"source": [
- "header = ['Parameter', \n",
- " \"New constant\", \"Old constant \", \n",
- " \"New constant\", \"Old constant \", \n",
+ "header = ['Parameter',\n",
+ " \"New constant\", \"Old constant \",\n",
+ " \"New constant\", \"Old constant \",\n",
" \"New constant\", \"Old constant \",\n",
" \"New constant\", \"Old constant \"]\n",
"\n",
@@ -897,15 +897,15 @@
" ['Offset', 'Noise', 'ThresholdsDark']\n",
"\n",
"for const in constants:\n",
- " \n",
+ "\n",
" if const == 'ThresholdsDark':\n",
" table = [['','HG-MG threshold', 'HG-MG threshold', 'MG-LG threshold', 'MG-LG threshold']]\n",
- " else: \n",
+ " else:\n",
" table = [['','High gain', 'High gain', 'Medium gain', 'Medium gain', 'Low gain', 'Low gain']]\n",
"\n",
" for qm in res.keys():\n",
" data = np.copy(res[qm][const])\n",
- " \n",
+ "\n",
" if const == 'ThresholdsDark':\n",
" data[...,0][res[qm]['BadPixelsDark'][...,0]>0] = np.nan\n",
" data[...,1][res[qm]['BadPixelsDark'][...,1]>0] = np.nan\n",
@@ -938,7 +938,7 @@
" table.append(line)\n",
"\n",
" display(Markdown('### {} [ADU], good pixels only ###'.format(const)))\n",
- " md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=header))) "
+ " md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=header)))"
]
}
],
--
GitLab
From e4c9b1266c4d61ccbb4b7b64f79d6e4d4e0d8e2e Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Tue, 2 Mar 2021 14:11:51 +0100
Subject: [PATCH 09/72] Use IntEnum for convenience later
---
cal_tools/cal_tools/enums.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/cal_tools/cal_tools/enums.py b/cal_tools/cal_tools/enums.py
index 27ce86511..4c71c1cc8 100644
--- a/cal_tools/cal_tools/enums.py
+++ b/cal_tools/cal_tools/enums.py
@@ -1,4 +1,4 @@
-from enum import Enum
+from enum import Enum, IntEnum
class BadPixels(Enum):
@@ -50,7 +50,7 @@ class SnowResolution(Enum):
INTERPOLATE = "interpolate"
-class AgipdGainMode(Enum):
+class AgipdGainMode(IntEnum):
"""Encoding added to distinguish between adaptive and fixed gain"""
ADAPTIVE_GAIN = 0 # adaptive is default (if gain mode missing in slow data)
--
GitLab
From 327fcb67256244f908305c352ecc42c5de5853fe Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Wed, 3 Mar 2021 20:22:53 +0100
Subject: [PATCH 10/72] Moving get_gain_mode to agipdlib
---
cal_tools/cal_tools/agipdlib.py | 12 +++++++++++-
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 18 ++----------------
2 files changed, 13 insertions(+), 17 deletions(-)
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 9b40845e0..1c14701f1 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -6,7 +6,7 @@ import h5py
import numpy as np
import sharedmem
from cal_tools.agipdutils import *
-from cal_tools.enums import BadPixels, SnowResolution
+from cal_tools.enums import AgipdGainMode, BadPixels, SnowResolution
from cal_tools.tools import get_constant_from_db_and_time
from iCalibrationDB import Conditions, Constants, Detectors
@@ -118,6 +118,16 @@ def get_gain_setting(fname: str, h5path_ctrl: str) -> int:
else:
raise ValueError('Could not derive gain setting from setupr and patternTypeIndex') # noqa
+def get_gain_mode(fname: str, h5path_ctrl: str) -> AgipdGainMode:
+ """Returns the gain mode (adaptive or fixed) from slow data"""
+
+ h5path_run = h5path_ctrl.replace("CONTROL/", "RUN/", 1)
+ h5path_gainmode = f'{h5path_run}/gainModeIndex/value'
+ with h5py.File(fname, 'r') as fd:
+ if h5path_gainmode in fd:
+ return AgipdGainMode(fd[h5path_gainmode][0])
+ return AgipdGainMode.ADAPTIVE_GAIN
+
def get_bias_voltage(fname: str, karabo_id_control: str,
module: Optional[int] = 0) -> int:
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index ca2c3ba7b..0138ddf88 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -97,7 +97,7 @@
"\n",
"import multiprocessing\n",
"\n",
- "from cal_tools.agipdlib import (get_acq_rate, get_bias_voltage,\n",
+ "from cal_tools.agipdlib import (get_acq_rate, get_bias_voltage, get_gain_mode,\n",
" get_gain_setting, get_num_cells)\n",
"from cal_tools.enums import AgipdGainMode\n",
"from cal_tools.plotting import (create_constant_overview, plot_badpix_3d,\n",
@@ -110,20 +110,6 @@
"from iCalibrationDB import Conditions, Constants, Detectors"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "def get_gain_mode(h5fn):\n",
- " h5path_gainmode = f'{h5path_ctrl.replace(\"/CONTROL/\", \"/RUN/\", 1)}/gainModeIndex/value'\n",
- " with h5py.File(h5fn, 'r') as fd:\n",
- " if h5path_gainmode in fd:\n",
- " return AgipdGainMode(fd[h5path_gainmode][0])\n",
- " return AgipdGainMode.ADAPTIVE_GAIN"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -168,7 +154,7 @@
"control_names = [f'{in_folder}/r{r:04d}/RAW-R{r:04d}-{karabo_da_control}-S00000.h5'\n",
" for r in (run_high, run_med, run_low)]\n",
"\n",
- "run_gain_modes = [get_gain_mode(fn) for fn in control_names]\n",
+ "run_gain_modes = [get_gain_mode(fn, h5path_ctrl) for fn in control_names]\n",
"if all(gm == AgipdGainMode.ADAPTIVE_GAIN for gm in run_gain_modes):\n",
" fixed_gain_mode = False\n",
"elif run_gain_modes == [AgipdGainMode.FIXED_HIGH_GAIN, AgipdGainMode.FIXED_MEDIUM_GAIN, AgipdGainMode.FIXED_LOW_GAIN]:\n",
--
GitLab
From f33eea0eca6c8daad5b62394bd4414df6da7b473 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Wed, 3 Mar 2021 20:23:38 +0100
Subject: [PATCH 11/72] Stripping trailing space, unused variable
---
cal_tools/cal_tools/agipdlib.py | 6 +++---
notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb | 4 +---
2 files changed, 4 insertions(+), 6 deletions(-)
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 1c14701f1..48156b022 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -392,7 +392,7 @@ class AgipdCorrections:
Both corrections are iterative and requires 4 iterations.
Correction is performed in chunks of (e.g. 512 images).
- A complete array of data from one file
+ A complete array of data from one file
(256 trains, 352 cells) will take
256 * 352 * 128 * 512 * 4 // 1024**3 = 22 Gb in memory
@@ -494,7 +494,7 @@ class AgipdCorrections:
self.shared_dict[i_proc]['t0_rgain'][first:last] = \
rawgain / t0[cellid, ...]
self.shared_dict[i_proc]['raw_data'][first:last] = np.copy(data)
-
+
# Often most pixels are in high-gain, so it's more efficient to
# set the whole output block to zero than select the right pixels.
gain[:] = 0
@@ -524,7 +524,7 @@ class AgipdCorrections:
def baseline_correction(self, i_proc:int, first:int, last:int):
"""
- Perform image-wise base-line shift correction for
+ Perform image-wise base-line shift correction for
data in shared memory via histogram or stripe
:param first: Index of the first image to be corrected
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 0138ddf88..2a719adff 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -119,8 +119,6 @@
"# insert control device if format string (does nothing otherwise)\n",
"h5path_ctrl = h5path_ctrl.format(karabo_id_control)\n",
"\n",
- "gains = np.arange(3)\n",
- "\n",
"max_cells = mem_cells\n",
"\n",
"offset_runs = OrderedDict()\n",
@@ -558,7 +556,7 @@
" acquisition_rate=acq_rate,\n",
" gain_setting=gain_setting)\n",
"\n",
- " # This should be used in case of running notebook \n",
+ " # This should be used in case of running notebook\n",
" # by a different method other than myMDC which already\n",
" # sends CalCat info.\n",
" # TODO: Set db_module to \"\" by default in the first cell\n",
--
GitLab
From 298ca0b6494c09f0295c81acf2586ea060e9372b Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Wed, 3 Mar 2021 20:25:08 +0100
Subject: [PATCH 12/72] Skip thresholds if fixed gain mode
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 48 ++++++++++++-------
1 file changed, 30 insertions(+), 18 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 2a719adff..f078a19ef 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -331,32 +331,41 @@
" first_index = int(first[status != 0][0])\n",
" im = np.array(infile[f\"{h5path_f}/data\"][first_index:last_index,...])\n",
" cellIds = np.squeeze(infile[f\"{h5path_f}/cellId\"][first_index:last_index,...])\n",
- "\n",
+ " \n",
" if interlaced:\n",
- " ga = im[1::2, 0, ...]\n",
+ " if not fixed_gain_mode:\n",
+ " ga = im[1::2, 0, ...]\n",
" im = im[0::2, 0, ...].astype(np.float32)\n",
" cellIds = cellIds[::2]\n",
" else:\n",
- " ga = im[:, 1, ...]\n",
+ " if not fixed_gain_mode:\n",
+ " ga = im[:, 1, ...]\n",
" im = im[:, 0, ...].astype(np.float32)\n",
"\n",
" im = np.rollaxis(im, 2)\n",
" im = np.rollaxis(im, 2, 1)\n",
"\n",
- " ga = np.rollaxis(ga, 2)\n",
- " ga = np.rollaxis(ga, 2, 1)\n",
- "\n",
+ " if not fixed_gain_mode:\n",
+ " ga = np.rollaxis(ga, 2)\n",
+ " ga = np.rollaxis(ga, 2, 1)\n",
+ " \n",
" offset = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
- " gains = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
" noise = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
- " gains_std = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
+ "\n",
+ " if fixed_gain_mode:\n",
+ " gains = None\n",
+ " gains_std = None\n",
+ " else:\n",
+ " gains = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
+ " gains_std = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
"\n",
" for cc in np.unique(cellIds[cellIds < num_cells]):\n",
" cellidx = cellIds == cc\n",
" offset[...,cc] = np.median(im[..., cellidx], axis=2)\n",
" noise[...,cc] = np.std(im[..., cellidx], axis=2)\n",
- " gains[...,cc] = np.median(ga[..., cellidx], axis=2)\n",
- " gains_std[...,cc] = np.std(ga[..., cellidx], axis=2)\n",
+ " if not fixed_gain_mode:\n",
+ " gains[...,cc] = np.median(ga[..., cellidx], axis=2)\n",
+ " gains_std[...,cc] = np.std(ga[..., cellidx], axis=2)\n",
"\n",
" # bad pixels\n",
" bp = np.zeros(offset.shape, np.uint32)\n",
@@ -389,10 +398,11 @@
"source": [
"offset_g = OrderedDict()\n",
"noise_g = OrderedDict()\n",
- "gain_g = OrderedDict()\n",
- "gainstd_g = OrderedDict()\n",
"badpix_g = OrderedDict()\n",
- "\n",
+ "if not fixed_gain_mode:\n",
+ " gain_g = OrderedDict()\n",
+ " gainstd_g = OrderedDict()\n",
+ " \n",
"start = datetime.now()\n",
"all_cells = []\n",
"all_acq_rate = []\n",
@@ -421,15 +431,17 @@
" if qm not in offset_g:\n",
" offset_g[qm] = np.zeros((offset.shape[0], offset.shape[1], offset.shape[2], 3))\n",
" noise_g[qm] = np.zeros_like(offset_g[qm])\n",
- " gain_g[qm] = np.zeros_like(offset_g[qm])\n",
- " gainstd_g[qm] = np.zeros_like(offset_g[qm])\n",
" badpix_g[qm] = np.zeros_like(offset_g[qm], np.uint32)\n",
+ " if not fixed_gain_mode:\n",
+ " gain_g[qm] = np.zeros_like(offset_g[qm])\n",
+ " gainstd_g[qm] = np.zeros_like(offset_g[qm])\n",
"\n",
" offset_g[qm][...,gg] = offset\n",
" noise_g[qm][...,gg] = noise\n",
- " gain_g[qm][...,gg] = gains\n",
- " gainstd_g[qm][..., gg] = gains_std\n",
" badpix_g[qm][...,gg] = bp\n",
+ " if not fixed_gain_mode:\n",
+ " gain_g[qm][...,gg] = gains\n",
+ " gainstd_g[qm][..., gg] = gains_std\n",
"\n",
"\n",
"duration = (datetime.now() - start).total_seconds()\n",
@@ -878,7 +890,7 @@
"if fixed_gain_mode:\n",
" constants = ['Offset', 'Noise']\n",
"else:\n",
- " ['Offset', 'Noise', 'ThresholdsDark']\n",
+ " constants = ['Offset', 'Noise', 'ThresholdsDark']\n",
"\n",
"for const in constants:\n",
"\n",
--
GitLab
From f799da67f2a321b907c5315dd927c694c71e8051 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Wed, 3 Mar 2021 20:25:31 +0100
Subject: [PATCH 13/72] Add gain mode to condition
This requires the pending MR for cal_db_interactive. Tested with e79f2c9d (current head of the
branch of that MR).
---
notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index f078a19ef..2f6e45c84 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -566,7 +566,8 @@
" condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
" bias_voltage=bias_voltage,\n",
" acquisition_rate=acq_rate,\n",
- " gain_setting=gain_setting)\n",
+ " gain_setting=gain_setting,\n",
+ " gain_mode=(True if fixed_gain_mode else None))\n",
"\n",
" # This should be used in case of running notebook\n",
" # by a different method other than myMDC which already\n",
@@ -615,7 +616,8 @@
" condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
" bias_voltage=bias_voltage,\n",
" acquisition_rate=acq_rate,\n",
- " gain_setting=gain_setting)\n",
+ " gain_setting=gain_setting,\n",
+ " gain_mode=(True if fixed_gain_mode else None))\n",
" if db_output:\n",
" md = send_to_db(db_module, karabo_id, dconst, condition, file_loc,\n",
" report, cal_db_interface, creation_time=creation_time,\n",
--
GitLab
From 5accaf0da49d82c6175aac401d22e37aa98e47b8 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Mon, 8 Mar 2021 17:36:18 +0100
Subject: [PATCH 14/72] Only define the condition once
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 29 ++++++++++---------
1 file changed, 15 insertions(+), 14 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 2f6e45c84..8ea3880e6 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -538,6 +538,21 @@
" }"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# set the operating condition\n",
+ "# note: iCalibrationDB only adds gain_mode if it is truthy, so we don't need to handle None\n",
+ "condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
+ " bias_voltage=bias_voltage,\n",
+ " acquisition_rate=acq_rate,\n",
+ " gain_setting=gain_setting,\n",
+ " gain_mode=(True if fixed_gain_mode else None))"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -554,7 +569,6 @@
"detinst = getattr(Detectors, dinstance)\n",
"\n",
"print('Retrieve pre-existing constants for comparison.')\n",
- "\n",
"for qm in res:\n",
" qm_db = qm_dict[qm]\n",
" karabo_da = qm_db[\"karabo_da\"]\n",
@@ -562,13 +576,6 @@
" dconst = getattr(Constants.AGIPD, const)()\n",
" dconst.data = res[qm][const]\n",
"\n",
- " # Setting conditions\n",
- " condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
- " bias_voltage=bias_voltage,\n",
- " acquisition_rate=acq_rate,\n",
- " gain_setting=gain_setting,\n",
- " gain_mode=(True if fixed_gain_mode else None))\n",
- "\n",
" # This should be used in case of running notebook\n",
" # by a different method other than myMDC which already\n",
" # sends CalCat info.\n",
@@ -612,12 +619,6 @@
" dconst = getattr(Constants.AGIPD, const)()\n",
" dconst.data = res[qm][const]\n",
"\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
- " bias_voltage=bias_voltage,\n",
- " acquisition_rate=acq_rate,\n",
- " gain_setting=gain_setting,\n",
- " gain_mode=(True if fixed_gain_mode else None))\n",
" if db_output:\n",
" md = send_to_db(db_module, karabo_id, dconst, condition, file_loc,\n",
" report, cal_db_interface, creation_time=creation_time,\n",
--
GitLab
From 260ea9754e18825928dbfd11a32ad9c9e1d805f4 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Mon, 8 Mar 2021 17:39:42 +0100
Subject: [PATCH 15/72] Moving index_to_qm to tools
---
cal_tools/cal_tools/tools.py | 9 +++++++-
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 22 +++++--------------
2 files changed, 14 insertions(+), 17 deletions(-)
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index f112c3cf0..d2f2375b8 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -78,7 +78,7 @@ def map_modules_from_folder(in_folder, run, path_template, karabo_da,
sequences_qm = {}
for inset in karabo_da:
module_idx = int(inset[-2:])
- name = f"Q{module_idx // 4 + 1}M{module_idx % 4 + 1}"
+ name = module_index_to_qm(module_idx)
module_files[name] = Queue()
sequences_qm[name] = 0
mod_ids[name] = module_idx
@@ -671,6 +671,13 @@ def get_constant_from_db_and_time(karabo_id: str, karabo_da: str,
return data, None
+def module_index_to_qm(index: int, total_modules: int = 16):
+ """Maps module index (0-indexed) to quadrant + module string (1-indexed)"""
+ modules_per_quad = total_modules // 4
+ quad, mod = divmod(index, modules_per_quad)
+ return f"Q{quad+1}M{mod+1}"
+
+
class CalibrationMetadata(dict):
"""Convenience class: dictionary stored in metadata YAML file
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 8ea3880e6..abbc7a72b 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -104,7 +104,7 @@
" show_overview, show_processed_modules)\n",
"from cal_tools.tools import (get_dir_creation_date, get_from_db,\n",
" get_pdu_from_db, get_random_db_interface,\n",
- " get_report, map_gain_stages,\n",
+ " get_report, map_gain_stages, module_index_to_qm,\n",
" run_prop_seq_from_path, save_const_to_h5,\n",
" send_to_db)\n",
"from iCalibrationDB import Conditions, Constants, Detectors"
@@ -252,16 +252,6 @@
"print(f\"Will process a total of {total_sequences} files.\")"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "def index_to_qm(i: int):\n",
- " return f\"Q{i//4+1}M{i%4+1}\""
- ]
- },
{
"cell_type": "markdown",
"metadata": {},
@@ -411,7 +401,7 @@
"for gg, (gain, mapped_files) in enumerate(gain_mapped_files.items()):\n",
" dones = []\n",
" for i in modules:\n",
- " qm = index_to_qm(i)\n",
+ " qm = module_index_to_qm(i)\n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
" fname_in = mapped_files[qm].get()\n",
" print(\"Process file: \", fname_in)\n",
@@ -427,7 +417,7 @@
" all_cells.append(thiscell)\n",
" all_acq_rate.append(thisacq)\n",
" for i in modules:\n",
- " qm = index_to_qm(i)\n",
+ " qm = module_index_to_qm(i)\n",
" if qm not in offset_g:\n",
" offset_g[qm] = np.zeros((offset.shape[0], offset.shape[1], offset.shape[2], 3))\n",
" noise_g[qm] = np.zeros_like(offset_g[qm])\n",
@@ -498,7 +488,7 @@
"source": [
"res = OrderedDict()\n",
"for i in modules:\n",
- " qm = index_to_qm(i)\n",
+ " qm = module_index_to_qm(i)\n",
" res[qm] = {\n",
" 'Offset': offset_g[qm],\n",
" 'Noise': noise_g[qm],\n",
@@ -531,7 +521,7 @@
"# Create the modules dict of karabo_das and PDUs\n",
"qm_dict = OrderedDict()\n",
"for i, k_da in zip(modules, karabo_da):\n",
- " qm = index_to_qm(i)\n",
+ " qm = module_index_to_qm(i)\n",
" qm_dict[qm] = {\n",
" \"karabo_da\": k_da,\n",
" \"db_module\": \"\"\n",
@@ -643,7 +633,7 @@
"source": [
"mnames=[]\n",
"for i in modules:\n",
- " qm = index_to_qm(i)\n",
+ " qm = module_index_to_qm(i)\n",
" mnames.append(qm)\n",
" display(Markdown(f'## Position of the module {qm} and its ASICs##'))\n",
"show_processed_modules(dinstance, constants=None, mnames=mnames, mode=\"position\")"
--
GitLab
From bedec5fad1cdd2e30cabafba28ca91818c90cd5a Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Tue, 9 Mar 2021 14:43:57 +0100
Subject: [PATCH 16/72] Adding back operation_mode parameter (only used for
warning so far)
---
notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index abbc7a72b..7a3f8784c 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -26,6 +26,7 @@
"run_high = 9985 # run number in which high gain data was recorded, required\n",
"run_med = 9984 # run number in which medium gain data was recorded, required\n",
"run_low = 9983 # run number in which low gain data was recorded, required\n",
+ "operation_mode = \"ADAPTIVE_GAIN\" # Detector operation mode, optional (defaults to \"ADAPTIVE_GAIN\")\n",
"\n",
"karabo_id = \"HED_DET_AGIPD500K2G\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
@@ -152,10 +153,16 @@
"control_names = [f'{in_folder}/r{r:04d}/RAW-R{r:04d}-{karabo_da_control}-S00000.h5'\n",
" for r in (run_high, run_med, run_low)]\n",
"\n",
+ "if operation_mode not in (\"ADAPTIVE_GAIN\", \"FIXED_GAIN\"):\n",
+ " print(f\"WARNING: unknown operation_mode \\\"{operation_mode}\\\" parameter set\")\n",
"run_gain_modes = [get_gain_mode(fn, h5path_ctrl) for fn in control_names]\n",
"if all(gm == AgipdGainMode.ADAPTIVE_GAIN for gm in run_gain_modes):\n",
" fixed_gain_mode = False\n",
+ " if operation_mode == \"FIXED_GAIN\":\n",
+ " print(\"WARNING: operation_mode parameter is FIXED_GAIN, slow data indicates adaptive gain\")\n",
"elif run_gain_modes == [AgipdGainMode.FIXED_HIGH_GAIN, AgipdGainMode.FIXED_MEDIUM_GAIN, AgipdGainMode.FIXED_LOW_GAIN]:\n",
+ " if operation_mode == \"ADAPTIVE_GAIN\":\n",
+ " print(\"WARNING: operation_mode parameter ix ADAPTIVE_GAIN, slow data indicates fixed gain\")\n",
" fixed_gain_mode = True\n",
"else:\n",
" print(f'Something is clearly wrong; slow data indicates gain modes {run_gain_modes}')\n",
--
GitLab
From 2d93c446fcf5d6b8b875ce8d85287b4dad9a7673 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 11 Mar 2021 13:43:29 +0100
Subject: [PATCH 17/72] Dropping driver="core" when opening h5py file
The relevant data is copied into memory in pretty large chunks, so the core driver is not expected
to do much for us here. In some brief testing, using the default driver instead resulted in
approximately the same (or slightly better) runtime and significantly lower peak memory consumption.
---
notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 7a3f8784c..8c17aac08 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -312,7 +312,7 @@
" h5path_f = h5path.format(channel)\n",
" h5path_idx_f = h5path_idx.format(channel)\n",
"\n",
- " with h5py.File(fast_data_filename, \"r\", driver=\"core\") as infile:\n",
+ " with h5py.File(fast_data_filename, \"r\") as infile:\n",
" if rawversion == 2:\n",
" count = np.squeeze(infile[f\"{h5path_idx_f}/count\"])\n",
" first = np.squeeze(infile[f\"{h5path_idx_f}/first\"])\n",
--
GitLab
From e0ce9a414e59f1db726456086772e1b76f4aaa65 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 11 Mar 2021 14:40:51 +0100
Subject: [PATCH 18/72] Adding specific threshold parameters for fixed gain
mode
Currently, there is no way to specify most parameters when requesting darks via MyMDC - values are
just loaded from the calibration configurations repository. To allow for using different offset
thresholds in fixed gain mode versus adaptive gain mode, adding these extra parameters is a
pragmatic if not elegant solution for now.
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 43 ++++++++++++++-----
1 file changed, 32 insertions(+), 11 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 8c17aac08..44987c67b 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -52,10 +52,13 @@
"rawversion = 2 # RAW file format version\n",
"\n",
"thresholds_offset_sigma = 3. # offset sigma thresholds for offset deduced bad pixels\n",
- "thresholds_offset_hard = [0, 0] # For setting the same threshold offset for the 3 gains. Left for backcompatability. Default [0, 0] to take the following parameters.\n",
- "thresholds_offset_hard_hg = [3000, 7000] # High-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
- "thresholds_offset_hard_mg = [6000, 10000] # Medium-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
- "thresholds_offset_hard_lg = [6000, 10000] # Low-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
+ "thresholds_offset_hard = [0, 0] # For setting the same threshold offset for the 3 gains. Left for backcompatability. Default [0, 0] to take the following parameters.\n",
+ "thresholds_offset_hard_hg = [3000, 7000] # High-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
+ "thresholds_offset_hard_mg = [6000, 10000] # Medium-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
+ "thresholds_offset_hard_lg = [6000, 10000] # Low-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
+ "thresholds_offset_hard_hg_fixed = [3500, 6500] # Same as thresholds_offset_hard_hg, but for fixed gain operation\n",
+ "thresholds_offset_hard_mg_fixed = [3500, 6500] # Same as thresholds_offset_hard_mg, but for fixed gain operation\n",
+ "thresholds_offset_hard_lg_fixed = [3500, 6500] # Same as thresholds_offset_hard_lg, but for fixed gain operation\n",
"\n",
"thresholds_noise_sigma = 5. # noise sigma thresholds for offset deduced bad pixels\n",
"thresholds_noise_hard = [0, 0] # For setting the same threshold noise for the 3 gains. Left for backcompatability. Default [0, 0] to take the following parameters.\n",
@@ -274,15 +277,33 @@
"metadata": {},
"outputs": [],
"source": [
- "if thresholds_offset_hard == [0, 0]:\n",
- " thresholds_offset_hard = [thresholds_offset_hard_hg, thresholds_offset_hard_mg, thresholds_offset_hard_lg]\n",
- "else:\n",
+ "if thresholds_offset_hard != [0, 0]:\n",
+ " # if set, this will override the individual parameters\n",
" thresholds_offset_hard = [thresholds_offset_hard] * 3\n",
- "\n",
- "if thresholds_noise_hard == [0, 0]:\n",
- " thresholds_noise_hard = [thresholds_noise_hard_hg, thresholds_noise_hard_mg, thresholds_noise_hard_lg]\n",
+ "elif fixed_gain_mode:\n",
+ " thresholds_offset_hard = [\n",
+ " thresholds_offset_hard_hg_fixed,\n",
+ " thresholds_offset_hard_mg_fixed,\n",
+ " thresholds_offset_hard_lg_fixed,\n",
+ " ]\n",
+ "else:\n",
+ " thresholds_offset_hard = [\n",
+ " thresholds_offset_hard_hg,\n",
+ " thresholds_offset_hard_mg,\n",
+ " thresholds_offset_hard_lg,\n",
+ " ]\n",
+ "print(f\"Will use the following hard offset thresholds\")\n",
+ "for name, value in zip((\"High\", \"Medium\", \"Low\"), thresholds_offset_hard):\n",
+ " print(f\"- {name} gain: {value}\")\n",
+ "\n",
+ "if thresholds_noise_hard != [0, 0]:\n",
+ " thresholds_noise_hard = [thresholds_noise_hard] * 3\n",
"else:\n",
- " thresholds_noise_hard = [thresholds_noise_hard] * 3"
+ " thresholds_noise_hard = [\n",
+ " thresholds_noise_hard_hg,\n",
+ " thresholds_noise_hard_mg,\n",
+ " thresholds_noise_hard_lg,\n",
+ " ]"
]
},
{
--
GitLab
From c040abc76bee387e4583ca797b0b42ed39840c0e Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 11 Mar 2021 14:53:21 +0100
Subject: [PATCH 19/72] We don't need to omit the gain mode from the condition
here
---
notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 44987c67b..23a1bd6b2 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -568,7 +568,7 @@
" bias_voltage=bias_voltage,\n",
" acquisition_rate=acq_rate,\n",
" gain_setting=gain_setting,\n",
- " gain_mode=(True if fixed_gain_mode else None))"
+ " gain_mode=fixed_gain_mode)"
]
},
{
--
GitLab
From aee296b2f508019e88c1ea2e6fbca8d98b635852 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Fri, 12 Mar 2021 13:56:40 +0100
Subject: [PATCH 20/72] Initial changes to use new stuff from base branch
---
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 26 +++++++++++--------
1 file changed, 15 insertions(+), 11 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 926498bd0..82f22b590 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -142,14 +142,14 @@
"sns.set_style(\"ticks\")\n",
"\n",
"import seaborn as sns\n",
- "from cal_tools.agipdlib import (AgipdCorrections, get_acq_rate,\n",
+ "from cal_tools.agipdlib import (AgipdCorrections, get_acq_rate, get_gain_mode,\n",
" get_gain_setting, get_num_cells)\n",
"from cal_tools.ana_tools import get_range\n",
"from cal_tools.cython import agipdalgs as calgs\n",
- "from cal_tools.enums import BadPixels\n",
+ "from cal_tools.enums import AgipdGainMode, BadPixels\n",
"from cal_tools.step_timing import StepTimer\n",
"from cal_tools.tools import (CalibrationMetadata, get_dir_creation_date,\n",
- " map_modules_from_folder)\n",
+ " map_modules_from_folder, module_index_to_qm)\n",
"\n",
"sns.set()\n",
"sns.set_context(\"paper\", font_scale=1.4)\n",
@@ -256,11 +256,8 @@
"else:\n",
" modules = [int(x[-2:]) for x in karabo_da]\n",
" \n",
- "def mod_name(modno):\n",
- " return f\"Q{modno // 4 + 1}M{modno % 4 + 1}\"\n",
- "\n",
"print(\"Process modules: \", ', '.join(\n",
- " [mod_name(x) for x in modules]))\n",
+ " [module_index_to_qm(x) for x in modules]))\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"print(f\"Instrument {instrument}\")\n",
"print(f\"Detector instance {dinstance}\")"
@@ -369,7 +366,9 @@
" print(e)\n",
" print(\"Set gain setting to 0\")\n",
" gain_setting = 0\n",
- " "
+ "\n",
+ "# Evaluate gain mode (operation mode)\n",
+ "gain_mode = get_gain_mode(control_fn, h5path_ctrl)"
]
},
{
@@ -379,8 +378,13 @@
"outputs": [],
"source": [
"print(f\"Using {creation_time} as creation time\")\n",
- "print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Memory cells: {mem_cells_db}\\n\"\n",
- " f\"• Acquisition rate: {acq_rate}\\n• Gain setting: {gain_setting}\\n• Photon Energy: {photon_energy}\\n\")"
+ "print(\"Operating conditions are:\")\n",
+ "print(f\"• Bias voltage: {bias_voltage}\")\n",
+ "print(f\"• Memory cells: {mem_cells_db}\")\n",
+ "print(f\"• Acquisition rate: {acq_rate}\")\n",
+ "print(f\"• Gain setting: {gain_setting}\")\n",
+ "print(f\"• Gain mode: {gain_mode.name}\")\n",
+ "print(f\"• Photon Energy: {photon_energy}\")"
]
},
{
@@ -591,7 +595,7 @@
"timestamps = {}\n",
"\n",
"for i, (error, modno, when, k_da) in enumerate(const_out):\n",
- " qm = mod_name(modno)\n",
+ " qm = module_index_to_qm(modno)\n",
" # expose errors while applying correction\n",
" if error:\n",
" print(\"Error: {}\".format(error) )\n",
--
GitLab
From b86ab3f5b32d0cff2810f76580057d11b1482b92 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Fri, 12 Mar 2021 15:13:15 +0100
Subject: [PATCH 21/72] Moving output regarding gain mode to other
condition-related notes
---
notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 23a1bd6b2..9caca3bb9 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -172,8 +172,7 @@
"\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"print(f\"Instrument {instrument}\")\n",
- "print(f\"Detector instance {dinstance}\")\n",
- "print(f\"Operation mode is {'fixed' if fixed_gain_mode else 'adaptive'} gain mode\")"
+ "print(f\"Detector instance {dinstance}\")"
]
},
{
@@ -239,7 +238,8 @@
"print(f\"Input: {in_folder}\")\n",
"print(f\"Output: {out_folder}\")\n",
"print(f\"Bias voltage: {bias_voltage}V\")\n",
- "print(f\"Gain setting: {gain_setting}\")"
+ "print(f\"Gain setting: {gain_setting}\")\n",
+ "print(f\"Operation mode is {'fixed' if fixed_gain_mode else 'adaptive'} gain mode\")"
]
},
{
--
GitLab
From 8085ce8ed27411eb0a4671ca86f8b92f76b6ada6 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Fri, 12 Mar 2021 16:28:21 +0100
Subject: [PATCH 22/72] Initial refactoring and cleanup
nbstripout, some flake8 (removing unused variables, imports), isort, simplifying retrieve_constants
as it does not need to work under ipyparallel
---
...IPD_Retrieve_Constants_Precorrection.ipynb | 226 ++++++++----------
1 file changed, 100 insertions(+), 126 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 6ce5f20bf..677fe4d1f 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -14,15 +14,9 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-21T11:30:06.730220Z",
- "start_time": "2019-02-21T11:30:06.658286Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
- "cluster_profile = \"noDB\"\n",
"in_folder = \"/gpfs/exfel/exp/SPB/202030/p900119/raw\" # the folder to read data from, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_\" # the folder to output to, required\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
@@ -91,12 +85,8 @@
"metadata": {},
"outputs": [],
"source": [
- "import sys\n",
- "from collections import OrderedDict\n",
- "from functools import partial\n",
"from typing import List, Tuple\n",
"\n",
- "import h5py\n",
"import matplotlib\n",
"import numpy as np\n",
"\n",
@@ -105,10 +95,7 @@
"from datetime import timedelta\n",
"from pathlib import Path\n",
"\n",
- "import matplotlib.pyplot as plt\n",
- "from cal_tools.agipdlib import get_gain_setting\n",
- "from cal_tools.tools import (CalibrationMetadata, get_dir_creation_date,\n",
- " map_modules_from_folder)\n",
+ "from cal_tools import agipdlib, tools\n",
"from dateutil import parser\n",
"from iCalibrationDB import Conditions, Constants, Detectors"
]
@@ -122,7 +109,7 @@
"# slopes_ff_from_files left as str for now\n",
"in_folder = Path(in_folder)\n",
"out_folder = Path(out_folder)\n",
- "metadata = CalibrationMetadata(out_folder)"
+ "metadata = tools.CalibrationMetadata(out_folder)"
]
},
{
@@ -135,7 +122,7 @@
"\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(str(in_folder), run)\n",
+ " creation_time = tools.get_dir_creation_date(str(in_folder), run)\n",
" offset = parser.parse(creation_date_offset)\n",
" delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
" creation_time += delta\n",
@@ -171,7 +158,7 @@
" gain_setting = None\n",
" else:\n",
" try:\n",
- " gain_setting = get_gain_setting(str(control_fn), h5path_ctrl)\n",
+ " gain_setting = agipdlib.get_gain_setting(str(control_fn), h5path_ctrl)\n",
" except Exception as e:\n",
" print(f'ERROR: while reading gain setting from: \\n{control_fn}')\n",
" print(e)\n",
@@ -207,24 +194,6 @@
" modules = [int(x[-2:]) for x in karabo_da]"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-21T11:30:07.974174Z",
- "start_time": "2019-02-21T11:30:07.914832Z"
- }
- },
- "outputs": [],
- "source": [
- "# set everything up filewise\n",
- "print(f\"Checking the files before retrieving constants\")\n",
- "mmf = map_modules_from_folder(str(in_folder), run, path_template, karabo_da, sequences)\n",
- "\n",
- "mapped_files, mod_ids, total_sequences, sequences_qm, _ = mmf"
- ]
- },
{
"cell_type": "markdown",
"metadata": {},
@@ -238,30 +207,15 @@
"metadata": {},
"outputs": [],
"source": [
- "def retrieve_constants(karabo_id: str, bias_voltage: int, max_cells: float,\n",
- " acq_rate: float, gain_setting: float, photon_energy: float,\n",
- " only_dark: bool, nodb_with_dark: bool, \n",
- " cal_db_interface: str, creation_time: str, \n",
- " corr_bools: dict, pc_bools: List[bool],\n",
- " inp: Tuple[str, str, str, int]\n",
- " ) -> Tuple[str, str, float, float, str, dict]:\n",
+ "def retrieve_constants(\n",
+ " qm_files: List[Path],\n",
+ " qm: str,\n",
+ " karabo_da: str,\n",
+ " idx: int\n",
+ ") -> Tuple[str, str, float, float, str, dict]:\n",
" \"\"\"\n",
- " Retreive constant for each module in parallel and produce a dictionary\n",
- " with the creation-time and constant file path.\n",
+ " Retreive constants for a module.\n",
" \n",
- " :param karabo_id: (STR) Karabo ID\n",
- " :param bias_voltage: (FLOAT) Bias Voltage\n",
- " :param max_cells: (INT) Memory cells\n",
- " :param acq_rate: (FLOAT) Acquisition Rate\n",
- " :param gain_setting: (FLOAT) Gain setting\n",
- " :param photon_energy: (FLOAT) Photon Energy\n",
- " :param only_dark: (BOOL) only retrieve dark constants\n",
- " :param nodb_with_dark: (BOOL) no constant retrieval even for dark\n",
- " :param cal_db_interface: (STR) the database interface port\n",
- " :param creation_time: (STR) raw data creation time\n",
- " :param corr_bools: (DICT) A dictionary with bools for applying requested corrections\n",
- " :param pc_bools: (LIST) list of bools to retrieve pulse capacitor constants\n",
- " :param inp: (LIST) input for the parallel cluster of the partial function\n",
" :return:\n",
" qm: module virtual name i.e. Q1M1.\n",
" karabo_da: karabo data aggregator.\n",
@@ -271,41 +225,38 @@
" mdata_dict: (DICT) dictionary with the metadata for the retrieved constants.\n",
" \"\"\"\n",
"\n",
- " import sys\n",
- " import traceback\n",
- "\n",
- " import numpy as np\n",
- " from cal_tools.agipdlib import get_acq_rate, get_num_cells\n",
- " from cal_tools.agipdutils import assemble_constant_dict\n",
- " from cal_tools.tools import get_from_db\n",
- " from iCalibrationDB import Conditions, Constants, Detectors\n",
- "\n",
" err = None\n",
- "\n",
- " qm_files, qm, karabo_da, idx = inp\n",
- " # get number of memory cells from a sequence file with image data\n",
- " for f in qm_files:\n",
- " if not max_cells:\n",
- " max_cells = get_num_cells(f, karabo_id, idx)\n",
- " if max_cells is None:\n",
- " if f != qm_files[-1]:\n",
- " continue\n",
- " else:\n",
- " raise ValueError(f\"No raw images found for {qm} for all sequences\")\n",
- " else:\n",
- " cells = np.arange(max_cells)\n",
- " # get out of the loop,\n",
- " # if max_cells is successfully calculated. \n",
+ " if max_cells != 0:\n",
+ " # either use overriding notebook parameter\n",
+ " local_max_cells = max_cells\n",
+ " else:\n",
+ " # or look around in sequence files\n",
+ " for f in qm_files:\n",
+ " local_max_cells = agipdlib.get_num_cells(f, karabo_id, idx)\n",
+ " if local_max_cells is not None:\n",
" break\n",
+ " # maybe we never found this in a sequence file...\n",
+ " if local_max_cells is None:\n",
+ " raise ValueError(f\"No raw images found for {qm} for all sequences\")\n",
"\n",
- " if acq_rate == 0.:\n",
- " acq_rate = get_acq_rate((f, karabo_id, idx))\n",
+ " if acq_rate == 0: \n",
+ " local_acq_rate = agipdlib.get_acq_rate(fast_paths=(f, karabo_id, idx))\n",
+ " else:\n",
+ " local_acq_rate = acq_rate\n",
"\n",
" # avoid creating retireving constant, if requested.\n",
" if not nodb_with_dark:\n",
- " const_dict = assemble_constant_dict(corr_bools, pc_bools, max_cells, bias_voltage,\n",
- " gain_setting, acq_rate, photon_energy,\n",
- " beam_energy=None, only_dark=only_dark)\n",
+ " const_dict = agipdlib.assemble_constant_dict(\n",
+ " corr_bools,\n",
+ " pc_bools,\n",
+ " local_max_cells,\n",
+ " bias_voltage,\n",
+ " gain_setting,\n",
+ " local_acq_rate,\n",
+ " photon_energy,\n",
+ " beam_energy=None,\n",
+ " only_dark=only_dark,\n",
+ " )\n",
"\n",
" # Retrieve multiple constants through an input dictionary\n",
" # to return a dict of useful metadata.\n",
@@ -321,28 +272,42 @@
" mdata_dict['constants'][cname][\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
" mdata_dict['constants'][cname][\"creation-time\"] = \"00:00:00\"\n",
" else:\n",
- " try:\n",
- " condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
- " co, mdata = \\\n",
- " get_from_db(karabo_id, karabo_da, getattr(Constants.AGIPD, cname)(),\n",
- " condition, getattr(np, cval[0])(cval[1]),\n",
- " cal_db_interface, creation_time, meta_only=True, verbosity=0)\n",
- " mdata_const = mdata.calibration_constant_version\n",
- " device_name = mdata.calibration_constant_version.device_name\n",
- " # check if constant was sucessfully retrieved.\n",
- " if mdata.comm_db_success:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
- " f\"{mdata_const.filename}\"\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
- " mdata_dict['physical-detector-unit'] = mdata_const.device_name\n",
- " else:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = const_dict[cname][:2]\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = None\n",
- " except Exception as e:\n",
- " err = f\"Error: {e}, Traceback: {traceback.format_exc()}\"\n",
- " print(err)\n",
- "\n",
- " return qm, mdata_dict, karabo_da, acq_rate, max_cells, err\n",
+ " condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
+ " co, mdata = tools.get_from_db(\n",
+ " karabo_id,\n",
+ " karabo_da,\n",
+ " getattr(Constants.AGIPD, cname)(),\n",
+ " condition,\n",
+ " getattr(np, cval[0])(cval[1]),\n",
+ " cal_db_interface,\n",
+ " creation_time,\n",
+ " meta_only=True,\n",
+ " verbosity=1,\n",
+ " )\n",
+ " mdata_const = mdata.calibration_constant_version\n",
+ " # check if constant was sucessfully retrieved.\n",
+ " if mdata.comm_db_success:\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
+ " f\"{mdata_const.filename}\"\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
+ " mdata_dict['physical-detector-unit'] = mdata_const.device_name\n",
+ " else:\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = const_dict[cname][:2]\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = None\n",
+ "\n",
+ " return qm, mdata_dict, karabo_da, acq_rate, local_max_cells, err"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# set everything up filewise\n",
+ "mapped_files, _, _, _, _ = tools.map_modules_from_folder(\n",
+ " str(in_folder), run, path_template, karabo_da, sequences\n",
+ ")\n",
"\n",
"pc_bools = [corr_bools.get(\"rel_gain\"),\n",
" corr_bools.get(\"adjust_mg_baseline\"),\n",
@@ -361,26 +326,35 @@
"\n",
"# A dict to connect virtual device\n",
"# to actual device name.\n",
- "for i, k_da in zip(modules, karabo_da):\n",
- " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ "for module_index, k_da in zip(modules, karabo_da):\n",
+ " qm = tools.module_index_to_qm(module_index)\n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
" device = getattr(getattr(Detectors, dinstance), qm)\n",
- " qm_files = [str(mapped_files[qm].get()) for _ in range(mapped_files[qm].qsize())]\n",
- "\n",
+ " # TODO: make map_modules_from_folder just return list(s)\n",
+ " qm_files = [Path(mapped_files[qm].get()) for _ in range(mapped_files[qm].qsize())]\n",
" else:\n",
" print(f\"Skipping {qm}\")\n",
" continue\n",
"\n",
- " inp.append((qm_files, qm, k_da, i))\n",
- "\n",
- "p = partial(retrieve_constants, karabo_id, bias_voltage, max_cells, \n",
- " acq_rate, gain_setting, photon_energy, only_dark, nodb_with_dark, \n",
- " cal_db_interface, creation_time, \n",
- " corr_bools, pc_bools)\n",
- "\n",
- "with mp.Pool(processes=nmods) as pool:\n",
- " results = pool.map(p, inp)\n",
- "\n",
+ " inp.append((qm_files, qm, k_da, module_index))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with multiprocessing.Pool(processes=nmods) as pool:\n",
+ " results = pool.starmap(p, inp)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
"mod_dev = dict()\n",
"mdata_dict = dict()\n",
"for r in results:\n",
@@ -393,12 +367,12 @@
"# check if it is requested not to retrieve any constants from the database\n",
"if not nodb_with_dark:\n",
" metadata.update({\"retrieved-constants\": mdata_dict})\n",
- " \n",
+ "\n",
" print(\"\\nRetrieved constants for modules: \",\n",
" f\"{[', '.join([f'Q{x//4+1}M{x%4+1}' for x in modules])]}\")\n",
" print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Memory cells: {max_cells}\\n\"\n",
" f\"• Acquisition rate: {acq_rate}\\n• Gain setting: {gain_setting}\\n• Photon Energy: {photon_energy}\\n\")\n",
- " print(f\"Constant metadata is saved under \\\"retrieved-constants\\\" in calibration_metadata.yml\\n\")\n",
+ " print(\"Constant metadata is saved under \\\"retrieved-constants\\\" in calibration_metadata.yml\\n\")\n",
"else:\n",
" print(\"No constants were retrieved as calibrated files will be used.\")"
]
@@ -425,7 +399,7 @@
" # Store few time stamps if exists\n",
" # Add NA to keep array structure\n",
" for cname in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
- " if not k_da in mdata_dict or dinfo[\"err\"]:\n",
+ " if k_da not in mdata_dict or dinfo[\"err\"]:\n",
" module_timestamps[cname] = \"Err\"\n",
" else:\n",
" if cname in mdata_dict[k_da]:\n",
--
GitLab
From 8428963865a226eba8435f233269fee9dd10236a Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Fri, 12 Mar 2021 16:44:30 +0100
Subject: [PATCH 23/72] Bump up version of cal_db_interactive
---
requirements.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/requirements.txt b/requirements.txt
index ea6991b7d..2f82d5569 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-git+file:///gpfs/exfel/sw/calsoft/git/cal_db_interactive@2.0.1
+git+file:///gpfs/exfel/sw/calsoft/git/cal_db_interactive@2.0.3
git+file:///gpfs/exfel/sw/calsoft/git/nbparameterise@0.3
git+file:///gpfs/exfel/sw/calsoft/git/pyDetLib@2.5.6-2.10.0#subdirectory=lib
astcheck == 0.2.5
--
GitLab
From 0284bae9f6aa7e14bfd1b5701e33bf275f344789 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Fri, 12 Mar 2021 17:27:13 +0100
Subject: [PATCH 24/72] Additional cleanup, starting to add gain_mode in
agipdutils
---
cal_tools/cal_tools/agipdutils.py | 51 ++++---
...IPD_Retrieve_Constants_Precorrection.ipynb | 143 ++++++++++--------
2 files changed, 105 insertions(+), 89 deletions(-)
diff --git a/cal_tools/cal_tools/agipdutils.py b/cal_tools/cal_tools/agipdutils.py
index 9533a6571..3cb51e378 100644
--- a/cal_tools/cal_tools/agipdutils.py
+++ b/cal_tools/cal_tools/agipdutils.py
@@ -1,18 +1,27 @@
import copy
import numpy as np
-from cal_tools.enums import BadPixels, SnowResolution
+from cal_tools.enums import AgipdGainMode, BadPixels, SnowResolution
from scipy.signal import cwt, find_peaks_cwt, ricker
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import StandardScaler
-def assemble_constant_dict(corr_bools, pc_bools, memory_cells, bias_voltage,
- gain_setting, acquisition_rate,
- photon_energy, beam_energy=None, only_dark=False):
+def assemble_constant_dict(
+ corr_bools,
+ pc_bools,
+ memory_cells,
+ bias_voltage,
+ gain_setting,
+ acquisition_rate,
+ photon_energy,
+ beam_energy=None,
+ only_dark=False,
+ gain_mode=AgipdGainMode.ADAPTIVE_GAIN,
+):
"""
Assemble a dictionary with the iCalibrationDB constant names and
- the operating conditions for retrieveing the required constants
+ the operating conditions for retrieving the required constants
for correction.
:param corr_bools: (Dict) A dict of booleans for applying
@@ -24,20 +33,23 @@ def assemble_constant_dict(corr_bools, pc_bools, memory_cells, bias_voltage,
:param acquisition_rate: (Float) Acquisition rate
:param photon_energy: (Float) Photong energy
:param beam_energy: (Float) Beam Energy
- :param only_dark: (Bool) Indicating a retrieval for dark
- constants only from db
+ :param only_dark: (Bool) Indicating a retrieval for dark constants only from db
+ :param gain_mode: Operation mode of the detector (default to adaptive gain)
:return: const_dict: (Dict) An assembeld dictionary that can be used
to retrieve the required constants
"""
darkcond = [
- "Dark", {
+ "Dark",
+ {
"memory_cells": memory_cells,
"bias_voltage": bias_voltage,
"acquisition_rate": acquisition_rate,
"gain_setting": gain_setting,
+ "gain_mode": gain_mode,
"pixels_x": 512,
- "pixels_y": 128, }
+ "pixels_y": 128,
+ },
]
const_dict = {
"Offset": ["zeros", (128, 512, memory_cells, 3), darkcond],
@@ -46,28 +58,21 @@ def assemble_constant_dict(corr_bools, pc_bools, memory_cells, bias_voltage,
"BadPixelsDark": ["zeros", (128, 512, memory_cells, 3), darkcond],
}
- if not (corr_bools.get('only_offset') or only_dark):
-
+ if not (corr_bools.get("only_offset") or only_dark):
if any(pc_bools):
- const_dict["BadPixelsPC"] = \
- ["zeros", (memory_cells, 128, 512), darkcond]
- const_dict["SlopesPC"] = \
- ["ones", (128, 512, memory_cells, 10), darkcond]
+ const_dict["BadPixelsPC"] = ["zeros", (memory_cells, 128, 512), darkcond]
+ const_dict["SlopesPC"] = ["ones", (128, 512, memory_cells, 10), darkcond]
if corr_bools.get("xray_corr"):
# Add illuminated conditions
illumcond = [
- "Illuminated", {
- "beam_energy": beam_energy,
- "photon_energy": photon_energy
- }
+ "Illuminated",
+ {"beam_energy": beam_energy, "photon_energy": photon_energy},
]
illumcond[1].update(darkcond[1])
- const_dict["BadPixelsFF"] = ["zeros", (128, 512, memory_cells),
- illumcond]
- const_dict["SlopesFF"] = ["ones", (128, 512, memory_cells, 2),
- illumcond]
+ const_dict["BadPixelsFF"] = ["zeros", (128, 512, memory_cells), illumcond]
+ const_dict["SlopesFF"] = ["ones", (128, 512, memory_cells, 2), illumcond]
return const_dict
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 677fe4d1f..73e3ed6d4 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -63,8 +63,7 @@
"outputs": [],
"source": [
"# Fill dictionaries comprising bools and arguments for correction and data analysis\n",
- "\n",
- "# Here the herarichy and dependability for correction booleans are defined \n",
+ "# Here the hierarichy and dependencies for correction booleans are defined \n",
"corr_bools = {}\n",
"\n",
"# offset is at the bottom of AGIPD correction pyramid.\n",
@@ -91,7 +90,7 @@
"import numpy as np\n",
"\n",
"matplotlib.use(\"agg\")\n",
- "import multiprocessing as mp\n",
+ "import multiprocessing\n",
"from datetime import timedelta\n",
"from pathlib import Path\n",
"\n",
@@ -138,9 +137,7 @@
"\n",
"warnings.filterwarnings('ignore')\n",
"\n",
- "from cal_tools.agipdlib import SnowResolution\n",
- "\n",
- "melt_snow = False if corr_bools[\"only_offset\"] else SnowResolution.NONE"
+ "melt_snow = False if corr_bools[\"only_offset\"] else agipdlib.SnowResolution.NONE"
]
},
{
@@ -165,7 +162,11 @@
" print(\"Set gain setting to 0\")\n",
" gain_setting = 0\n",
"\n",
+ "# Evaluate gain mode (operation mode)\n",
+ "gain_mode = agipdlib.get_gain_mode(control_fn, h5path_ctrl)\n",
+ " \n",
"print(f\"Gain setting: {gain_setting}\")\n",
+ "print(f\"Gain mode: {gain_mode.name}\")\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"\n",
"\n",
@@ -214,7 +215,7 @@
" idx: int\n",
") -> Tuple[str, str, float, float, str, dict]:\n",
" \"\"\"\n",
- " Retreive constants for a module.\n",
+ " Retrieve constants for a module.\n",
" \n",
" :return:\n",
" qm: module virtual name i.e. Q1M1.\n",
@@ -244,56 +245,61 @@
" else:\n",
" local_acq_rate = acq_rate\n",
"\n",
- " # avoid creating retireving constant, if requested.\n",
- " if not nodb_with_dark:\n",
- " const_dict = agipdlib.assemble_constant_dict(\n",
- " corr_bools,\n",
- " pc_bools,\n",
- " local_max_cells,\n",
- " bias_voltage,\n",
- " gain_setting,\n",
- " local_acq_rate,\n",
- " photon_energy,\n",
- " beam_energy=None,\n",
- " only_dark=only_dark,\n",
- " )\n",
- "\n",
- " # Retrieve multiple constants through an input dictionary\n",
- " # to return a dict of useful metadata.\n",
- " mdata_dict = dict()\n",
- " mdata_dict['constants'] = dict()\n",
- " mdata_dict['physical-detector-unit'] = None # initialization\n",
- "\n",
- " for cname, cval in const_dict.items():\n",
- " # saving metadata in a dict\n",
- " mdata_dict['constants'][cname] = dict()\n",
- "\n",
- " if slopes_ff_from_files and cname in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = \"00:00:00\"\n",
+ " # avoid retrieving constant, if requested.\n",
+ " if nodb_with_dark:\n",
+ " return\n",
+ " \n",
+ " const_dict = agipdlib.assemble_constant_dict(\n",
+ " corr_bools,\n",
+ " pc_bools,\n",
+ " local_max_cells,\n",
+ " bias_voltage,\n",
+ " gain_setting,\n",
+ " local_acq_rate,\n",
+ " photon_energy,\n",
+ " gain_mode=gain_mode,\n",
+ " beam_energy=None,\n",
+ " only_dark=only_dark,\n",
+ " )\n",
+ "\n",
+ " # Retrieve multiple constants through an input dictionary\n",
+ " # to return a dict of useful metadata.\n",
+ " mdata_dict = dict()\n",
+ " mdata_dict['constants'] = dict()\n",
+ " mdata_dict['physical-detector-unit'] = None # initialization\n",
+ "\n",
+ " for cname, cval in const_dict.items():\n",
+ " print(cname)\n",
+ " print(cval)\n",
+ " # saving metadata in a dict\n",
+ " mdata_dict['constants'][cname] = dict()\n",
+ "\n",
+ " if slopes_ff_from_files and cname in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = \"00:00:00\"\n",
+ " else:\n",
+ " condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
+ " co, mdata = tools.get_from_db(\n",
+ " karabo_id,\n",
+ " karabo_da,\n",
+ " getattr(Constants.AGIPD, cname)(),\n",
+ " condition,\n",
+ " getattr(np, cval[0])(cval[1]),\n",
+ " cal_db_interface,\n",
+ " creation_time,\n",
+ " meta_only=True,\n",
+ " verbosity=1,\n",
+ " )\n",
+ " mdata_const = mdata.calibration_constant_version\n",
+ " # check if constant was sucessfully retrieved.\n",
+ " if mdata.comm_db_success:\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
+ " f\"{mdata_const.filename}\"\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
+ " mdata_dict['physical-detector-unit'] = mdata_const.device_name\n",
" else:\n",
- " condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
- " co, mdata = tools.get_from_db(\n",
- " karabo_id,\n",
- " karabo_da,\n",
- " getattr(Constants.AGIPD, cname)(),\n",
- " condition,\n",
- " getattr(np, cval[0])(cval[1]),\n",
- " cal_db_interface,\n",
- " creation_time,\n",
- " meta_only=True,\n",
- " verbosity=1,\n",
- " )\n",
- " mdata_const = mdata.calibration_constant_version\n",
- " # check if constant was sucessfully retrieved.\n",
- " if mdata.comm_db_success:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
- " f\"{mdata_const.filename}\"\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
- " mdata_dict['physical-detector-unit'] = mdata_const.device_name\n",
- " else:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = const_dict[cname][:2]\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = None\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = const_dict[cname][:2]\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = None\n",
"\n",
" return qm, mdata_dict, karabo_da, acq_rate, local_max_cells, err"
]
@@ -320,7 +326,7 @@
"only_dark = False\n",
"nodb_with_dark = False\n",
"if not nodb:\n",
- " only_dark=(calfile != \"\")\n",
+ " only_dark = (calfile != \"\")\n",
"if calfile != \"\" and not corr_bools[\"only_offset\"]:\n",
" nodb_with_dark = nodb\n",
"\n",
@@ -346,7 +352,7 @@
"outputs": [],
"source": [
"with multiprocessing.Pool(processes=nmods) as pool:\n",
- " results = pool.starmap(p, inp)"
+ " results = pool.starmap(retrieve_constants, inp)"
]
},
{
@@ -365,16 +371,21 @@
" print(f\"Error for module {qm}: {err}\")\n",
" mdata_dict[karabo_da] = md_dict\n",
"# check if it is requested not to retrieve any constants from the database\n",
- "if not nodb_with_dark:\n",
+ "if nodb_with_dark:\n",
+ " print(\"No constants were retrieved as calibrated files will be used.\")\n",
+ "else:\n",
" metadata.update({\"retrieved-constants\": mdata_dict})\n",
"\n",
- " print(\"\\nRetrieved constants for modules: \",\n",
- " f\"{[', '.join([f'Q{x//4+1}M{x%4+1}' for x in modules])]}\")\n",
- " print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Memory cells: {max_cells}\\n\"\n",
- " f\"• Acquisition rate: {acq_rate}\\n• Gain setting: {gain_setting}\\n• Photon Energy: {photon_energy}\\n\")\n",
- " print(\"Constant metadata is saved under \\\"retrieved-constants\\\" in calibration_metadata.yml\\n\")\n",
- "else:\n",
- " print(\"No constants were retrieved as calibrated files will be used.\")"
+ " print(\"\\nRetrieved constants for modules:\",\n",
+ " ', '.join([tools.module_index_to_qm(x) for x in modules]))\n",
+ " print(f\"Operating conditions are:\")\n",
+ " print(f\"• Bias voltage: {bias_voltage}\")\n",
+ " print(f\"• Memory cells: {max_cells}\\n\")\n",
+ " print(f\"• Acquisition rate: {acq_rate}\")\n",
+ " print(f\"• Gain mode: {gain_mode.name}\")\n",
+ " print(f\"• Gain setting: {gain_setting}\")\n",
+ " print(f\"• Photon Energy: {photon_energy}\")\n",
+ " print(\"Constant metadata is saved under \\\"retrieved-constants\\\" in calibration_metadata.yml\\n\")"
]
},
{
--
GitLab
From ef0a008aebdeb4c203f2bbba4fc7903d106bb711 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Mon, 15 Mar 2021 15:45:04 +0100
Subject: [PATCH 25/72] Flake8 and related fixes
- trailing whitespace
- unused variables
- unused imports
- misc formatting
---
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 197 ++++++------------
...IPD_Retrieve_Constants_Precorrection.ipynb | 6 +-
2 files changed, 68 insertions(+), 135 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 82f22b590..f13e87851 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -14,12 +14,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-21T11:30:06.730220Z",
- "start_time": "2019-02-21T11:30:06.658286Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"in_folder = \"/gpfs/exfel/exp/HED/202031/p900174/raw\" # the folder to read data from, required\n",
@@ -51,7 +46,7 @@
"gain_setting = 0.1 # the gain setting, use 0.1 to try to auto-determine\n",
"photon_energy = 9.2 # photon energy in keV\n",
"overwrite = True # set to True if existing data should be overwritten\n",
- "max_pulses = [0, 500, 1] # range list [st, end, step] of maximum pulse indices within a train. 3 allowed maximum list input elements. \n",
+ "max_pulses = [0, 500, 1] # range list [st, end, step] of maximum pulse indices within a train. 3 allowed maximum list input elements.\n",
"mem_cells_db = 0 # set to a value different than 0 to use this value for DB queries\n",
"cell_id_preview = 1 # cell Id used for preview in single-shot plots\n",
"\n",
@@ -71,7 +66,7 @@
"xray_gain = False # do relative gain correction based on xray data\n",
"blc_noise = False # if set, baseline correction via noise peak location is attempted\n",
"blc_stripes = False # if set, baseline corrected via stripes\n",
- "blc_hmatch = False # if set, base line correction via histogram matching is attempted \n",
+ "blc_hmatch = False # if set, base line correction via histogram matching is attempted\n",
"match_asics = False # if set, inner ASIC borders are matched to the same signal level\n",
"adjust_mg_baseline = False # adjust medium gain baseline to match highest high gain value\n",
"zero_nans = False # set NaN values in corrected data to 0\n",
@@ -104,8 +99,6 @@
"metadata": {},
"outputs": [],
"source": [
- "import copy\n",
- "import gc\n",
"import itertools\n",
"import math\n",
"import re\n",
@@ -114,11 +107,11 @@
"from datetime import timedelta\n",
"from multiprocessing import Pool\n",
"from pathlib import Path\n",
- "from time import perf_counter, sleep, time\n",
+ "from time import perf_counter\n",
"\n",
"import tabulate\n",
"from dateutil import parser\n",
- "from IPython.display import HTML, Latex, Markdown, display\n",
+ "from IPython.display import Latex, Markdown, display\n",
"\n",
"warnings.filterwarnings('ignore')\n",
"import matplotlib\n",
@@ -126,11 +119,8 @@
"import yaml\n",
"from extra_data import RunDirectory, stack_detector_data\n",
"from extra_geom import AGIPD_1MGeometry, AGIPD_500K2GGeometry\n",
- "from iCalibrationDB import Detectors\n",
"from matplotlib import cm as colormap\n",
"from matplotlib.colors import LogNorm\n",
- "from matplotlib.ticker import FormatStrFormatter, LinearLocator\n",
- "from mpl_toolkits.mplot3d import Axes3D\n",
"\n",
"matplotlib.use(\"agg\")\n",
"%matplotlib inline\n",
@@ -148,8 +138,7 @@
"from cal_tools.cython import agipdalgs as calgs\n",
"from cal_tools.enums import AgipdGainMode, BadPixels\n",
"from cal_tools.step_timing import StepTimer\n",
- "from cal_tools.tools import (CalibrationMetadata, get_dir_creation_date,\n",
- " map_modules_from_folder, module_index_to_qm)\n",
+ "from cal_tools.tools import (get_dir_creation_date, map_modules_from_folder, module_index_to_qm)\n",
"\n",
"sns.set()\n",
"sns.set_context(\"paper\", font_scale=1.4)\n",
@@ -255,7 +244,7 @@
" karabo_da = [\"AGIPD{:02d}\".format(i) for i in modules]\n",
"else:\n",
" modules = [int(x[-2:]) for x in karabo_da]\n",
- " \n",
+ "\n",
"print(\"Process modules: \", ', '.join(\n",
" [module_index_to_qm(x) for x in modules]))\n",
"print(f\"Detector in use is {karabo_id}\")\n",
@@ -270,17 +259,17 @@
"outputs": [],
"source": [
"# Display Information about the selected pulses indices for correction.\n",
- "pulses_lst = list(range(*max_pulses)) if not (len(max_pulses)==1 and max_pulses[0]==0) else max_pulses \n",
+ "pulses_lst = list(range(*max_pulses)) if not (len(max_pulses)==1 and max_pulses[0]==0) else max_pulses\n",
"\n",
"try:\n",
- " if len(pulses_lst) > 1: \n",
+ " if len(pulses_lst) > 1:\n",
" print(\"A range of {} pulse indices is selected: from {} to {} with a step of {}\"\n",
" .format(len(pulses_lst), pulses_lst[0] , pulses_lst[-1] + (pulses_lst[1] - pulses_lst[0]),\n",
" pulses_lst[1] - pulses_lst[0]))\n",
" else:\n",
- " print(\"one pulse is selected: a pulse of idx {}\".format(pulses_lst[0]))\n",
+ " print(f\"one pulse is selected: a pulse of idx {pulses_lst[0]}\")\n",
"except Exception as e:\n",
- " raise ValueError('max_pulses input Error: {}'.format(e))"
+ " raise ValueError(f\"max_pulses input Error: {e}\")"
]
},
{
@@ -352,7 +341,7 @@
" delta = timedelta(hours=offset.hour,\n",
" minutes=offset.minute, seconds=offset.second)\n",
" creation_time += delta\n",
- " \n",
+ "\n",
"# Evaluate gain setting\n",
"if gain_setting == 0.1:\n",
" if creation_time.replace(tzinfo=None) < parser.parse('2020-01-31'):\n",
@@ -434,11 +423,10 @@
"def retrieve_constants(mod):\n",
" \"\"\"\n",
" Retrieve calibration constants and load them to shared memory\n",
- " \n",
+ "\n",
" Metadata for constants is taken from yml file or retrieved from the DB\n",
" \"\"\"\n",
- " err = ''\n",
- " # TODO: parallelize over modules\n",
+ " err = \"\"\n",
" k_da = karabo_da[mod]\n",
" try:\n",
" # check if there is a yaml file in out_folder that has the device constants.\n",
@@ -446,8 +434,19 @@
" when = agipd_corr.initialize_from_yaml(k_da, const_yaml, mod)\n",
" else:\n",
" # TODO: should we save what is found here in metadata?\n",
- " when = agipd_corr.initialize_from_db(karabo_id, k_da, cal_db_interface, creation_time, mem_cells_db, bias_voltage,\n",
- " photon_energy, gain_setting, acq_rate, mod, False)\n",
+ " when = agipd_corr.initialize_from_db(\n",
+ " karabo_id,\n",
+ " k_da,\n",
+ " cal_db_interface,\n",
+ " creation_time,\n",
+ " mem_cells_db,\n",
+ " bias_voltage,\n",
+ " photon_energy,\n",
+ " gain_setting,\n",
+ " acq_rate,\n",
+ " mod,\n",
+ " False,\n",
+ " )\n",
" except Exception as e:\n",
" err = f\"Error: {e}\\nError traceback: {traceback.format_exc()}\"\n",
" when = None\n",
@@ -494,7 +493,7 @@
"source": [
"def imagewise_chunks(img_counts):\n",
" \"\"\"Break up the loaded data into chunks of up to chunk_size\n",
- " \n",
+ "\n",
" Yields (file data slot, start index, stop index)\n",
" \"\"\"\n",
" for i_proc, n_img in enumerate(img_counts):\n",
@@ -515,9 +514,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"with Pool() as pool:\n",
@@ -546,7 +543,7 @@
" # Perform image-wise correction\n",
" pool.starmap(agipd_corr.baseline_correction, imagewise_chunks(img_counts))\n",
" step_timer.done_step(\"Base-line shift correction\")\n",
- " \n",
+ "\n",
" if common_mode:\n",
" # Perform cross-file correction parallel over asics\n",
" pool.starmap(agipd_corr.cm_correction, itertools.product(\n",
@@ -584,9 +581,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"# if the yml file contains \"retrieved-constants\", that means a leading\n",
@@ -604,7 +599,7 @@
" if fst_print:\n",
" print(\"Constants are retrieved with creation time: \")\n",
" fst_print = False\n",
- " \n",
+ "\n",
" module_timestamps = {}\n",
"\n",
" # If correction is crashed\n",
@@ -652,8 +647,7 @@
" Z = data.T\n",
"\n",
" # Plot the surface.\n",
- " surf = ax.plot_surface(X, Y, Z, cmap=colormap.coolwarm,\n",
- " linewidth=0, antialiased=False)\n",
+ " ax.plot_surface(X, Y, Z, cmap=colormap.coolwarm, linewidth=0, antialiased=False)\n",
" ax.set_xlabel(x_axis)\n",
" ax.set_ylabel(y_axis)\n",
" ax.set_zlabel(\"Counts\")\n",
@@ -680,7 +674,7 @@
"source": [
"def get_trains_data(run_folder, source, include, detector_id, tid=None, modules=16, fillvalue=np.nan):\n",
" \"\"\"Load single train for all module\n",
- " \n",
+ "\n",
" :param run_folder: Path to folder with data\n",
" :param source: Data source to be loaded\n",
" :param include: Inset of file name to be considered\n",
@@ -693,7 +687,7 @@
" tid, data = run_data.select(f'{detector_id}/DET/*', source).train_from_id(tid)\n",
" else:\n",
" tid, data = next(iter(run_data.select(f'{detector_id}/DET/*', source).trains(require_all=True)))\n",
- " \n",
+ "\n",
" return tid, stack_detector_data(train=data, data=source, fillvalue=fillvalue, modules=modules)"
]
},
@@ -791,7 +785,7 @@
"print(f\"Gain statistics in %\")\n",
"table = [[f'{gains[gains==0].size/gains.size*100:.02f}',\n",
" f'{gains[gains==1].size/gains.size*100:.03f}',\n",
- " f'{gains[gains==2].size/gains.size*100:.03f}']] \n",
+ " f'{gains[gains==2].size/gains.size*100:.03f}']]\n",
"md = display(Latex(tabulate.tabulate(table, tablefmt='latex',\n",
" headers=[\"High\", \"Medium\", \"Low\"])))"
]
@@ -806,9 +800,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"pulse_range = [np.min(pulseId[pulseId>=0]), np.max(pulseId[pulseId>=0])]\n",
@@ -882,12 +874,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:33.226396Z",
- "start_time": "2019-02-18T17:29:27.027758Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
@@ -923,12 +910,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:33.761015Z",
- "start_time": "2019-02-18T17:29:33.227922Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
@@ -941,24 +923,19 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:35.903487Z",
- "start_time": "2019-02-18T17:29:33.762568Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
"ax = fig.add_subplot(111)\n",
"vmin, vmax = get_range(corrected[cell_id_preview], 5, -50)\n",
"nbins = np.int((vmax + 50) / 2)\n",
- "h = ax.hist(corrected[cell_id_preview].flatten(), \n",
- " bins=nbins, range=(-50, vmax), \n",
+ "h = ax.hist(corrected[cell_id_preview].flatten(),\n",
+ " bins=nbins, range=(-50, vmax),\n",
" histtype='stepfilled', log=True)\n",
- "_ = plt.xlabel('[ADU]')\n",
- "_ = plt.ylabel('Counts')\n",
- "_ = ax.grid()"
+ "plt.xlabel('[ADU]')\n",
+ "plt.ylabel('Counts')\n",
+ "ax.grid()"
]
},
{
@@ -968,18 +945,13 @@
"outputs": [],
"source": [
"display(Markdown('### Mean CORRECTED Preview ###\\n'))\n",
- "display(Markdown(f'A mean across one train \\n'))"
+ "display(Markdown(f'A mean across one train\\n'))"
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:39.369686Z",
- "start_time": "2019-02-18T17:29:35.905152Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
@@ -992,12 +964,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:49.217848Z",
- "start_time": "2019-02-18T17:29:39.371232Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
@@ -1009,16 +976,16 @@
"nbins = np.int((vmax + 100) / 5)\n",
"h = ax.hist(corrected.flatten(), bins=nbins,\n",
" range=(-100, vmax), histtype='step', log=True, label = 'All')\n",
- "_ = ax.hist(corrected[gains == 0].flatten(), bins=nbins, range=(-100, vmax),\n",
- " alpha=0.5, log=True, label='High gain', color='green')\n",
- "_ = ax.hist(corrected[gains == 1].flatten(), bins=nbins, range=(-100, vmax),\n",
- " alpha=0.5, log=True, label='Medium gain', color='red')\n",
- "_ = ax.hist(corrected[gains == 2].flatten(), bins=nbins,\n",
- " range=(-100, vmax), alpha=0.5, log=True, label='Low gain', color='yellow')\n",
- "_ = ax.legend()\n",
- "_ = ax.grid()\n",
- "_ = plt.xlabel('[ADU]')\n",
- "_ = plt.ylabel('Counts')"
+ "ax.hist(corrected[gains == 0].flatten(), bins=nbins, range=(-100, vmax),\n",
+ " alpha=0.5, log=True, label='High gain', color='green')\n",
+ "ax.hist(corrected[gains == 1].flatten(), bins=nbins, range=(-100, vmax),\n",
+ " alpha=0.5, log=True, label='Medium gain', color='red')\n",
+ "ax.hist(corrected[gains == 2].flatten(), bins=nbins, range=(-100, vmax),\n",
+ " alpha=0.5, log=True, label='Low gain', color='yellow')\n",
+ "ax.legend()\n",
+ "ax.grid()\n",
+ "plt.xlabel('[ADU]')\n",
+ "plt.ylabel('Counts')"
]
},
{
@@ -1034,12 +1001,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:49.641675Z",
- "start_time": "2019-02-18T17:29:49.224167Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
@@ -1050,9 +1012,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
"source": [
"## Bad Pixels ##\n",
"The mask contains dedicated entries for all pixels and memory cells as well as all three gains stages. Each mask entry is encoded in 32 bits as:"
@@ -1061,12 +1021,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:49.651913Z",
- "start_time": "2019-02-18T17:29:49.643556Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"table = []\n",
@@ -1089,24 +1044,17 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:50.086169Z",
- "start_time": "2019-02-18T17:29:49.653391Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
"ax = fig.add_subplot(111)\n",
- "ax = geom.plot_data_fast(np.log2(mask[cell_id_preview]), ax=ax, vmin=0, vmax=32, cmap=\"jet\")"
+ "geom.plot_data_fast(np.log2(mask[cell_id_preview]), ax=ax, vmin=0, vmax=32, cmap=\"jet\")"
]
},
{
"cell_type": "markdown",
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
"source": [
"### Percentage of Bad Pixels across one train ###"
]
@@ -1114,18 +1062,12 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:51.686562Z",
- "start_time": "2019-02-18T17:29:50.088883Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
"ax = fig.add_subplot(111)\n",
- "ax = geom.plot_data_fast(np.mean(mask>0, axis=0),\n",
- " vmin=0, ax=ax, vmax=1, cmap=\"jet\")"
+ "geom.plot_data_fast(np.mean(mask>0, axis=0), vmin=0, ax=ax, vmax=1, cmap=\"jet\")"
]
},
{
@@ -1138,12 +1080,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:55.483270Z",
- "start_time": "2019-02-18T17:29:53.664226Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 73e3ed6d4..f6b40e513 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -133,10 +133,6 @@
"print(f\"Outputting to {out_folder}\")\n",
"out_folder.mkdir(parents=True, exist_ok=True)\n",
"\n",
- "import warnings\n",
- "\n",
- "warnings.filterwarnings('ignore')\n",
- "\n",
"melt_snow = False if corr_bools[\"only_offset\"] else agipdlib.SnowResolution.NONE"
]
},
@@ -380,7 +376,7 @@
" ', '.join([tools.module_index_to_qm(x) for x in modules]))\n",
" print(f\"Operating conditions are:\")\n",
" print(f\"• Bias voltage: {bias_voltage}\")\n",
- " print(f\"• Memory cells: {max_cells}\\n\")\n",
+ " print(f\"• Memory cells: {max_cells}\")\n",
" print(f\"• Acquisition rate: {acq_rate}\")\n",
" print(f\"• Gain mode: {gain_mode.name}\")\n",
" print(f\"• Gain setting: {gain_setting}\")\n",
--
GitLab
From a324b12f9a3c26715b26ee1e76d1e9a47627cd06 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 18 Mar 2021 12:25:04 +0100
Subject: [PATCH 26/72] Cleanup
---
...IPD_Retrieve_Constants_Precorrection.ipynb | 39 ++++++-------------
1 file changed, 12 insertions(+), 27 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index f6b40e513..664f20837 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -51,7 +51,7 @@
"xray_gain = True # do relative gain correction based on xray data\n",
"blc_noise = False # if set, baseline correction via noise peak location is attempted\n",
"blc_stripes = False # if set, baseline corrected via stripes\n",
- "blc_hmatch = False # if set, base line correction via histogram matching is attempted \n",
+ "blc_hmatch = False # if set, base line correction via histogram matching is attempted\n",
"match_asics = False # if set, inner ASIC borders are matched to the same signal level\n",
"adjust_mg_baseline = False # adjust medium gain baseline to match highest high gain value"
]
@@ -205,14 +205,11 @@
"outputs": [],
"source": [
"def retrieve_constants(\n",
- " qm_files: List[Path],\n",
- " qm: str,\n",
- " karabo_da: str,\n",
- " idx: int\n",
+ " qm_files: List[Path], qm: str, karabo_da: str, idx: int\n",
") -> Tuple[str, str, float, float, str, dict]:\n",
" \"\"\"\n",
" Retrieve constants for a module.\n",
- " \n",
+ "\n",
" :return:\n",
" qm: module virtual name i.e. Q1M1.\n",
" karabo_da: karabo data aggregator.\n",
@@ -236,7 +233,7 @@
" if local_max_cells is None:\n",
" raise ValueError(f\"No raw images found for {qm} for all sequences\")\n",
"\n",
- " if acq_rate == 0: \n",
+ " if acq_rate == 0:\n",
" local_acq_rate = agipdlib.get_acq_rate(fast_paths=(f, karabo_id, idx))\n",
" else:\n",
" local_acq_rate = acq_rate\n",
@@ -244,7 +241,7 @@
" # avoid retrieving constant, if requested.\n",
" if nodb_with_dark:\n",
" return\n",
- " \n",
+ "\n",
" const_dict = agipdlib.assemble_constant_dict(\n",
" corr_bools,\n",
" pc_bools,\n",
@@ -359,13 +356,11 @@
"source": [
"mod_dev = dict()\n",
"mdata_dict = dict()\n",
- "for r in results:\n",
- " if r:\n",
- " qm, md_dict, karabo_da, acq_rate, max_cells, err = r\n",
- " mod_dev[karabo_da] = {\"mod\": qm, \"err\": err}\n",
- " if err:\n",
- " print(f\"Error for module {qm}: {err}\")\n",
- " mdata_dict[karabo_da] = md_dict\n",
+ "for qm, md_dict, karabo_da, acq_rate, max_cells, err in results:\n",
+ " mod_dev[karabo_da] = {\"mod\": qm, \"err\": err}\n",
+ " if err:\n",
+ " print(f\"Error for module {qm}: {err}\")\n",
+ " mdata_dict[karabo_da] = md_dict\n",
"# check if it is requested not to retrieve any constants from the database\n",
"if nodb_with_dark:\n",
" print(\"No constants were retrieved as calibrated files will be used.\")\n",
@@ -390,12 +385,11 @@
"metadata": {},
"outputs": [],
"source": [
- "print(\"Constants are retrieved with creation time: \")\n",
- "i = 0\n",
+ "print(\"Constants are retrieved with creation time:\")\n",
"timestamps = {}\n",
"\n",
"for k_da, dinfo in mod_dev.items():\n",
- " print(dinfo[\"mod\"], \":\")\n",
+ " print(f\"{dinfo['mod']}:\")\n",
" module_timestamps = {}\n",
" module_name = dinfo[\"mod\"]\n",
" if k_da in mdata_dict:\n",
@@ -403,8 +397,6 @@
" if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
" mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
" print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
- " # Store few time stamps if exists\n",
- " # Add NA to keep array structure\n",
" for cname in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
" if k_da not in mdata_dict or dinfo[\"err\"]:\n",
" module_timestamps[cname] = \"Err\"\n",
@@ -418,13 +410,6 @@
" module_timestamps[cname] = \"NA\"\n",
" timestamps[module_name] = module_timestamps\n",
"\n",
- " i += 1\n",
- " if sequences:\n",
- " seq_num = sequences[0]\n",
- " else:\n",
- " # if sequences[0] changed to None as it was -1\n",
- " seq_num = 0\n",
- "\n",
"time_summary = metadata.setdefault(\"retrieved-constants\", {}).setdefault(\"time-summary\", {})\n",
"time_summary[\"SAll\"] = timestamps\n",
"\n",
--
GitLab
From 80de4713ef0f5d9c92692975a12e2c07e0eb5bab Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 18 Mar 2021 12:29:32 +0100
Subject: [PATCH 27/72] Hack: for FG, get some regular constants, disable some
corrections
---
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 76 ++++++++++++------
...IPD_Retrieve_Constants_Precorrection.ipynb | 77 +++++++++++--------
2 files changed, 99 insertions(+), 54 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index f13e87851..c66a23a15 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -131,6 +131,7 @@
"sns.set_context(\"paper\", font_scale=1.4)\n",
"sns.set_style(\"ticks\")\n",
"\n",
+ "import cal_tools\n",
"import seaborn as sns\n",
"from cal_tools.agipdlib import (AgipdCorrections, get_acq_rate, get_gain_mode,\n",
" get_gain_setting, get_num_cells)\n",
@@ -138,7 +139,6 @@
"from cal_tools.cython import agipdalgs as calgs\n",
"from cal_tools.enums import AgipdGainMode, BadPixels\n",
"from cal_tools.step_timing import StepTimer\n",
- "from cal_tools.tools import (get_dir_creation_date, map_modules_from_folder, module_index_to_qm)\n",
"\n",
"sns.set()\n",
"sns.set_context(\"paper\", font_scale=1.4)\n",
@@ -170,7 +170,7 @@
"source": [
"# Fill dictionaries comprising bools and arguments for correction and data analysis\n",
"\n",
- "# Here the herarichy and dependability for correction booleans are defined\n",
+ "# Here the hierarchy and dependability for correction booleans are defined\n",
"corr_bools = {}\n",
"\n",
"# offset is at the bottom of AGIPD correction pyramid.\n",
@@ -195,8 +195,16 @@
" corr_bools[\"common_mode\"] = common_mode\n",
" corr_bools[\"melt_snow\"] = melt_snow\n",
" corr_bools[\"mask_zero_std\"] = mask_zero_std\n",
- " corr_bools[\"low_medium_gap\"] = low_medium_gap \n",
- " "
+ " corr_bools[\"low_medium_gap\"] = low_medium_gap\n",
+ "\n",
+ "# Many corrections don't apply to fixed gain mode; will explicitly disable later if detected\n",
+ "disable_for_fixed_gain = [\n",
+ " \"adjust_mg_baseline\",\n",
+ " \"blc_set_min\",\n",
+ " \"force_hg_if_below\",\n",
+ " \"force_mg_if_below\",\n",
+ " \"low_medium_gap\"\n",
+ "]"
]
},
{
@@ -245,8 +253,7 @@
"else:\n",
" modules = [int(x[-2:]) for x in karabo_da]\n",
"\n",
- "print(\"Process modules: \", ', '.join(\n",
- " [module_index_to_qm(x) for x in modules]))\n",
+ "print(\"Process modules:\", ', '.join(cal_tools.tools.module_index_to_qm(x) for x in modules))\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"print(f\"Instrument {instrument}\")\n",
"print(f\"Detector instance {dinstance}\")"
@@ -279,9 +286,9 @@
"outputs": [],
"source": [
"# set everything up filewise\n",
- "mmf = map_modules_from_folder(str(in_folder), run, path_template,\n",
- " karabo_da, sequences)\n",
- "mapped_files, mod_ids, total_sequences, sequences_qm, _ = mmf\n",
+ "mapped_files, _, total_sequences, _, _ = cal_tools.tools.map_modules_from_folder(\n",
+ " str(in_folder), run, path_template, karabo_da, sequences\n",
+ ")\n",
"file_list = []\n",
"\n",
"# ToDo: Split table over pages\n",
@@ -336,10 +343,9 @@
"# Evaluate creation time\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(str(in_folder), run)\n",
+ " creation_time = cal_tools.tools.get_dir_creation_date(str(in_folder), run)\n",
" offset = parser.parse(creation_date_offset)\n",
- " delta = timedelta(hours=offset.hour,\n",
- " minutes=offset.minute, seconds=offset.second)\n",
+ " delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
" creation_time += delta\n",
"\n",
"# Evaluate gain setting\n",
@@ -376,6 +382,19 @@
"print(f\"• Photon Energy: {photon_energy}\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if gain_mode:\n",
+ " for to_disable in disable_for_fixed_gain:\n",
+ " if corr_bools.get(to_disable, False):\n",
+ " print(f\"Warning: {to_disable} correction was requested, but does not apply to fixed gain mode\")\n",
+ " corr_bools[to_disable] = False"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -389,10 +408,13 @@
"metadata": {},
"outputs": [],
"source": [
- "agipd_corr = AgipdCorrections(max_cells, max_pulses,\n",
- " h5_data_path=h5path,\n",
- " h5_index_path=h5path_idx,\n",
- " corr_bools=corr_bools)\n",
+ "agipd_corr = AgipdCorrections(\n",
+ " max_cells,\n",
+ " max_pulses,\n",
+ " h5_data_path=h5path,\n",
+ " h5_index_path=h5path_idx,\n",
+ " corr_bools=corr_bools\n",
+ ")\n",
"\n",
"agipd_corr.baseline_corr_noise_threshold = -blc_noise_threshold\n",
"agipd_corr.hg_hard_threshold = hg_hard_threshold\n",
@@ -406,6 +428,15 @@
"agipd_corr.ff_gain = ff_gain"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "module_index_to_karabo_da = {mod: da for (mod, da) in zip(modules, karabo_da)}"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -415,11 +446,10 @@
"# Retrieve calibration constants to RAM\n",
"agipd_corr.allocate_constants(modules, (3, mem_cells_db, 512, 128))\n",
"\n",
- "metadata = CalibrationMetadata(out_folder)\n",
+ "metadata = cal_tools.tools.CalibrationMetadata(out_folder)\n",
"# NOTE: this notebook will not overwrite calibration metadata file\n",
"const_yaml = metadata.get(\"retrieved-constants\", {})\n",
"\n",
- "# retrive constants\n",
"def retrieve_constants(mod):\n",
" \"\"\"\n",
" Retrieve calibration constants and load them to shared memory\n",
@@ -427,13 +457,15 @@
" Metadata for constants is taken from yml file or retrieved from the DB\n",
" \"\"\"\n",
" err = \"\"\n",
- " k_da = karabo_da[mod]\n",
+ " k_da = module_index_to_karabo_da[mod]\n",
" try:\n",
" # check if there is a yaml file in out_folder that has the device constants.\n",
" if k_da in const_yaml:\n",
+ " print(f\"Pre-correction notebook already found constants for {k_da}\")\n",
" when = agipd_corr.initialize_from_yaml(k_da, const_yaml, mod)\n",
" else:\n",
- " # TODO: should we save what is found here in metadata?\n",
+ " print(f\"Have to query database for constants for {k_da}\")\n",
+ " # TODO: replace with proper retrieval (as done in pre-correction)\n",
" when = agipd_corr.initialize_from_db(\n",
" karabo_id,\n",
" k_da,\n",
@@ -466,7 +498,7 @@
"outputs": [],
"source": [
"# allocate memory for images and hists\n",
- "n_images_max = max_cells*256\n",
+ "n_images_max = max_cells * 256\n",
"data_shape = (n_images_max, 512, 128)\n",
"agipd_corr.allocate_images(data_shape, n_cores_files)"
]
@@ -590,7 +622,7 @@
"timestamps = {}\n",
"\n",
"for i, (error, modno, when, k_da) in enumerate(const_out):\n",
- " qm = module_index_to_qm(modno)\n",
+ " qm = cal_tools.tools.module_index_to_qm(modno)\n",
" # expose errors while applying correction\n",
" if error:\n",
" print(\"Error: {}\".format(error) )\n",
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 664f20837..d579581bf 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -258,41 +258,54 @@
" # Retrieve multiple constants through an input dictionary\n",
" # to return a dict of useful metadata.\n",
" mdata_dict = dict()\n",
- " mdata_dict['constants'] = dict()\n",
- " mdata_dict['physical-detector-unit'] = None # initialization\n",
- "\n",
- " for cname, cval in const_dict.items():\n",
- " print(cname)\n",
- " print(cval)\n",
+ " mdata_dict[\"constants\"] = dict()\n",
+ " mdata_dict[\"physical-detector-unit\"] = None # initialization\n",
+ "\n",
+ " for const_name, (const_init_fun, const_shape, (cond_type, cond_param)) in const_dict.items():\n",
+ " if gain_mode and const_name in (\"ThresholdsDark\",):\n",
+ " print(f\"Note: skipping {const_name} for fixed gain mode\")\n",
+ " continue\n",
+ " \n",
" # saving metadata in a dict\n",
- " mdata_dict['constants'][cname] = dict()\n",
- "\n",
- " if slopes_ff_from_files and cname in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = \"00:00:00\"\n",
+ " const_mdata = dict()\n",
+ " mdata_dict[\"constants\"][const_name] = const_mdata\n",
+ "\n",
+ " if slopes_ff_from_files and const_name in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
+ " const_mdata[\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
+ " const_mdata[\"creation-time\"] = \"00:00:00\"\n",
+ " continue\n",
+ " \n",
+ " if gain_mode and const_name in (\"BadPixelsPC\", \"SlopesPC\", \"BadPixelsFF\", \"SlopesFF\"):\n",
+ " param_copy = cond_param.copy()\n",
+ " del param_copy[\"gain_mode\"]\n",
+ " condition = getattr(Conditions, cond_type).AGIPD(**param_copy)\n",
+ " print(f\"Note: {const_name} based on adaptive gain mode constants will be retrieved\")\n",
" else:\n",
- " condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
- " co, mdata = tools.get_from_db(\n",
- " karabo_id,\n",
- " karabo_da,\n",
- " getattr(Constants.AGIPD, cname)(),\n",
- " condition,\n",
- " getattr(np, cval[0])(cval[1]),\n",
- " cal_db_interface,\n",
- " creation_time,\n",
- " meta_only=True,\n",
- " verbosity=1,\n",
+ " condition = getattr(Conditions, cond_type).AGIPD(**cond_param)\n",
+ "\n",
+ " _, mdata = tools.get_from_db(\n",
+ " karabo_id,\n",
+ " karabo_da,\n",
+ " getattr(Constants.AGIPD, const_name)(),\n",
+ " condition,\n",
+ " getattr(np, const_init_fun)(const_shape),\n",
+ " cal_db_interface,\n",
+ " creation_time,\n",
+ " meta_only=True,\n",
+ " verbosity=1,\n",
+ " )\n",
+ " mdata_const = mdata.calibration_constant_version\n",
+ " # check if constant was sucessfully retrieved.\n",
+ " if mdata.comm_db_success:\n",
+ " const_mdata[\"file-path\"] = (\n",
+ " f\"{mdata_const.hdf5path}\" f\"{mdata_const.filename}\"\n",
" )\n",
- " mdata_const = mdata.calibration_constant_version\n",
- " # check if constant was sucessfully retrieved.\n",
- " if mdata.comm_db_success:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
- " f\"{mdata_const.filename}\"\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
- " mdata_dict['physical-detector-unit'] = mdata_const.device_name\n",
- " else:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = const_dict[cname][:2]\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = None\n",
+ " const_mdata[\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
+ " mdata_dict[\"physical-detector-unit\"] = mdata_const.device_name\n",
+ " else:\n",
+ " const_mdata[\"file-path\"] = const_dict[const_name][:2]\n",
+ " const_mdata[\"creation-time\"] = None\n",
+ " print(const_mdata)\n",
"\n",
" return qm, mdata_dict, karabo_da, acq_rate, local_max_cells, err"
]
--
GitLab
From fdd0338dc2abe0f1087dcda4f82b53c0eab552f5 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 18 Mar 2021 19:20:14 +0100
Subject: [PATCH 28/72] Formatting + minor typos
---
cal_tools/cal_tools/agipdlib.py | 58 +++++++++++++++------------------
1 file changed, 26 insertions(+), 32 deletions(-)
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 48156b022..83f8dcb0d 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -15,8 +15,7 @@ from cal_tools.cython import agipdalgs as calgs
def get_num_cells(fname, loc, module):
with h5py.File(fname, "r") as f:
- cells = \
- f[f"INSTRUMENT/{loc}/DET/{module}CH0:xtdf/image/cellId"][()]
+ cells = f[f"INSTRUMENT/{loc}/DET/{module}CH0:xtdf/image/cellId"][()]
if cells.shape[0] == 0:
return None
maxcell = np.max(cells)
@@ -88,7 +87,7 @@ def get_gain_setting(fname: str, h5path_ctrl: str) -> int:
gain-setting 1: setupr@dark=8, setupr@slopespc=40
gain-setting 0: setupr@dark=0, setupr@slopespc=32
- patternTypeIndex 1: High-gian
+ patternTypeIndex 1: High-gain
patternTypeIndex 2: Medium-gain
patternTypeIndex 3: Low-gain
patternTypeIndex 4: SlopesPC
@@ -187,9 +186,7 @@ class AgipdCorrections:
const_yaml = metadata["retrieved-constants"]
for mod in modules:
- qm = f"Q{mod // 4 + 1}M{mod % 4 + 1}"
- agipd_corr.initialize_from_yaml(karabo_da,
- const_yaml, mod)
+ agipd_corr.initialize_from_yaml(karabo_da, const_yaml, mod)
data_shape = (n_images_max, 512, 128)
agipd_corr.allocate_images(data_shape, n_cores_files)
@@ -315,8 +312,7 @@ class AgipdCorrections:
data_dict['rawgain'][:n_img] = raw_data[:, 1]
data_dict['cellId'][:n_img] = allcells[firange]
data_dict['pulseId'][:n_img] = allpulses[firange]
- data_dict['trainId'][:n_img] = np.squeeze(
- group['trainId'][:][firange])
+ data_dict['trainId'][:n_img] = np.squeeze(group['trainId'][:][firange])
except Exception as e:
print(f'Error during reading data from file {file_name}: {e}')
print(f'Error traceback: {traceback.format_exc()}')
@@ -354,10 +350,10 @@ class AgipdCorrections:
# Copy any other data from the input file.
# This includes indexes, so it's important that the corrected data
# we write is aligned with the raw data.
- with h5py.File(file_name, 'r') as infile:
- self.copy_and_sanitize_non_cal_data(infile, outfile,
- agipd_base,
- idx_base, trains)
+ with h5py.File(file_name, "r") as infile:
+ self.copy_and_sanitize_non_cal_data(
+ infile, outfile, agipd_base, idx_base, trains
+ )
# All corrected data goes in a /INSTRUMENT/.../image group
image_grp = outfile[data_path]
@@ -545,9 +541,9 @@ class AgipdCorrections:
cellid = self.shared_dict[i_proc]['cellId'][first:last]
# output is saved in sharedmem to pass for correct_agipd()
# as this function takes about 3 seconds.
- self.shared_dict[i_proc]['msk'][first:last] = \
- calgs.gain_choose_int(gain,
- self.mask[module_idx][:, cellid]) # noqa
+ self.shared_dict[i_proc]["msk"][first:last] = calgs.gain_choose_int(
+ gain, self.mask[module_idx][:, cellid]
+ )
if hasattr(self, "rel_gain"):
# Get the correct rel_gain depending on cell-id
@@ -620,14 +616,12 @@ class AgipdCorrections:
# if baseline correction was not requested
# msk and rel_corr will still be empty shared_mem arrays
if not any(self.blc_bools):
- msk = calgs.gain_choose_int(gain,
- self.mask[module_idx][:, cellid])
+ msk = calgs.gain_choose_int(gain, self.mask[module_idx][:, cellid])
# same for relative gain and then bad pixel mask
if hasattr(self, "rel_gain"):
# Get the correct rel_gain depending on cell-id
- rel_corr = calgs.gain_choose(gain,
- self.rel_gain[module_idx][:, cellid]) # noqa
+ rel_corr = calgs.gain_choose(gain, self.rel_gain[module_idx][:, cellid])
# Correct for relative gain
if self.corr_bools.get("rel_gain") and hasattr(self, "rel_gain"):
@@ -690,11 +684,9 @@ class AgipdCorrections:
# Copy the data across into the existing shared-memory array
mask[...] = msk[...]
- def get_valid_image_idx(self, idx_base: str, infile: str,
- index_v: Optional[int] = 2):
- """ Return the indices of valid data
- """
- if index_v == 2:
+ def get_valid_image_idx(self, idx_base: str, infile: str, raw_format_version: int = 2):
+ """Return the indices of valid data"""
+ if raw_format_version == 2:
count = np.squeeze(infile[idx_base + "image/count"])
first = np.squeeze(infile[idx_base + "image/first"])
if np.count_nonzero(count != 0) == 0:
@@ -719,13 +711,16 @@ class AgipdCorrections:
# Creating an array of validated indices.
# If all indices were validated this array will be the same,
# as what is stored at /DET/image/trainId
- valid_indices = np.concatenate([np.arange(validf[i],
- validf[i]+validc[i])
- for i in range(validf.size)],
- axis=0)
+ valid_indices = np.concatenate(
+ [
+ np.arange(validf[i], validf[i] + validc[i])
+ for i in range(validf.size)
+ ],
+ axis=0,
+ )
valid_indices = np.squeeze(valid_indices).astype(np.int32)
- elif index_v == 1:
+ elif raw_format_version == 1:
status = np.squeeze(infile[idx_base + "image/status"])
if np.count_nonzero(status != 0) == 0:
raise IOError(f"File {infile} has no valid counts")
@@ -743,10 +738,9 @@ class AgipdCorrections:
valid_indices = None
else:
raise AttributeError(
- f"Not a known raw format version: {index_v}")
+ f"Not a known raw format version: {raw_format_version}")
- return (valid, first_index, last_index, idxtrains,
- valid_indices)
+ return (valid, first_index, last_index, idxtrains, valid_indices)
def apply_selected_pulses(self, i_proc: int) -> int:
"""Select sharedmem data indices to correct based on selected
--
GitLab
From 56214c75e096e9003a117d614d19ae7cb6a751a1 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 18 Mar 2021 20:16:11 +0100
Subject: [PATCH 29/72] Adding gain mode handling to AgipdCorrections
---
cal_tools/cal_tools/agipdlib.py | 105 ++++++++++++++++----------------
1 file changed, 52 insertions(+), 53 deletions(-)
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 83f8dcb0d..8daaa94e0 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -154,10 +154,15 @@ def get_bias_voltage(fname: str, karabo_id_control: str,
class AgipdCorrections:
- def __init__(self, max_cells, max_pulses,
- h5_data_path="INSTRUMENT/SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf/",
- h5_index_path="INDEX/SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf/",
- corr_bools: Optional[dict] = None):
+ def __init__(
+ self,
+ max_cells,
+ max_pulses,
+ h5_data_path="INSTRUMENT/SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf/",
+ h5_index_path="INDEX/SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf/",
+ corr_bools: Optional[dict] = None,
+ gain_mode: AgipdGainMode = AgipdGainMode.ADAPTIVE_GAIN,
+ ):
"""
Initialize an AgipdCorrections Class
@@ -212,6 +217,7 @@ class AgipdCorrections:
self.pulses_lst = list(range(*max_pulses)) \
if not (len(max_pulses) == 1 and max_pulses[0] == 0) else max_pulses #noqa
self.max_cells = max_cells
+ self.gain_mode = gain_mode
# Correction parameters
self.baseline_corr_noise_threshold = -1000
@@ -480,35 +486,35 @@ class AgipdCorrections:
gain = self.shared_dict[i_proc]['gain'][first:last]
cellid = self.shared_dict[i_proc]['cellId'][first:last]
- # first evaluate the gain into 0, 1, 2 --> high, medium, low
- t0 = self.thresholds[module_idx][0]
- t1 = self.thresholds[module_idx][1]
+ if self.gain_mode is AgipdGainMode.ADAPTIVE_GAIN:
+ # first evaluate the gain into 0, 1, 2 --> high, medium, low
+ t0 = self.thresholds[module_idx][0]
+ t1 = self.thresholds[module_idx][1]
- # load raw_data and rgain to be used during gain_correction,
- # if requested
- if self.corr_bools.get('melt_snow'):
- self.shared_dict[i_proc]['t0_rgain'][first:last] = \
- rawgain / t0[cellid, ...]
- self.shared_dict[i_proc]['raw_data'][first:last] = np.copy(data)
-
- # Often most pixels are in high-gain, so it's more efficient to
- # set the whole output block to zero than select the right pixels.
- gain[:] = 0
- # exceeding first threshold means data is medium or low gain
- gain[rawgain > t0[cellid, ...]] = 1
- # exceeding also second threshold means data is low gain
- gain[rawgain > t1[cellid, ...]] = 2
+ # load raw_data and rgain to be used during gain_correction if requested
+ if self.corr_bools.get("melt_snow"):
+ self.shared_dict[i_proc]["t0_rgain"][first:last] = rawgain / t0[cellid, ...]
+ self.shared_dict[i_proc]["raw_data"][first:last] = np.copy(data)
+
+ # Often most pixels are in high-gain, so it's more efficient to
+ # set the whole output block to zero than select the right pixels.
+ gain[:] = 0
+ # exceeding first threshold means data is medium or low gain
+ gain[rawgain > t0[cellid, ...]] = 1
+ # exceeding also second threshold means data is low gain
+ gain[rawgain > t1[cellid, ...]] = 2
+ else:
+ # the enum values map 1, 2, 3 to (fixed) gain modes
+ gain[:] = self.gain_mode - 1
offsetb = self.offset[module_idx][:, cellid]
# force into high or medium gain if requested
- if self.corr_bools.get('force_mg_if_below'):
- gain[(gain == 2) & (
- (data - offsetb[1]) < self.mg_hard_threshold)] = 1
+ if self.corr_bools.get("force_mg_if_below"):
+ gain[(gain == 2) & ((data - offsetb[1]) < self.mg_hard_threshold)] = 1
- if self.corr_bools.get('force_hg_if_below'):
- gain[(gain > 0) & (
- (data - offsetb[0]) < self.hg_hard_threshold)] = 0
+ if self.corr_bools.get("force_hg_if_below"):
+ gain[(gain > 0) & ((data - offsetb[0]) < self.hg_hard_threshold)] = 0
# choose constants according to gain setting
off = calgs.gain_choose(gain, offsetb)
@@ -1079,7 +1085,8 @@ class AgipdCorrections:
self.offset[module_idx][...] = cons_data["Offset"].transpose()[...]
self.noise[module_idx][...] = cons_data["Noise"].transpose()[...]
- self.thresholds[module_idx][...] = cons_data["ThresholdsDark"].transpose()[:3,...] # noqa
+ if self.gain_mode is AgipdGainMode.ADAPTIVE_GAIN:
+ self.thresholds[module_idx][...] = cons_data["ThresholdsDark"].transpose()[:3,...] # noqa
if self.corr_bools.get("low_medium_gap"):
t0 = self.thresholds[module_idx][0]
@@ -1221,20 +1228,18 @@ class AgipdCorrections:
return
- def initialize_from_yaml(self, karabo_da: str,
- const_yaml: Dict[str, Any],
- module_idx: int
- ) -> Dict[str, Any]:
+ def initialize_from_yaml(
+ self, karabo_da: str, const_yaml: Dict[str, Any], module_idx: int
+ ) -> Dict[str, Any]:
"""Initialize calibration constants from a yaml file
- :param karabo-da: karabo data aggerator
- :param const_yaml: (Dict) from the "retrieved-constants" part of a yaml
- file in pre-notebook, which consists of metadata of either the constant
+ :param karabo_da: a karabo data aggregator
+ :param const_yaml: from the "retrieved-constants" part of a yaml
+ file from pre-notebook, which consists of metadata of either the constant
file path or the empty constant shape, and the creation-time of the
retrieved constants
:param module_idx: Index of module.
- :return when: Dictionary of retrieved constants with
- their creation-time.
+ :return when: Dictionary of retrieved constants with their creation-time.
"""
# string of the device name.
@@ -1340,23 +1345,17 @@ class AgipdCorrections:
:param constant_shape: Shape of expected constants (gain, cells, x, y)
"""
for module_idx in modules:
- self.offset[module_idx] = sharedmem.empty(constant_shape,
- dtype='f4')
- self.thresholds[module_idx] = sharedmem.empty(constant_shape,
- dtype='f4')
- self.noise[module_idx] = sharedmem.empty(constant_shape,
- dtype='f4')
-
- self.md_additional_offset[module_idx] = sharedmem.empty(
- constant_shape[1:], dtype='f4')
- self.rel_gain[module_idx] = sharedmem.empty(constant_shape,
- dtype='f4')
- self.frac_high_med[module_idx] = sharedmem.empty(constant_shape[1],
- dtype='f4')
+ self.offset[module_idx] = sharedmem.empty(constant_shape, dtype="f4")
+ if self.gain_mode is AgipdGainMode.ADAPTIVE_GAIN:
+ self.thresholds[module_idx] = sharedmem.empty(constant_shape, dtype="f4")
+ self.noise[module_idx] = sharedmem.empty(constant_shape, dtype="f4")
+
+ self.md_additional_offset[module_idx] = sharedmem.empty(constant_shape[1:], dtype="f4")
+ self.rel_gain[module_idx] = sharedmem.empty(constant_shape, dtype="f4")
+ self.frac_high_med[module_idx] = sharedmem.empty(constant_shape[1], dtype="f4")
- self.mask[module_idx] = sharedmem.empty(constant_shape, dtype='i4')
- self.xray_cor[module_idx] = sharedmem.empty(constant_shape[1:],
- dtype='f4')
+ self.mask[module_idx] = sharedmem.empty(constant_shape, dtype="i4")
+ self.xray_cor[module_idx] = sharedmem.empty(constant_shape[1:], dtype="f4")
def allocate_images(self, shape, n_cores_files):
"""
--
GitLab
From 5b1d6e47ac2be427b4ff72c61e3e3c682b864450 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Fri, 19 Mar 2021 16:06:45 +0100
Subject: [PATCH 30/72] Formatting, absolute multiprocessing import, typos
---
cal_tools/cal_tools/agipdlib.py | 68 ++++++++-----------
cal_tools/cal_tools/agipdutils.py | 4 +-
cal_tools/cal_tools/tools.py | 3 +-
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 6 +-
4 files changed, 36 insertions(+), 45 deletions(-)
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 8daaa94e0..019808b84 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -952,13 +952,11 @@ class AgipdCorrections:
# Extract parameters through identifying
# unique trains, index and numbers.
- uq, fidxv, cntsv = np.unique(trains, return_index=True,
- return_counts=True)
+ uq, fidxv, cntsv = np.unique(trains, return_index=True, return_counts=True)
# Validate calculated CORR INDEX contents by checking
# difference between trainId stored in RAW data and trains from
- train_diff = np.isin(np.array(infile["/INDEX/trainId"]), uq,
- invert=True)
+ train_diff = np.isin(np.array(infile["/INDEX/trainId"]), uq, invert=True)
# Insert zeros for missing trains.
# fidxv and cntsv should have same length as
@@ -1249,10 +1247,6 @@ class AgipdCorrections:
for cname, mdata in const_yaml[karabo_da]["constants"].items():
when[cname] = mdata["creation-time"]
if when[cname]:
- # This path is only used when testing new flat fields from
- # file during development: it takes ages to test using all
- # cells. Consequently, the shape needs to be fixed when less
- # cells are used.
with h5py.File(mdata["file-path"], "r") as cf:
cons_data[cname] = np.copy(cf[f"{db_module}/{cname}/0/data"]) # noqa
else:
@@ -1323,15 +1317,21 @@ class AgipdCorrections:
"""
- const_dict = \
- assemble_constant_dict(self.corr_bools, self.pc_bools,
- memory_cells, bias_voltage, gain_setting,
- acquisition_rate, photon_energy,
- beam_energy=None, only_dark=only_dark)
+ const_dict = assemble_constant_dict(
+ self.corr_bools,
+ self.pc_bools,
+ memory_cells,
+ bias_voltage,
+ gain_setting,
+ acquisition_rate,
+ photon_energy,
+ beam_energy=None,
+ only_dark=only_dark,
+ )
- cons_data, when = \
- self.retrieve_constant_and_time(karabo_id, karabo_da, const_dict,
- cal_db_interface, creation_time)
+ cons_data, when = self.retrieve_constant_and_time(
+ karabo_id, karabo_da, const_dict, cal_db_interface, creation_time
+ )
self.init_constants(cons_data, when, module_idx)
@@ -1369,26 +1369,18 @@ class AgipdCorrections:
self.shared_dict = []
for i in range(n_cores_files):
self.shared_dict.append({})
- self.shared_dict[i]['cellId'] = sharedmem.empty(shape[0],
- dtype='u2')
- self.shared_dict[i]['pulseId'] = sharedmem.empty(shape[0],
- dtype='u8')
- self.shared_dict[i]['trainId'] = sharedmem.empty(shape[0],
- dtype='u8')
- self.shared_dict[i]['moduleIdx'] = sharedmem.empty(1, dtype='i4')
- self.shared_dict[i]['nImg'] = sharedmem.empty(1, dtype='i4')
- self.shared_dict[i]['mask'] = sharedmem.empty(shape, dtype='u4')
- self.shared_dict[i]['data'] = sharedmem.empty(shape, dtype='f4')
- self.shared_dict[i]['rawgain'] = sharedmem.empty(shape,
- dtype='u2')
- self.shared_dict[i]['gain'] = sharedmem.empty(shape, dtype='u1')
- self.shared_dict[i]['blShift'] = sharedmem.empty(shape[0],
- dtype='f4')
+ self.shared_dict[i]["cellId"] = sharedmem.empty(shape[0], dtype="u2")
+ self.shared_dict[i]["pulseId"] = sharedmem.empty(shape[0], dtype="u8")
+ self.shared_dict[i]["trainId"] = sharedmem.empty(shape[0], dtype="u8")
+ self.shared_dict[i]["moduleIdx"] = sharedmem.empty(1, dtype="i4")
+ self.shared_dict[i]["nImg"] = sharedmem.empty(1, dtype="i4")
+ self.shared_dict[i]["mask"] = sharedmem.empty(shape, dtype="u4")
+ self.shared_dict[i]["data"] = sharedmem.empty(shape, dtype="f4")
+ self.shared_dict[i]["rawgain"] = sharedmem.empty(shape, dtype="u2")
+ self.shared_dict[i]["gain"] = sharedmem.empty(shape, dtype="u1")
+ self.shared_dict[i]["blShift"] = sharedmem.empty(shape[0], dtype="f4")
# Parameters shared between image-wise correction functions
- self.shared_dict[i]['msk'] = sharedmem.empty(shape, dtype='i4')
- self.shared_dict[i]['raw_data'] = sharedmem.empty(shape,
- dtype='f4')
- self.shared_dict[i]['rel_corr'] = sharedmem.empty(shape,
- dtype='f4')
- self.shared_dict[i]['t0_rgain'] = sharedmem.empty(shape,
- dtype='u2')
+ self.shared_dict[i]["msk"] = sharedmem.empty(shape, dtype="i4")
+ self.shared_dict[i]["raw_data"] = sharedmem.empty(shape, dtype="f4")
+ self.shared_dict[i]["rel_corr"] = sharedmem.empty(shape, dtype="f4")
+ self.shared_dict[i]["t0_rgain"] = sharedmem.empty(shape, dtype="u2")
diff --git a/cal_tools/cal_tools/agipdutils.py b/cal_tools/cal_tools/agipdutils.py
index 3cb51e378..1281834e5 100644
--- a/cal_tools/cal_tools/agipdutils.py
+++ b/cal_tools/cal_tools/agipdutils.py
@@ -31,11 +31,11 @@ def assemble_constant_dict(
:param bias_voltage: (Int) Bias Voltage
:param gain_setting: (Float) Gain setting
:param acquisition_rate: (Float) Acquisition rate
- :param photon_energy: (Float) Photong energy
+ :param photon_energy: (Float) Photon energy
:param beam_energy: (Float) Beam Energy
:param only_dark: (Bool) Indicating a retrieval for dark constants only from db
:param gain_mode: Operation mode of the detector (default to adaptive gain)
- :return: const_dict: (Dict) An assembeld dictionary that can be used
+ :return: const_dict: (Dict) An assembled dictionary that can be used
to retrieve the required constants
"""
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index d2f2375b8..2a2adf59f 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -537,8 +537,7 @@ def get_from_db(karabo_id: str, karabo_da: str,
if ntries > 0:
if load_data and meta_only:
mdata_const = metadata.calibration_constant_version
- fpath = Path(mdata_const.hdf5path,
- mdata_const.filename)
+ fpath = Path(mdata_const.hdf5path, mdata_const.filename)
with h5py.File(fpath, "r") as f:
arr = f[f"{mdata_const.h5path}/data"][()]
metadata.calibration_constant.data = arr
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index c66a23a15..25abd5e19 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -101,11 +101,11 @@
"source": [
"import itertools\n",
"import math\n",
+ "import multiprocessing\n",
"import re\n",
"import traceback\n",
"import warnings\n",
"from datetime import timedelta\n",
- "from multiprocessing import Pool\n",
"from pathlib import Path\n",
"from time import perf_counter\n",
"\n",
@@ -486,7 +486,7 @@
"\n",
"\n",
"ts = perf_counter()\n",
- "with Pool(processes=len(modules)) as pool:\n",
+ "with multiprocessing.Pool(processes=len(modules)) as pool:\n",
" const_out = pool.map(retrieve_constants, modules)\n",
"print(f\"Constants were loaded in {perf_counter()-ts:.01f}s\")"
]
@@ -549,7 +549,7 @@
"metadata": {},
"outputs": [],
"source": [
- "with Pool() as pool:\n",
+ "with multiprocessing.Pool() as pool:\n",
" for file_batch in batches(file_list, n_cores_files):\n",
" # TODO: Move some printed output to logging or similar\n",
" print(f\"Processing next {len(file_batch)} files:\")\n",
--
GitLab
From 945b6568ebbde76b00243d5c4f1fba9a34062c0f Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Fri, 19 Mar 2021 16:07:14 +0100
Subject: [PATCH 31/72] Skip melt_snow, remember gain_mode
---
notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 25abd5e19..bad008604 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -203,7 +203,8 @@
" \"blc_set_min\",\n",
" \"force_hg_if_below\",\n",
" \"force_mg_if_below\",\n",
- " \"low_medium_gap\"\n",
+ " \"low_medium_gap\",\n",
+ " \"melt_snow\"\n",
"]"
]
},
@@ -413,7 +414,8 @@
" max_pulses,\n",
" h5_data_path=h5path,\n",
" h5_index_path=h5path_idx,\n",
- " corr_bools=corr_bools\n",
+ " corr_bools=corr_bools,\n",
+ " gain_mode=gain_mode\n",
")\n",
"\n",
"agipd_corr.baseline_corr_noise_threshold = -blc_noise_threshold\n",
--
GitLab
From bd98a3c6b5a10f7347ba8a74d0ce895565db9ca5 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Mon, 22 Mar 2021 14:03:48 +0100
Subject: [PATCH 32/72] Removed unused variable, import
---
notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 9caca3bb9..8fb07bd61 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -85,7 +85,7 @@
"warnings.filterwarnings('ignore')\n",
"import os\n",
"from collections import OrderedDict\n",
- "from typing import List, Tuple\n",
+ "from typing import Tuple\n",
"\n",
"import h5py\n",
"import matplotlib\n",
@@ -578,10 +578,6 @@
"outputs": [],
"source": [
"# Retrieve existing constants for comparison\n",
- "if fixed_gain_mode:\n",
- " clist = [\"Offset\", \"Noise\", \"BadPixelsDark\"]\n",
- "else:\n",
- " clist = [\"Offset\", \"Noise\", \"ThresholdsDark\", \"BadPixelsDark\"]\n",
"old_const = {}\n",
"old_mdata = {}\n",
"detinst = getattr(Detectors, dinstance)\n",
--
GitLab
From 127de3605c114487b6dd25cf04a1043f595afd49 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Mon, 22 Mar 2021 19:22:19 +0100
Subject: [PATCH 33/72] Explicitly set read mode on h5py.File
---
cal_tools/cal_tools/agipdlib.py | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 019808b84..9fa3e19a0 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -67,7 +67,7 @@ def get_acq_rate(fast_paths: Tuple[str, str, int],
fast_data_file = Path(fast_data_file)
if fast_data_file.is_file():
fast_data_path = f'INSTRUMENT/{karabo_id}/DET/{module}CH0:xtdf/image/pulseId' # noqa
- with h5py.File(fast_data_file) as fin:
+ with h5py.File(fast_data_file, "r") as fin:
if fast_data_path in fin:
# pulses is of shape (NNNN, 1), of type uint8.
# Squeeze out the data, and subtract the 3rd entry from the 2nd
@@ -122,7 +122,7 @@ def get_gain_mode(fname: str, h5path_ctrl: str) -> AgipdGainMode:
h5path_run = h5path_ctrl.replace("CONTROL/", "RUN/", 1)
h5path_gainmode = f'{h5path_run}/gainModeIndex/value'
- with h5py.File(fname, 'r') as fd:
+ with h5py.File(fname, "r") as fd:
if h5path_gainmode in fd:
return AgipdGainMode(fd[h5path_gainmode][0])
return AgipdGainMode.ADAPTIVE_GAIN
@@ -289,20 +289,20 @@ class AgipdCorrections:
data_dict = self.shared_dict[i_proc]
data_dict['moduleIdx'][0] = module_idx
try:
- f = h5py.File(file_name, 'r')
+ f = h5py.File(file_name, "r")
group = f[agipd_base]["image"]
- (_, first_index, last_index,
+ (_, first_index, last_index,
_, valid_indices) = self.get_valid_image_idx(idx_base, f)
allcells = np.squeeze(group['cellId'])
allpulses = np.squeeze(group['pulseId'])
-
+
firange = self.gen_valid_range(first_index, last_index,
self.max_cells, allcells,
allpulses, valid_indices,
apply_sel_pulses)
-
+
n_img = firange.shape[0]
data_dict['nImg'][0] = n_img
if np.all(np.diff(firange) == 1):
@@ -352,7 +352,7 @@ class AgipdCorrections:
return
trains = data_dict['trainId'][:n_img]
- with h5py.File(ofile_name, 'w') as outfile:
+ with h5py.File(ofile_name, "w") as outfile:
# Copy any other data from the input file.
# This includes indexes, so it's important that the corrected data
# we write is aligned with the raw data.
--
GitLab
From 54344a2b8c1fd35ca141d81ef34c202f99a09e7b Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Wed, 24 Feb 2021 12:19:21 +0100
Subject: [PATCH 34/72] Start refactoring of darks notebook
Applying isort and nbstripout. Changing from ipyparallel to multiprocessing - and while doing so,
simplifying the call signature of characterize_module.
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 278 ++++++------------
1 file changed, 89 insertions(+), 189 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 8d11bae69..09ae91c62 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -16,22 +16,16 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-20T12:42:51.255184Z",
- "start_time": "2019-02-20T12:42:51.225500Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
- "cluster_profile = \"noDB\" # The ipcluster profile to use\n",
- "in_folder = \"/gpfs/exfel/d/raw/DETLAB/202031/p900172/\" # path to input data, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/miniHalfAGIPD\" # path to output to, required\n",
+ "in_folder = \"/gpfs/exfel/d/raw/CALLAB/202031/p900113\" # path to input data, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/hammerd/agipd-fixed-gain\" # path to output to, required\n",
"sequences = [0] # sequence files to evaluate.\n",
"modules = [-1] # list of modules to evaluate, RANGE ALLOWED\n",
- "run_high = 84 # run number in which high gain data was recorded, required\n",
- "run_med = 87 # run number in which medium gain data was recorded, required\n",
- "run_low = 88 # run number in which low gain data was recorded, required\n",
+ "run_high = 9985 # run number in which high gain data was recorded, required\n",
+ "run_med = 9984 # run number in which medium gain data was recorded, required\n",
+ "run_low = 9983 # run number in which low gain data was recorded, required\n",
"operation_mode = 'ADAPTIVE_GAIN' # Detector operation mode, optional\n",
"\n",
"karabo_id = \"HED_DET_AGIPD500K2G\" # karabo karabo_id\n",
@@ -40,9 +34,9 @@
"path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # the template to use to access data\n",
"h5path = '/INSTRUMENT/{}/DET/{}:xtdf/image' # path in the HDF5 file to images\n",
"h5path_idx = '/INDEX/{}/DET/{}:xtdf/image' # path in the HDF5 file to images\n",
- "h5path_ctrl = '/CONTROL/{}/MDL/FPGA_COMP_TEST' # path to control information\n",
- "karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for control device '\n",
- "karabo_da_control = \"AGIPD1MCTRL00\" # karabo DA for control infromation\n",
+ "h5path_ctrl = '/CONTROL/{}/MDL/FPGA_COMP' # path to control information\n",
+ "karabo_id_control = \"HED_EXP_AGIPD500K2G\" # karabo-id for control device '\n",
+ "karabo_da_control = \"AGIPD500K2G00\" # karabo DA for control infromation\n",
"\n",
"use_dir_creation_date = True # use dir creation date as data production reference date\n",
"cal_db_interface = \"tcp://max-exfl016:8020\" # the database interface to use\n",
@@ -77,30 +71,24 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-20T12:42:52.599660Z",
- "start_time": "2019-02-20T12:42:51.472138Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
- "import warnings\n",
- "\n",
"# imports and things that do not usually need to be changed\n",
"from datetime import datetime\n",
"\n",
"import dateutil.parser\n",
"\n",
- "warnings.filterwarnings('ignore')\n",
- "import os\n",
"from collections import OrderedDict\n",
+ "from functools import partial\n",
"from typing import List, Tuple\n",
"\n",
"import h5py\n",
"import matplotlib\n",
"import numpy as np\n",
"import tabulate\n",
+ "from cal_tools.agipdlib import get_acq_rate, get_num_cells\n",
+ "from cal_tools.enums import BadPixels\n",
"\n",
"matplotlib.use('agg')\n",
"import matplotlib.pyplot as plt\n",
@@ -108,36 +96,27 @@
"\n",
"%matplotlib inline\n",
"\n",
+ "import multiprocessing\n",
+ "\n",
"from cal_tools.agipdlib import get_bias_voltage, get_gain_setting\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.plotting import (\n",
- " create_constant_overview,\n",
- " plot_badpix_3d,\n",
- " show_overview,\n",
- " show_processed_modules,\n",
- ")\n",
- "from cal_tools.tools import (\n",
- " get_dir_creation_date,\n",
- " get_from_db,\n",
- " get_notebook_name,\n",
- " get_pdu_from_db,\n",
- " get_random_db_interface,\n",
- " get_report,\n",
- " map_gain_stages,\n",
- " parse_runs,\n",
- " run_prop_seq_from_path,\n",
- " save_const_to_h5,\n",
- " send_to_db,\n",
- ")\n",
- "\n",
- "# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
- "from ipyparallel import Client\n",
- "\n",
- "view = Client(profile=cluster_profile)[:]\n",
- "view.use_dill()\n",
- "\n",
- "from iCalibrationDB import Conditions, Constants, Detectors, Versions\n",
- "\n",
+ "from cal_tools.plotting import (create_constant_overview, plot_badpix_3d,\n",
+ " show_overview, show_processed_modules)\n",
+ "from cal_tools.tools import (get_dir_creation_date, get_from_db,\n",
+ " get_notebook_name, get_pdu_from_db,\n",
+ " get_random_db_interface, get_report,\n",
+ " map_gain_stages, parse_runs,\n",
+ " run_prop_seq_from_path, save_const_to_h5,\n",
+ " send_to_db)\n",
+ "from iCalibrationDB import Conditions, Constants, Detectors, Versions"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
"gains = np.arange(3)\n",
"\n",
"IL_MODE = interlaced\n",
@@ -218,12 +197,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-20T12:42:52.608214Z",
- "start_time": "2019-02-20T12:42:52.601257Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"if karabo_da[0] == '-1':\n",
@@ -264,12 +238,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-20T12:42:54.024731Z",
- "start_time": "2019-02-20T12:42:53.901555Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"# set everything up filewise\n",
@@ -291,74 +260,50 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-20T10:50:55.839958Z",
- "start_time": "2019-02-20T10:50:55.468134Z"
- },
- "scrolled": true
- },
+ "metadata": {},
"outputs": [],
"source": [
- "import copy\n",
- "from functools import partial\n",
- "\n",
+ "def characterize_module(inp: Tuple[str, int, int]) -> Tuple[np.array, np.array, np.array, np.array, int, np.array, int, float]:\n",
+ " fast_data_filename, channel, gg = inp\n",
"\n",
- "def characterize_module(il_mode: bool,\n",
- " cells: int,\n",
- " bp_thresh: Tuple[List[int], float, List[int], float], \n",
- " rawversion: int,\n",
- " loc: str, \n",
- " acq_rate: float,\n",
- " h5path: str,\n",
- " h5path_idx: str,\n",
- " control_names: List[str],\n",
- " karabo_id_control: str,\n",
- " inp: Tuple[str, int, int]) -> Tuple[np.array, np.array, np.array, np.array, int, np.array, int, float]:\n",
- " import copy\n",
- "\n",
- " import h5py\n",
- " import numpy as np\n",
- " from cal_tools.agipdlib import get_acq_rate, get_num_cells\n",
- " from cal_tools.enums import BadPixels\n",
+ " if max_cells == 0:\n",
+ " num_cells = get_num_cells(fast_data_filename, karabo_id, channel)\n",
+ " else:\n",
+ " num_cells = max_cells\n",
"\n",
- " fast_data_filename, channel, gg = inp\n",
- " \n",
- " if cells == 0:\n",
- " cells = get_num_cells(fast_data_filename, loc, channel)\n",
+ " print(f\"Using {num_cells} memory cells\")\n",
"\n",
- " print(f\"Using {cells} memory cells\")\n",
- " \n",
" if acq_rate == 0.:\n",
" slow_paths = control_names[gg], karabo_id_control\n",
- " fast_paths = fast_data_filename, loc, channel\n",
- " acq_rate = get_acq_rate(fast_paths, slow_paths)\n",
+ " fast_paths = fast_data_filename, karabo_id, channel\n",
+ " local_acq_rate = get_acq_rate(fast_paths, slow_paths)\n",
+ " else:\n",
+ " local_acq_rate = acq_rate\n",
+ "\n",
+ " local_thresholds_offset_hard = thresholds_offset_hard[gg]\n",
+ " local_thresholds_noise_hard = thresholds_noise_hard[gg]\n",
+ "\n",
+ " h5path_f = h5path.format(channel)\n",
+ " h5path_idx_f = h5path_idx.format(channel)\n",
"\n",
- " thresholds_offset, thresholds_offset_sigma, thresholds_noise, thresholds_noise_sigma = bp_thresh \n",
- " thresholds_offset_hard = thresholds_offset[gg]\n",
- " thresholds_noise_hard = thresholds_noise[gg]\n",
- " \n",
- " h5path = h5path.format(channel)\n",
- " h5path_idx = h5path_idx.format(channel)\n",
- " \n",
" with h5py.File(fast_data_filename, \"r\", driver=\"core\") as infile:\n",
" if rawversion == 2:\n",
- " count = np.squeeze(infile[f\"{h5path_idx}/count\"])\n",
- " first = np.squeeze(infile[f\"{h5path_idx}/first\"])\n",
+ " count = np.squeeze(infile[f\"{h5path_idx_f}/count\"])\n",
+ " first = np.squeeze(infile[f\"{h5path_idx_f}/first\"])\n",
" last_index = int(first[count != 0][-1]+count[count != 0][-1])\n",
" first_index = int(first[count != 0][0])\n",
" else:\n",
- " status = np.squeeze(infile[f\"{h5path_idx}/status\"])\n",
+ " status = np.squeeze(infile[f\"{h5path_idx_f}/status\"])\n",
" if np.count_nonzero(status != 0) == 0:\n",
" return\n",
- " last = np.squeeze(infile[f\"{h5path_idx}/last\"])\n",
- " first = np.squeeze(infile[f\"{h5path_idx}/first\"])\n",
+ " last = np.squeeze(infile[f\"{h5path_idx_f}/last\"])\n",
+ " first = np.squeeze(infile[f\"{h5path_idx_f}/first\"])\n",
" last_index = int(last[status != 0][-1]) + 1\n",
" first_index = int(first[status != 0][0])\n",
- " im = np.array(infile[f\"{h5path}/data\"][first_index:last_index,...]) \n",
- " cellIds = np.squeeze(infile[f\"{h5path}/cellId\"][first_index:last_index,...]) \n",
+ " im = np.array(infile[f\"{h5path_f}/data\"][first_index:last_index,...])\n",
+ " cellIds = np.squeeze(infile[f\"{h5path_f}/cellId\"][first_index:last_index,...])\n",
"\n",
- " if il_mode:\n",
+ " if IL_MODE:\n",
" ga = im[1::2, 0, ...]\n",
" im = im[0::2, 0, ...].astype(np.float32)\n",
" cellIds = cellIds[::2]\n",
@@ -372,13 +317,12 @@
" ga = np.rollaxis(ga, 2)\n",
" ga = np.rollaxis(ga, 2, 1)\n",
"\n",
- " mcells = cells #max(cells, np.max(cellIds)+1)\n",
- " offset = np.zeros((im.shape[0], im.shape[1], mcells))\n",
- " gains = np.zeros((im.shape[0], im.shape[1], mcells))\n",
- " noise = np.zeros((im.shape[0], im.shape[1], mcells))\n",
- " gains_std = np.zeros((im.shape[0], im.shape[1], mcells))\n",
- " \n",
- " for cc in np.unique(cellIds[cellIds < mcells]):\n",
+ " offset = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
+ " gains = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
+ " noise = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
+ " gains_std = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
+ "\n",
+ " for cc in np.unique(cellIds[cellIds < num_cells]):\n",
" cellidx = cellIds == cc\n",
" offset[...,cc] = np.median(im[..., cellidx], axis=2)\n",
" noise[...,cc] = np.std(im[..., cellidx], axis=2)\n",
@@ -393,19 +337,19 @@
"\n",
" bp[(offset < offset_mn-thresholds_offset_sigma*offset_std) |\n",
" (offset > offset_mn+thresholds_offset_sigma*offset_std)] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
- " bp[(offset < thresholds_offset_hard[0]) | (\n",
- " offset > thresholds_offset_hard[1])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
+ " bp[(offset < local_thresholds_offset_hard[0]) | (\n",
+ " offset > local_thresholds_offset_hard[1])] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
" bp[~np.isfinite(offset)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
"\n",
" # noise related bad pixels\n",
" noise_mn = np.nanmedian(noise, axis=(0,1))\n",
- " noise_std = np.nanstd(noise, axis=(0,1)) \n",
+ " noise_std = np.nanstd(noise, axis=(0,1))\n",
" bp[(noise < noise_mn-thresholds_noise_sigma*noise_std) |\n",
" (noise > noise_mn+thresholds_noise_sigma*noise_std)] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
- " bp[(noise < thresholds_noise_hard[0]) | (noise > thresholds_noise_hard[1])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
+ " bp[(noise < local_thresholds_noise_hard[0]) | (noise > local_thresholds_noise_hard[1])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
" bp[~np.isfinite(noise)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
"\n",
- " return offset, noise, gains, gains_std, gg, bp, cells, acq_rate\n",
+ " return offset, noise, gains, gains_std, gg, bp, num_cells, local_acq_rate\n",
"\n",
"offset_g = OrderedDict()\n",
"noise_g = OrderedDict()\n",
@@ -428,7 +372,7 @@
"else:\n",
" thresholds_noise_hard = [thresholds_noise_hard] * 3\n",
"\n",
- " \n",
+ "\n",
"inp = []\n",
"for gain, mapped_files in gain_mapped_files.items():\n",
" dones = []\n",
@@ -441,18 +385,11 @@
" else:\n",
" continue\n",
" inp.append((fname_in, i, gg))\n",
- " \n",
- " gg += 1\n",
"\n",
- "p = partial(characterize_module, IL_MODE, max_cells,\n",
- " (thresholds_offset_hard, thresholds_offset_sigma,\n",
- " thresholds_noise_hard, thresholds_noise_sigma),\n",
- " rawversion, karabo_id, acq_rate, h5path, h5path_idx,\n",
- " control_names, karabo_id_control)\n",
+ " gg += 1\n",
"\n",
- "# Don't remove. Used for Debugging.\n",
- "#results = list(map(p, inp))\n",
- "results = view.map_sync(p, inp)\n",
+ "with multiprocessing.Pool() as pool:\n",
+ " results = pool.map(p, inp)\n",
"\n",
"for ii, r in enumerate(results):\n",
" offset, noise, gains, gains_std, gg, bp, thiscell, thisacq = r\n",
@@ -472,7 +409,7 @@
" gain_g[qm][...,gg] = gains\n",
" gainstd_g[qm][..., gg] = gains_std\n",
" badpix_g[qm][...,gg] = bp\n",
- " \n",
+ "\n",
"\n",
"duration = (datetime.now() - start).total_seconds()\n",
"\n",
@@ -506,12 +443,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2018-12-06T09:38:18.220833Z",
- "start_time": "2018-12-06T09:38:17.926616Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"thresholds_g = {}\n",
@@ -526,12 +458,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2018-12-06T09:38:18.234582Z",
- "start_time": "2018-12-06T09:38:18.222838Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"res = OrderedDict()\n",
@@ -575,9 +502,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Retrieve existing constants for comparison\n",
@@ -632,12 +557,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2018-12-06T09:49:32.449330Z",
- "start_time": "2018-12-06T09:49:20.231607Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"md = None\n",
@@ -673,9 +593,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"mnames=[]\n",
@@ -705,13 +623,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2018-12-06T09:49:14.540552Z",
- "start_time": "2018-12-06T09:49:13.009674Z"
- },
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"cell = 3\n",
@@ -729,9 +641,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"cell = 3\n",
@@ -749,9 +659,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"cell = 3\n",
@@ -762,9 +670,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"cols = {BadPixels.NOISE_OUT_OF_THRESHOLD.value: (BadPixels.NOISE_OUT_OF_THRESHOLD.name, '#FF000080'),\n",
@@ -808,9 +714,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"create_constant_overview(offset_g, \"Offset (ADU)\", max_cells, 4000, 8000,\n",
@@ -820,9 +724,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"create_constant_overview(noise_g, \"Noise (ADU)\", max_cells, 0, 100,\n",
@@ -832,9 +734,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Plot only three gain threshold maps.\n",
@@ -1015,5 +915,5 @@
}
},
"nbformat": 4,
- "nbformat_minor": 1
+ "nbformat_minor": 4
}
--
GitLab
From aa0d7bc153fa91d5b4740c1066e2bbb90a0f5b04 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Wed, 24 Feb 2021 14:48:30 +0100
Subject: [PATCH 35/72] Cleaning up imports
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 25 ++++++-------------
1 file changed, 8 insertions(+), 17 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 09ae91c62..dfab44ce1 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -79,15 +79,14 @@
"\n",
"import dateutil.parser\n",
"\n",
+ "import os\n",
"from collections import OrderedDict\n",
- "from functools import partial\n",
"from typing import List, Tuple\n",
"\n",
"import h5py\n",
"import matplotlib\n",
"import numpy as np\n",
"import tabulate\n",
- "from cal_tools.agipdlib import get_acq_rate, get_num_cells\n",
"from cal_tools.enums import BadPixels\n",
"\n",
"matplotlib.use('agg')\n",
@@ -98,17 +97,16 @@
"\n",
"import multiprocessing\n",
"\n",
- "from cal_tools.agipdlib import get_bias_voltage, get_gain_setting\n",
- "from cal_tools.enums import BadPixels\n",
+ "from cal_tools.agipdlib import (get_acq_rate, get_bias_voltage,\n",
+ " get_gain_setting, get_num_cells)\n",
"from cal_tools.plotting import (create_constant_overview, plot_badpix_3d,\n",
" show_overview, show_processed_modules)\n",
"from cal_tools.tools import (get_dir_creation_date, get_from_db,\n",
- " get_notebook_name, get_pdu_from_db,\n",
- " get_random_db_interface, get_report,\n",
- " map_gain_stages, parse_runs,\n",
+ " get_pdu_from_db, get_random_db_interface,\n",
+ " get_report, map_gain_stages,\n",
" run_prop_seq_from_path, save_const_to_h5,\n",
" send_to_db)\n",
- "from iCalibrationDB import Conditions, Constants, Detectors, Versions"
+ "from iCalibrationDB import Conditions, Constants, Detectors"
]
},
{
@@ -129,7 +127,7 @@
"\n",
"creation_time=None\n",
"if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run_high)\n",
+ " creation_time = get_dir_creation_date(in_folder, run_high, verbosity=2)\n",
"\n",
"print(f\"Using {creation_time} as creation time of constant.\")\n",
"\n",
@@ -389,7 +387,7 @@
" gg += 1\n",
"\n",
"with multiprocessing.Pool() as pool:\n",
- " results = pool.map(p, inp)\n",
+ " results = pool.map(characterize_module, inp)\n",
"\n",
"for ii, r in enumerate(results):\n",
" offset, noise, gains, gains_std, gg, bp, thiscell, thisacq = r\n",
@@ -886,13 +884,6 @@
" display(Markdown('### {} [ADU], good pixels only ###'.format(const)))\n",
" md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=header))) "
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
--
GitLab
From de94a3041e891222f3110047d0c56e7fe3060c58 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 25 Feb 2021 13:57:22 +0100
Subject: [PATCH 36/72] Additional refactoring
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 72 +++++++++++--------
1 file changed, 41 insertions(+), 31 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index dfab44ce1..4672a39e2 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -117,7 +117,6 @@
"source": [
"gains = np.arange(3)\n",
"\n",
- "IL_MODE = interlaced\n",
"max_cells = mem_cells\n",
" \n",
"offset_runs = OrderedDict()\n",
@@ -127,7 +126,7 @@
"\n",
"creation_time=None\n",
"if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run_high, verbosity=2)\n",
+ " creation_time = get_dir_creation_date(in_folder, run_high)\n",
"\n",
"print(f\"Using {creation_time} as creation time of constant.\")\n",
"\n",
@@ -165,8 +164,8 @@
"source": [
"runs = [run_high, run_med, run_low]\n",
"\n",
- "if \"{\" in h5path_ctrl:\n",
- " h5path_ctrl = h5path_ctrl.format(karabo_id_control)\n",
+ "# insert control device if format string (does nothing otherwise)\n",
+ "h5path_ctrl = h5path_ctrl.format(karabo_id_control)\n",
"\n",
"if gain_setting == 0.1:\n",
" if creation_time.replace(tzinfo=None) < dateutil.parser.parse('2020-01-31'):\n",
@@ -218,7 +217,7 @@
"print(f\"Memory cells: {mem_cells}/{max_cells}\")\n",
"print(\"Runs: {}\".format([ v for v in offset_runs.values()]))\n",
"print(f\"Sequences: {sequences}\")\n",
- "print(f\"Interlaced mode: {IL_MODE}\")\n",
+ "print(f\"Interlaced mode: {interlaced}\")\n",
"print(f\"Using DB: {db_output}\")\n",
"print(f\"Input: {in_folder}\")\n",
"print(f\"Output: {out_folder}\")\n",
@@ -246,6 +245,16 @@
"print(f\"Will process a total of {total_sequences} files.\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def index_to_qm(i: int):\n",
+ " return f\"Q{i//4+1}M{i%4+1}\""
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -261,9 +270,24 @@
"metadata": {},
"outputs": [],
"source": [
- "def characterize_module(inp: Tuple[str, int, int]) -> Tuple[np.array, np.array, np.array, np.array, int, np.array, int, float]:\n",
- " fast_data_filename, channel, gg = inp\n",
+ "if thresholds_offset_hard == [0, 0]:\n",
+ " thresholds_offset_hard = [thresholds_offset_hard_hg, thresholds_offset_hard_mg, thresholds_offset_hard_lg]\n",
+ "else:\n",
+ " thresholds_offset_hard = [thresholds_offset_hard] * 3\n",
"\n",
+ "if thresholds_noise_hard == [0, 0]:\n",
+ " thresholds_noise_hard = [thresholds_noise_hard_hg, thresholds_noise_hard_mg, thresholds_noise_hard_lg]\n",
+ "else:\n",
+ " thresholds_noise_hard = [thresholds_noise_hard] * 3"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def characterize_module(fast_data_filename: str, channel: int, gg: int) -> Tuple[np.array, np.array, np.array, np.array, int, np.array, int, float]:\n",
" if max_cells == 0:\n",
" num_cells = get_num_cells(fast_data_filename, karabo_id, channel)\n",
" else:\n",
@@ -301,7 +325,7 @@
" im = np.array(infile[f\"{h5path_f}/data\"][first_index:last_index,...])\n",
" cellIds = np.squeeze(infile[f\"{h5path_f}/cellId\"][first_index:last_index,...])\n",
"\n",
- " if IL_MODE:\n",
+ " if interlaced:\n",
" ga = im[1::2, 0, ...]\n",
" im = im[0::2, 0, ...].astype(np.float32)\n",
" cellIds = cellIds[::2]\n",
@@ -349,33 +373,22 @@
"\n",
" return offset, noise, gains, gains_std, gg, bp, num_cells, local_acq_rate\n",
"\n",
+ "\n",
"offset_g = OrderedDict()\n",
"noise_g = OrderedDict()\n",
"gain_g = OrderedDict()\n",
"gainstd_g = OrderedDict()\n",
"badpix_g = OrderedDict()\n",
- "gg = 0\n",
"\n",
"start = datetime.now()\n",
"all_cells = []\n",
"all_acq_rate = []\n",
"\n",
- "if thresholds_offset_hard == [0, 0]:\n",
- " thresholds_offset_hard = [thresholds_offset_hard_hg, thresholds_offset_hard_mg, thresholds_offset_hard_lg]\n",
- "else:\n",
- " thresholds_offset_hard = [thresholds_offset_hard] * 3\n",
- "\n",
- "if thresholds_noise_hard == [0, 0]:\n",
- " thresholds_noise_hard = [thresholds_noise_hard_hg, thresholds_noise_hard_mg, thresholds_noise_hard_lg]\n",
- "else:\n",
- " thresholds_noise_hard = [thresholds_noise_hard] * 3\n",
- "\n",
- "\n",
"inp = []\n",
- "for gain, mapped_files in gain_mapped_files.items():\n",
+ "for gg, (gain, mapped_files) in enumerate(gain_mapped_files.items()):\n",
" dones = []\n",
" for i in modules:\n",
- " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ " qm = index_to_qm(i)\n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
" fname_in = mapped_files[qm].get()\n",
" print(\"Process file: \", fname_in)\n",
@@ -384,17 +397,14 @@
" continue\n",
" inp.append((fname_in, i, gg))\n",
"\n",
- " gg += 1\n",
- "\n",
"with multiprocessing.Pool() as pool:\n",
- " results = pool.map(characterize_module, inp)\n",
+ " results = pool.starmap(characterize_module, inp)\n",
"\n",
- "for ii, r in enumerate(results):\n",
- " offset, noise, gains, gains_std, gg, bp, thiscell, thisacq = r\n",
+ "for offset, noise, gains, gains_std, gg, bp, thiscell, thisacq in results:\n",
" all_cells.append(thiscell)\n",
" all_acq_rate.append(thisacq)\n",
" for i in modules:\n",
- " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ " qm = index_to_qm(i)\n",
" if qm not in offset_g:\n",
" offset_g[qm] = np.zeros((offset.shape[0], offset.shape[1], offset.shape[2], 3))\n",
" noise_g[qm] = np.zeros_like(offset_g[qm])\n",
@@ -461,7 +471,7 @@
"source": [
"res = OrderedDict()\n",
"for i in modules:\n",
- " qm = \"Q{}M{}\".format(i//4+1, i%4+1)\n",
+ " qm = index_to_qm(i)\n",
" res[qm] = {'Offset': offset_g[qm],\n",
" 'Noise': noise_g[qm],\n",
" 'ThresholdsDark': thresholds_g[qm],\n",
@@ -492,7 +502,7 @@
"# Create the modules dict of karabo_das and PDUs\n",
"qm_dict = OrderedDict()\n",
"for i, k_da in zip(modules, karabo_da):\n",
- " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ " qm = index_to_qm(i)\n",
" qm_dict[qm] = {\"karabo_da\": k_da,\n",
" \"db_module\": \"\"}"
]
@@ -596,7 +606,7 @@
"source": [
"mnames=[]\n",
"for i in modules:\n",
- " qm = f\"Q{i//4+1}M{i % 4+1}\"\n",
+ " qm = index_to_qm(i)\n",
" mnames.append(qm)\n",
" display(Markdown(f'## Position of the module {qm} and its ASICs##'))\n",
"show_processed_modules(dinstance, constants=None, mnames=mnames, mode=\"position\")"
--
GitLab
From 47f79f003c064c330e9319b3fc26149073ea1811 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 25 Feb 2021 16:18:28 +0100
Subject: [PATCH 37/72] Adding enum encoding AGIPD gain mode
---
cal_tools/cal_tools/enums.py | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/cal_tools/cal_tools/enums.py b/cal_tools/cal_tools/enums.py
index 4abafdc10..186e17208 100644
--- a/cal_tools/cal_tools/enums.py
+++ b/cal_tools/cal_tools/enums.py
@@ -1,4 +1,4 @@
-from enum import Enum, IntFlag
+from enum import Enum, IntEnum, IntFlag
class BadPixels(IntFlag):
@@ -48,3 +48,12 @@ class SnowResolution(Enum):
"""
NONE = "none"
INTERPOLATE = "interpolate"
+
+
+class AgipdGainMode(IntEnum):
+ """Encoding added to distinguish between adaptive and fixed gain"""
+
+ ADAPTIVE_GAIN = 0 # adaptive is default (if gain mode missing in slow data)
+ FIXED_HIGH_GAIN = 1 # non-zero means fixed gain
+ FIXED_MEDIUM_GAIN = 2
+ FIXED_LOW_GAIN = 3
--
GitLab
From 5f464dc1d90a917f8dff9371d5ba4472610cbd1c Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Fri, 26 Feb 2021 17:19:59 +0100
Subject: [PATCH 38/72] Adding prototype gain mode detection from slow data
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 34 +++++++++++++++----
1 file changed, 28 insertions(+), 6 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 4672a39e2..c15bbfba5 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -26,7 +26,6 @@
"run_high = 9985 # run number in which high gain data was recorded, required\n",
"run_med = 9984 # run number in which medium gain data was recorded, required\n",
"run_low = 9983 # run number in which low gain data was recorded, required\n",
- "operation_mode = 'ADAPTIVE_GAIN' # Detector operation mode, optional\n",
"\n",
"karabo_id = \"HED_DET_AGIPD500K2G\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
@@ -99,6 +98,7 @@
"\n",
"from cal_tools.agipdlib import (get_acq_rate, get_bias_voltage,\n",
" get_gain_setting, get_num_cells)\n",
+ "from cal_tools.enums import AgipdGainMode\n",
"from cal_tools.plotting import (create_constant_overview, plot_badpix_3d,\n",
" show_overview, show_processed_modules)\n",
"from cal_tools.tools import (get_dir_creation_date, get_from_db,\n",
@@ -115,6 +115,23 @@
"metadata": {},
"outputs": [],
"source": [
+ "def get_gain_mode(h5fn):\n",
+ " h5path_gainmode = f'{h5path_ctrl.replace(\"/CONTROL/\", \"/RUN/\", 1)}/gainModeIndex/value'\n",
+ " with h5py.File(h5fn, 'r') as fd:\n",
+ " if h5path_gainmode in fd:\n",
+ " return AgipdGainMode(fd[h5path_gainmode][0])\n",
+ " return AgipdGainMode.ADAPTIVE_GAIN"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# insert control device if format string (does nothing otherwise)\n",
+ "h5path_ctrl = h5path_ctrl.format(karabo_id_control)\n",
+ "\n",
"gains = np.arange(3)\n",
"\n",
"max_cells = mem_cells\n",
@@ -148,12 +165,20 @@
" nmods = 8\n",
"\n",
"control_names = [f'{in_folder}/r{r:04d}/RAW-R{r:04d}-{karabo_da_control}-S00000.h5'\n",
- " for r in (run_high, run_med, run_low)] \n",
+ " for r in (run_high, run_med, run_low)]\n",
+ "\n",
+ "run_gain_modes = [get_gain_mode(fn) for fn in control_names]\n",
+ "if all(gm == AgipdGainMode.ADAPTIVE_GAIN for gm in run_gain_modes):\n",
+ " fixed_gain_mode = False\n",
+ "elif run_gain_modes == [AgipdGainMode.FIXED_HIGH_GAIN, AgipdGainMode.FIXED_MEDIUM_GAIN, AgipdGainMode.FIXED_LOW_GAIN]:\n",
+ " fixed_gain_mode = True\n",
+ "else:\n",
+ " print(f'Something is clearly wrong; slow data indicates gain modes {run_gain_modes}')\n",
"\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"print(f\"Instrument {instrument}\")\n",
"print(f\"Detector instance {dinstance}\")\n",
- "print(f\"Operation mode is {operation_mode}\")"
+ "print(f\"Operation mode is {'fixed' if fixed_gain_mode else 'adaptive'} gain mode\")"
]
},
{
@@ -164,9 +189,6 @@
"source": [
"runs = [run_high, run_med, run_low]\n",
"\n",
- "# insert control device if format string (does nothing otherwise)\n",
- "h5path_ctrl = h5path_ctrl.format(karabo_id_control)\n",
- "\n",
"if gain_setting == 0.1:\n",
" if creation_time.replace(tzinfo=None) < dateutil.parser.parse('2020-01-31'):\n",
" print(\"Set gain-setting to None for runs taken before 2020-01-31\")\n",
--
GitLab
From 926d27a15567096b5ce6b198b3c68841f3a275de Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Tue, 2 Mar 2021 09:20:45 +0100
Subject: [PATCH 39/72] Formatting, splitting a cell
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 29 ++++++++++++-------
1 file changed, 19 insertions(+), 10 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index c15bbfba5..dfbeb5b60 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -393,9 +393,15 @@
" bp[(noise < local_thresholds_noise_hard[0]) | (noise > local_thresholds_noise_hard[1])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
" bp[~np.isfinite(noise)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
"\n",
- " return offset, noise, gains, gains_std, gg, bp, num_cells, local_acq_rate\n",
- "\n",
- "\n",
+ " return offset, noise, gains, gains_std, gg, bp, num_cells, local_acq_rate"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
"offset_g = OrderedDict()\n",
"noise_g = OrderedDict()\n",
"gain_g = OrderedDict()\n",
@@ -494,11 +500,12 @@
"res = OrderedDict()\n",
"for i in modules:\n",
" qm = index_to_qm(i)\n",
- " res[qm] = {'Offset': offset_g[qm],\n",
- " 'Noise': noise_g[qm],\n",
- " 'ThresholdsDark': thresholds_g[qm],\n",
- " 'BadPixelsDark': badpix_g[qm] \n",
- " }"
+ " res[qm] = {\n",
+ " 'Offset': offset_g[qm],\n",
+ " 'Noise': noise_g[qm],\n",
+ " 'ThresholdsDark': thresholds_g[qm],\n",
+ " 'BadPixelsDark': badpix_g[qm]\n",
+ " }"
]
},
{
@@ -525,8 +532,10 @@
"qm_dict = OrderedDict()\n",
"for i, k_da in zip(modules, karabo_da):\n",
" qm = index_to_qm(i)\n",
- " qm_dict[qm] = {\"karabo_da\": k_da,\n",
- " \"db_module\": \"\"}"
+ " qm_dict[qm] = {\n",
+ " \"karabo_da\": k_da,\n",
+ " \"db_module\": \"\"\n",
+ " }"
]
},
{
--
GitLab
From 6d06707e1e0456108764e3afd9f13f79b0720e78 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Tue, 2 Mar 2021 10:02:12 +0100
Subject: [PATCH 40/72] Skipping thresholds, gain separation bp for fixed gain
mode
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 76 +++++++++++--------
1 file changed, 45 insertions(+), 31 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index dfbeb5b60..58aaf3e52 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -462,11 +462,12 @@
"outputs": [],
"source": [
"# Add a badpixel due to bad gain separation\n",
- "for g in range(2):\n",
- " # Bad pixels during bad gain separation.\n",
- " # Fraction of pixels in the module with separation lower than \"thresholds_gain_sigma\".\n",
- " bad_sep = (gain_g[qm][..., g+1] - gain_g[qm][..., g]) / np.sqrt(gainstd_g[qm][..., g+1]**2 + gainstd_g[qm][..., g]**2)\n",
- " badpix_g[qm][...,g+1][(bad_sep)<thresholds_gain_sigma]|= BadPixels.GAIN_THRESHOLDING_ERROR.value"
+ "if not fixed_gain_mode:\n",
+ " for g in range(2):\n",
+ " # Bad pixels during bad gain separation.\n",
+ " # Fraction of pixels in the module with separation lower than \"thresholds_gain_sigma\".\n",
+ " bad_sep = (gain_g[qm][..., g+1] - gain_g[qm][..., g]) / np.sqrt(gainstd_g[qm][..., g+1]**2 + gainstd_g[qm][..., g]**2)\n",
+ " badpix_g[qm][...,g+1][(bad_sep)<thresholds_gain_sigma]|= BadPixels.GAIN_THRESHOLDING_ERROR.value"
]
},
{
@@ -482,13 +483,14 @@
"metadata": {},
"outputs": [],
"source": [
- "thresholds_g = {}\n",
- "for qm in gain_g.keys():\n",
- " thresholds_g[qm] = np.zeros((gain_g[qm].shape[0], gain_g[qm].shape[1], gain_g[qm].shape[2], 5))\n",
- " thresholds_g[qm][...,0] = (gain_g[qm][...,1]+gain_g[qm][...,0])/2\n",
- " thresholds_g[qm][...,1] = (gain_g[qm][...,2]+gain_g[qm][...,1])/2\n",
- " for i in range(3):\n",
- " thresholds_g[qm][...,2+i] = gain_g[qm][...,i]"
+ "if not fixed_gain_mode:\n",
+ " thresholds_g = {}\n",
+ " for qm in gain_g.keys():\n",
+ " thresholds_g[qm] = np.zeros((gain_g[qm].shape[0], gain_g[qm].shape[1], gain_g[qm].shape[2], 5))\n",
+ " thresholds_g[qm][...,0] = (gain_g[qm][...,1]+gain_g[qm][...,0])/2\n",
+ " thresholds_g[qm][...,1] = (gain_g[qm][...,2]+gain_g[qm][...,1])/2\n",
+ " for i in range(3):\n",
+ " thresholds_g[qm][...,2+i] = gain_g[qm][...,i]"
]
},
{
@@ -503,9 +505,10 @@
" res[qm] = {\n",
" 'Offset': offset_g[qm],\n",
" 'Noise': noise_g[qm],\n",
- " 'ThresholdsDark': thresholds_g[qm],\n",
" 'BadPixelsDark': badpix_g[qm]\n",
- " }"
+ " }\n",
+ " if not fixed_gain_mode:\n",
+ " res[qm]['ThresholdsDark'] = thresholds_g[qm]"
]
},
{
@@ -545,7 +548,10 @@
"outputs": [],
"source": [
"# Retrieve existing constants for comparison\n",
- "clist = [\"Offset\", \"Noise\", \"ThresholdsDark\", \"BadPixelsDark\"]\n",
+ "if fixed_gain_mode:\n",
+ " clist = [\"Offset\", \"Noise\", \"BadPixelsDark\"]\n",
+ "else:\n",
+ " clist = [\"Offset\", \"Noise\", \"ThresholdsDark\", \"BadPixelsDark\"]\n",
"old_const = {}\n",
"old_mdata = {}\n",
"detinst = getattr(Detectors, dinstance)\n",
@@ -776,19 +782,20 @@
"metadata": {},
"outputs": [],
"source": [
- "# Plot only three gain threshold maps.\n",
- "bp_thresh = OrderedDict()\n",
- "for mod, con in badpix_g.items():\n",
- " bp_thresh[mod] = np.zeros((con.shape[0], con.shape[1], con.shape[2], 5), dtype=con.dtype)\n",
- " bp_thresh[mod][...,:2] = con[...,:2]\n",
- " bp_thresh[mod][...,2:] = con\n",
+ "if not fixed_gain_mode:\n",
+ " # Plot only three gain threshold maps.\n",
+ " bp_thresh = OrderedDict()\n",
+ " for mod, con in badpix_g.items():\n",
+ " bp_thresh[mod] = np.zeros((con.shape[0], con.shape[1], con.shape[2], 5), dtype=con.dtype)\n",
+ " bp_thresh[mod][...,:2] = con[...,:2]\n",
+ " bp_thresh[mod][...,2:] = con\n",
"\n",
"\n",
- "create_constant_overview(thresholds_g, \"Threshold (ADU)\", max_cells, 4000, 10000, 5,\n",
- " badpixels=[bp_thresh, np.nan],\n",
- " gmap=['HG-MG Threshold', 'MG-LG Threshold', 'High gain', 'Medium gain', 'low gain'],\n",
- " marker=['d','d','','','']\n",
- " )"
+ " create_constant_overview(thresholds_g, \"Threshold (ADU)\", max_cells, 4000, 10000, 5,\n",
+ " badpixels=[bp_thresh, np.nan],\n",
+ " gmap=['HG-MG Threshold', 'MG-LG Threshold', 'High gain', 'Medium gain', 'low gain'],\n",
+ " marker=['d','d','','','']\n",
+ " )"
]
},
{
@@ -883,14 +890,21 @@
" \"New constant\", \"Old constant \",\n",
" \"New constant\", \"Old constant \"]\n",
"\n",
- "for const in ['Offset', 'Noise', 'ThresholdsDark']:\n",
- " if const != 'ThresholdsDark':\n",
- " table = [['','High gain', 'High gain', 'Medium gain', 'Medium gain', 'Low gain', 'Low gain']]\n",
- " else:\n",
+ "if fixed_gain_mode:\n",
+ " constants = ['Offset', 'Noise']\n",
+ "else:\n",
+ " ['Offset', 'Noise', 'ThresholdsDark']\n",
+ "\n",
+ "for const in constants:\n",
+ " \n",
+ " if const == 'ThresholdsDark':\n",
" table = [['','HG-MG threshold', 'HG-MG threshold', 'MG-LG threshold', 'MG-LG threshold']]\n",
- " for qm in res.keys():\n",
+ " else: \n",
+ " table = [['','High gain', 'High gain', 'Medium gain', 'Medium gain', 'Low gain', 'Low gain']]\n",
"\n",
+ " for qm in res.keys():\n",
" data = np.copy(res[qm][const])\n",
+ " \n",
" if const == 'ThresholdsDark':\n",
" data[...,0][res[qm]['BadPixelsDark'][...,0]>0] = np.nan\n",
" data[...,1][res[qm]['BadPixelsDark'][...,1]>0] = np.nan\n",
--
GitLab
From 5d452dca21d125560b3ec184a213e3d2d5b9b6b1 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Tue, 2 Mar 2021 10:09:06 +0100
Subject: [PATCH 41/72] Stripping some trailing spaces
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 48 +++++++++----------
1 file changed, 24 insertions(+), 24 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 58aaf3e52..9dc02f12d 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -135,7 +135,7 @@
"gains = np.arange(3)\n",
"\n",
"max_cells = mem_cells\n",
- " \n",
+ "\n",
"offset_runs = OrderedDict()\n",
"offset_runs[\"high\"] = run_high\n",
"offset_runs[\"med\"] = run_med\n",
@@ -203,7 +203,7 @@
" gsettings.append(get_gain_setting(control_fname, h5path_ctrl))\n",
" if not all(g == gsettings[0] for g in gsettings):\n",
" raise ValueError(f\"Different gain settings for the 3 input runs {gsettings}\")\n",
- " gain_setting = gsettings[0] \n",
+ " gain_setting = gsettings[0]\n",
" except Exception as e:\n",
" print(f'Error while reading gain setting from: \\n{control_fname}')\n",
" print(f'Error: {e}')\n",
@@ -228,7 +228,7 @@
"h5path = h5path.format(karabo_id, receiver_id)\n",
"h5path_idx = h5path_idx.format(karabo_id, receiver_id)\n",
"\n",
- "if bias_voltage == 0: \n",
+ "if bias_voltage == 0:\n",
" # Read the bias voltage from files, if recorded.\n",
" # If not available, make use of the historical voltage the detector is running at\n",
" bias_voltage = get_bias_voltage(control_names[0], karabo_id_control)\n",
@@ -620,15 +620,15 @@
" acquisition_rate=acq_rate,\n",
" gain_setting=gain_setting)\n",
" if db_output:\n",
- " md = send_to_db(db_module, karabo_id, dconst, condition, file_loc, \n",
+ " md = send_to_db(db_module, karabo_id, dconst, condition, file_loc,\n",
" report, cal_db_interface, creation_time=creation_time,\n",
" timeout=cal_db_timeout)\n",
"\n",
" if local_output:\n",
- " md = save_const_to_h5(db_module, karabo_id, dconst, condition, dconst.data, \n",
+ " md = save_const_to_h5(db_module, karabo_id, dconst, condition, dconst.data,\n",
" file_loc, report, creation_time, out_folder)\n",
" print(f\"Calibration constant {const} is stored locally.\\n\")\n",
- " \n",
+ "\n",
" print(\"Constants parameter conditions are:\\n\")\n",
" print(f\"• memory_cells: {max_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
" f\"• acquisition_rate: {acq_rate}\\n• gain_setting: {gain_setting}\\n\"\n",
@@ -723,18 +723,18 @@
" BadPixels.OFFSET_OUT_OF_THRESHOLD.value: (BadPixels.OFFSET_OUT_OF_THRESHOLD.name, '#00FF0080'),\n",
" BadPixels.GAIN_THRESHOLDING_ERROR.value: (BadPixels.GAIN_THRESHOLDING_ERROR.name, '#FF40FF40'),\n",
" BadPixels.OFFSET_OUT_OF_THRESHOLD.value | BadPixels.NOISE_OUT_OF_THRESHOLD.value: ('OFFSET_OUT_OF_THRESHOLD + NOISE_OUT_OF_THRESHOLD', '#DD00DD80'),\n",
- " BadPixels.OFFSET_OUT_OF_THRESHOLD.value | BadPixels.NOISE_OUT_OF_THRESHOLD.value | \n",
+ " BadPixels.OFFSET_OUT_OF_THRESHOLD.value | BadPixels.NOISE_OUT_OF_THRESHOLD.value |\n",
" BadPixels.GAIN_THRESHOLDING_ERROR.value: ('MIXED', '#BFDF009F')}\n",
"\n",
"if high_res_badpix_3d:\n",
" display(Markdown(\"\"\"\n",
- " \n",
+ "\n",
" ## Global Bad Pixel Behaviour ##\n",
"\n",
- " The following plots show the results of bad pixel evaluation for all evaluated memory cells. \n",
- " Cells are stacked in the Z-dimension, while pixels values in x/y are rebinned with a factor of 2. \n",
- " This excludes single bad pixels present only in disconnected pixels. \n",
- " Hence, any bad pixels spanning at least 4 pixels in the x/y-plane, or across at least two memory cells are indicated. \n",
+ " The following plots show the results of bad pixel evaluation for all evaluated memory cells.\n",
+ " Cells are stacked in the Z-dimension, while pixels values in x/y are rebinned with a factor of 2.\n",
+ " This excludes single bad pixels present only in disconnected pixels.\n",
+ " Hence, any bad pixels spanning at least 4 pixels in the x/y-plane, or across at least two memory cells are indicated.\n",
" Colors encode the bad pixel type, or mixed type.\n",
"\n",
" \"\"\"))\n",
@@ -847,10 +847,10 @@
" for bit in bits:\n",
" l_data_old.append(np.count_nonzero(old_const['BadPixelsDark'][:, :, :, gain] & bit.value))\n",
"\n",
- " l_data_name = ['All bad pixels', 'NOISE_OUT_OF_THRESHOLD', \n",
+ " l_data_name = ['All bad pixels', 'NOISE_OUT_OF_THRESHOLD',\n",
" 'OFFSET_OUT_OF_THRESHOLD', 'OFFSET_NOISE_EVAL_ERROR', 'GAIN_THRESHOLDING_ERROR']\n",
"\n",
- " l_threshold = ['', f'{thresholds_noise_sigma}' f'{thresholds_noise_hard[gain]}', \n",
+ " l_threshold = ['', f'{thresholds_noise_sigma}' f'{thresholds_noise_hard[gain]}',\n",
" f'{thresholds_offset_sigma}' f'{thresholds_offset_hard[gain]}',\n",
" '', f'{thresholds_gain_sigma}']\n",
"\n",
@@ -873,9 +873,9 @@
"\n",
"'''))\n",
"if len(table)>0:\n",
- " md = display(Latex(tabulate.tabulate(table, tablefmt='latex', \n",
- " headers=[\"Pixel type\", \"Threshold\", \n",
- " \"New constant\", \"Old constant\"]))) "
+ " md = display(Latex(tabulate.tabulate(table, tablefmt='latex',\n",
+ " headers=[\"Pixel type\", \"Threshold\",\n",
+ " \"New constant\", \"Old constant\"])))"
]
},
{
@@ -884,9 +884,9 @@
"metadata": {},
"outputs": [],
"source": [
- "header = ['Parameter', \n",
- " \"New constant\", \"Old constant \", \n",
- " \"New constant\", \"Old constant \", \n",
+ "header = ['Parameter',\n",
+ " \"New constant\", \"Old constant \",\n",
+ " \"New constant\", \"Old constant \",\n",
" \"New constant\", \"Old constant \",\n",
" \"New constant\", \"Old constant \"]\n",
"\n",
@@ -896,15 +896,15 @@
" ['Offset', 'Noise', 'ThresholdsDark']\n",
"\n",
"for const in constants:\n",
- " \n",
+ "\n",
" if const == 'ThresholdsDark':\n",
" table = [['','HG-MG threshold', 'HG-MG threshold', 'MG-LG threshold', 'MG-LG threshold']]\n",
- " else: \n",
+ " else:\n",
" table = [['','High gain', 'High gain', 'Medium gain', 'Medium gain', 'Low gain', 'Low gain']]\n",
"\n",
" for qm in res.keys():\n",
" data = np.copy(res[qm][const])\n",
- " \n",
+ "\n",
" if const == 'ThresholdsDark':\n",
" data[...,0][res[qm]['BadPixelsDark'][...,0]>0] = np.nan\n",
" data[...,1][res[qm]['BadPixelsDark'][...,1]>0] = np.nan\n",
@@ -937,7 +937,7 @@
" table.append(line)\n",
"\n",
" display(Markdown('### {} [ADU], good pixels only ###'.format(const)))\n",
- " md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=header))) "
+ " md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=header)))"
]
}
],
--
GitLab
From 5c6530e793ceecb840106a23296851c6e4c64c81 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Wed, 3 Mar 2021 20:22:53 +0100
Subject: [PATCH 42/72] Moving get_gain_mode to agipdlib
---
cal_tools/cal_tools/agipdlib.py | 12 +++++++++++-
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 18 ++----------------
2 files changed, 13 insertions(+), 17 deletions(-)
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 073c106be..1f3af4b12 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -15,7 +15,7 @@ from cal_tools.agipdutils import (
match_asic_borders,
melt_snowy_pixels,
)
-from cal_tools.enums import BadPixels, SnowResolution
+from cal_tools.enums import AgipdGainMode, BadPixels, SnowResolution
from cal_tools.tools import get_constant_from_db_and_time
@@ -125,6 +125,16 @@ def get_gain_setting(fname: str, h5path_ctrl: str) -> int:
else:
raise ValueError('Could not derive gain setting from setupr and patternTypeIndex') # noqa
+def get_gain_mode(fname: str, h5path_ctrl: str) -> AgipdGainMode:
+ """Returns the gain mode (adaptive or fixed) from slow data"""
+
+ h5path_run = h5path_ctrl.replace("CONTROL/", "RUN/", 1)
+ h5path_gainmode = f'{h5path_run}/gainModeIndex/value'
+ with h5py.File(fname, 'r') as fd:
+ if h5path_gainmode in fd:
+ return AgipdGainMode(fd[h5path_gainmode][0])
+ return AgipdGainMode.ADAPTIVE_GAIN
+
def get_bias_voltage(fname: str, karabo_id_control: str,
module: Optional[int] = 0) -> int:
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 9dc02f12d..e2714006d 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -96,7 +96,7 @@
"\n",
"import multiprocessing\n",
"\n",
- "from cal_tools.agipdlib import (get_acq_rate, get_bias_voltage,\n",
+ "from cal_tools.agipdlib import (get_acq_rate, get_bias_voltage, get_gain_mode,\n",
" get_gain_setting, get_num_cells)\n",
"from cal_tools.enums import AgipdGainMode\n",
"from cal_tools.plotting import (create_constant_overview, plot_badpix_3d,\n",
@@ -109,20 +109,6 @@
"from iCalibrationDB import Conditions, Constants, Detectors"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "def get_gain_mode(h5fn):\n",
- " h5path_gainmode = f'{h5path_ctrl.replace(\"/CONTROL/\", \"/RUN/\", 1)}/gainModeIndex/value'\n",
- " with h5py.File(h5fn, 'r') as fd:\n",
- " if h5path_gainmode in fd:\n",
- " return AgipdGainMode(fd[h5path_gainmode][0])\n",
- " return AgipdGainMode.ADAPTIVE_GAIN"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -167,7 +153,7 @@
"control_names = [f'{in_folder}/r{r:04d}/RAW-R{r:04d}-{karabo_da_control}-S00000.h5'\n",
" for r in (run_high, run_med, run_low)]\n",
"\n",
- "run_gain_modes = [get_gain_mode(fn) for fn in control_names]\n",
+ "run_gain_modes = [get_gain_mode(fn, h5path_ctrl) for fn in control_names]\n",
"if all(gm == AgipdGainMode.ADAPTIVE_GAIN for gm in run_gain_modes):\n",
" fixed_gain_mode = False\n",
"elif run_gain_modes == [AgipdGainMode.FIXED_HIGH_GAIN, AgipdGainMode.FIXED_MEDIUM_GAIN, AgipdGainMode.FIXED_LOW_GAIN]:\n",
--
GitLab
From 3e7bd9a6de64725665e92a1e3a02cd9a50a54e27 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Wed, 3 Mar 2021 20:23:38 +0100
Subject: [PATCH 43/72] Stripping trailing space, unused variable
---
notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index e2714006d..6760fdb25 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -118,8 +118,6 @@
"# insert control device if format string (does nothing otherwise)\n",
"h5path_ctrl = h5path_ctrl.format(karabo_id_control)\n",
"\n",
- "gains = np.arange(3)\n",
- "\n",
"max_cells = mem_cells\n",
"\n",
"offset_runs = OrderedDict()\n",
@@ -557,7 +555,7 @@
" acquisition_rate=acq_rate,\n",
" gain_setting=gain_setting)\n",
"\n",
- " # This should be used in case of running notebook \n",
+ " # This should be used in case of running notebook\n",
" # by a different method other than myMDC which already\n",
" # sends CalCat info.\n",
" # TODO: Set db_module to \"\" by default in the first cell\n",
--
GitLab
From 657d1a34c15f9dc40c7b0c554bbc1c9d97857166 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Wed, 3 Mar 2021 20:25:08 +0100
Subject: [PATCH 44/72] Skip thresholds if fixed gain mode
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 48 ++++++++++++-------
1 file changed, 30 insertions(+), 18 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 6760fdb25..1e4d4ded8 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -330,32 +330,41 @@
" first_index = int(first[status != 0][0])\n",
" im = np.array(infile[f\"{h5path_f}/data\"][first_index:last_index,...])\n",
" cellIds = np.squeeze(infile[f\"{h5path_f}/cellId\"][first_index:last_index,...])\n",
- "\n",
+ " \n",
" if interlaced:\n",
- " ga = im[1::2, 0, ...]\n",
+ " if not fixed_gain_mode:\n",
+ " ga = im[1::2, 0, ...]\n",
" im = im[0::2, 0, ...].astype(np.float32)\n",
" cellIds = cellIds[::2]\n",
" else:\n",
- " ga = im[:, 1, ...]\n",
+ " if not fixed_gain_mode:\n",
+ " ga = im[:, 1, ...]\n",
" im = im[:, 0, ...].astype(np.float32)\n",
"\n",
" im = np.rollaxis(im, 2)\n",
" im = np.rollaxis(im, 2, 1)\n",
"\n",
- " ga = np.rollaxis(ga, 2)\n",
- " ga = np.rollaxis(ga, 2, 1)\n",
- "\n",
+ " if not fixed_gain_mode:\n",
+ " ga = np.rollaxis(ga, 2)\n",
+ " ga = np.rollaxis(ga, 2, 1)\n",
+ " \n",
" offset = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
- " gains = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
" noise = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
- " gains_std = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
+ "\n",
+ " if fixed_gain_mode:\n",
+ " gains = None\n",
+ " gains_std = None\n",
+ " else:\n",
+ " gains = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
+ " gains_std = np.zeros((im.shape[0], im.shape[1], num_cells))\n",
"\n",
" for cc in np.unique(cellIds[cellIds < num_cells]):\n",
" cellidx = cellIds == cc\n",
" offset[...,cc] = np.median(im[..., cellidx], axis=2)\n",
" noise[...,cc] = np.std(im[..., cellidx], axis=2)\n",
- " gains[...,cc] = np.median(ga[..., cellidx], axis=2)\n",
- " gains_std[...,cc] = np.std(ga[..., cellidx], axis=2)\n",
+ " if not fixed_gain_mode:\n",
+ " gains[...,cc] = np.median(ga[..., cellidx], axis=2)\n",
+ " gains_std[...,cc] = np.std(ga[..., cellidx], axis=2)\n",
"\n",
" # bad pixels\n",
" bp = np.zeros(offset.shape, np.uint32)\n",
@@ -388,10 +397,11 @@
"source": [
"offset_g = OrderedDict()\n",
"noise_g = OrderedDict()\n",
- "gain_g = OrderedDict()\n",
- "gainstd_g = OrderedDict()\n",
"badpix_g = OrderedDict()\n",
- "\n",
+ "if not fixed_gain_mode:\n",
+ " gain_g = OrderedDict()\n",
+ " gainstd_g = OrderedDict()\n",
+ " \n",
"start = datetime.now()\n",
"all_cells = []\n",
"all_acq_rate = []\n",
@@ -420,15 +430,17 @@
" if qm not in offset_g:\n",
" offset_g[qm] = np.zeros((offset.shape[0], offset.shape[1], offset.shape[2], 3))\n",
" noise_g[qm] = np.zeros_like(offset_g[qm])\n",
- " gain_g[qm] = np.zeros_like(offset_g[qm])\n",
- " gainstd_g[qm] = np.zeros_like(offset_g[qm])\n",
" badpix_g[qm] = np.zeros_like(offset_g[qm], np.uint32)\n",
+ " if not fixed_gain_mode:\n",
+ " gain_g[qm] = np.zeros_like(offset_g[qm])\n",
+ " gainstd_g[qm] = np.zeros_like(offset_g[qm])\n",
"\n",
" offset_g[qm][...,gg] = offset\n",
" noise_g[qm][...,gg] = noise\n",
- " gain_g[qm][...,gg] = gains\n",
- " gainstd_g[qm][..., gg] = gains_std\n",
" badpix_g[qm][...,gg] = bp\n",
+ " if not fixed_gain_mode:\n",
+ " gain_g[qm][...,gg] = gains\n",
+ " gainstd_g[qm][..., gg] = gains_std\n",
"\n",
"\n",
"duration = (datetime.now() - start).total_seconds()\n",
@@ -877,7 +889,7 @@
"if fixed_gain_mode:\n",
" constants = ['Offset', 'Noise']\n",
"else:\n",
- " ['Offset', 'Noise', 'ThresholdsDark']\n",
+ " constants = ['Offset', 'Noise', 'ThresholdsDark']\n",
"\n",
"for const in constants:\n",
"\n",
--
GitLab
From bd6137af90153c326a2298712656499d52c1b622 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Wed, 3 Mar 2021 20:25:31 +0100
Subject: [PATCH 45/72] Add gain mode to condition
This requires the pending MR for cal_db_interactive. Tested with e79f2c9d (current head of the
branch of that MR).
---
notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 1e4d4ded8..a14dea206 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -565,7 +565,8 @@
" condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
" bias_voltage=bias_voltage,\n",
" acquisition_rate=acq_rate,\n",
- " gain_setting=gain_setting)\n",
+ " gain_setting=gain_setting,\n",
+ " gain_mode=(True if fixed_gain_mode else None))\n",
"\n",
" # This should be used in case of running notebook\n",
" # by a different method other than myMDC which already\n",
@@ -614,7 +615,8 @@
" condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
" bias_voltage=bias_voltage,\n",
" acquisition_rate=acq_rate,\n",
- " gain_setting=gain_setting)\n",
+ " gain_setting=gain_setting,\n",
+ " gain_mode=(True if fixed_gain_mode else None))\n",
" if db_output:\n",
" md = send_to_db(db_module, karabo_id, dconst, condition, file_loc,\n",
" report, cal_db_interface, creation_time=creation_time,\n",
--
GitLab
From d134cb40da3a158b0c4dfb7d447045df6e9d80e9 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Mon, 8 Mar 2021 17:36:18 +0100
Subject: [PATCH 46/72] Only define the condition once
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 29 ++++++++++---------
1 file changed, 15 insertions(+), 14 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index a14dea206..bec0026e2 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -537,6 +537,21 @@
" }"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# set the operating condition\n",
+ "# note: iCalibrationDB only adds gain_mode if it is truthy, so we don't need to handle None\n",
+ "condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
+ " bias_voltage=bias_voltage,\n",
+ " acquisition_rate=acq_rate,\n",
+ " gain_setting=gain_setting,\n",
+ " gain_mode=(True if fixed_gain_mode else None))"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -553,7 +568,6 @@
"detinst = getattr(Detectors, dinstance)\n",
"\n",
"print('Retrieve pre-existing constants for comparison.')\n",
- "\n",
"for qm in res:\n",
" qm_db = qm_dict[qm]\n",
" karabo_da = qm_db[\"karabo_da\"]\n",
@@ -561,13 +575,6 @@
" dconst = getattr(Constants.AGIPD, const)()\n",
" dconst.data = res[qm][const]\n",
"\n",
- " # Setting conditions\n",
- " condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
- " bias_voltage=bias_voltage,\n",
- " acquisition_rate=acq_rate,\n",
- " gain_setting=gain_setting,\n",
- " gain_mode=(True if fixed_gain_mode else None))\n",
- "\n",
" # This should be used in case of running notebook\n",
" # by a different method other than myMDC which already\n",
" # sends CalCat info.\n",
@@ -611,12 +618,6 @@
" dconst = getattr(Constants.AGIPD, const)()\n",
" dconst.data = res[qm][const]\n",
"\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
- " bias_voltage=bias_voltage,\n",
- " acquisition_rate=acq_rate,\n",
- " gain_setting=gain_setting,\n",
- " gain_mode=(True if fixed_gain_mode else None))\n",
" if db_output:\n",
" md = send_to_db(db_module, karabo_id, dconst, condition, file_loc,\n",
" report, cal_db_interface, creation_time=creation_time,\n",
--
GitLab
From 67de8cc9996774b53a36314e3a649dbcf45cf916 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Mon, 8 Mar 2021 17:39:42 +0100
Subject: [PATCH 47/72] Moving index_to_qm to tools
---
cal_tools/cal_tools/tools.py | 9 +++++++-
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 22 +++++--------------
2 files changed, 14 insertions(+), 17 deletions(-)
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index d4015a558..a6aa3f5e8 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -78,7 +78,7 @@ def map_modules_from_folder(in_folder, run, path_template, karabo_da,
sequences_qm = {}
for inset in karabo_da:
module_idx = int(inset[-2:])
- name = f"Q{module_idx // 4 + 1}M{module_idx % 4 + 1}"
+ name = module_index_to_qm(module_idx)
module_files[name] = Queue()
sequences_qm[name] = 0
mod_ids[name] = module_idx
@@ -673,6 +673,13 @@ def get_constant_from_db_and_time(karabo_id: str, karabo_da: str,
return data, None
+def module_index_to_qm(index: int, total_modules: int = 16):
+ """Maps module index (0-indexed) to quadrant + module string (1-indexed)"""
+ modules_per_quad = total_modules // 4
+ quad, mod = divmod(index, modules_per_quad)
+ return f"Q{quad+1}M{mod+1}"
+
+
class CalibrationMetadata(dict):
"""Convenience class: dictionary stored in metadata YAML file
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index bec0026e2..d715b75b2 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -103,7 +103,7 @@
" show_overview, show_processed_modules)\n",
"from cal_tools.tools import (get_dir_creation_date, get_from_db,\n",
" get_pdu_from_db, get_random_db_interface,\n",
- " get_report, map_gain_stages,\n",
+ " get_report, map_gain_stages, module_index_to_qm,\n",
" run_prop_seq_from_path, save_const_to_h5,\n",
" send_to_db)\n",
"from iCalibrationDB import Conditions, Constants, Detectors"
@@ -251,16 +251,6 @@
"print(f\"Will process a total of {total_sequences} files.\")"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "def index_to_qm(i: int):\n",
- " return f\"Q{i//4+1}M{i%4+1}\""
- ]
- },
{
"cell_type": "markdown",
"metadata": {},
@@ -410,7 +400,7 @@
"for gg, (gain, mapped_files) in enumerate(gain_mapped_files.items()):\n",
" dones = []\n",
" for i in modules:\n",
- " qm = index_to_qm(i)\n",
+ " qm = module_index_to_qm(i)\n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
" fname_in = mapped_files[qm].get()\n",
" print(\"Process file: \", fname_in)\n",
@@ -426,7 +416,7 @@
" all_cells.append(thiscell)\n",
" all_acq_rate.append(thisacq)\n",
" for i in modules:\n",
- " qm = index_to_qm(i)\n",
+ " qm = module_index_to_qm(i)\n",
" if qm not in offset_g:\n",
" offset_g[qm] = np.zeros((offset.shape[0], offset.shape[1], offset.shape[2], 3))\n",
" noise_g[qm] = np.zeros_like(offset_g[qm])\n",
@@ -497,7 +487,7 @@
"source": [
"res = OrderedDict()\n",
"for i in modules:\n",
- " qm = index_to_qm(i)\n",
+ " qm = module_index_to_qm(i)\n",
" res[qm] = {\n",
" 'Offset': offset_g[qm],\n",
" 'Noise': noise_g[qm],\n",
@@ -530,7 +520,7 @@
"# Create the modules dict of karabo_das and PDUs\n",
"qm_dict = OrderedDict()\n",
"for i, k_da in zip(modules, karabo_da):\n",
- " qm = index_to_qm(i)\n",
+ " qm = module_index_to_qm(i)\n",
" qm_dict[qm] = {\n",
" \"karabo_da\": k_da,\n",
" \"db_module\": \"\"\n",
@@ -642,7 +632,7 @@
"source": [
"mnames=[]\n",
"for i in modules:\n",
- " qm = index_to_qm(i)\n",
+ " qm = module_index_to_qm(i)\n",
" mnames.append(qm)\n",
" display(Markdown(f'## Position of the module {qm} and its ASICs##'))\n",
"show_processed_modules(dinstance, constants=None, mnames=mnames, mode=\"position\")"
--
GitLab
From 1b7030d9d492f50cf7d4ab46f4d54f4d0fdc1b16 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Tue, 9 Mar 2021 14:43:57 +0100
Subject: [PATCH 48/72] Adding back operation_mode parameter (only used for
warning so far)
---
notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index d715b75b2..b1298821c 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -26,6 +26,7 @@
"run_high = 9985 # run number in which high gain data was recorded, required\n",
"run_med = 9984 # run number in which medium gain data was recorded, required\n",
"run_low = 9983 # run number in which low gain data was recorded, required\n",
+ "operation_mode = \"ADAPTIVE_GAIN\" # Detector operation mode, optional (defaults to \"ADAPTIVE_GAIN\")\n",
"\n",
"karabo_id = \"HED_DET_AGIPD500K2G\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
@@ -151,10 +152,16 @@
"control_names = [f'{in_folder}/r{r:04d}/RAW-R{r:04d}-{karabo_da_control}-S00000.h5'\n",
" for r in (run_high, run_med, run_low)]\n",
"\n",
+ "if operation_mode not in (\"ADAPTIVE_GAIN\", \"FIXED_GAIN\"):\n",
+ " print(f\"WARNING: unknown operation_mode \\\"{operation_mode}\\\" parameter set\")\n",
"run_gain_modes = [get_gain_mode(fn, h5path_ctrl) for fn in control_names]\n",
"if all(gm == AgipdGainMode.ADAPTIVE_GAIN for gm in run_gain_modes):\n",
" fixed_gain_mode = False\n",
+ " if operation_mode == \"FIXED_GAIN\":\n",
+ " print(\"WARNING: operation_mode parameter is FIXED_GAIN, slow data indicates adaptive gain\")\n",
"elif run_gain_modes == [AgipdGainMode.FIXED_HIGH_GAIN, AgipdGainMode.FIXED_MEDIUM_GAIN, AgipdGainMode.FIXED_LOW_GAIN]:\n",
+ " if operation_mode == \"ADAPTIVE_GAIN\":\n",
+ " print(\"WARNING: operation_mode parameter ix ADAPTIVE_GAIN, slow data indicates fixed gain\")\n",
" fixed_gain_mode = True\n",
"else:\n",
" print(f'Something is clearly wrong; slow data indicates gain modes {run_gain_modes}')\n",
--
GitLab
From 4926afd454db2dc58091dbeaf763e0da549b9df2 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 11 Mar 2021 13:43:29 +0100
Subject: [PATCH 49/72] Dropping driver="core" when opening h5py file
The relevant data is copied into memory in pretty large chunks, so the core driver is not expected
to do much for us here. In some brief testing, using the default driver instead resulted in
approximately the same (or slightly better) runtime and significantly lower peak memory consumption.
---
notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index b1298821c..db3fee539 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -311,7 +311,7 @@
" h5path_f = h5path.format(channel)\n",
" h5path_idx_f = h5path_idx.format(channel)\n",
"\n",
- " with h5py.File(fast_data_filename, \"r\", driver=\"core\") as infile:\n",
+ " with h5py.File(fast_data_filename, \"r\") as infile:\n",
" if rawversion == 2:\n",
" count = np.squeeze(infile[f\"{h5path_idx_f}/count\"])\n",
" first = np.squeeze(infile[f\"{h5path_idx_f}/first\"])\n",
--
GitLab
From df2671d362b15fa70ca114600a0c245f38791ddc Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 11 Mar 2021 14:40:51 +0100
Subject: [PATCH 50/72] Adding specific threshold parameters for fixed gain
mode
Currently, there is no way to specify most parameters when requesting darks via MyMDC - values are
just loaded from the calibration configurations repository. To allow for using different offset
thresholds in fixed gain mode versus adaptive gain mode, adding these extra parameters is a
pragmatic if not elegant solution for now.
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 43 ++++++++++++++-----
1 file changed, 32 insertions(+), 11 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index db3fee539..eae874835 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -52,10 +52,13 @@
"rawversion = 2 # RAW file format version\n",
"\n",
"thresholds_offset_sigma = 3. # offset sigma thresholds for offset deduced bad pixels\n",
- "thresholds_offset_hard = [0, 0] # For setting the same threshold offset for the 3 gains. Left for backcompatability. Default [0, 0] to take the following parameters.\n",
- "thresholds_offset_hard_hg = [3000, 7000] # High-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
- "thresholds_offset_hard_mg = [6000, 10000] # Medium-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
- "thresholds_offset_hard_lg = [6000, 10000] # Low-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
+ "thresholds_offset_hard = [0, 0] # For setting the same threshold offset for the 3 gains. Left for backcompatability. Default [0, 0] to take the following parameters.\n",
+ "thresholds_offset_hard_hg = [3000, 7000] # High-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
+ "thresholds_offset_hard_mg = [6000, 10000] # Medium-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
+ "thresholds_offset_hard_lg = [6000, 10000] # Low-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
+ "thresholds_offset_hard_hg_fixed = [3500, 6500] # Same as thresholds_offset_hard_hg, but for fixed gain operation\n",
+ "thresholds_offset_hard_mg_fixed = [3500, 6500] # Same as thresholds_offset_hard_mg, but for fixed gain operation\n",
+ "thresholds_offset_hard_lg_fixed = [3500, 6500] # Same as thresholds_offset_hard_lg, but for fixed gain operation\n",
"\n",
"thresholds_noise_sigma = 5. # noise sigma thresholds for offset deduced bad pixels\n",
"thresholds_noise_hard = [0, 0] # For setting the same threshold noise for the 3 gains. Left for backcompatability. Default [0, 0] to take the following parameters.\n",
@@ -273,15 +276,33 @@
"metadata": {},
"outputs": [],
"source": [
- "if thresholds_offset_hard == [0, 0]:\n",
- " thresholds_offset_hard = [thresholds_offset_hard_hg, thresholds_offset_hard_mg, thresholds_offset_hard_lg]\n",
- "else:\n",
+ "if thresholds_offset_hard != [0, 0]:\n",
+ " # if set, this will override the individual parameters\n",
" thresholds_offset_hard = [thresholds_offset_hard] * 3\n",
- "\n",
- "if thresholds_noise_hard == [0, 0]:\n",
- " thresholds_noise_hard = [thresholds_noise_hard_hg, thresholds_noise_hard_mg, thresholds_noise_hard_lg]\n",
+ "elif fixed_gain_mode:\n",
+ " thresholds_offset_hard = [\n",
+ " thresholds_offset_hard_hg_fixed,\n",
+ " thresholds_offset_hard_mg_fixed,\n",
+ " thresholds_offset_hard_lg_fixed,\n",
+ " ]\n",
+ "else:\n",
+ " thresholds_offset_hard = [\n",
+ " thresholds_offset_hard_hg,\n",
+ " thresholds_offset_hard_mg,\n",
+ " thresholds_offset_hard_lg,\n",
+ " ]\n",
+ "print(f\"Will use the following hard offset thresholds\")\n",
+ "for name, value in zip((\"High\", \"Medium\", \"Low\"), thresholds_offset_hard):\n",
+ " print(f\"- {name} gain: {value}\")\n",
+ "\n",
+ "if thresholds_noise_hard != [0, 0]:\n",
+ " thresholds_noise_hard = [thresholds_noise_hard] * 3\n",
"else:\n",
- " thresholds_noise_hard = [thresholds_noise_hard] * 3"
+ " thresholds_noise_hard = [\n",
+ " thresholds_noise_hard_hg,\n",
+ " thresholds_noise_hard_mg,\n",
+ " thresholds_noise_hard_lg,\n",
+ " ]"
]
},
{
--
GitLab
From 967902379f65042f4e468e0980830271d7cf5b4b Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 11 Mar 2021 14:53:21 +0100
Subject: [PATCH 51/72] We don't need to omit the gain mode from the condition
here
---
notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index eae874835..e44b990c1 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -567,7 +567,7 @@
" bias_voltage=bias_voltage,\n",
" acquisition_rate=acq_rate,\n",
" gain_setting=gain_setting,\n",
- " gain_mode=(True if fixed_gain_mode else None))"
+ " gain_mode=fixed_gain_mode)"
]
},
{
--
GitLab
From 9c1934ec152e67efc74ad330a9c03f92b6ed6c1d Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Fri, 12 Mar 2021 15:13:15 +0100
Subject: [PATCH 52/72] Moving output regarding gain mode to other
condition-related notes
---
notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index e44b990c1..d96004389 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -171,8 +171,7 @@
"\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"print(f\"Instrument {instrument}\")\n",
- "print(f\"Detector instance {dinstance}\")\n",
- "print(f\"Operation mode is {'fixed' if fixed_gain_mode else 'adaptive'} gain mode\")"
+ "print(f\"Detector instance {dinstance}\")"
]
},
{
@@ -238,7 +237,8 @@
"print(f\"Input: {in_folder}\")\n",
"print(f\"Output: {out_folder}\")\n",
"print(f\"Bias voltage: {bias_voltage}V\")\n",
- "print(f\"Gain setting: {gain_setting}\")"
+ "print(f\"Gain setting: {gain_setting}\")\n",
+ "print(f\"Operation mode is {'fixed' if fixed_gain_mode else 'adaptive'} gain mode\")"
]
},
{
--
GitLab
From 6e632ff3a004fc3d4791b7e01a64bc8e0ca69d93 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Mon, 22 Mar 2021 14:03:48 +0100
Subject: [PATCH 53/72] Removed unused variable, import
---
notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb | 6 +-----
1 file changed, 1 insertion(+), 5 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index d96004389..b8e05be88 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -84,7 +84,7 @@
"\n",
"import os\n",
"from collections import OrderedDict\n",
- "from typing import List, Tuple\n",
+ "from typing import Tuple\n",
"\n",
"import h5py\n",
"import matplotlib\n",
@@ -577,10 +577,6 @@
"outputs": [],
"source": [
"# Retrieve existing constants for comparison\n",
- "if fixed_gain_mode:\n",
- " clist = [\"Offset\", \"Noise\", \"BadPixelsDark\"]\n",
- "else:\n",
- " clist = [\"Offset\", \"Noise\", \"ThresholdsDark\", \"BadPixelsDark\"]\n",
"old_const = {}\n",
"old_mdata = {}\n",
"detinst = getattr(Detectors, dinstance)\n",
--
GitLab
From 14abd503bc5c4d3db3da10bfe8c3b4c67bcf21f5 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Tue, 23 Mar 2021 14:41:08 +0100
Subject: [PATCH 54/72] Missed two conflicts from rebase
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 26 -------------------
1 file changed, 26 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index d9664f4fd..b1a033fa3 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -434,37 +434,11 @@
" else:\n",
" continue\n",
" inp.append((fname_in, i, gg))\n",
-<<<<<<< HEAD
-||||||| 335796a
- " \n",
- " gg += 1\n",
- "\n",
- "p = partial(characterize_module, IL_MODE, max_cells,\n",
- " (thresholds_offset_hard, thresholds_offset_sigma,\n",
- " thresholds_noise_hard, thresholds_noise_sigma),\n",
- " rawversion, karabo_id, acq_rate, h5path, h5path_idx,\n",
- " control_names, karabo_id_control)\n",
-=======
"\n",
"with multiprocessing.Pool() as pool:\n",
" results = pool.starmap(characterize_module, inp)\n",
->>>>>>> bd98a3c6b5a10f7347ba8a74d0ce895565db9ca5
"\n",
-<<<<<<< HEAD
- "with multiprocessing.Pool() as pool:\n",
- " results = pool.starmap(characterize_module, inp)\n",
- "\n",
- "for offset, noise, gains, gains_std, gg, bp, thiscell, thisacq in results:\n",
-||||||| 335796a
- "# Don't remove. Used for Debugging.\n",
- "#results = list(map(p, inp))\n",
- "results = view.map_sync(p, inp)\n",
- "\n",
- "for ii, r in enumerate(results):\n",
- " offset, noise, gains, gains_std, gg, bp, thiscell, thisacq = r\n",
-=======
"for offset, noise, gains, gains_std, gg, bp, thiscell, thisacq in results:\n",
->>>>>>> bd98a3c6b5a10f7347ba8a74d0ce895565db9ca5
" all_cells.append(thiscell)\n",
" all_acq_rate.append(thisacq)\n",
" for i in modules:\n",
--
GitLab
From 91d5f17911e6af706fea055792480120bc3c8b0c Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Tue, 23 Mar 2021 14:42:30 +0100
Subject: [PATCH 55/72] isort
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 39 +++++++++++++------
1 file changed, 27 insertions(+), 12 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index b1a033fa3..22ee77702 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -77,13 +77,12 @@
"metadata": {},
"outputs": [],
"source": [
- "from datetime import datetime\n",
- "\n",
- "import dateutil.parser\n",
"import os\n",
"from collections import OrderedDict\n",
+ "from datetime import datetime\n",
"from typing import Tuple\n",
"\n",
+ "import dateutil.parser\n",
"import h5py\n",
"import matplotlib\n",
"import numpy as np\n",
@@ -98,16 +97,32 @@
"\n",
"import multiprocessing\n",
"\n",
- "from cal_tools.agipdlib import (get_acq_rate, get_bias_voltage, get_gain_mode,\n",
- " get_gain_setting, get_num_cells)\n",
+ "from cal_tools.agipdlib import (\n",
+ " get_acq_rate,\n",
+ " get_bias_voltage,\n",
+ " get_gain_mode,\n",
+ " get_gain_setting,\n",
+ " get_num_cells,\n",
+ ")\n",
"from cal_tools.enums import AgipdGainMode\n",
- "from cal_tools.plotting import (create_constant_overview, plot_badpix_3d,\n",
- " show_overview, show_processed_modules)\n",
- "from cal_tools.tools import (get_dir_creation_date, get_from_db,\n",
- " get_pdu_from_db, get_random_db_interface,\n",
- " get_report, map_gain_stages, module_index_to_qm,\n",
- " run_prop_seq_from_path, save_const_to_h5,\n",
- " send_to_db)\n",
+ "from cal_tools.plotting import (\n",
+ " create_constant_overview,\n",
+ " plot_badpix_3d,\n",
+ " show_overview,\n",
+ " show_processed_modules,\n",
+ ")\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " get_from_db,\n",
+ " get_pdu_from_db,\n",
+ " get_random_db_interface,\n",
+ " get_report,\n",
+ " map_gain_stages,\n",
+ " module_index_to_qm,\n",
+ " run_prop_seq_from_path,\n",
+ " save_const_to_h5,\n",
+ " send_to_db,\n",
+ ")\n",
"from iCalibrationDB import Conditions, Constants, Detectors"
]
},
--
GitLab
From 832ad5b3911d40074a966b219a7852050b646ad3 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Fri, 12 Mar 2021 13:56:40 +0100
Subject: [PATCH 56/72] Initial changes to use new stuff from base branch
---
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 37 +++++++++----------
1 file changed, 17 insertions(+), 20 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index c4cbf1ef2..82f22b590 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -142,21 +142,14 @@
"sns.set_style(\"ticks\")\n",
"\n",
"import seaborn as sns\n",
- "from cal_tools.agipdlib import (\n",
- " AgipdCorrections,\n",
- " get_acq_rate,\n",
- " get_gain_setting,\n",
- " get_num_cells,\n",
- ")\n",
+ "from cal_tools.agipdlib import (AgipdCorrections, get_acq_rate, get_gain_mode,\n",
+ " get_gain_setting, get_num_cells)\n",
"from cal_tools.ana_tools import get_range\n",
"from cal_tools.cython import agipdalgs as calgs\n",
- "from cal_tools.enums import BadPixels\n",
+ "from cal_tools.enums import AgipdGainMode, BadPixels\n",
"from cal_tools.step_timing import StepTimer\n",
- "from cal_tools.tools import (\n",
- " CalibrationMetadata,\n",
- " get_dir_creation_date,\n",
- " map_modules_from_folder,\n",
- ")\n",
+ "from cal_tools.tools import (CalibrationMetadata, get_dir_creation_date,\n",
+ " map_modules_from_folder, module_index_to_qm)\n",
"\n",
"sns.set()\n",
"sns.set_context(\"paper\", font_scale=1.4)\n",
@@ -263,11 +256,8 @@
"else:\n",
" modules = [int(x[-2:]) for x in karabo_da]\n",
" \n",
- "def mod_name(modno):\n",
- " return f\"Q{modno // 4 + 1}M{modno % 4 + 1}\"\n",
- "\n",
"print(\"Process modules: \", ', '.join(\n",
- " [mod_name(x) for x in modules]))\n",
+ " [module_index_to_qm(x) for x in modules]))\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"print(f\"Instrument {instrument}\")\n",
"print(f\"Detector instance {dinstance}\")"
@@ -376,7 +366,9 @@
" print(e)\n",
" print(\"Set gain setting to 0\")\n",
" gain_setting = 0\n",
- " "
+ "\n",
+ "# Evaluate gain mode (operation mode)\n",
+ "gain_mode = get_gain_mode(control_fn, h5path_ctrl)"
]
},
{
@@ -386,8 +378,13 @@
"outputs": [],
"source": [
"print(f\"Using {creation_time} as creation time\")\n",
- "print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Memory cells: {mem_cells_db}\\n\"\n",
- " f\"• Acquisition rate: {acq_rate}\\n• Gain setting: {gain_setting}\\n• Photon Energy: {photon_energy}\\n\")"
+ "print(\"Operating conditions are:\")\n",
+ "print(f\"• Bias voltage: {bias_voltage}\")\n",
+ "print(f\"• Memory cells: {mem_cells_db}\")\n",
+ "print(f\"• Acquisition rate: {acq_rate}\")\n",
+ "print(f\"• Gain setting: {gain_setting}\")\n",
+ "print(f\"• Gain mode: {gain_mode.name}\")\n",
+ "print(f\"• Photon Energy: {photon_energy}\")"
]
},
{
@@ -598,7 +595,7 @@
"timestamps = {}\n",
"\n",
"for i, (error, modno, when, k_da) in enumerate(const_out):\n",
- " qm = mod_name(modno)\n",
+ " qm = module_index_to_qm(modno)\n",
" # expose errors while applying correction\n",
" if error:\n",
" print(\"Error: {}\".format(error) )\n",
--
GitLab
From ded58b8165c22d1acc2474ce63c0f33a901b4a9f Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Fri, 12 Mar 2021 16:28:21 +0100
Subject: [PATCH 57/72] Initial refactoring and cleanup
nbstripout, some flake8 (removing unused variables, imports), isort, simplifying retrieve_constants
as it does not need to work under ipyparallel
---
...IPD_Retrieve_Constants_Precorrection.ipynb | 228 ++++++++----------
1 file changed, 100 insertions(+), 128 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 8126a0a01..78386a95e 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -14,15 +14,9 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-21T11:30:06.730220Z",
- "start_time": "2019-02-21T11:30:06.658286Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
- "cluster_profile = \"noDB\"\n",
"in_folder = \"/gpfs/exfel/exp/SPB/202030/p900119/raw\" # the folder to read data from, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_\" # the folder to output to, required\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
@@ -91,12 +85,8 @@
"metadata": {},
"outputs": [],
"source": [
- "import sys\n",
- "from collections import OrderedDict\n",
- "from functools import partial\n",
"from typing import List, Tuple\n",
"\n",
- "import h5py\n",
"import matplotlib\n",
"import numpy as np\n",
"\n",
@@ -106,12 +96,7 @@
"from pathlib import Path\n",
"\n",
"import matplotlib.pyplot as plt\n",
- "from cal_tools.agipdlib import get_gain_setting\n",
- "from cal_tools.tools import (\n",
- " CalibrationMetadata,\n",
- " get_dir_creation_date,\n",
- " map_modules_from_folder,\n",
- ")\n",
+ "from cal_tools import agipdlib, tools\n",
"from dateutil import parser\n",
"from iCalibrationDB import Conditions, Constants, Detectors"
]
@@ -125,7 +110,7 @@
"# slopes_ff_from_files left as str for now\n",
"in_folder = Path(in_folder)\n",
"out_folder = Path(out_folder)\n",
- "metadata = CalibrationMetadata(out_folder)"
+ "metadata = tools.CalibrationMetadata(out_folder)"
]
},
{
@@ -138,7 +123,7 @@
"\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(str(in_folder), run)\n",
+ " creation_time = tools.get_dir_creation_date(str(in_folder), run)\n",
" offset = parser.parse(creation_date_offset)\n",
" delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
" creation_time += delta\n",
@@ -174,7 +159,7 @@
" gain_setting = None\n",
" else:\n",
" try:\n",
- " gain_setting = get_gain_setting(str(control_fn), h5path_ctrl)\n",
+ " gain_setting = agipdlib.get_gain_setting(str(control_fn), h5path_ctrl)\n",
" except Exception as e:\n",
" print(f'ERROR: while reading gain setting from: \\n{control_fn}')\n",
" print(e)\n",
@@ -210,24 +195,6 @@
" modules = [int(x[-2:]) for x in karabo_da]"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-21T11:30:07.974174Z",
- "start_time": "2019-02-21T11:30:07.914832Z"
- }
- },
- "outputs": [],
- "source": [
- "# set everything up filewise\n",
- "print(f\"Checking the files before retrieving constants\")\n",
- "mmf = map_modules_from_folder(str(in_folder), run, path_template, karabo_da, sequences)\n",
- "\n",
- "mapped_files, mod_ids, total_sequences, sequences_qm, _ = mmf"
- ]
- },
{
"cell_type": "markdown",
"metadata": {},
@@ -241,30 +208,15 @@
"metadata": {},
"outputs": [],
"source": [
- "def retrieve_constants(karabo_id: str, bias_voltage: int, max_cells: float,\n",
- " acq_rate: float, gain_setting: float, photon_energy: float,\n",
- " only_dark: bool, nodb_with_dark: bool, \n",
- " cal_db_interface: str, creation_time: str, \n",
- " corr_bools: dict, pc_bools: List[bool],\n",
- " inp: Tuple[str, str, str, int]\n",
- " ) -> Tuple[str, str, float, float, str, dict]:\n",
+ "def retrieve_constants(\n",
+ " qm_files: List[Path],\n",
+ " qm: str,\n",
+ " karabo_da: str,\n",
+ " idx: int\n",
+ ") -> Tuple[str, str, float, float, str, dict]:\n",
" \"\"\"\n",
- " Retreive constant for each module in parallel and produce a dictionary\n",
- " with the creation-time and constant file path.\n",
+ " Retreive constants for a module.\n",
" \n",
- " :param karabo_id: (STR) Karabo ID\n",
- " :param bias_voltage: (FLOAT) Bias Voltage\n",
- " :param max_cells: (INT) Memory cells\n",
- " :param acq_rate: (FLOAT) Acquisition Rate\n",
- " :param gain_setting: (FLOAT) Gain setting\n",
- " :param photon_energy: (FLOAT) Photon Energy\n",
- " :param only_dark: (BOOL) only retrieve dark constants\n",
- " :param nodb_with_dark: (BOOL) no constant retrieval even for dark\n",
- " :param cal_db_interface: (STR) the database interface port\n",
- " :param creation_time: (STR) raw data creation time\n",
- " :param corr_bools: (DICT) A dictionary with bools for applying requested corrections\n",
- " :param pc_bools: (LIST) list of bools to retrieve pulse capacitor constants\n",
- " :param inp: (LIST) input for the parallel cluster of the partial function\n",
" :return:\n",
" qm: module virtual name i.e. Q1M1.\n",
" karabo_da: karabo data aggregator.\n",
@@ -274,41 +226,38 @@
" mdata_dict: (DICT) dictionary with the metadata for the retrieved constants.\n",
" \"\"\"\n",
"\n",
- " import sys\n",
- " import traceback\n",
- "\n",
- " import numpy as np\n",
- " from cal_tools.agipdlib import get_acq_rate, get_num_cells\n",
- " from cal_tools.agipdutils import assemble_constant_dict\n",
- " from cal_tools.tools import get_from_db\n",
- " from iCalibrationDB import Conditions, Constants, Detectors\n",
- "\n",
" err = None\n",
- "\n",
- " qm_files, qm, karabo_da, idx = inp\n",
- " # get number of memory cells from a sequence file with image data\n",
- " for f in qm_files:\n",
- " if not max_cells:\n",
- " max_cells = get_num_cells(f, karabo_id, idx)\n",
- " if max_cells is None:\n",
- " if f != qm_files[-1]:\n",
- " continue\n",
- " else:\n",
- " raise ValueError(f\"No raw images found for {qm} for all sequences\")\n",
- " else:\n",
- " cells = np.arange(max_cells)\n",
- " # get out of the loop,\n",
- " # if max_cells is successfully calculated. \n",
+ " if max_cells != 0:\n",
+ " # either use overriding notebook parameter\n",
+ " local_max_cells = max_cells\n",
+ " else:\n",
+ " # or look around in sequence files\n",
+ " for f in qm_files:\n",
+ " local_max_cells = agipdlib.get_num_cells(f, karabo_id, idx)\n",
+ " if local_max_cells is not None:\n",
" break\n",
+ " # maybe we never found this in a sequence file...\n",
+ " if local_max_cells is None:\n",
+ " raise ValueError(f\"No raw images found for {qm} for all sequences\")\n",
"\n",
- " if acq_rate == 0.:\n",
- " acq_rate = get_acq_rate((f, karabo_id, idx))\n",
+ " if acq_rate == 0: \n",
+ " local_acq_rate = agipdlib.get_acq_rate(fast_paths=(f, karabo_id, idx))\n",
+ " else:\n",
+ " local_acq_rate = acq_rate\n",
"\n",
" # avoid creating retireving constant, if requested.\n",
" if not nodb_with_dark:\n",
- " const_dict = assemble_constant_dict(corr_bools, pc_bools, max_cells, bias_voltage,\n",
- " gain_setting, acq_rate, photon_energy,\n",
- " beam_energy=None, only_dark=only_dark)\n",
+ " const_dict = agipdlib.assemble_constant_dict(\n",
+ " corr_bools,\n",
+ " pc_bools,\n",
+ " local_max_cells,\n",
+ " bias_voltage,\n",
+ " gain_setting,\n",
+ " local_acq_rate,\n",
+ " photon_energy,\n",
+ " beam_energy=None,\n",
+ " only_dark=only_dark,\n",
+ " )\n",
"\n",
" # Retrieve multiple constants through an input dictionary\n",
" # to return a dict of useful metadata.\n",
@@ -324,28 +273,42 @@
" mdata_dict['constants'][cname][\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
" mdata_dict['constants'][cname][\"creation-time\"] = \"00:00:00\"\n",
" else:\n",
- " try:\n",
- " condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
- " co, mdata = \\\n",
- " get_from_db(karabo_id, karabo_da, getattr(Constants.AGIPD, cname)(),\n",
- " condition, getattr(np, cval[0])(cval[1]),\n",
- " cal_db_interface, creation_time, meta_only=True, verbosity=0)\n",
- " mdata_const = mdata.calibration_constant_version\n",
- " device_name = mdata.calibration_constant_version.device_name\n",
- " # check if constant was sucessfully retrieved.\n",
- " if mdata.comm_db_success:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
- " f\"{mdata_const.filename}\"\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
- " mdata_dict['physical-detector-unit'] = mdata_const.device_name\n",
- " else:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = const_dict[cname][:2]\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = None\n",
- " except Exception as e:\n",
- " err = f\"Error: {e}, Traceback: {traceback.format_exc()}\"\n",
- " print(err)\n",
- "\n",
- " return qm, mdata_dict, karabo_da, acq_rate, max_cells, err\n",
+ " condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
+ " co, mdata = tools.get_from_db(\n",
+ " karabo_id,\n",
+ " karabo_da,\n",
+ " getattr(Constants.AGIPD, cname)(),\n",
+ " condition,\n",
+ " getattr(np, cval[0])(cval[1]),\n",
+ " cal_db_interface,\n",
+ " creation_time,\n",
+ " meta_only=True,\n",
+ " verbosity=1,\n",
+ " )\n",
+ " mdata_const = mdata.calibration_constant_version\n",
+ " # check if constant was sucessfully retrieved.\n",
+ " if mdata.comm_db_success:\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
+ " f\"{mdata_const.filename}\"\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
+ " mdata_dict['physical-detector-unit'] = mdata_const.device_name\n",
+ " else:\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = const_dict[cname][:2]\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = None\n",
+ "\n",
+ " return qm, mdata_dict, karabo_da, acq_rate, local_max_cells, err"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# set everything up filewise\n",
+ "mapped_files, _, _, _, _ = tools.map_modules_from_folder(\n",
+ " str(in_folder), run, path_template, karabo_da, sequences\n",
+ ")\n",
"\n",
"pc_bools = [corr_bools.get(\"rel_gain\"),\n",
" corr_bools.get(\"adjust_mg_baseline\"),\n",
@@ -364,26 +327,35 @@
"\n",
"# A dict to connect virtual device\n",
"# to actual device name.\n",
- "for i, k_da in zip(modules, karabo_da):\n",
- " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ "for module_index, k_da in zip(modules, karabo_da):\n",
+ " qm = tools.module_index_to_qm(module_index)\n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
" device = getattr(getattr(Detectors, dinstance), qm)\n",
- " qm_files = [str(mapped_files[qm].get()) for _ in range(mapped_files[qm].qsize())]\n",
- "\n",
+ " # TODO: make map_modules_from_folder just return list(s)\n",
+ " qm_files = [Path(mapped_files[qm].get()) for _ in range(mapped_files[qm].qsize())]\n",
" else:\n",
" print(f\"Skipping {qm}\")\n",
" continue\n",
"\n",
- " inp.append((qm_files, qm, k_da, i))\n",
- "\n",
- "p = partial(retrieve_constants, karabo_id, bias_voltage, max_cells, \n",
- " acq_rate, gain_setting, photon_energy, only_dark, nodb_with_dark, \n",
- " cal_db_interface, creation_time, \n",
- " corr_bools, pc_bools)\n",
- "\n",
- "with mp.Pool(processes=nmods) as pool:\n",
- " results = pool.map(p, inp)\n",
- "\n",
+ " inp.append((qm_files, qm, k_da, module_index))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with multiprocessing.Pool(processes=nmods) as pool:\n",
+ " results = pool.starmap(p, inp)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
"mod_dev = dict()\n",
"mdata_dict = dict()\n",
"for r in results:\n",
@@ -396,12 +368,12 @@
"# check if it is requested not to retrieve any constants from the database\n",
"if not nodb_with_dark:\n",
" metadata.update({\"retrieved-constants\": mdata_dict})\n",
- " \n",
+ "\n",
" print(\"\\nRetrieved constants for modules: \",\n",
" f\"{[', '.join([f'Q{x//4+1}M{x%4+1}' for x in modules])]}\")\n",
" print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Memory cells: {max_cells}\\n\"\n",
" f\"• Acquisition rate: {acq_rate}\\n• Gain setting: {gain_setting}\\n• Photon Energy: {photon_energy}\\n\")\n",
- " print(f\"Constant metadata is saved under \\\"retrieved-constants\\\" in calibration_metadata.yml\\n\")\n",
+ " print(\"Constant metadata is saved under \\\"retrieved-constants\\\" in calibration_metadata.yml\\n\")\n",
"else:\n",
" print(\"No constants were retrieved as calibrated files will be used.\")"
]
@@ -428,7 +400,7 @@
" # Store few time stamps if exists\n",
" # Add NA to keep array structure\n",
" for cname in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
- " if not k_da in mdata_dict or dinfo[\"err\"]:\n",
+ " if k_da not in mdata_dict or dinfo[\"err\"]:\n",
" module_timestamps[cname] = \"Err\"\n",
" else:\n",
" if cname in mdata_dict[k_da]:\n",
--
GitLab
From bd9ec4965b4057fea80ef21b298280660a099a6b Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Fri, 12 Mar 2021 17:27:13 +0100
Subject: [PATCH 58/72] Additional cleanup, starting to add gain_mode in
agipdutils
---
cal_tools/cal_tools/agipdutils.py | 53 ++++---
...IPD_Retrieve_Constants_Precorrection.ipynb | 143 ++++++++++--------
2 files changed, 106 insertions(+), 90 deletions(-)
diff --git a/cal_tools/cal_tools/agipdutils.py b/cal_tools/cal_tools/agipdutils.py
index 9d58ccf83..aec289dd6 100644
--- a/cal_tools/cal_tools/agipdutils.py
+++ b/cal_tools/cal_tools/agipdutils.py
@@ -2,18 +2,27 @@ import copy
from typing import Tuple
import numpy as np
-from cal_tools.enums import BadPixels, SnowResolution
-from scipy.signal import cwt, ricker
+from cal_tools.enums import AgipdGainMode, BadPixels, SnowResolution
+from scipy.signal import cwt, find_peaks_cwt, ricker
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import StandardScaler
-def assemble_constant_dict(corr_bools, pc_bools, memory_cells, bias_voltage,
- gain_setting, acquisition_rate,
- photon_energy, beam_energy=None, only_dark=False):
+def assemble_constant_dict(
+ corr_bools,
+ pc_bools,
+ memory_cells,
+ bias_voltage,
+ gain_setting,
+ acquisition_rate,
+ photon_energy,
+ beam_energy=None,
+ only_dark=False,
+ gain_mode=AgipdGainMode.ADAPTIVE_GAIN,
+):
"""
Assemble a dictionary with the iCalibrationDB constant names and
- the operating conditions for retrieveing the required constants
+ the operating conditions for retrieving the required constants
for correction.
:param corr_bools: (Dict) A dict of booleans for applying
@@ -25,20 +34,23 @@ def assemble_constant_dict(corr_bools, pc_bools, memory_cells, bias_voltage,
:param acquisition_rate: (Float) Acquisition rate
:param photon_energy: (Float) Photong energy
:param beam_energy: (Float) Beam Energy
- :param only_dark: (Bool) Indicating a retrieval for dark
- constants only from db
+ :param only_dark: (Bool) Indicating a retrieval for dark constants only from db
+ :param gain_mode: Operation mode of the detector (default to adaptive gain)
:return: const_dict: (Dict) An assembeld dictionary that can be used
to retrieve the required constants
"""
darkcond = [
- "Dark", {
+ "Dark",
+ {
"memory_cells": memory_cells,
"bias_voltage": bias_voltage,
"acquisition_rate": acquisition_rate,
"gain_setting": gain_setting,
+ "gain_mode": gain_mode,
"pixels_x": 512,
- "pixels_y": 128, }
+ "pixels_y": 128,
+ },
]
const_dict = {
"Offset": ["zeros", (128, 512, memory_cells, 3), darkcond],
@@ -47,28 +59,21 @@ def assemble_constant_dict(corr_bools, pc_bools, memory_cells, bias_voltage,
"BadPixelsDark": ["zeros", (128, 512, memory_cells, 3), darkcond],
}
- if not (corr_bools.get('only_offset') or only_dark):
-
+ if not (corr_bools.get("only_offset") or only_dark):
if any(pc_bools):
- const_dict["BadPixelsPC"] = \
- ["zeros", (memory_cells, 128, 512), darkcond]
- const_dict["SlopesPC"] = \
- ["ones", (128, 512, memory_cells, 10), darkcond]
+ const_dict["BadPixelsPC"] = ["zeros", (memory_cells, 128, 512), darkcond]
+ const_dict["SlopesPC"] = ["ones", (128, 512, memory_cells, 10), darkcond]
if corr_bools.get("xray_corr"):
# Add illuminated conditions
illumcond = [
- "Illuminated", {
- "beam_energy": beam_energy,
- "photon_energy": photon_energy
- }
+ "Illuminated",
+ {"beam_energy": beam_energy, "photon_energy": photon_energy},
]
illumcond[1].update(darkcond[1])
- const_dict["BadPixelsFF"] = ["zeros", (128, 512, memory_cells),
- illumcond]
- const_dict["SlopesFF"] = ["ones", (128, 512, memory_cells, 2),
- illumcond]
+ const_dict["BadPixelsFF"] = ["zeros", (128, 512, memory_cells), illumcond]
+ const_dict["SlopesFF"] = ["ones", (128, 512, memory_cells, 2), illumcond]
return const_dict
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 78386a95e..ed1574138 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -63,8 +63,7 @@
"outputs": [],
"source": [
"# Fill dictionaries comprising bools and arguments for correction and data analysis\n",
- "\n",
- "# Here the herarichy and dependability for correction booleans are defined \n",
+ "# Here the hierarichy and dependencies for correction booleans are defined \n",
"corr_bools = {}\n",
"\n",
"# offset is at the bottom of AGIPD correction pyramid.\n",
@@ -91,7 +90,7 @@
"import numpy as np\n",
"\n",
"matplotlib.use(\"agg\")\n",
- "import multiprocessing as mp\n",
+ "import multiprocessing\n",
"from datetime import timedelta\n",
"from pathlib import Path\n",
"\n",
@@ -139,9 +138,7 @@
"\n",
"warnings.filterwarnings('ignore')\n",
"\n",
- "from cal_tools.agipdlib import SnowResolution\n",
- "\n",
- "melt_snow = False if corr_bools[\"only_offset\"] else SnowResolution.NONE"
+ "melt_snow = False if corr_bools[\"only_offset\"] else agipdlib.SnowResolution.NONE"
]
},
{
@@ -166,7 +163,11 @@
" print(\"Set gain setting to 0\")\n",
" gain_setting = 0\n",
"\n",
+ "# Evaluate gain mode (operation mode)\n",
+ "gain_mode = agipdlib.get_gain_mode(control_fn, h5path_ctrl)\n",
+ " \n",
"print(f\"Gain setting: {gain_setting}\")\n",
+ "print(f\"Gain mode: {gain_mode.name}\")\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"\n",
"\n",
@@ -215,7 +216,7 @@
" idx: int\n",
") -> Tuple[str, str, float, float, str, dict]:\n",
" \"\"\"\n",
- " Retreive constants for a module.\n",
+ " Retrieve constants for a module.\n",
" \n",
" :return:\n",
" qm: module virtual name i.e. Q1M1.\n",
@@ -245,56 +246,61 @@
" else:\n",
" local_acq_rate = acq_rate\n",
"\n",
- " # avoid creating retireving constant, if requested.\n",
- " if not nodb_with_dark:\n",
- " const_dict = agipdlib.assemble_constant_dict(\n",
- " corr_bools,\n",
- " pc_bools,\n",
- " local_max_cells,\n",
- " bias_voltage,\n",
- " gain_setting,\n",
- " local_acq_rate,\n",
- " photon_energy,\n",
- " beam_energy=None,\n",
- " only_dark=only_dark,\n",
- " )\n",
- "\n",
- " # Retrieve multiple constants through an input dictionary\n",
- " # to return a dict of useful metadata.\n",
- " mdata_dict = dict()\n",
- " mdata_dict['constants'] = dict()\n",
- " mdata_dict['physical-detector-unit'] = None # initialization\n",
- "\n",
- " for cname, cval in const_dict.items():\n",
- " # saving metadata in a dict\n",
- " mdata_dict['constants'][cname] = dict()\n",
- "\n",
- " if slopes_ff_from_files and cname in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = \"00:00:00\"\n",
+ " # avoid retrieving constant, if requested.\n",
+ " if nodb_with_dark:\n",
+ " return\n",
+ " \n",
+ " const_dict = agipdlib.assemble_constant_dict(\n",
+ " corr_bools,\n",
+ " pc_bools,\n",
+ " local_max_cells,\n",
+ " bias_voltage,\n",
+ " gain_setting,\n",
+ " local_acq_rate,\n",
+ " photon_energy,\n",
+ " gain_mode=gain_mode,\n",
+ " beam_energy=None,\n",
+ " only_dark=only_dark,\n",
+ " )\n",
+ "\n",
+ " # Retrieve multiple constants through an input dictionary\n",
+ " # to return a dict of useful metadata.\n",
+ " mdata_dict = dict()\n",
+ " mdata_dict['constants'] = dict()\n",
+ " mdata_dict['physical-detector-unit'] = None # initialization\n",
+ "\n",
+ " for cname, cval in const_dict.items():\n",
+ " print(cname)\n",
+ " print(cval)\n",
+ " # saving metadata in a dict\n",
+ " mdata_dict['constants'][cname] = dict()\n",
+ "\n",
+ " if slopes_ff_from_files and cname in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = \"00:00:00\"\n",
+ " else:\n",
+ " condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
+ " co, mdata = tools.get_from_db(\n",
+ " karabo_id,\n",
+ " karabo_da,\n",
+ " getattr(Constants.AGIPD, cname)(),\n",
+ " condition,\n",
+ " getattr(np, cval[0])(cval[1]),\n",
+ " cal_db_interface,\n",
+ " creation_time,\n",
+ " meta_only=True,\n",
+ " verbosity=1,\n",
+ " )\n",
+ " mdata_const = mdata.calibration_constant_version\n",
+ " # check if constant was sucessfully retrieved.\n",
+ " if mdata.comm_db_success:\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
+ " f\"{mdata_const.filename}\"\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
+ " mdata_dict['physical-detector-unit'] = mdata_const.device_name\n",
" else:\n",
- " condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
- " co, mdata = tools.get_from_db(\n",
- " karabo_id,\n",
- " karabo_da,\n",
- " getattr(Constants.AGIPD, cname)(),\n",
- " condition,\n",
- " getattr(np, cval[0])(cval[1]),\n",
- " cal_db_interface,\n",
- " creation_time,\n",
- " meta_only=True,\n",
- " verbosity=1,\n",
- " )\n",
- " mdata_const = mdata.calibration_constant_version\n",
- " # check if constant was sucessfully retrieved.\n",
- " if mdata.comm_db_success:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
- " f\"{mdata_const.filename}\"\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
- " mdata_dict['physical-detector-unit'] = mdata_const.device_name\n",
- " else:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = const_dict[cname][:2]\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = None\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = const_dict[cname][:2]\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = None\n",
"\n",
" return qm, mdata_dict, karabo_da, acq_rate, local_max_cells, err"
]
@@ -321,7 +327,7 @@
"only_dark = False\n",
"nodb_with_dark = False\n",
"if not nodb:\n",
- " only_dark=(calfile != \"\")\n",
+ " only_dark = (calfile != \"\")\n",
"if calfile != \"\" and not corr_bools[\"only_offset\"]:\n",
" nodb_with_dark = nodb\n",
"\n",
@@ -347,7 +353,7 @@
"outputs": [],
"source": [
"with multiprocessing.Pool(processes=nmods) as pool:\n",
- " results = pool.starmap(p, inp)"
+ " results = pool.starmap(retrieve_constants, inp)"
]
},
{
@@ -366,16 +372,21 @@
" print(f\"Error for module {qm}: {err}\")\n",
" mdata_dict[karabo_da] = md_dict\n",
"# check if it is requested not to retrieve any constants from the database\n",
- "if not nodb_with_dark:\n",
+ "if nodb_with_dark:\n",
+ " print(\"No constants were retrieved as calibrated files will be used.\")\n",
+ "else:\n",
" metadata.update({\"retrieved-constants\": mdata_dict})\n",
"\n",
- " print(\"\\nRetrieved constants for modules: \",\n",
- " f\"{[', '.join([f'Q{x//4+1}M{x%4+1}' for x in modules])]}\")\n",
- " print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Memory cells: {max_cells}\\n\"\n",
- " f\"• Acquisition rate: {acq_rate}\\n• Gain setting: {gain_setting}\\n• Photon Energy: {photon_energy}\\n\")\n",
- " print(\"Constant metadata is saved under \\\"retrieved-constants\\\" in calibration_metadata.yml\\n\")\n",
- "else:\n",
- " print(\"No constants were retrieved as calibrated files will be used.\")"
+ " print(\"\\nRetrieved constants for modules:\",\n",
+ " ', '.join([tools.module_index_to_qm(x) for x in modules]))\n",
+ " print(f\"Operating conditions are:\")\n",
+ " print(f\"• Bias voltage: {bias_voltage}\")\n",
+ " print(f\"• Memory cells: {max_cells}\\n\")\n",
+ " print(f\"• Acquisition rate: {acq_rate}\")\n",
+ " print(f\"• Gain mode: {gain_mode.name}\")\n",
+ " print(f\"• Gain setting: {gain_setting}\")\n",
+ " print(f\"• Photon Energy: {photon_energy}\")\n",
+ " print(\"Constant metadata is saved under \\\"retrieved-constants\\\" in calibration_metadata.yml\\n\")"
]
},
{
--
GitLab
From 4375aa50343e0e220cb4987fdaa60b514334fca6 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Mon, 15 Mar 2021 15:45:04 +0100
Subject: [PATCH 59/72] Flake8 and related fixes
- trailing whitespace
- unused variables
- unused imports
- misc formatting
---
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 197 ++++++------------
...IPD_Retrieve_Constants_Precorrection.ipynb | 6 +-
2 files changed, 68 insertions(+), 135 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 82f22b590..f13e87851 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -14,12 +14,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-21T11:30:06.730220Z",
- "start_time": "2019-02-21T11:30:06.658286Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"in_folder = \"/gpfs/exfel/exp/HED/202031/p900174/raw\" # the folder to read data from, required\n",
@@ -51,7 +46,7 @@
"gain_setting = 0.1 # the gain setting, use 0.1 to try to auto-determine\n",
"photon_energy = 9.2 # photon energy in keV\n",
"overwrite = True # set to True if existing data should be overwritten\n",
- "max_pulses = [0, 500, 1] # range list [st, end, step] of maximum pulse indices within a train. 3 allowed maximum list input elements. \n",
+ "max_pulses = [0, 500, 1] # range list [st, end, step] of maximum pulse indices within a train. 3 allowed maximum list input elements.\n",
"mem_cells_db = 0 # set to a value different than 0 to use this value for DB queries\n",
"cell_id_preview = 1 # cell Id used for preview in single-shot plots\n",
"\n",
@@ -71,7 +66,7 @@
"xray_gain = False # do relative gain correction based on xray data\n",
"blc_noise = False # if set, baseline correction via noise peak location is attempted\n",
"blc_stripes = False # if set, baseline corrected via stripes\n",
- "blc_hmatch = False # if set, base line correction via histogram matching is attempted \n",
+ "blc_hmatch = False # if set, base line correction via histogram matching is attempted\n",
"match_asics = False # if set, inner ASIC borders are matched to the same signal level\n",
"adjust_mg_baseline = False # adjust medium gain baseline to match highest high gain value\n",
"zero_nans = False # set NaN values in corrected data to 0\n",
@@ -104,8 +99,6 @@
"metadata": {},
"outputs": [],
"source": [
- "import copy\n",
- "import gc\n",
"import itertools\n",
"import math\n",
"import re\n",
@@ -114,11 +107,11 @@
"from datetime import timedelta\n",
"from multiprocessing import Pool\n",
"from pathlib import Path\n",
- "from time import perf_counter, sleep, time\n",
+ "from time import perf_counter\n",
"\n",
"import tabulate\n",
"from dateutil import parser\n",
- "from IPython.display import HTML, Latex, Markdown, display\n",
+ "from IPython.display import Latex, Markdown, display\n",
"\n",
"warnings.filterwarnings('ignore')\n",
"import matplotlib\n",
@@ -126,11 +119,8 @@
"import yaml\n",
"from extra_data import RunDirectory, stack_detector_data\n",
"from extra_geom import AGIPD_1MGeometry, AGIPD_500K2GGeometry\n",
- "from iCalibrationDB import Detectors\n",
"from matplotlib import cm as colormap\n",
"from matplotlib.colors import LogNorm\n",
- "from matplotlib.ticker import FormatStrFormatter, LinearLocator\n",
- "from mpl_toolkits.mplot3d import Axes3D\n",
"\n",
"matplotlib.use(\"agg\")\n",
"%matplotlib inline\n",
@@ -148,8 +138,7 @@
"from cal_tools.cython import agipdalgs as calgs\n",
"from cal_tools.enums import AgipdGainMode, BadPixels\n",
"from cal_tools.step_timing import StepTimer\n",
- "from cal_tools.tools import (CalibrationMetadata, get_dir_creation_date,\n",
- " map_modules_from_folder, module_index_to_qm)\n",
+ "from cal_tools.tools import (get_dir_creation_date, map_modules_from_folder, module_index_to_qm)\n",
"\n",
"sns.set()\n",
"sns.set_context(\"paper\", font_scale=1.4)\n",
@@ -255,7 +244,7 @@
" karabo_da = [\"AGIPD{:02d}\".format(i) for i in modules]\n",
"else:\n",
" modules = [int(x[-2:]) for x in karabo_da]\n",
- " \n",
+ "\n",
"print(\"Process modules: \", ', '.join(\n",
" [module_index_to_qm(x) for x in modules]))\n",
"print(f\"Detector in use is {karabo_id}\")\n",
@@ -270,17 +259,17 @@
"outputs": [],
"source": [
"# Display Information about the selected pulses indices for correction.\n",
- "pulses_lst = list(range(*max_pulses)) if not (len(max_pulses)==1 and max_pulses[0]==0) else max_pulses \n",
+ "pulses_lst = list(range(*max_pulses)) if not (len(max_pulses)==1 and max_pulses[0]==0) else max_pulses\n",
"\n",
"try:\n",
- " if len(pulses_lst) > 1: \n",
+ " if len(pulses_lst) > 1:\n",
" print(\"A range of {} pulse indices is selected: from {} to {} with a step of {}\"\n",
" .format(len(pulses_lst), pulses_lst[0] , pulses_lst[-1] + (pulses_lst[1] - pulses_lst[0]),\n",
" pulses_lst[1] - pulses_lst[0]))\n",
" else:\n",
- " print(\"one pulse is selected: a pulse of idx {}\".format(pulses_lst[0]))\n",
+ " print(f\"one pulse is selected: a pulse of idx {pulses_lst[0]}\")\n",
"except Exception as e:\n",
- " raise ValueError('max_pulses input Error: {}'.format(e))"
+ " raise ValueError(f\"max_pulses input Error: {e}\")"
]
},
{
@@ -352,7 +341,7 @@
" delta = timedelta(hours=offset.hour,\n",
" minutes=offset.minute, seconds=offset.second)\n",
" creation_time += delta\n",
- " \n",
+ "\n",
"# Evaluate gain setting\n",
"if gain_setting == 0.1:\n",
" if creation_time.replace(tzinfo=None) < parser.parse('2020-01-31'):\n",
@@ -434,11 +423,10 @@
"def retrieve_constants(mod):\n",
" \"\"\"\n",
" Retrieve calibration constants and load them to shared memory\n",
- " \n",
+ "\n",
" Metadata for constants is taken from yml file or retrieved from the DB\n",
" \"\"\"\n",
- " err = ''\n",
- " # TODO: parallelize over modules\n",
+ " err = \"\"\n",
" k_da = karabo_da[mod]\n",
" try:\n",
" # check if there is a yaml file in out_folder that has the device constants.\n",
@@ -446,8 +434,19 @@
" when = agipd_corr.initialize_from_yaml(k_da, const_yaml, mod)\n",
" else:\n",
" # TODO: should we save what is found here in metadata?\n",
- " when = agipd_corr.initialize_from_db(karabo_id, k_da, cal_db_interface, creation_time, mem_cells_db, bias_voltage,\n",
- " photon_energy, gain_setting, acq_rate, mod, False)\n",
+ " when = agipd_corr.initialize_from_db(\n",
+ " karabo_id,\n",
+ " k_da,\n",
+ " cal_db_interface,\n",
+ " creation_time,\n",
+ " mem_cells_db,\n",
+ " bias_voltage,\n",
+ " photon_energy,\n",
+ " gain_setting,\n",
+ " acq_rate,\n",
+ " mod,\n",
+ " False,\n",
+ " )\n",
" except Exception as e:\n",
" err = f\"Error: {e}\\nError traceback: {traceback.format_exc()}\"\n",
" when = None\n",
@@ -494,7 +493,7 @@
"source": [
"def imagewise_chunks(img_counts):\n",
" \"\"\"Break up the loaded data into chunks of up to chunk_size\n",
- " \n",
+ "\n",
" Yields (file data slot, start index, stop index)\n",
" \"\"\"\n",
" for i_proc, n_img in enumerate(img_counts):\n",
@@ -515,9 +514,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"with Pool() as pool:\n",
@@ -546,7 +543,7 @@
" # Perform image-wise correction\n",
" pool.starmap(agipd_corr.baseline_correction, imagewise_chunks(img_counts))\n",
" step_timer.done_step(\"Base-line shift correction\")\n",
- " \n",
+ "\n",
" if common_mode:\n",
" # Perform cross-file correction parallel over asics\n",
" pool.starmap(agipd_corr.cm_correction, itertools.product(\n",
@@ -584,9 +581,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"# if the yml file contains \"retrieved-constants\", that means a leading\n",
@@ -604,7 +599,7 @@
" if fst_print:\n",
" print(\"Constants are retrieved with creation time: \")\n",
" fst_print = False\n",
- " \n",
+ "\n",
" module_timestamps = {}\n",
"\n",
" # If correction is crashed\n",
@@ -652,8 +647,7 @@
" Z = data.T\n",
"\n",
" # Plot the surface.\n",
- " surf = ax.plot_surface(X, Y, Z, cmap=colormap.coolwarm,\n",
- " linewidth=0, antialiased=False)\n",
+ " ax.plot_surface(X, Y, Z, cmap=colormap.coolwarm, linewidth=0, antialiased=False)\n",
" ax.set_xlabel(x_axis)\n",
" ax.set_ylabel(y_axis)\n",
" ax.set_zlabel(\"Counts\")\n",
@@ -680,7 +674,7 @@
"source": [
"def get_trains_data(run_folder, source, include, detector_id, tid=None, modules=16, fillvalue=np.nan):\n",
" \"\"\"Load single train for all module\n",
- " \n",
+ "\n",
" :param run_folder: Path to folder with data\n",
" :param source: Data source to be loaded\n",
" :param include: Inset of file name to be considered\n",
@@ -693,7 +687,7 @@
" tid, data = run_data.select(f'{detector_id}/DET/*', source).train_from_id(tid)\n",
" else:\n",
" tid, data = next(iter(run_data.select(f'{detector_id}/DET/*', source).trains(require_all=True)))\n",
- " \n",
+ "\n",
" return tid, stack_detector_data(train=data, data=source, fillvalue=fillvalue, modules=modules)"
]
},
@@ -791,7 +785,7 @@
"print(f\"Gain statistics in %\")\n",
"table = [[f'{gains[gains==0].size/gains.size*100:.02f}',\n",
" f'{gains[gains==1].size/gains.size*100:.03f}',\n",
- " f'{gains[gains==2].size/gains.size*100:.03f}']] \n",
+ " f'{gains[gains==2].size/gains.size*100:.03f}']]\n",
"md = display(Latex(tabulate.tabulate(table, tablefmt='latex',\n",
" headers=[\"High\", \"Medium\", \"Low\"])))"
]
@@ -806,9 +800,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"pulse_range = [np.min(pulseId[pulseId>=0]), np.max(pulseId[pulseId>=0])]\n",
@@ -882,12 +874,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:33.226396Z",
- "start_time": "2019-02-18T17:29:27.027758Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
@@ -923,12 +910,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:33.761015Z",
- "start_time": "2019-02-18T17:29:33.227922Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
@@ -941,24 +923,19 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:35.903487Z",
- "start_time": "2019-02-18T17:29:33.762568Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
"ax = fig.add_subplot(111)\n",
"vmin, vmax = get_range(corrected[cell_id_preview], 5, -50)\n",
"nbins = np.int((vmax + 50) / 2)\n",
- "h = ax.hist(corrected[cell_id_preview].flatten(), \n",
- " bins=nbins, range=(-50, vmax), \n",
+ "h = ax.hist(corrected[cell_id_preview].flatten(),\n",
+ " bins=nbins, range=(-50, vmax),\n",
" histtype='stepfilled', log=True)\n",
- "_ = plt.xlabel('[ADU]')\n",
- "_ = plt.ylabel('Counts')\n",
- "_ = ax.grid()"
+ "plt.xlabel('[ADU]')\n",
+ "plt.ylabel('Counts')\n",
+ "ax.grid()"
]
},
{
@@ -968,18 +945,13 @@
"outputs": [],
"source": [
"display(Markdown('### Mean CORRECTED Preview ###\\n'))\n",
- "display(Markdown(f'A mean across one train \\n'))"
+ "display(Markdown(f'A mean across one train\\n'))"
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:39.369686Z",
- "start_time": "2019-02-18T17:29:35.905152Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
@@ -992,12 +964,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:49.217848Z",
- "start_time": "2019-02-18T17:29:39.371232Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
@@ -1009,16 +976,16 @@
"nbins = np.int((vmax + 100) / 5)\n",
"h = ax.hist(corrected.flatten(), bins=nbins,\n",
" range=(-100, vmax), histtype='step', log=True, label = 'All')\n",
- "_ = ax.hist(corrected[gains == 0].flatten(), bins=nbins, range=(-100, vmax),\n",
- " alpha=0.5, log=True, label='High gain', color='green')\n",
- "_ = ax.hist(corrected[gains == 1].flatten(), bins=nbins, range=(-100, vmax),\n",
- " alpha=0.5, log=True, label='Medium gain', color='red')\n",
- "_ = ax.hist(corrected[gains == 2].flatten(), bins=nbins,\n",
- " range=(-100, vmax), alpha=0.5, log=True, label='Low gain', color='yellow')\n",
- "_ = ax.legend()\n",
- "_ = ax.grid()\n",
- "_ = plt.xlabel('[ADU]')\n",
- "_ = plt.ylabel('Counts')"
+ "ax.hist(corrected[gains == 0].flatten(), bins=nbins, range=(-100, vmax),\n",
+ " alpha=0.5, log=True, label='High gain', color='green')\n",
+ "ax.hist(corrected[gains == 1].flatten(), bins=nbins, range=(-100, vmax),\n",
+ " alpha=0.5, log=True, label='Medium gain', color='red')\n",
+ "ax.hist(corrected[gains == 2].flatten(), bins=nbins, range=(-100, vmax),\n",
+ " alpha=0.5, log=True, label='Low gain', color='yellow')\n",
+ "ax.legend()\n",
+ "ax.grid()\n",
+ "plt.xlabel('[ADU]')\n",
+ "plt.ylabel('Counts')"
]
},
{
@@ -1034,12 +1001,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:49.641675Z",
- "start_time": "2019-02-18T17:29:49.224167Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
@@ -1050,9 +1012,7 @@
},
{
"cell_type": "markdown",
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
"source": [
"## Bad Pixels ##\n",
"The mask contains dedicated entries for all pixels and memory cells as well as all three gains stages. Each mask entry is encoded in 32 bits as:"
@@ -1061,12 +1021,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:49.651913Z",
- "start_time": "2019-02-18T17:29:49.643556Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"table = []\n",
@@ -1089,24 +1044,17 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:50.086169Z",
- "start_time": "2019-02-18T17:29:49.653391Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
"ax = fig.add_subplot(111)\n",
- "ax = geom.plot_data_fast(np.log2(mask[cell_id_preview]), ax=ax, vmin=0, vmax=32, cmap=\"jet\")"
+ "geom.plot_data_fast(np.log2(mask[cell_id_preview]), ax=ax, vmin=0, vmax=32, cmap=\"jet\")"
]
},
{
"cell_type": "markdown",
- "metadata": {
- "collapsed": true
- },
+ "metadata": {},
"source": [
"### Percentage of Bad Pixels across one train ###"
]
@@ -1114,18 +1062,12 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:51.686562Z",
- "start_time": "2019-02-18T17:29:50.088883Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
"ax = fig.add_subplot(111)\n",
- "ax = geom.plot_data_fast(np.mean(mask>0, axis=0),\n",
- " vmin=0, ax=ax, vmax=1, cmap=\"jet\")"
+ "geom.plot_data_fast(np.mean(mask>0, axis=0), vmin=0, ax=ax, vmax=1, cmap=\"jet\")"
]
},
{
@@ -1138,12 +1080,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:29:55.483270Z",
- "start_time": "2019-02-18T17:29:53.664226Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(20, 10))\n",
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index ed1574138..9921ec22e 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -134,10 +134,6 @@
"print(f\"Outputting to {out_folder}\")\n",
"out_folder.mkdir(parents=True, exist_ok=True)\n",
"\n",
- "import warnings\n",
- "\n",
- "warnings.filterwarnings('ignore')\n",
- "\n",
"melt_snow = False if corr_bools[\"only_offset\"] else agipdlib.SnowResolution.NONE"
]
},
@@ -381,7 +377,7 @@
" ', '.join([tools.module_index_to_qm(x) for x in modules]))\n",
" print(f\"Operating conditions are:\")\n",
" print(f\"• Bias voltage: {bias_voltage}\")\n",
- " print(f\"• Memory cells: {max_cells}\\n\")\n",
+ " print(f\"• Memory cells: {max_cells}\")\n",
" print(f\"• Acquisition rate: {acq_rate}\")\n",
" print(f\"• Gain mode: {gain_mode.name}\")\n",
" print(f\"• Gain setting: {gain_setting}\")\n",
--
GitLab
From ec2301a275ac09b1e16c44f532166eafd0cc7640 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 18 Mar 2021 12:25:04 +0100
Subject: [PATCH 60/72] Cleanup
---
...IPD_Retrieve_Constants_Precorrection.ipynb | 39 ++++++-------------
1 file changed, 12 insertions(+), 27 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 9921ec22e..5a6300766 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -51,7 +51,7 @@
"xray_gain = True # do relative gain correction based on xray data\n",
"blc_noise = False # if set, baseline correction via noise peak location is attempted\n",
"blc_stripes = False # if set, baseline corrected via stripes\n",
- "blc_hmatch = False # if set, base line correction via histogram matching is attempted \n",
+ "blc_hmatch = False # if set, base line correction via histogram matching is attempted\n",
"match_asics = False # if set, inner ASIC borders are matched to the same signal level\n",
"adjust_mg_baseline = False # adjust medium gain baseline to match highest high gain value"
]
@@ -206,14 +206,11 @@
"outputs": [],
"source": [
"def retrieve_constants(\n",
- " qm_files: List[Path],\n",
- " qm: str,\n",
- " karabo_da: str,\n",
- " idx: int\n",
+ " qm_files: List[Path], qm: str, karabo_da: str, idx: int\n",
") -> Tuple[str, str, float, float, str, dict]:\n",
" \"\"\"\n",
" Retrieve constants for a module.\n",
- " \n",
+ "\n",
" :return:\n",
" qm: module virtual name i.e. Q1M1.\n",
" karabo_da: karabo data aggregator.\n",
@@ -237,7 +234,7 @@
" if local_max_cells is None:\n",
" raise ValueError(f\"No raw images found for {qm} for all sequences\")\n",
"\n",
- " if acq_rate == 0: \n",
+ " if acq_rate == 0:\n",
" local_acq_rate = agipdlib.get_acq_rate(fast_paths=(f, karabo_id, idx))\n",
" else:\n",
" local_acq_rate = acq_rate\n",
@@ -245,7 +242,7 @@
" # avoid retrieving constant, if requested.\n",
" if nodb_with_dark:\n",
" return\n",
- " \n",
+ "\n",
" const_dict = agipdlib.assemble_constant_dict(\n",
" corr_bools,\n",
" pc_bools,\n",
@@ -360,13 +357,11 @@
"source": [
"mod_dev = dict()\n",
"mdata_dict = dict()\n",
- "for r in results:\n",
- " if r:\n",
- " qm, md_dict, karabo_da, acq_rate, max_cells, err = r\n",
- " mod_dev[karabo_da] = {\"mod\": qm, \"err\": err}\n",
- " if err:\n",
- " print(f\"Error for module {qm}: {err}\")\n",
- " mdata_dict[karabo_da] = md_dict\n",
+ "for qm, md_dict, karabo_da, acq_rate, max_cells, err in results:\n",
+ " mod_dev[karabo_da] = {\"mod\": qm, \"err\": err}\n",
+ " if err:\n",
+ " print(f\"Error for module {qm}: {err}\")\n",
+ " mdata_dict[karabo_da] = md_dict\n",
"# check if it is requested not to retrieve any constants from the database\n",
"if nodb_with_dark:\n",
" print(\"No constants were retrieved as calibrated files will be used.\")\n",
@@ -391,12 +386,11 @@
"metadata": {},
"outputs": [],
"source": [
- "print(\"Constants are retrieved with creation time: \")\n",
- "i = 0\n",
+ "print(\"Constants are retrieved with creation time:\")\n",
"timestamps = {}\n",
"\n",
"for k_da, dinfo in mod_dev.items():\n",
- " print(dinfo[\"mod\"], \":\")\n",
+ " print(f\"{dinfo['mod']}:\")\n",
" module_timestamps = {}\n",
" module_name = dinfo[\"mod\"]\n",
" if k_da in mdata_dict:\n",
@@ -404,8 +398,6 @@
" if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
" mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
" print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
- " # Store few time stamps if exists\n",
- " # Add NA to keep array structure\n",
" for cname in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
" if k_da not in mdata_dict or dinfo[\"err\"]:\n",
" module_timestamps[cname] = \"Err\"\n",
@@ -419,13 +411,6 @@
" module_timestamps[cname] = \"NA\"\n",
" timestamps[module_name] = module_timestamps\n",
"\n",
- " i += 1\n",
- " if sequences:\n",
- " seq_num = sequences[0]\n",
- " else:\n",
- " # if sequences[0] changed to None as it was -1\n",
- " seq_num = 0\n",
- "\n",
"time_summary = metadata.setdefault(\"retrieved-constants\", {}).setdefault(\"time-summary\", {})\n",
"time_summary[\"SAll\"] = timestamps\n",
"\n",
--
GitLab
From 8119989e2d59a7668cdfa73ec6cabac252b8b42a Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 18 Mar 2021 12:29:32 +0100
Subject: [PATCH 61/72] Hack: for FG, get some regular constants, disable some
corrections
---
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 76 ++++++++++++------
...IPD_Retrieve_Constants_Precorrection.ipynb | 77 +++++++++++--------
2 files changed, 99 insertions(+), 54 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index f13e87851..c66a23a15 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -131,6 +131,7 @@
"sns.set_context(\"paper\", font_scale=1.4)\n",
"sns.set_style(\"ticks\")\n",
"\n",
+ "import cal_tools\n",
"import seaborn as sns\n",
"from cal_tools.agipdlib import (AgipdCorrections, get_acq_rate, get_gain_mode,\n",
" get_gain_setting, get_num_cells)\n",
@@ -138,7 +139,6 @@
"from cal_tools.cython import agipdalgs as calgs\n",
"from cal_tools.enums import AgipdGainMode, BadPixels\n",
"from cal_tools.step_timing import StepTimer\n",
- "from cal_tools.tools import (get_dir_creation_date, map_modules_from_folder, module_index_to_qm)\n",
"\n",
"sns.set()\n",
"sns.set_context(\"paper\", font_scale=1.4)\n",
@@ -170,7 +170,7 @@
"source": [
"# Fill dictionaries comprising bools and arguments for correction and data analysis\n",
"\n",
- "# Here the herarichy and dependability for correction booleans are defined\n",
+ "# Here the hierarchy and dependability for correction booleans are defined\n",
"corr_bools = {}\n",
"\n",
"# offset is at the bottom of AGIPD correction pyramid.\n",
@@ -195,8 +195,16 @@
" corr_bools[\"common_mode\"] = common_mode\n",
" corr_bools[\"melt_snow\"] = melt_snow\n",
" corr_bools[\"mask_zero_std\"] = mask_zero_std\n",
- " corr_bools[\"low_medium_gap\"] = low_medium_gap \n",
- " "
+ " corr_bools[\"low_medium_gap\"] = low_medium_gap\n",
+ "\n",
+ "# Many corrections don't apply to fixed gain mode; will explicitly disable later if detected\n",
+ "disable_for_fixed_gain = [\n",
+ " \"adjust_mg_baseline\",\n",
+ " \"blc_set_min\",\n",
+ " \"force_hg_if_below\",\n",
+ " \"force_mg_if_below\",\n",
+ " \"low_medium_gap\"\n",
+ "]"
]
},
{
@@ -245,8 +253,7 @@
"else:\n",
" modules = [int(x[-2:]) for x in karabo_da]\n",
"\n",
- "print(\"Process modules: \", ', '.join(\n",
- " [module_index_to_qm(x) for x in modules]))\n",
+ "print(\"Process modules:\", ', '.join(cal_tools.tools.module_index_to_qm(x) for x in modules))\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"print(f\"Instrument {instrument}\")\n",
"print(f\"Detector instance {dinstance}\")"
@@ -279,9 +286,9 @@
"outputs": [],
"source": [
"# set everything up filewise\n",
- "mmf = map_modules_from_folder(str(in_folder), run, path_template,\n",
- " karabo_da, sequences)\n",
- "mapped_files, mod_ids, total_sequences, sequences_qm, _ = mmf\n",
+ "mapped_files, _, total_sequences, _, _ = cal_tools.tools.map_modules_from_folder(\n",
+ " str(in_folder), run, path_template, karabo_da, sequences\n",
+ ")\n",
"file_list = []\n",
"\n",
"# ToDo: Split table over pages\n",
@@ -336,10 +343,9 @@
"# Evaluate creation time\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(str(in_folder), run)\n",
+ " creation_time = cal_tools.tools.get_dir_creation_date(str(in_folder), run)\n",
" offset = parser.parse(creation_date_offset)\n",
- " delta = timedelta(hours=offset.hour,\n",
- " minutes=offset.minute, seconds=offset.second)\n",
+ " delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
" creation_time += delta\n",
"\n",
"# Evaluate gain setting\n",
@@ -376,6 +382,19 @@
"print(f\"• Photon Energy: {photon_energy}\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if gain_mode:\n",
+ " for to_disable in disable_for_fixed_gain:\n",
+ " if corr_bools.get(to_disable, False):\n",
+ " print(f\"Warning: {to_disable} correction was requested, but does not apply to fixed gain mode\")\n",
+ " corr_bools[to_disable] = False"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -389,10 +408,13 @@
"metadata": {},
"outputs": [],
"source": [
- "agipd_corr = AgipdCorrections(max_cells, max_pulses,\n",
- " h5_data_path=h5path,\n",
- " h5_index_path=h5path_idx,\n",
- " corr_bools=corr_bools)\n",
+ "agipd_corr = AgipdCorrections(\n",
+ " max_cells,\n",
+ " max_pulses,\n",
+ " h5_data_path=h5path,\n",
+ " h5_index_path=h5path_idx,\n",
+ " corr_bools=corr_bools\n",
+ ")\n",
"\n",
"agipd_corr.baseline_corr_noise_threshold = -blc_noise_threshold\n",
"agipd_corr.hg_hard_threshold = hg_hard_threshold\n",
@@ -406,6 +428,15 @@
"agipd_corr.ff_gain = ff_gain"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "module_index_to_karabo_da = {mod: da for (mod, da) in zip(modules, karabo_da)}"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -415,11 +446,10 @@
"# Retrieve calibration constants to RAM\n",
"agipd_corr.allocate_constants(modules, (3, mem_cells_db, 512, 128))\n",
"\n",
- "metadata = CalibrationMetadata(out_folder)\n",
+ "metadata = cal_tools.tools.CalibrationMetadata(out_folder)\n",
"# NOTE: this notebook will not overwrite calibration metadata file\n",
"const_yaml = metadata.get(\"retrieved-constants\", {})\n",
"\n",
- "# retrive constants\n",
"def retrieve_constants(mod):\n",
" \"\"\"\n",
" Retrieve calibration constants and load them to shared memory\n",
@@ -427,13 +457,15 @@
" Metadata for constants is taken from yml file or retrieved from the DB\n",
" \"\"\"\n",
" err = \"\"\n",
- " k_da = karabo_da[mod]\n",
+ " k_da = module_index_to_karabo_da[mod]\n",
" try:\n",
" # check if there is a yaml file in out_folder that has the device constants.\n",
" if k_da in const_yaml:\n",
+ " print(f\"Pre-correction notebook already found constants for {k_da}\")\n",
" when = agipd_corr.initialize_from_yaml(k_da, const_yaml, mod)\n",
" else:\n",
- " # TODO: should we save what is found here in metadata?\n",
+ " print(f\"Have to query database for constants for {k_da}\")\n",
+ " # TODO: replace with proper retrieval (as done in pre-correction)\n",
" when = agipd_corr.initialize_from_db(\n",
" karabo_id,\n",
" k_da,\n",
@@ -466,7 +498,7 @@
"outputs": [],
"source": [
"# allocate memory for images and hists\n",
- "n_images_max = max_cells*256\n",
+ "n_images_max = max_cells * 256\n",
"data_shape = (n_images_max, 512, 128)\n",
"agipd_corr.allocate_images(data_shape, n_cores_files)"
]
@@ -590,7 +622,7 @@
"timestamps = {}\n",
"\n",
"for i, (error, modno, when, k_da) in enumerate(const_out):\n",
- " qm = module_index_to_qm(modno)\n",
+ " qm = cal_tools.tools.module_index_to_qm(modno)\n",
" # expose errors while applying correction\n",
" if error:\n",
" print(\"Error: {}\".format(error) )\n",
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 5a6300766..0a2583b17 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -259,41 +259,54 @@
" # Retrieve multiple constants through an input dictionary\n",
" # to return a dict of useful metadata.\n",
" mdata_dict = dict()\n",
- " mdata_dict['constants'] = dict()\n",
- " mdata_dict['physical-detector-unit'] = None # initialization\n",
- "\n",
- " for cname, cval in const_dict.items():\n",
- " print(cname)\n",
- " print(cval)\n",
+ " mdata_dict[\"constants\"] = dict()\n",
+ " mdata_dict[\"physical-detector-unit\"] = None # initialization\n",
+ "\n",
+ " for const_name, (const_init_fun, const_shape, (cond_type, cond_param)) in const_dict.items():\n",
+ " if gain_mode and const_name in (\"ThresholdsDark\",):\n",
+ " print(f\"Note: skipping {const_name} for fixed gain mode\")\n",
+ " continue\n",
+ " \n",
" # saving metadata in a dict\n",
- " mdata_dict['constants'][cname] = dict()\n",
- "\n",
- " if slopes_ff_from_files and cname in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = \"00:00:00\"\n",
+ " const_mdata = dict()\n",
+ " mdata_dict[\"constants\"][const_name] = const_mdata\n",
+ "\n",
+ " if slopes_ff_from_files and const_name in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
+ " const_mdata[\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
+ " const_mdata[\"creation-time\"] = \"00:00:00\"\n",
+ " continue\n",
+ " \n",
+ " if gain_mode and const_name in (\"BadPixelsPC\", \"SlopesPC\", \"BadPixelsFF\", \"SlopesFF\"):\n",
+ " param_copy = cond_param.copy()\n",
+ " del param_copy[\"gain_mode\"]\n",
+ " condition = getattr(Conditions, cond_type).AGIPD(**param_copy)\n",
+ " print(f\"Note: {const_name} based on adaptive gain mode constants will be retrieved\")\n",
" else:\n",
- " condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
- " co, mdata = tools.get_from_db(\n",
- " karabo_id,\n",
- " karabo_da,\n",
- " getattr(Constants.AGIPD, cname)(),\n",
- " condition,\n",
- " getattr(np, cval[0])(cval[1]),\n",
- " cal_db_interface,\n",
- " creation_time,\n",
- " meta_only=True,\n",
- " verbosity=1,\n",
+ " condition = getattr(Conditions, cond_type).AGIPD(**cond_param)\n",
+ "\n",
+ " _, mdata = tools.get_from_db(\n",
+ " karabo_id,\n",
+ " karabo_da,\n",
+ " getattr(Constants.AGIPD, const_name)(),\n",
+ " condition,\n",
+ " getattr(np, const_init_fun)(const_shape),\n",
+ " cal_db_interface,\n",
+ " creation_time,\n",
+ " meta_only=True,\n",
+ " verbosity=1,\n",
+ " )\n",
+ " mdata_const = mdata.calibration_constant_version\n",
+ " # check if constant was sucessfully retrieved.\n",
+ " if mdata.comm_db_success:\n",
+ " const_mdata[\"file-path\"] = (\n",
+ " f\"{mdata_const.hdf5path}\" f\"{mdata_const.filename}\"\n",
" )\n",
- " mdata_const = mdata.calibration_constant_version\n",
- " # check if constant was sucessfully retrieved.\n",
- " if mdata.comm_db_success:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
- " f\"{mdata_const.filename}\"\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
- " mdata_dict['physical-detector-unit'] = mdata_const.device_name\n",
- " else:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = const_dict[cname][:2]\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = None\n",
+ " const_mdata[\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
+ " mdata_dict[\"physical-detector-unit\"] = mdata_const.device_name\n",
+ " else:\n",
+ " const_mdata[\"file-path\"] = const_dict[const_name][:2]\n",
+ " const_mdata[\"creation-time\"] = None\n",
+ " print(const_mdata)\n",
"\n",
" return qm, mdata_dict, karabo_da, acq_rate, local_max_cells, err"
]
--
GitLab
From 4ae7ae8b389885923a385d58e0245111f57a6646 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 18 Mar 2021 19:20:14 +0100
Subject: [PATCH 62/72] Formatting + minor typos
---
cal_tools/cal_tools/agipdlib.py | 57 +++++++++++++++------------------
1 file changed, 26 insertions(+), 31 deletions(-)
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 8b1fd3102..4f6200fa0 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -23,8 +23,7 @@ from iCalibrationDB import Conditions
def get_num_cells(fname, loc, module):
with h5py.File(fname, "r") as f:
- cells = \
- f[f"INSTRUMENT/{loc}/DET/{module}CH0:xtdf/image/cellId"][()]
+ cells = f[f"INSTRUMENT/{loc}/DET/{module}CH0:xtdf/image/cellId"][()]
if cells.shape[0] == 0:
return None
maxcell = np.max(cells)
@@ -97,7 +96,7 @@ def get_gain_setting(fname: str, h5path_ctrl: str) -> int:
gain-setting 1: setupr@dark=8, setupr@slopespc=40
gain-setting 0: setupr@dark=0, setupr@slopespc=32
- patternTypeIndex 1: High-gian
+ patternTypeIndex 1: High-gain
patternTypeIndex 2: Medium-gain
patternTypeIndex 3: Low-gain
patternTypeIndex 4: SlopesPC
@@ -197,9 +196,7 @@ class AgipdCorrections:
const_yaml = metadata["retrieved-constants"]
for mod in modules:
- qm = f"Q{mod // 4 + 1}M{mod % 4 + 1}"
- agipd_corr.initialize_from_yaml(karabo_da,
- const_yaml, mod)
+ agipd_corr.initialize_from_yaml(karabo_da, const_yaml, mod)
data_shape = (n_images_max, 512, 128)
agipd_corr.allocate_images(data_shape, n_cores_files)
@@ -325,8 +322,7 @@ class AgipdCorrections:
data_dict['rawgain'][:n_img] = raw_data[:, 1]
data_dict['cellId'][:n_img] = allcells[firange]
data_dict['pulseId'][:n_img] = allpulses[firange]
- data_dict['trainId'][:n_img] = np.squeeze(
- group['trainId'][:][firange])
+ data_dict['trainId'][:n_img] = np.squeeze(group['trainId'][:][firange])
except Exception as e:
print(f'Error during reading data from file {file_name}: {e}')
print(f'Error traceback: {traceback.format_exc()}')
@@ -364,10 +360,10 @@ class AgipdCorrections:
# Copy any other data from the input file.
# This includes indexes, so it's important that the corrected data
# we write is aligned with the raw data.
- with h5py.File(file_name, 'r') as infile:
- self.copy_and_sanitize_non_cal_data(infile, outfile,
- agipd_base,
- idx_base, trains)
+ with h5py.File(file_name, "r") as infile:
+ self.copy_and_sanitize_non_cal_data(
+ infile, outfile, agipd_base, idx_base, trains
+ )
# All corrected data goes in a /INSTRUMENT/.../image group
image_grp = outfile[data_path]
@@ -555,8 +551,9 @@ class AgipdCorrections:
cellid = self.shared_dict[i_proc]['cellId'][first:last]
# output is saved in sharedmem to pass for correct_agipd()
# as this function takes about 3 seconds.
- self.shared_dict[i_proc]['msk'][first:last] = \
- calgs.gain_choose_int(gain, self.mask[module_idx][:, cellid])
+ self.shared_dict[i_proc]["msk"][first:last] = calgs.gain_choose_int(
+ gain, self.mask[module_idx][:, cellid]
+ )
if hasattr(self, "rel_gain"):
# Get the correct rel_gain depending on cell-id
@@ -628,14 +625,12 @@ class AgipdCorrections:
# if baseline correction was not requested
# msk and rel_corr will still be empty shared_mem arrays
if not any(self.blc_bools):
- msk = calgs.gain_choose_int(gain,
- self.mask[module_idx][:, cellid])
+ msk = calgs.gain_choose_int(gain, self.mask[module_idx][:, cellid])
# same for relative gain and then bad pixel mask
if hasattr(self, "rel_gain"):
# Get the correct rel_gain depending on cell-id
- rel_corr = calgs.gain_choose(gain,
- self.rel_gain[module_idx][:, cellid]) # noqa
+ rel_corr = calgs.gain_choose(gain, self.rel_gain[module_idx][:, cellid])
# Correct for relative gain
if self.corr_bools.get("rel_gain") and hasattr(self, "rel_gain"):
@@ -698,11 +693,9 @@ class AgipdCorrections:
# Copy the data across into the existing shared-memory array
mask[...] = msk[...]
- def get_valid_image_idx(self, idx_base: str, infile: str,
- index_v: Optional[int] = 2):
- """ Return the indices of valid data
- """
- if index_v == 2:
+ def get_valid_image_idx(self, idx_base: str, infile: str, raw_format_version: int = 2):
+ """Return the indices of valid data"""
+ if raw_format_version == 2:
count = np.squeeze(infile[idx_base + "image/count"])
first = np.squeeze(infile[idx_base + "image/first"])
if np.count_nonzero(count != 0) == 0:
@@ -727,13 +720,16 @@ class AgipdCorrections:
# Creating an array of validated indices.
# If all indices were validated this array will be the same,
# as what is stored at /DET/image/trainId
- valid_indices = np.concatenate([np.arange(validf[i],
- validf[i]+validc[i])
- for i in range(validf.size)],
- axis=0)
+ valid_indices = np.concatenate(
+ [
+ np.arange(validf[i], validf[i] + validc[i])
+ for i in range(validf.size)
+ ],
+ axis=0,
+ )
valid_indices = np.squeeze(valid_indices).astype(np.int32)
- elif index_v == 1:
+ elif raw_format_version == 1:
status = np.squeeze(infile[idx_base + "image/status"])
if np.count_nonzero(status != 0) == 0:
raise IOError(f"File {infile} has no valid counts")
@@ -751,10 +747,9 @@ class AgipdCorrections:
valid_indices = None
else:
raise AttributeError(
- f"Not a known raw format version: {index_v}")
+ f"Not a known raw format version: {raw_format_version}")
- return (valid, first_index, last_index, idxtrains,
- valid_indices)
+ return (valid, first_index, last_index, idxtrains, valid_indices)
def apply_selected_pulses(self, i_proc: int) -> int:
"""Select sharedmem data indices to correct based on selected
--
GitLab
From f7f0353284efc83e2cddbf18f0df1e0227828705 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 18 Mar 2021 20:16:11 +0100
Subject: [PATCH 63/72] Adding gain mode handling to AgipdCorrections
---
cal_tools/cal_tools/agipdlib.py | 105 ++++++++++++++++----------------
1 file changed, 52 insertions(+), 53 deletions(-)
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 4f6200fa0..6eb9d6f60 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -164,10 +164,15 @@ def get_bias_voltage(fname: str, karabo_id_control: str,
class AgipdCorrections:
- def __init__(self, max_cells, max_pulses,
- h5_data_path="INSTRUMENT/SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf/",
- h5_index_path="INDEX/SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf/",
- corr_bools: Optional[dict] = None):
+ def __init__(
+ self,
+ max_cells,
+ max_pulses,
+ h5_data_path="INSTRUMENT/SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf/",
+ h5_index_path="INDEX/SPB_DET_AGIPD1M-1/DET/{}CH0:xtdf/",
+ corr_bools: Optional[dict] = None,
+ gain_mode: AgipdGainMode = AgipdGainMode.ADAPTIVE_GAIN,
+ ):
"""
Initialize an AgipdCorrections Class
@@ -222,6 +227,7 @@ class AgipdCorrections:
self.pulses_lst = list(range(*max_pulses)) \
if not (len(max_pulses) == 1 and max_pulses[0] == 0) else max_pulses # noqa
self.max_cells = max_cells
+ self.gain_mode = gain_mode
# Correction parameters
self.baseline_corr_noise_threshold = -1000
@@ -490,35 +496,35 @@ class AgipdCorrections:
gain = self.shared_dict[i_proc]['gain'][first:last]
cellid = self.shared_dict[i_proc]['cellId'][first:last]
- # first evaluate the gain into 0, 1, 2 --> high, medium, low
- t0 = self.thresholds[module_idx][0]
- t1 = self.thresholds[module_idx][1]
+ if self.gain_mode is AgipdGainMode.ADAPTIVE_GAIN:
+ # first evaluate the gain into 0, 1, 2 --> high, medium, low
+ t0 = self.thresholds[module_idx][0]
+ t1 = self.thresholds[module_idx][1]
- # load raw_data and rgain to be used during gain_correction,
- # if requested
- if self.corr_bools.get('melt_snow'):
- self.shared_dict[i_proc]['t0_rgain'][first:last] = \
- rawgain / t0[cellid, ...]
- self.shared_dict[i_proc]['raw_data'][first:last] = np.copy(data)
-
- # Often most pixels are in high-gain, so it's more efficient to
- # set the whole output block to zero than select the right pixels.
- gain[:] = 0
- # exceeding first threshold means data is medium or low gain
- gain[rawgain > t0[cellid, ...]] = 1
- # exceeding also second threshold means data is low gain
- gain[rawgain > t1[cellid, ...]] = 2
+ # load raw_data and rgain to be used during gain_correction if requested
+ if self.corr_bools.get("melt_snow"):
+ self.shared_dict[i_proc]["t0_rgain"][first:last] = rawgain / t0[cellid, ...]
+ self.shared_dict[i_proc]["raw_data"][first:last] = np.copy(data)
+
+ # Often most pixels are in high-gain, so it's more efficient to
+ # set the whole output block to zero than select the right pixels.
+ gain[:] = 0
+ # exceeding first threshold means data is medium or low gain
+ gain[rawgain > t0[cellid, ...]] = 1
+ # exceeding also second threshold means data is low gain
+ gain[rawgain > t1[cellid, ...]] = 2
+ else:
+ # the enum values map 1, 2, 3 to (fixed) gain modes
+ gain[:] = self.gain_mode - 1
offsetb = self.offset[module_idx][:, cellid]
# force into high or medium gain if requested
- if self.corr_bools.get('force_mg_if_below'):
- gain[(gain == 2) & (
- (data - offsetb[1]) < self.mg_hard_threshold)] = 1
+ if self.corr_bools.get("force_mg_if_below"):
+ gain[(gain == 2) & ((data - offsetb[1]) < self.mg_hard_threshold)] = 1
- if self.corr_bools.get('force_hg_if_below'):
- gain[(gain > 0) & (
- (data - offsetb[0]) < self.hg_hard_threshold)] = 0
+ if self.corr_bools.get("force_hg_if_below"):
+ gain[(gain > 0) & ((data - offsetb[0]) < self.hg_hard_threshold)] = 0
# choose constants according to gain setting
off = calgs.gain_choose(gain, offsetb)
@@ -1088,7 +1094,8 @@ class AgipdCorrections:
self.offset[module_idx][...] = cons_data["Offset"].transpose()[...]
self.noise[module_idx][...] = cons_data["Noise"].transpose()[...]
- self.thresholds[module_idx][...] = cons_data["ThresholdsDark"].transpose()[:3, ...] # noqa
+ if self.gain_mode is AgipdGainMode.ADAPTIVE_GAIN:
+ self.thresholds[module_idx][...] = cons_data["ThresholdsDark"].transpose()[:3,...] # noqa
if self.corr_bools.get("low_medium_gap"):
t0 = self.thresholds[module_idx][0]
@@ -1232,20 +1239,18 @@ class AgipdCorrections:
return
- def initialize_from_yaml(self, karabo_da: str,
- const_yaml: Dict[str, Any],
- module_idx: int
- ) -> Dict[str, Any]:
+ def initialize_from_yaml(
+ self, karabo_da: str, const_yaml: Dict[str, Any], module_idx: int
+ ) -> Dict[str, Any]:
"""Initialize calibration constants from a yaml file
- :param karabo-da: karabo data aggerator
- :param const_yaml: (Dict) from the "retrieved-constants" part of a yaml
- file in pre-notebook, which consists of metadata of either the constant
+ :param karabo_da: a karabo data aggregator
+ :param const_yaml: from the "retrieved-constants" part of a yaml
+ file from pre-notebook, which consists of metadata of either the constant
file path or the empty constant shape, and the creation-time of the
retrieved constants
:param module_idx: Index of module.
- :return when: Dictionary of retrieved constants with
- their creation-time.
+ :return when: Dictionary of retrieved constants with their creation-time.
"""
# string of the device name.
@@ -1351,23 +1356,17 @@ class AgipdCorrections:
:param constant_shape: Shape of expected constants (gain, cells, x, y)
"""
for module_idx in modules:
- self.offset[module_idx] = sharedmem.empty(constant_shape,
- dtype='f4')
- self.thresholds[module_idx] = sharedmem.empty(constant_shape,
- dtype='f4')
- self.noise[module_idx] = sharedmem.empty(constant_shape,
- dtype='f4')
-
- self.md_additional_offset[module_idx] = sharedmem.empty(
- constant_shape[1:], dtype='f4')
- self.rel_gain[module_idx] = sharedmem.empty(constant_shape,
- dtype='f4')
- self.frac_high_med[module_idx] = sharedmem.empty(constant_shape[1],
- dtype='f4')
+ self.offset[module_idx] = sharedmem.empty(constant_shape, dtype="f4")
+ if self.gain_mode is AgipdGainMode.ADAPTIVE_GAIN:
+ self.thresholds[module_idx] = sharedmem.empty(constant_shape, dtype="f4")
+ self.noise[module_idx] = sharedmem.empty(constant_shape, dtype="f4")
+
+ self.md_additional_offset[module_idx] = sharedmem.empty(constant_shape[1:], dtype="f4")
+ self.rel_gain[module_idx] = sharedmem.empty(constant_shape, dtype="f4")
+ self.frac_high_med[module_idx] = sharedmem.empty(constant_shape[1], dtype="f4")
- self.mask[module_idx] = sharedmem.empty(constant_shape, dtype='i4')
- self.xray_cor[module_idx] = sharedmem.empty(constant_shape[1:],
- dtype='f4')
+ self.mask[module_idx] = sharedmem.empty(constant_shape, dtype="i4")
+ self.xray_cor[module_idx] = sharedmem.empty(constant_shape[1:], dtype="f4")
def allocate_images(self, shape, n_cores_files):
"""
--
GitLab
From 32ec3759284718cd5b653bfd14e3282c91bcab15 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Fri, 19 Mar 2021 16:06:45 +0100
Subject: [PATCH 64/72] Formatting, absolute multiprocessing import, typos
---
cal_tools/cal_tools/agipdlib.py | 68 ++++++++-----------
cal_tools/cal_tools/agipdutils.py | 4 +-
cal_tools/cal_tools/tools.py | 3 +-
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 6 +-
4 files changed, 36 insertions(+), 45 deletions(-)
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 6eb9d6f60..e22b228d7 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -961,13 +961,11 @@ class AgipdCorrections:
# Extract parameters through identifying
# unique trains, index and numbers.
- uq, fidxv, cntsv = np.unique(trains, return_index=True,
- return_counts=True)
+ uq, fidxv, cntsv = np.unique(trains, return_index=True, return_counts=True)
# Validate calculated CORR INDEX contents by checking
# difference between trainId stored in RAW data and trains from
- train_diff = np.isin(np.array(infile["/INDEX/trainId"]), uq,
- invert=True)
+ train_diff = np.isin(np.array(infile["/INDEX/trainId"]), uq, invert=True)
# Insert zeros for missing trains.
# fidxv and cntsv should have same length as
@@ -1260,10 +1258,6 @@ class AgipdCorrections:
for cname, mdata in const_yaml[karabo_da]["constants"].items():
when[cname] = mdata["creation-time"]
if when[cname]:
- # This path is only used when testing new flat fields from
- # file during development: it takes ages to test using all
- # cells. Consequently, the shape needs to be fixed when less
- # cells are used.
with h5py.File(mdata["file-path"], "r") as cf:
cons_data[cname] = np.copy(cf[f"{db_module}/{cname}/0/data"]) # noqa
else:
@@ -1334,15 +1328,21 @@ class AgipdCorrections:
"""
- const_dict = \
- assemble_constant_dict(self.corr_bools, self.pc_bools,
- memory_cells, bias_voltage, gain_setting,
- acquisition_rate, photon_energy,
- beam_energy=None, only_dark=only_dark)
+ const_dict = assemble_constant_dict(
+ self.corr_bools,
+ self.pc_bools,
+ memory_cells,
+ bias_voltage,
+ gain_setting,
+ acquisition_rate,
+ photon_energy,
+ beam_energy=None,
+ only_dark=only_dark,
+ )
- cons_data, when = \
- self.retrieve_constant_and_time(karabo_id, karabo_da, const_dict,
- cal_db_interface, creation_time)
+ cons_data, when = self.retrieve_constant_and_time(
+ karabo_id, karabo_da, const_dict, cal_db_interface, creation_time
+ )
self.init_constants(cons_data, when, module_idx)
@@ -1380,26 +1380,18 @@ class AgipdCorrections:
self.shared_dict = []
for i in range(n_cores_files):
self.shared_dict.append({})
- self.shared_dict[i]['cellId'] = sharedmem.empty(shape[0],
- dtype='u2')
- self.shared_dict[i]['pulseId'] = sharedmem.empty(shape[0],
- dtype='u8')
- self.shared_dict[i]['trainId'] = sharedmem.empty(shape[0],
- dtype='u8')
- self.shared_dict[i]['moduleIdx'] = sharedmem.empty(1, dtype='i4')
- self.shared_dict[i]['nImg'] = sharedmem.empty(1, dtype='i4')
- self.shared_dict[i]['mask'] = sharedmem.empty(shape, dtype='u4')
- self.shared_dict[i]['data'] = sharedmem.empty(shape, dtype='f4')
- self.shared_dict[i]['rawgain'] = sharedmem.empty(shape,
- dtype='u2')
- self.shared_dict[i]['gain'] = sharedmem.empty(shape, dtype='u1')
- self.shared_dict[i]['blShift'] = sharedmem.empty(shape[0],
- dtype='f4')
+ self.shared_dict[i]["cellId"] = sharedmem.empty(shape[0], dtype="u2")
+ self.shared_dict[i]["pulseId"] = sharedmem.empty(shape[0], dtype="u8")
+ self.shared_dict[i]["trainId"] = sharedmem.empty(shape[0], dtype="u8")
+ self.shared_dict[i]["moduleIdx"] = sharedmem.empty(1, dtype="i4")
+ self.shared_dict[i]["nImg"] = sharedmem.empty(1, dtype="i4")
+ self.shared_dict[i]["mask"] = sharedmem.empty(shape, dtype="u4")
+ self.shared_dict[i]["data"] = sharedmem.empty(shape, dtype="f4")
+ self.shared_dict[i]["rawgain"] = sharedmem.empty(shape, dtype="u2")
+ self.shared_dict[i]["gain"] = sharedmem.empty(shape, dtype="u1")
+ self.shared_dict[i]["blShift"] = sharedmem.empty(shape[0], dtype="f4")
# Parameters shared between image-wise correction functions
- self.shared_dict[i]['msk'] = sharedmem.empty(shape, dtype='i4')
- self.shared_dict[i]['raw_data'] = sharedmem.empty(shape,
- dtype='f4')
- self.shared_dict[i]['rel_corr'] = sharedmem.empty(shape,
- dtype='f4')
- self.shared_dict[i]['t0_rgain'] = sharedmem.empty(shape,
- dtype='u2')
+ self.shared_dict[i]["msk"] = sharedmem.empty(shape, dtype="i4")
+ self.shared_dict[i]["raw_data"] = sharedmem.empty(shape, dtype="f4")
+ self.shared_dict[i]["rel_corr"] = sharedmem.empty(shape, dtype="f4")
+ self.shared_dict[i]["t0_rgain"] = sharedmem.empty(shape, dtype="u2")
diff --git a/cal_tools/cal_tools/agipdutils.py b/cal_tools/cal_tools/agipdutils.py
index aec289dd6..44cac4c87 100644
--- a/cal_tools/cal_tools/agipdutils.py
+++ b/cal_tools/cal_tools/agipdutils.py
@@ -32,11 +32,11 @@ def assemble_constant_dict(
:param bias_voltage: (Int) Bias Voltage
:param gain_setting: (Float) Gain setting
:param acquisition_rate: (Float) Acquisition rate
- :param photon_energy: (Float) Photong energy
+ :param photon_energy: (Float) Photon energy
:param beam_energy: (Float) Beam Energy
:param only_dark: (Bool) Indicating a retrieval for dark constants only from db
:param gain_mode: Operation mode of the detector (default to adaptive gain)
- :return: const_dict: (Dict) An assembeld dictionary that can be used
+ :return: const_dict: (Dict) An assembled dictionary that can be used
to retrieve the required constants
"""
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index a6aa3f5e8..4e5575146 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -539,8 +539,7 @@ def get_from_db(karabo_id: str, karabo_da: str,
if ntries > 0:
if load_data and meta_only:
mdata_const = metadata.calibration_constant_version
- fpath = Path(mdata_const.hdf5path,
- mdata_const.filename)
+ fpath = Path(mdata_const.hdf5path, mdata_const.filename)
with h5py.File(fpath, "r") as f:
arr = f[f"{mdata_const.h5path}/data"][()]
metadata.calibration_constant.data = arr
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index c66a23a15..25abd5e19 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -101,11 +101,11 @@
"source": [
"import itertools\n",
"import math\n",
+ "import multiprocessing\n",
"import re\n",
"import traceback\n",
"import warnings\n",
"from datetime import timedelta\n",
- "from multiprocessing import Pool\n",
"from pathlib import Path\n",
"from time import perf_counter\n",
"\n",
@@ -486,7 +486,7 @@
"\n",
"\n",
"ts = perf_counter()\n",
- "with Pool(processes=len(modules)) as pool:\n",
+ "with multiprocessing.Pool(processes=len(modules)) as pool:\n",
" const_out = pool.map(retrieve_constants, modules)\n",
"print(f\"Constants were loaded in {perf_counter()-ts:.01f}s\")"
]
@@ -549,7 +549,7 @@
"metadata": {},
"outputs": [],
"source": [
- "with Pool() as pool:\n",
+ "with multiprocessing.Pool() as pool:\n",
" for file_batch in batches(file_list, n_cores_files):\n",
" # TODO: Move some printed output to logging or similar\n",
" print(f\"Processing next {len(file_batch)} files:\")\n",
--
GitLab
From 4628ab144793db70ce90345d2ac6f255d1902517 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Fri, 19 Mar 2021 16:07:14 +0100
Subject: [PATCH 65/72] Skip melt_snow, remember gain_mode
---
notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 25abd5e19..bad008604 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -203,7 +203,8 @@
" \"blc_set_min\",\n",
" \"force_hg_if_below\",\n",
" \"force_mg_if_below\",\n",
- " \"low_medium_gap\"\n",
+ " \"low_medium_gap\",\n",
+ " \"melt_snow\"\n",
"]"
]
},
@@ -413,7 +414,8 @@
" max_pulses,\n",
" h5_data_path=h5path,\n",
" h5_index_path=h5path_idx,\n",
- " corr_bools=corr_bools\n",
+ " corr_bools=corr_bools,\n",
+ " gain_mode=gain_mode\n",
")\n",
"\n",
"agipd_corr.baseline_corr_noise_threshold = -blc_noise_threshold\n",
--
GitLab
From 58e90a84bc3b003fc602b58e7e51258828fea8d7 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Mon, 22 Mar 2021 19:22:19 +0100
Subject: [PATCH 66/72] Explicitly set read mode on h5py.File
---
cal_tools/cal_tools/agipdlib.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index e22b228d7..285315908 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -75,7 +75,7 @@ def get_acq_rate(fast_paths: Tuple[str, str, int],
fast_data_file = Path(fast_data_file)
if fast_data_file.is_file():
fast_data_path = f'INSTRUMENT/{karabo_id}/DET/{module}CH0:xtdf/image/pulseId' # noqa
- with h5py.File(fast_data_file) as fin:
+ with h5py.File(fast_data_file, "r") as fin:
if fast_data_path in fin:
# pulses is of shape (NNNN, 1), of type uint8.
# Squeeze out the data, and subtract the 3rd entry from the 2nd
@@ -132,7 +132,7 @@ def get_gain_mode(fname: str, h5path_ctrl: str) -> AgipdGainMode:
h5path_run = h5path_ctrl.replace("CONTROL/", "RUN/", 1)
h5path_gainmode = f'{h5path_run}/gainModeIndex/value'
- with h5py.File(fname, 'r') as fd:
+ with h5py.File(fname, "r") as fd:
if h5path_gainmode in fd:
return AgipdGainMode(fd[h5path_gainmode][0])
return AgipdGainMode.ADAPTIVE_GAIN
@@ -299,7 +299,7 @@ class AgipdCorrections:
data_dict = self.shared_dict[i_proc]
data_dict['moduleIdx'][0] = module_idx
try:
- f = h5py.File(file_name, 'r')
+ f = h5py.File(file_name, "r")
group = f[agipd_base]["image"]
(_, first_index, last_index,
@@ -362,7 +362,7 @@ class AgipdCorrections:
return
trains = data_dict['trainId'][:n_img]
- with h5py.File(ofile_name, 'w') as outfile:
+ with h5py.File(ofile_name, "w") as outfile:
# Copy any other data from the input file.
# This includes indexes, so it's important that the corrected data
# we write is aligned with the raw data.
--
GitLab
From bfa5cb601f268e39a849e0ebeef87151b93bd267 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Wed, 24 Mar 2021 13:04:31 +0100
Subject: [PATCH 67/72] Bump cal_db_interactive version
---
requirements.txt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/requirements.txt b/requirements.txt
index ea6991b7d..83437d4e6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-git+file:///gpfs/exfel/sw/calsoft/git/cal_db_interactive@2.0.1
+git+file:///gpfs/exfel/sw/calsoft/git/cal_db_interactive@2.0.4
git+file:///gpfs/exfel/sw/calsoft/git/nbparameterise@0.3
git+file:///gpfs/exfel/sw/calsoft/git/pyDetLib@2.5.6-2.10.0#subdirectory=lib
astcheck == 0.2.5
--
GitLab
From aa274057ef6f284869158ddf1445e7f6f2bf1be2 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Wed, 24 Mar 2021 16:56:08 +0100
Subject: [PATCH 68/72] Now hopefully applying the correct version of isort
---
cal_tools/cal_tools/agipdlib.py | 3 ++-
notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb | 9 +++++++--
2 files changed, 9 insertions(+), 3 deletions(-)
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 285315908..103bbc162 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -15,11 +15,12 @@ from cal_tools.agipdutils import (
match_asic_borders,
melt_snowy_pixels,
)
-from cal_tools.cython import agipdalgs as calgs
from cal_tools.enums import AgipdGainMode, BadPixels, SnowResolution
from cal_tools.tools import get_constant_from_db_and_time
from iCalibrationDB import Conditions
+from cal_tools.cython import agipdalgs as calgs
+
def get_num_cells(fname, loc, module):
with h5py.File(fname, "r") as f:
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index bad008604..bfee6b55d 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -133,8 +133,13 @@
"\n",
"import cal_tools\n",
"import seaborn as sns\n",
- "from cal_tools.agipdlib import (AgipdCorrections, get_acq_rate, get_gain_mode,\n",
- " get_gain_setting, get_num_cells)\n",
+ "from cal_tools.agipdlib import (\n",
+ " AgipdCorrections,\n",
+ " get_acq_rate,\n",
+ " get_gain_mode,\n",
+ " get_gain_setting,\n",
+ " get_num_cells,\n",
+ ")\n",
"from cal_tools.ana_tools import get_range\n",
"from cal_tools.cython import agipdalgs as calgs\n",
"from cal_tools.enums import AgipdGainMode, BadPixels\n",
--
GitLab
From e894d0bd2be9b8614cdb270bdf42adf7b580f118 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 25 Mar 2021 09:15:09 +0100
Subject: [PATCH 69/72] isort
---
cal_tools/cal_tools/agipdlib.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 8b1fd3102..be8033e5a 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -15,11 +15,12 @@ from cal_tools.agipdutils import (
match_asic_borders,
melt_snowy_pixels,
)
-from cal_tools.cython import agipdalgs as calgs
from cal_tools.enums import AgipdGainMode, BadPixels, SnowResolution
from cal_tools.tools import get_constant_from_db_and_time
from iCalibrationDB import Conditions
+from cal_tools.cython import agipdalgs as calgs
+
def get_num_cells(fname, loc, module):
with h5py.File(fname, "r") as f:
--
GitLab
From b7a4c1652a6d8982935c48838d0e7090beb2acfb Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 25 Mar 2021 09:27:54 +0100
Subject: [PATCH 70/72] Decreasing verbosity
---
notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb | 6 +-----
.../AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb | 6 +-----
2 files changed, 2 insertions(+), 10 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index bfee6b55d..27de08239 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -468,10 +468,8 @@
" try:\n",
" # check if there is a yaml file in out_folder that has the device constants.\n",
" if k_da in const_yaml:\n",
- " print(f\"Pre-correction notebook already found constants for {k_da}\")\n",
" when = agipd_corr.initialize_from_yaml(k_da, const_yaml, mod)\n",
" else:\n",
- " print(f\"Have to query database for constants for {k_da}\")\n",
" # TODO: replace with proper retrieval (as done in pre-correction)\n",
" when = agipd_corr.initialize_from_db(\n",
" karabo_id,\n",
@@ -559,9 +557,7 @@
"with multiprocessing.Pool() as pool:\n",
" for file_batch in batches(file_list, n_cores_files):\n",
" # TODO: Move some printed output to logging or similar\n",
- " print(f\"Processing next {len(file_batch)} files:\")\n",
- " for file_name in file_batch:\n",
- " print(\" \", file_name)\n",
+ " print(f\"Processing next {len(file_batch)} files\")\n",
" step_timer.start()\n",
" img_counts = pool.starmap(agipd_corr.read_file, zip(range(len(file_batch)), file_batch,\n",
" [not common_mode]*len(file_batch)))\n",
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 0a2583b17..b92a901a9 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -264,7 +264,6 @@
"\n",
" for const_name, (const_init_fun, const_shape, (cond_type, cond_param)) in const_dict.items():\n",
" if gain_mode and const_name in (\"ThresholdsDark\",):\n",
- " print(f\"Note: skipping {const_name} for fixed gain mode\")\n",
" continue\n",
" \n",
" # saving metadata in a dict\n",
@@ -280,7 +279,6 @@
" param_copy = cond_param.copy()\n",
" del param_copy[\"gain_mode\"]\n",
" condition = getattr(Conditions, cond_type).AGIPD(**param_copy)\n",
- " print(f\"Note: {const_name} based on adaptive gain mode constants will be retrieved\")\n",
" else:\n",
" condition = getattr(Conditions, cond_type).AGIPD(**cond_param)\n",
"\n",
@@ -293,7 +291,7 @@
" cal_db_interface,\n",
" creation_time,\n",
" meta_only=True,\n",
- " verbosity=1,\n",
+ " verbosity=0,\n",
" )\n",
" mdata_const = mdata.calibration_constant_version\n",
" # check if constant was sucessfully retrieved.\n",
@@ -306,7 +304,6 @@
" else:\n",
" const_mdata[\"file-path\"] = const_dict[const_name][:2]\n",
" const_mdata[\"creation-time\"] = None\n",
- " print(const_mdata)\n",
"\n",
" return qm, mdata_dict, karabo_da, acq_rate, local_max_cells, err"
]
@@ -346,7 +343,6 @@
" # TODO: make map_modules_from_folder just return list(s)\n",
" qm_files = [Path(mapped_files[qm].get()) for _ in range(mapped_files[qm].qsize())]\n",
" else:\n",
- " print(f\"Skipping {qm}\")\n",
" continue\n",
"\n",
" inp.append((qm_files, qm, k_da, module_index))"
--
GitLab
From d5c6b94287287992c6e9b6450c170bf0c453e58a Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Thu, 25 Mar 2021 10:36:49 +0100
Subject: [PATCH 71/72] Satisfying flake8 for agipdlib.py
---
cal_tools/cal_tools/agipdlib.py | 58 +++++++++++++++++----------------
1 file changed, 30 insertions(+), 28 deletions(-)
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 103bbc162..a098b07b8 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -329,7 +329,7 @@ class AgipdCorrections:
data_dict['rawgain'][:n_img] = raw_data[:, 1]
data_dict['cellId'][:n_img] = allcells[firange]
data_dict['pulseId'][:n_img] = allpulses[firange]
- data_dict['trainId'][:n_img] = np.squeeze(group['trainId'][:][firange])
+ data_dict['trainId'][:n_img] = np.squeeze(group['trainId'][:][firange]) # noqa
except Exception as e:
print(f'Error during reading data from file {file_name}: {e}')
print(f'Error traceback: {traceback.format_exc()}')
@@ -502,10 +502,10 @@ class AgipdCorrections:
t0 = self.thresholds[module_idx][0]
t1 = self.thresholds[module_idx][1]
- # load raw_data and rgain to be used during gain_correction if requested
if self.corr_bools.get("melt_snow"):
- self.shared_dict[i_proc]["t0_rgain"][first:last] = rawgain / t0[cellid, ...]
- self.shared_dict[i_proc]["raw_data"][first:last] = np.copy(data)
+ # load raw_data and rgain to be used during gain_correction
+ self.shared_dict[i_proc]["t0_rgain"][first:last] = rawgain / t0[cellid, ...] # noqa
+ self.shared_dict[i_proc]["raw_data"][first:last] = np.copy(data) # noqa
# Often most pixels are in high-gain, so it's more efficient to
# set the whole output block to zero than select the right pixels.
@@ -522,10 +522,10 @@ class AgipdCorrections:
# force into high or medium gain if requested
if self.corr_bools.get("force_mg_if_below"):
- gain[(gain == 2) & ((data - offsetb[1]) < self.mg_hard_threshold)] = 1
+ gain[(gain == 2) & ((data - offsetb[1]) < self.mg_hard_threshold)] = 1 # noqa
if self.corr_bools.get("force_hg_if_below"):
- gain[(gain > 0) & ((data - offsetb[0]) < self.hg_hard_threshold)] = 0
+ gain[(gain > 0) & ((data - offsetb[0]) < self.hg_hard_threshold)] = 0 # noqa
# choose constants according to gain setting
off = calgs.gain_choose(gain, offsetb)
@@ -637,7 +637,7 @@ class AgipdCorrections:
# same for relative gain and then bad pixel mask
if hasattr(self, "rel_gain"):
# Get the correct rel_gain depending on cell-id
- rel_corr = calgs.gain_choose(gain, self.rel_gain[module_idx][:, cellid])
+ rel_corr = calgs.gain_choose(gain, self.rel_gain[module_idx][:, cellid]) # noqa
# Correct for relative gain
if self.corr_bools.get("rel_gain") and hasattr(self, "rel_gain"):
@@ -700,7 +700,9 @@ class AgipdCorrections:
# Copy the data across into the existing shared-memory array
mask[...] = msk[...]
- def get_valid_image_idx(self, idx_base: str, infile: str, raw_format_version: int = 2):
+ def get_valid_image_idx(
+ self, idx_base: str, infile: str, raw_format_version: int = 2
+ ):
"""Return the indices of valid data"""
if raw_format_version == 2:
count = np.squeeze(infile[idx_base + "image/count"])
@@ -962,11 +964,11 @@ class AgipdCorrections:
# Extract parameters through identifying
# unique trains, index and numbers.
- uq, fidxv, cntsv = np.unique(trains, return_index=True, return_counts=True)
+ uq, fidxv, cntsv = np.unique(trains, return_index=True, return_counts=True) # noqa
# Validate calculated CORR INDEX contents by checking
# difference between trainId stored in RAW data and trains from
- train_diff = np.isin(np.array(infile["/INDEX/trainId"]), uq, invert=True)
+ train_diff = np.isin(np.array(infile["/INDEX/trainId"]), uq, invert=True) # noqa
# Insert zeros for missing trains.
# fidxv and cntsv should have same length as
@@ -1244,12 +1246,12 @@ class AgipdCorrections:
"""Initialize calibration constants from a yaml file
:param karabo_da: a karabo data aggregator
- :param const_yaml: from the "retrieved-constants" part of a yaml
- file from pre-notebook, which consists of metadata of either the constant
+ :param const_yaml: from the "retrieved-constants" part of a yaml file
+ from pre-notebook, which consists of metadata of either the constant
file path or the empty constant shape, and the creation-time of the
retrieved constants
- :param module_idx: Index of module.
- :return when: Dictionary of retrieved constants with their creation-time.
+ :param module_idx: Index of module
+ :return when: dict of retrieved constants with their creation-time
"""
# string of the device name.
@@ -1357,17 +1359,17 @@ class AgipdCorrections:
:param constant_shape: Shape of expected constants (gain, cells, x, y)
"""
for module_idx in modules:
- self.offset[module_idx] = sharedmem.empty(constant_shape, dtype="f4")
+ self.offset[module_idx] = sharedmem.empty(constant_shape, dtype="f4") # noqa
if self.gain_mode is AgipdGainMode.ADAPTIVE_GAIN:
- self.thresholds[module_idx] = sharedmem.empty(constant_shape, dtype="f4")
- self.noise[module_idx] = sharedmem.empty(constant_shape, dtype="f4")
+ self.thresholds[module_idx] = sharedmem.empty(constant_shape, dtype="f4") # noqa
+ self.noise[module_idx] = sharedmem.empty(constant_shape, dtype="f4") # noqa
- self.md_additional_offset[module_idx] = sharedmem.empty(constant_shape[1:], dtype="f4")
- self.rel_gain[module_idx] = sharedmem.empty(constant_shape, dtype="f4")
- self.frac_high_med[module_idx] = sharedmem.empty(constant_shape[1], dtype="f4")
+ self.md_additional_offset[module_idx] = sharedmem.empty(constant_shape[1:], dtype="f4") # noqa
+ self.rel_gain[module_idx] = sharedmem.empty(constant_shape, dtype="f4") # noqa
+ self.frac_high_med[module_idx] = sharedmem.empty(constant_shape[1], dtype="f4") # noqa
self.mask[module_idx] = sharedmem.empty(constant_shape, dtype="i4")
- self.xray_cor[module_idx] = sharedmem.empty(constant_shape[1:], dtype="f4")
+ self.xray_cor[module_idx] = sharedmem.empty(constant_shape[1:], dtype="f4") # noqa
def allocate_images(self, shape, n_cores_files):
"""
@@ -1381,18 +1383,18 @@ class AgipdCorrections:
self.shared_dict = []
for i in range(n_cores_files):
self.shared_dict.append({})
- self.shared_dict[i]["cellId"] = sharedmem.empty(shape[0], dtype="u2")
- self.shared_dict[i]["pulseId"] = sharedmem.empty(shape[0], dtype="u8")
- self.shared_dict[i]["trainId"] = sharedmem.empty(shape[0], dtype="u8")
+ self.shared_dict[i]["cellId"] = sharedmem.empty(shape[0], dtype="u2") # noqa
+ self.shared_dict[i]["pulseId"] = sharedmem.empty(shape[0], dtype="u8") # noqa
+ self.shared_dict[i]["trainId"] = sharedmem.empty(shape[0], dtype="u8") # noqa
self.shared_dict[i]["moduleIdx"] = sharedmem.empty(1, dtype="i4")
self.shared_dict[i]["nImg"] = sharedmem.empty(1, dtype="i4")
self.shared_dict[i]["mask"] = sharedmem.empty(shape, dtype="u4")
self.shared_dict[i]["data"] = sharedmem.empty(shape, dtype="f4")
self.shared_dict[i]["rawgain"] = sharedmem.empty(shape, dtype="u2")
self.shared_dict[i]["gain"] = sharedmem.empty(shape, dtype="u1")
- self.shared_dict[i]["blShift"] = sharedmem.empty(shape[0], dtype="f4")
+ self.shared_dict[i]["blShift"] = sharedmem.empty(shape[0], dtype="f4") # noqa
# Parameters shared between image-wise correction functions
self.shared_dict[i]["msk"] = sharedmem.empty(shape, dtype="i4")
- self.shared_dict[i]["raw_data"] = sharedmem.empty(shape, dtype="f4")
- self.shared_dict[i]["rel_corr"] = sharedmem.empty(shape, dtype="f4")
- self.shared_dict[i]["t0_rgain"] = sharedmem.empty(shape, dtype="u2")
+ self.shared_dict[i]["raw_data"] = sharedmem.empty(shape, dtype="f4") # noqa
+ self.shared_dict[i]["rel_corr"] = sharedmem.empty(shape, dtype="f4") # noqa
+ self.shared_dict[i]["t0_rgain"] = sharedmem.empty(shape, dtype="u2") # noqa
--
GitLab
From 52a1c748ef8be5d52e7771a0f5822c3d4fdb8c63 Mon Sep 17 00:00:00 2001
From: Cyril Danilevski <cyril.danilevski@xfel.eu>
Date: Mon, 29 Mar 2021 17:18:48 +0200
Subject: [PATCH 72/72] Add test for module_index_to_qm
---
cal_tools/cal_tools/enums.py | 6 +++---
cal_tools/cal_tools/tools.py | 1 +
tests/test_cal_tools.py | 18 +++++++++++++++++-
3 files changed, 21 insertions(+), 4 deletions(-)
diff --git a/cal_tools/cal_tools/enums.py b/cal_tools/cal_tools/enums.py
index 186e17208..21d4b4f08 100644
--- a/cal_tools/cal_tools/enums.py
+++ b/cal_tools/cal_tools/enums.py
@@ -51,9 +51,9 @@ class SnowResolution(Enum):
class AgipdGainMode(IntEnum):
- """Encoding added to distinguish between adaptive and fixed gain"""
+ """Gain Modes where the values is 0 if Adaptive, or High/Medium/Low."""
- ADAPTIVE_GAIN = 0 # adaptive is default (if gain mode missing in slow data)
- FIXED_HIGH_GAIN = 1 # non-zero means fixed gain
+ ADAPTIVE_GAIN = 0
+ FIXED_HIGH_GAIN = 1
FIXED_MEDIUM_GAIN = 2
FIXED_LOW_GAIN = 3
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index a6aa3f5e8..0ddf2c132 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -675,6 +675,7 @@ def get_constant_from_db_and_time(karabo_id: str, karabo_da: str,
def module_index_to_qm(index: int, total_modules: int = 16):
"""Maps module index (0-indexed) to quadrant + module string (1-indexed)"""
+ assert index < total_modules, f'{index} is greater than {total_modules}'
modules_per_quad = total_modules // 4
quad, mod = divmod(index, modules_per_quad)
return f"Q{quad+1}M{mod+1}"
diff --git a/tests/test_cal_tools.py b/tests/test_cal_tools.py
index 7fe26e12a..c7656d177 100644
--- a/tests/test_cal_tools.py
+++ b/tests/test_cal_tools.py
@@ -3,7 +3,7 @@ from pathlib import Path
import pytest
from cal_tools.plotting import show_processed_modules
-from cal_tools.tools import get_dir_creation_date
+from cal_tools.tools import get_dir_creation_date, module_index_to_qm
def test_show_processed_modules():
@@ -30,3 +30,19 @@ def test_dir_creation_date():
date = get_dir_creation_date(folder, 9999)
assert isinstance(date, datetime)
assert str(date) == '2019-12-16 08:52:25.196603'
+
+
+def test_module_index_to_qm():
+
+ assert module_index_to_qm(0) == 'Q1M1'
+ assert module_index_to_qm(1) == 'Q1M2'
+ assert module_index_to_qm(4) == 'Q2M1'
+ assert module_index_to_qm(6) == 'Q2M3'
+
+ assert module_index_to_qm(4, 5) == 'Q5M1'
+
+ with pytest.raises(AssertionError):
+ module_index_to_qm(18)
+
+ with pytest.raises(AssertionError):
+ module_index_to_qm(7, 5)
--
GitLab