From eeea4b4f2c9fafb69e742846ac6c3ef1c02c8ef5 Mon Sep 17 00:00:00 2001
From: Karim Ahmed <karim.ahmed@xfel.eu>
Date: Sat, 28 Mar 2020 13:19:22 +0100
Subject: [PATCH] add parallization on different gains
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 150 ++++++++++--------
.../overallmodules_Darks_Summary_NBC.ipynb | 5 +-
2 files changed, 85 insertions(+), 70 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index aea44da2d..b4242d6e9 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -29,9 +29,9 @@
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPDbad_sep64\" # path to output to, required\n",
"sequences = [0] # sequence files to evaluate.\n",
"modules = [-1] # list of modules to evaluate, RANGE ALLOWED\n",
- "run_high = 199 # run number in which high gain data was recorded, required\n",
- "run_med = 200 # run number in which medium gain data was recorded, required\n",
- "run_low = 201 # run number in which low gain data was recorded, required\n",
+ "run_high = 167 # run number in which high gain data was recorded, required\n",
+ "run_med = 168 # run number in which medium gain data was recorded, required\n",
+ "run_low = 169 # run number in which low gain data was recorded, required\n",
"\n",
"karabo_id = \"SPB_DET_AGIPD1M-1\" # karabo karabo_id\n",
"karabo_da = [-1] # data aggregators\n",
@@ -56,19 +56,19 @@
"interlaced = False # assume interlaced data format, for data prior to Dec. 2017\n",
"rawversion = 2 # RAW file format version\n",
"\n",
- "thresholds_offset_sigma = 3. # thresholds in terms of n sigma noise for offset deduced bad pixels\n",
- "thresholds_offset_hard = [0, 0] # This offset threshold is left for back compatability or for defining threshold for the 3 gains\n",
- "thresholds_offset_hard_hg = [4000, 8500] # High-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
- "thresholds_offset_hard_mg = [4000, 8500] # Medium-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
- "thresholds_offset_hard_lg = [4000, 8500] # Low-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
+ "thresholds_offset_sigma = 3. # offset sigma thresholds for offset deduced bad pixels\n",
+ "thresholds_offset_hard = [0, 0] # For setting the same threshold offset for the 3 gains. Left for backcompatability. Default [0, 0] to take the following parameters.\n",
+ "thresholds_offset_hard_hg = [3000, 7000] # High-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
+ "thresholds_offset_hard_mg = [6000, 10000] # Medium-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
+ "thresholds_offset_hard_lg = [6000, 10000] # Low-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
"\n",
- "thresholds_noise_sigma = 5. # thresholds in terms of n sigma noise for offset deduced bad pixels\n",
- "thresholds_noise_hard = [0, 0] # This noise threshold is left for back compatability or for defining threshold for the 3 gains\n",
+ "thresholds_noise_sigma = 5. # noise sigma thresholds for offset deduced bad pixels\n",
+ "thresholds_noise_hard = [0, 0] # For setting the same threshold noise for the 3 gains. Left for backcompatability. Default [0, 0] to take the following parameters.\n",
"thresholds_noise_hard_hg = [4, 20] # High-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
"thresholds_noise_hard_mg = [4, 20] # Medium-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
"thresholds_noise_hard_lg = [4, 20] # Low-gain thresholds in absolute ADU terms for offset deduced bad pixels\n",
"\n",
- "gain_sep_sigma = 5. # gain separation sigma\n",
+ "thresholds_gain_sigma = 5. # Gain separation sigma threshold\n",
"\n",
"high_res_badpix_3d = False # set this to True if you need high-resolution 3d bad pixel plots. Runtime: ~ 1h"
]
@@ -260,16 +260,15 @@
"source": [
"import copy\n",
"from functools import partial\n",
- "import iCalibrationDB\n",
"def characterize_module(il_mode, cells, bp_thresh, rawversion, loc, acq_rate,\n",
- " prev_gains, prev_noise, h5path, h5path_idx, inp):\n",
+ " h5path, h5path_idx, inp):\n",
" import numpy as np\n",
" import copy\n",
" import h5py\n",
" from cal_tools.enums import BadPixels\n",
" from cal_tools.agipdlib import get_num_cells, get_acq_rate\n",
"\n",
- " filename, filename_out, channel = inp\n",
+ " filename, channel, gg = inp\n",
" \n",
" if cells == 0:\n",
" cells = get_num_cells(filename, loc, channel)\n",
@@ -279,8 +278,9 @@
" if acq_rate == 0.:\n",
" acq_rate = get_acq_rate(filename, loc, channel)\n",
"\n",
- " thresholds_offset_hard, thresholds_offset_sigma, thresholds_noise_hard, thresholds_noise_sigma = bp_thresh \n",
- "\n",
+ " thresholds_offset, thresholds_offset_sigma, thresholds_noise, thresholds_noise_sigma = bp_thresh \n",
+ " thresholds_offset_hard = thresholds_offset[gg]\n",
+ " thresholds_noise_hard = thresholds_noise[gg]\n",
" infile = h5py.File(filename, \"r\", driver=\"core\")\n",
" \n",
" h5path = h5path.format(channel)\n",
@@ -322,12 +322,14 @@
" offset = np.zeros((im.shape[0], im.shape[1], mcells))\n",
" gains = np.zeros((im.shape[0], im.shape[1], mcells))\n",
" noise = np.zeros((im.shape[0], im.shape[1], mcells))\n",
- "\n",
+ " gains_std = np.zeros((im.shape[0], im.shape[1], mcells))\n",
+ " \n",
" for cc in np.unique(cellIds[cellIds < mcells]):\n",
" cellidx = cellIds == cc\n",
" offset[...,cc] = np.median(im[..., cellidx], axis=2)\n",
" noise[...,cc] = np.std(im[..., cellidx], axis=2)\n",
" gains[...,cc] = np.median(ga[..., cellidx], axis=2)\n",
+ " gains_std[...,cc] = np.std(ga[..., cellidx], axis=2)\n",
"\n",
" # bad pixels\n",
" bp = np.zeros(offset.shape, np.uint32)\n",
@@ -335,12 +337,6 @@
" offset_mn = np.nanmedian(offset, axis=(0,1))\n",
" offset_std = np.nanstd(offset, axis=(0,1))\n",
"\n",
- " # Bad pixels during bad gain separation.\n",
- " # Fraction of pixels in the module with separation lower than \"gain_sep_sigma\".\n",
- " if prev_gains is not None:\n",
- " bad_sep = (gains-prev_gains) / np.sqrt(noise**2 + prev_noise**2)\n",
- " bp[(bad_sep)<gain_sep_sigma]|= BadPixels.GAIN_THRESHOLDING_ERROR.value\n",
- "\n",
" bp[(offset < offset_mn-thresholds_offset_sigma*offset_std) |\n",
" (offset > offset_mn+thresholds_offset_sigma*offset_std)] |= BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
" bp[(offset < thresholds_offset_hard[0]) | (\n",
@@ -355,74 +351,71 @@
" bp[(noise < thresholds_noise_hard[0]) | (noise > thresholds_noise_hard[1])] |= BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
" bp[~np.isfinite(noise)] |= BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
"\n",
- " return offset, noise, gains, bp, cells, acq_rate\n",
+ " return offset, noise, gains, gains_std, gg, bp, cells, acq_rate\n",
"\n",
"offset_g = OrderedDict()\n",
"noise_g = OrderedDict()\n",
"gain_g = OrderedDict()\n",
+ "gainstd_g = OrderedDict()\n",
"badpix_g = OrderedDict()\n",
"gg = 0\n",
"\n",
"start = datetime.now()\n",
"all_cells = []\n",
"all_acq_rate = []\n",
- "gains = None\n",
- "noise = None\n",
- "\n",
- "prev_gains = None\n",
- "prev_noise = None\n",
"\n",
"if thresholds_offset_hard == [0, 0]:\n",
" thresholds_offset_hard = [thresholds_offset_hard_hg, thresholds_offset_hard_mg, thresholds_offset_hard_lg]\n",
"else:\n",
- " thresholds_offset_hard = [thresholds_offset_hard]*3\n",
+ " thresholds_offset_hard = [thresholds_offset_hard] * 3\n",
"\n",
"if thresholds_noise_hard == [0, 0]:\n",
" thresholds_noise_hard = [thresholds_noise_hard_hg, thresholds_noise_hard_mg, thresholds_noise_hard_lg]\n",
"else:\n",
- " thresholds_noise_hard = [thresholds_noise_hard]*3\n",
+ " thresholds_noise_hard = [thresholds_noise_hard] * 3\n",
"\n",
+ " \n",
+ "inp = []\n",
"for gain, mapped_files in gain_mapped_files.items():\n",
- " inp = []\n",
" dones = []\n",
" for i in modules:\n",
- " qm = \"Q{}M{}\".format(i//4 +1, i % 4 + 1)\n",
+ " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
" fname_in = mapped_files[qm].get()\n",
" dones.append(mapped_files[qm].empty())\n",
" else:\n",
" continue\n",
- " fout = os.path.abspath(\"{}/{}\".format(out_folder, (os.path.split(fname_in)[-1]).replace(\"RAW\", \"CORR\")))\n",
- " inp.append((fname_in, fout, i))\n",
- " first = False\n",
- " prev_gains = gains\n",
- " prev_noise = noise\n",
- " p = partial(characterize_module, IL_MODE, max_cells,\n",
- " (thresholds_offset_hard[gg], thresholds_offset_sigma,\n",
- " thresholds_noise_hard[gg], thresholds_noise_sigma),\n",
- " rawversion, karabo_id, acq_rate, prev_gains, prev_noise, h5path, h5path_idx)\n",
- "\n",
- " # Don't remove. Used for Debugging.\n",
- " #results = list(map(p, inp))\n",
- " results = view.map_sync(p, inp)\n",
- "\n",
- " for ii, r in enumerate(results):\n",
- " i = modules[ii]\n",
- " offset, noise, gains, bp, thiscell, thisacq = r\n",
- " all_cells.append(thiscell)\n",
- " all_acq_rate.append(thisacq)\n",
- " qm = \"Q{}M{}\".format(i//4 +1, i % 4 + 1)\n",
- " if qm not in offset_g:\n",
- " offset_g[qm] = np.zeros((offset.shape[0], offset.shape[1], offset.shape[2], 3))\n",
- " noise_g[qm] = np.zeros_like(offset_g[qm])\n",
- " gain_g[qm] = np.zeros_like(offset_g[qm])\n",
- " badpix_g[qm] = np.zeros_like(offset_g[qm], np.uint32)\n",
- "\n",
- " offset_g[qm][...,gg] = offset\n",
- " noise_g[qm][...,gg] = noise\n",
- " gain_g[qm][...,gg] = gains\n",
- " badpix_g[qm][...,gg] = bp\n",
- " gg +=1\n",
+ " inp.append((fname_in, i, gg))\n",
+ " \n",
+ " gg += 1\n",
+ "\n",
+ "p = partial(characterize_module, IL_MODE, max_cells,\n",
+ " (thresholds_offset_hard, thresholds_offset_sigma,\n",
+ " thresholds_noise_hard, thresholds_noise_sigma),\n",
+ " rawversion, karabo_id, acq_rate, h5path, h5path_idx)\n",
+ "\n",
+ "# Don't remove. Used for Debugging.\n",
+ "#results = list(map(p, inp))\n",
+ "results = view.map_sync(p, inp)\n",
+ "\n",
+ "for ii, r in enumerate(results):\n",
+ " offset, noise, gains, gains_std, gg, bp, thiscell, thisacq = r\n",
+ " all_cells.append(thiscell)\n",
+ " all_acq_rate.append(thisacq)\n",
+ " qm = \"Q{}M{}\".format(i//4 + 1, i % 4 + 1)\n",
+ " if qm not in offset_g:\n",
+ " offset_g[qm] = np.zeros((offset.shape[0], offset.shape[1], offset.shape[2], 3))\n",
+ " noise_g[qm] = np.zeros_like(offset_g[qm])\n",
+ " gain_g[qm] = np.zeros_like(offset_g[qm])\n",
+ " gainstd_g[qm] = np.zeros_like(offset_g[qm])\n",
+ " badpix_g[qm] = np.zeros_like(offset_g[qm], np.uint32)\n",
+ "\n",
+ " offset_g[qm][...,gg] = offset\n",
+ " noise_g[qm][...,gg] = noise\n",
+ " gain_g[qm][...,gg] = gains\n",
+ " gainstd_g[qm][..., gg] = gains_std\n",
+ " badpix_g[qm][...,gg] = bp\n",
+ " \n",
"\n",
"duration = (datetime.now()-start).total_seconds()\n",
"logger.runtime_summary_entry(success=True, runtime=duration,\n",
@@ -435,6 +428,20 @@
"print(f\"Using {acq_rate} MHz acquisition rate\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Add a badpixel due to bad gain separation\n",
+ "for g in range(2):\n",
+ " # Bad pixels during bad gain separation.\n",
+ " # Fraction of pixels in the module with separation lower than \"thresholds_gain_sigma\".\n",
+ " bad_sep = (gain_g[qm][..., g+1] - gain_g[qm][..., g]) / np.sqrt(gainstd_g[qm][..., g+1]**2 + gainstd_g[qm][..., g]**2)\n",
+ " badpix_g[qm][...,g+1][(bad_sep)<thresholds_gain_sigma]|= BadPixels.GAIN_THRESHOLDING_ERROR.value"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -564,8 +571,8 @@
"\n",
" print(f\"Injecting constants {const} with conditions:\\n\")\n",
" print(\"1. memory_cells: {}\\n2. bias_voltage: {}\\n\"\n",
- " \"3. acquisition_rate: {}\\n4. gain_setting: {}\\n\"\n",
- " \"5. creation_time: {}\\n\".format(max_cells, bias_voltage,\n",
+ " \"3. acquisition_rate: {}\\n4. gain_setting: {}\\n\"\n",
+ " \"5. creation_time: {}\\n\".format(max_cells, bias_voltage,\n",
" acq_rate, gain_setting,\n",
" creation_time))\n",
" for const in res[qm]:\n",
@@ -754,8 +761,8 @@
"source": [
"cols = {BadPixels.NOISE_OUT_OF_THRESHOLD.value: (BadPixels.NOISE_OUT_OF_THRESHOLD.name, '#FF000080'),\n",
" BadPixels.OFFSET_NOISE_EVAL_ERROR.value: (BadPixels.OFFSET_NOISE_EVAL_ERROR.name, '#0000FF80'),\n",
- " BadPixels.OFFSET_OUT_OF_THRESHOLD.value: (BadPixels.OFFSET_OUT_OF_THRESHOLD.name, '#00BF2080'),\n",
- " BadPixels.GAIN_THRESHOLDING_ERROR.value: (BadPixels.GAIN_THRESHOLDING_ERROR.name, '#FFDF0080'),\n",
+ " BadPixels.OFFSET_OUT_OF_THRESHOLD.value: (BadPixels.OFFSET_OUT_OF_THRESHOLD.name, '#FFDF0080'),\n",
+ " BadPixels.GAIN_THRESHOLDING_ERROR.value: (BadPixels.GAIN_THRESHOLDING_ERROR.name, '#00BF2080'),\n",
" BadPixels.OFFSET_OUT_OF_THRESHOLD.value | BadPixels.NOISE_OUT_OF_THRESHOLD.value: ('MIXED', '#DD00DD80')}\n",
"\n",
"rebin = 8 if not high_res_badpix_3d else 2\n",
@@ -994,6 +1001,13 @@
" md = display(Latex(tabulate.tabulate(table, tablefmt='latex', headers=header))) "
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb b/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
index 17c3a3902..6371a0d78 100644
--- a/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
+++ b/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
@@ -45,9 +45,10 @@
"\n",
"**\"GAIN_THRESHOLDING_ERROR\":**\n",
"\n",
- "Bad gain separated values with distance less than gain_separation_sigma \n",
+ "Bad gain separated pixels with sigma separation less than gain_separation_sigma_threshold\n",
"\n",
- "$$\\frac{\\mathrm{gain\\_offset\\_median} - \\mathrm{prev\\_gain\\_offset\\_median}}{\\sqrt{\\mathrm{noise\\_std}^\\mathrm{2} - \\mathrm{prev\\_noise\\_std}^\\mathrm{2}}} < \\mathrm{gain\\_separation\\_sigma} $$"
+ "$$ sigma\\_separation = \\frac{\\mathrm{gain\\_offset} - \\mathrm{previous\\_gain\\_offset}}{\\sqrt{\\mathrm{gain\\_offset_{std}}^\\mathrm{2} + \\mathrm{previuos\\_gain\\_offset_{std}}^\\mathrm{2}}}$$ \n",
+ "$$ Bad\\_separation = sigma\\_separation < \\mathrm{gain\\_separation\\_sigma\\_threshold} $$"
]
},
{
--
GitLab