diff --git a/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb b/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
index 0f1c2cd3858339186ecda900cdc05f910c73cc03..a1e0069c8afa24a682f42a47ae6312b6987bc0fb 100644
--- a/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
@@ -4,13 +4,14 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "# ePix100 Dark Characterization\n",
- "\n",
- "Author: European XFEL Detector Group, Version: 2.0\n",
- "\n",
- "The following notebook provides dark image analysis and calibration constants of the ePix100 detector.\n",
- "\n",
- "Dark characterization evaluates offset and noise of the detector and gives information about bad pixels. Resulting maps are saved as .h5 files for a latter use and injected to calibration DB."
+ "Updated version of the ePix100 Dark Characterization notebook:\n",
+ "\n",
+ "- Generates the bad pixel map.\n",
+ "- Added badpixel_threshold_sigma input variable: Number of standard deviations considered for bad pixel classification.\n",
+ "- Sensor size is read from hdf5 file instead of being manually input.\n",
+ "- Detector temperature can be read from hdf5 file instead of having a fixed value only.\n",
+ "- Removed the xcal.HistogramCalculator() function, which calculation output was not in use.\n",
+ "- Set db_module to \"\" by default in the first cell (TODO note of previous notebook version)."
]
},
{
@@ -38,10 +39,8 @@
"db_output = False # Output constants to the calibration database\n",
"local_output = True # output constants locally\n",
"\n",
- "# Conditions used for injected calibration constants.\n",
- "bias_voltage = 200 # bias voltage\n",
- "in_vacuum = False # detector operated in vacuum\n",
- "fix_temperature = 290. # fix temperature to this value\n",
+ "badpixel_threshold_sigma = 5. # bad pixels defined by values outside n times this std from median\n",
+ "number_dark_frames = 0 # number of images to be used, if set to 0 all available images are used\n",
"temp_limits = 5 # limit for parameter Operational temperature\n",
"\n",
"# Parameters used during selecting raw data trains.\n",
@@ -203,44 +202,175 @@
"metadata": {},
"outputs": [],
"source": [
- "#**************OFFSET MAP HISTOGRAM***********#\n",
- "ho, co = np.histogram(constant_maps['Offset'].flatten(), bins=700)\n",
+ "noise_mean = np.mean(constant_maps['Noise'].data.flatten())\n",
+ "noise_sigma = np.std(constant_maps['Noise'].data.flatten())\n",
+ "noise_median = np.median(constant_maps['Noise'].data.flatten())\n",
"\n",
+ "offset_mean = np.mean(constant_maps['Offset'].data.flatten())\n",
+ "offset_sigma = np.std(constant_maps['Offset'].data.flatten())\n",
+ "offset_median = np.median(constant_maps['Offset'].data.flatten())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "87b73246-9080-4303-adbd-d8708886e775",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#************** OFFSET HEAT MAP **************#\n",
+ "plt.figure(1)\n",
+ "fig1 = xana.heatmapPlot(constant_maps['Offset'][:, :, 0].data,\n",
+ " lut_label='[ADU]',\n",
+ " x_label = 'Column',\n",
+ " y_label = 'Row',\n",
+ " x_range=(0, sensorSize[1]),\n",
+ " y_range=(0, sensorSize[0]), \n",
+ " vmin=max(0,offset_median - badpixel_threshold_sigma*offset_sigma), \n",
+ " vmax=min(np.power(2,14)-1,offset_median + badpixel_threshold_sigma*offset_sigma))\n",
+ "\n",
+ "fig1.suptitle('Offset Map',x=.48,y=.9,fontsize=16)\n",
+ "fig1.set_size_inches(h=15,w=15)\n",
+ "\n",
+ "#************** OFFSET HISTOGRAM **************#\n",
+ "binwidth = offset_sigma/100\n",
+ "ho, co = np.histogram(constant_maps['Offset'].data.flatten(),\n",
+ " bins=np.arange((min(constant_maps['Offset'].data.flatten())),\n",
+ " max(constant_maps['Offset'].data.flatten()) + binwidth,\n",
+ " binwidth))\n",
"do = {'x': co[:-1],\n",
" 'y': ho,\n",
" 'y_err': np.sqrt(ho[:]),\n",
" 'drawstyle': 'bars',\n",
- " 'color': 'cornflowerblue',\n",
- " }\n",
+ " 'color': 'cornflowerblue'}\n",
+ "\n",
+ "fig2 = xana.simplePlot(do, \n",
+ " aspect=1.5,\n",
+ " x_label='Offset [ADU]',\n",
+ " y_label='Counts',\n",
+ " x_range=(max(0,offset_median - badpixel_threshold_sigma*offset_sigma), \n",
+ " min(np.power(2,14)-1,offset_median + badpixel_threshold_sigma*offset_sigma)),\n",
+ " y_range=(0,max(ho)*1.1),\n",
+ " y_log=True)\n",
"\n",
- "fig = xana.simplePlot(do, figsize='1col', aspect=2,\n",
- " x_label='Offset (ADU)',\n",
- " y_label=\"Counts\", y_log=True,\n",
- " )\n",
+ "fig2.suptitle('Offset Distribution',x=.5,y=.92,fontsize=16)\n",
"\n",
- "#*****NOISE MAP HISTOGRAM FROM THE OFFSET CORRECTED DATA*******#\n",
- "hn, cn = np.histogram(constant_maps['Noise'].flatten(), bins=200)\n",
+ "stats_str = 'mean : ' + \"{:.2f}\".format(np.round(offset_mean,2)) \\\n",
+ " + '\\nstd : ' + \"{:.2f}\".format(np.round(offset_sigma,2)) \\\n",
+ " + '\\nmedian: ' + \"{:.2f}\".format(np.round(offset_median,2)) \\\n",
+ " + '\\nmin: ' + \"{:.2f}\".format(np.min(constant_maps['Offset'].data.flatten())) \\\n",
+ " + '\\nmax: ' + \"{:.2f}\".format(np.max(constant_maps['Offset'].data.flatten()))\n",
+ "fig2.text(s=stats_str,x=.7,y=.7,fontsize=14,bbox=dict(facecolor='yellow', edgecolor='black', alpha=.1));\n",
"\n",
+ "#************** NOISE HEAT MAP **************#\n",
+ "fig3 = xana.heatmapPlot(constant_maps['Noise'][:, :, 0],\n",
+ " x_label='Columns', y_label='Rows',\n",
+ " lut_label='Noise [ADU]',\n",
+ " x_range=(0, sensorSize[1]),\n",
+ " y_range=(0, sensorSize[0]),\n",
+ " vmin=max(0,noise_median - badpixel_threshold_sigma*noise_sigma), \n",
+ " vmax=noise_median + badpixel_threshold_sigma*noise_sigma)\n",
+ "fig3.suptitle('Noise Map',x=.48,y=.9,fontsize=16)\n",
+ "fig3.set_size_inches(h=15,w=15)\n",
+ "\n",
+ "#************** NOISE HISTOGRAM **************#\n",
+ "binwidth = noise_sigma/100\n",
+ "hn, cn = np.histogram(constant_maps['Noise'].data.flatten(),\n",
+ " bins=np.arange((min(constant_maps['Noise'].data.flatten())),\n",
+ " max(constant_maps['Noise'].data.flatten()) + binwidth,\n",
+ " binwidth))\n",
"dn = {'x': cn[:-1],\n",
" 'y': hn,\n",
" 'y_err': np.sqrt(hn[:]),\n",
" 'drawstyle': 'bars',\n",
- " 'color': 'cornflowerblue',\n",
- " }\n",
- "\n",
- "fig = xana.simplePlot(dn, figsize='1col', aspect=2,\n",
- " x_label='Noise (ADU)',\n",
- " y_label=\"Counts\",\n",
+ " 'color': 'cornflowerblue'}\n",
+ "\n",
+ "fig4 = xana.simplePlot(dn,\n",
+ " aspect=1.5,\n",
+ " x_range=(max(0,noise_median - badpixel_threshold_sigma*noise_sigma), \n",
+ " noise_median + badpixel_threshold_sigma*noise_sigma),\n",
+ " y_range=(0,max(hn)*1.1),\n",
+ " x_label='Noise [ADU]',\n",
+ " y_label='Counts',\n",
" y_log=True)\n",
"\n",
- "#**************HEAT MAPS*******************#\n",
- "fig = xana.heatmapPlot(constant_maps['Offset'][:, :, 0],\n",
- " x_label='Columns', y_label='Rows',\n",
- " lut_label='Offset (ADU)',\n",
- " x_range=(0, pixels_y),\n",
- " y_range=(0, pixels_x), vmin=1000, vmax=4000)\n",
+ "fig4.suptitle('Noise Distribution',x=.5,y=.92,fontsize=16);\n",
+ "\n",
+ "stats_str = 'mean : ' + \"{:.2f}\".format(np.round(noise_mean,2)) \\\n",
+ " + '\\nstd : ' + \"{:.2f}\".format(np.round(noise_sigma,2)) \\\n",
+ " + '\\nmedian: ' + \"{:.2f}\".format(np.round(noise_median,2)) \\\n",
+ " + '\\nmin : ' + \"{:.2f}\".format(np.min(constant_maps['Noise'].data.flatten())) \\\n",
+ " + '\\nmax : ' + \"{:.2f}\".format(np.max(constant_maps['Noise'].data.flatten()))\n",
+ "fig4.text(s=stats_str,x=.72,y=.7,fontsize=14, bbox=dict(facecolor='yellow', edgecolor='black', alpha=.1));"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "02702ec6-0dbc-4cbf-9421-7cef2a6290c4",
+ "metadata": {},
+ "source": [
+ "## Bad Pixel Map ###\n",
+ "\n",
+ "The bad pixel map is deduced by comparing offset and noise of each pixel ($v_i$) against its median value:\n",
+ "\n",
+ "$$ \n",
+ "v_i > \\mathrm{median}(v_{k}) + n \\sigma_{v_{k}}\n",
+ "$$\n",
+ "or\n",
+ "$$\n",
+ "v_i < \\mathrm{median}(v_{k}) - n \\sigma_{v_{k}} \n",
+ "$$\n",
+ "\n",
+ "Values are encoded in a 32 bit mask, where for the dark image deduced bad pixels the following non-zero entries are relevant:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "0edad7cd-e719-4274-a1f0-9d0ed0937ef2",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def print_bp_entry(bp):\n",
+ " print(\"{:<30s} {:032b}\".format(bp.name, bp.value))\n",
+ "\n",
+ "print_bp_entry(BadPixels.OFFSET_OUT_OF_THRESHOLD)\n",
+ "print_bp_entry(BadPixels.NOISE_OUT_OF_THRESHOLD)\n",
+ "print_bp_entry(BadPixels.OFFSET_NOISE_EVAL_ERROR)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b71b82c2-206b-4862-a134-8ae3cc065780",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def eval_bpidx(d):\n",
+ "\n",
+ " mdn = np.nanmedian(d, axis=(0, 1))\n",
+ " std = np.nanstd(d, axis=(0, 1)) \n",
+ " idx = (d > mdn + badpixel_threshold_sigma*std) | (d < mdn - badpixel_threshold_sigma*std)\n",
+ "\n",
+ " return idx"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "f23ae7ca-cf5b-457c-9457-57e38f12a484",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "constant_maps['BadPixels'] = np.zeros(constant_maps['Offset'].shape, np.uint32)\n",
+ "\n",
+ "constant_maps['BadPixels'][eval_bpidx(constant_maps['Offset'])] = BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
+ "constant_maps['BadPixels'][eval_bpidx(constant_maps['Noise'])] = BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
+ "constant_maps['BadPixels'][~np.isfinite(constant_maps['Offset'])] = BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
+ "constant_maps['BadPixels'][~np.isfinite(constant_maps['Noise'])] = BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
"\n",
- "fig = xana.heatmapPlot(constant_maps['Noise'][:, :, 0],\n",
+ "#************** BAD PIXLES HEAT MAP **************#\n",
+ "fig5 = xana.heatmapPlot(constant_maps['BadPixels'][:, :, 0],\n",
" x_label='Columns', y_label='Rows',\n",
" lut_label='Noise (ADU)',\n",
" x_range=(0, pixels_y),\n",