{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Updated version of the ePix100 Dark Characterization notebook:\n",
"\n",
"- Generates the bad pixel map.\n",
"- Added badpixel_threshold_sigma input variable: Number of standard deviations considered for bad pixel classification.\n",
"- Sensor size is read from hdf5 file instead of being manually input.\n",
"- Detector temperature can be read from hdf5 file instead of having a fixed value only.\n",
"- Removed the xcal.HistogramCalculator() function, which calculation output was not in use.\n",
"- Set db_module to \"\" by default in the first cell (TODO note of previous notebook version)."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"in_folder = '/gpfs/exfel/exp/HED/202030/p900136/raw' # input folder, required\n",
"out_folder = '' # output folder, required\n",
"sequence = 0 # sequence file to use\n",
"run = 182 # which run to read data from, required\n",
"\n",
"# Parameters for accessing the raw data.\n",
"karabo_id = \"HED_IA1_EPX100-2\" # karabo karabo_id\n",
"karabo_da = [\"EPIX02\"] # data aggregators\n",
"receiver_template = \"RECEIVER\" # detector receiver template for accessing raw data files\n",
"path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
"instrument_source_template = '{}/DET/{}:daqOutput' # instrument detector data source in h5files\n",
"\n",
"# Parameters for the calibration database.\n",
"use_dir_creation_date = True\n",
"cal_db_interface = \"tcp://max-exfl016:8020\" # calibration DB interface to use\n",
"cal_db_timeout = 300000 # timeout on caldb requests\n",
"db_output = False # Output constants to the calibration database\n",
"local_output = True # output constants locally\n",
"\n",
"badpixel_threshold_sigma = 5. # bad pixels defined by values outside n times this std from median\n",
"number_dark_frames = 0 # number of images to be used, if set to 0 all available images are used\n",
"temp_limits = 5 # limit for parameter Operational temperature\n",
"\n",
"# Parameters used during selecting raw data trains.\n",
"min_trains = 1 # Minimum number of trains that should be available to process dark constants. Default 1.\n",
"max_trains = 1000 # Maximum number of trains to use for processing dark constants. Set to 0 to use all available trains.\n",
"\n",
"# Don't delete! myMDC sends this by default.\n",
"operation_mode = '' # Detector operation mode, optional\n",
"\n",
"# TODO: delete after removing from calibration_configurations\n",
"db_module = '' # ID of module in calibration database, this parameter is ignore in the notebook. TODO: remove from calibration_configurations."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"import warnings\n",
"\n",
"import numpy as np\n",
"import pasha as psh\n",
"from extra_data import RunDirectory\n",
"\n",
"import XFELDetAna.xfelprofiler as xprof\n",
"from XFELDetAna import xfelpyanatools as xana\n",
"from XFELDetAna.plotting.util import prettyPlotting\n",
"from cal_tools.tools import (\n",
" get_dir_creation_date,\n",
" get_pdu_from_db,\n",
" get_random_db_interface,\n",
" get_report,\n",
" save_const_to_h5,\n",
" send_to_db,\n",
")\n",
"from iCalibrationDB import Conditions, Constants"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline\n",
"\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"prettyPlotting = True\n",
"\n",
"profiler = xprof.Profiler()\n",
"profiler.disable()\n",
"\n",
"instrument_src = instrument_source_template.format(karabo_id, receiver_template)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Read report path and create file location tuple to add with the injection\n",
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
"file_loc = f'proposal:{proposal} runs:{run}'\n",
"pixels_x = 708\n",
"pixels_y = 768\n",
"report = get_report(out_folder)\n",
"\n",
"ped_dir = os.path.join(in_folder, f\"r{run:04d}\")\n",
"fp_name = path_template.format(run, karabo_da[0]).format(sequence)\n",
"run_dir = RunDirectory(ped_dir)\n",
"\n",
"print(f\"Run number: {run}\")\n",
"print(f\"Instrument H5File source: {instrument_src}\")\n",
"if use_dir_creation_date:\n",
" creation_time = get_dir_creation_date(in_folder, run)\n",
" print(f\"Using {creation_time.isoformat()} as creation time\")\n",
"os.makedirs(out_folder, exist_ok=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"pixel_data = (instrument_src, \"data.image.pixels\")\n",
"\n",
"# Specifies total number of images to proceed\n",
"n_trains = run_dir.get_data_counts(*pixel_data).shape[0]\n",
"\n",
"# Modify n_trains to process based on given maximum\n",
"# and minimun number of trains.\n",
"if max_trains:\n",
" n_trains = min(max_trains, n_trains)\n",
"\n",
"if n_trains < min_trains:\n",
" raise ValueError(\n",
" f\"Less than {min_trains} trains are available in RAW data.\"\n",
" \" Not enough data to process darks.\")\n",
"\n",
"print(f\"Number of dark images to analyze: {n_trains}.\")\n",
"\n",
"integration_time = int(run_dir.get_array(\n",
" f\"{karabo_id}/DET/CONTROL\",\n",
" \"expTime.value\")[0])\n",
"temperature = np.mean(run_dir.get_array(\n",
" instrument_src,\n",
" \"data.backTemp\").values) / 100.\n",
"\n",
"if fix_temperature != 0:\n",
" temperature_k = fix_temperature\n",
" print(\"Temperature is fixed!\")\n",
"else:\n",
" temperature_k = temperature + 273.15\n",
"\n",
"print(f\"Bias voltage is {bias_voltage} V\")\n",
"print(f\"Detector integration time is set to {integration_time}\")\n",
"print(f\"Mean temperature was {temperature:0.2f} °C / {temperature_k:0.2f} K\")\n",
"print(f\"Operated in vacuum: {in_vacuum} \")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"data_dc = run_dir.select(\n",
" *pixel_data, require_all=True).select_trains(np.s_[:n_trains])\n",
"print(f\"Reading data from: {data_dc.files}\\n\")\n",
"\n",
"data = data_dc[pixel_data].ndarray()\n",
"\n",
"noise_data = np.std(data, axis=0)\n",
"offset_data = np.mean(data, axis=0)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"constant_maps = {}\n",
"constant_maps['Offset'] = offset_data[..., np.newaxis]\n",
"constant_maps['Noise'] = noise_data[..., np.newaxis]\n",
"print(\"Initial constant maps are created\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"noise_mean = np.mean(constant_maps['Noise'].data.flatten())\n",
"noise_sigma = np.std(constant_maps['Noise'].data.flatten())\n",
"noise_median = np.median(constant_maps['Noise'].data.flatten())\n",
"\n",
"offset_mean = np.mean(constant_maps['Offset'].data.flatten())\n",
"offset_sigma = np.std(constant_maps['Offset'].data.flatten())\n",
"offset_median = np.median(constant_maps['Offset'].data.flatten())"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "87b73246-9080-4303-adbd-d8708886e775",
"metadata": {},
"outputs": [],
"source": [
"#************** OFFSET HEAT MAP **************#\n",
"plt.figure(1)\n",
"fig1 = xana.heatmapPlot(constant_maps['Offset'][:, :, 0].data,\n",
" lut_label='[ADU]',\n",
" x_label = 'Column',\n",
" y_label = 'Row',\n",
" x_range=(0, sensorSize[1]),\n",
" y_range=(0, sensorSize[0]), \n",
" vmin=max(0,offset_median - badpixel_threshold_sigma*offset_sigma), \n",
" vmax=min(np.power(2,14)-1,offset_median + badpixel_threshold_sigma*offset_sigma))\n",
"\n",
"fig1.suptitle('Offset Map',x=.48,y=.9,fontsize=16)\n",
"fig1.set_size_inches(h=15,w=15)\n",
"\n",
"#************** OFFSET HISTOGRAM **************#\n",
"binwidth = offset_sigma/100\n",
"ho, co = np.histogram(constant_maps['Offset'].data.flatten(),\n",
" bins=np.arange((min(constant_maps['Offset'].data.flatten())),\n",
" max(constant_maps['Offset'].data.flatten()) + binwidth,\n",
" binwidth))\n",
"do = {'x': co[:-1],\n",
" 'y': ho,\n",
" 'y_err': np.sqrt(ho[:]),\n",
" 'drawstyle': 'bars',\n",
" 'color': 'cornflowerblue'}\n",
"\n",
"fig2 = xana.simplePlot(do, \n",
" aspect=1.5,\n",
" x_label='Offset [ADU]',\n",
" y_label='Counts',\n",
" x_range=(max(0,offset_median - badpixel_threshold_sigma*offset_sigma), \n",
" min(np.power(2,14)-1,offset_median + badpixel_threshold_sigma*offset_sigma)),\n",
" y_range=(0,max(ho)*1.1),\n",
" y_log=True)\n",
"\n",
"fig2.suptitle('Offset Distribution',x=.5,y=.92,fontsize=16)\n",
"\n",
"stats_str = 'mean : ' + \"{:.2f}\".format(np.round(offset_mean,2)) \\\n",
" + '\\nstd : ' + \"{:.2f}\".format(np.round(offset_sigma,2)) \\\n",
" + '\\nmedian: ' + \"{:.2f}\".format(np.round(offset_median,2)) \\\n",
" + '\\nmin: ' + \"{:.2f}\".format(np.min(constant_maps['Offset'].data.flatten())) \\\n",
" + '\\nmax: ' + \"{:.2f}\".format(np.max(constant_maps['Offset'].data.flatten()))\n",
"fig2.text(s=stats_str,x=.7,y=.7,fontsize=14,bbox=dict(facecolor='yellow', edgecolor='black', alpha=.1));\n",
"\n",
"#************** NOISE HEAT MAP **************#\n",
"fig3 = xana.heatmapPlot(constant_maps['Noise'][:, :, 0],\n",
" x_label='Columns', y_label='Rows',\n",
" lut_label='Noise [ADU]',\n",
" x_range=(0, sensorSize[1]),\n",
" y_range=(0, sensorSize[0]),\n",
" vmin=max(0,noise_median - badpixel_threshold_sigma*noise_sigma), \n",
" vmax=noise_median + badpixel_threshold_sigma*noise_sigma)\n",
"fig3.suptitle('Noise Map',x=.48,y=.9,fontsize=16)\n",
"fig3.set_size_inches(h=15,w=15)\n",
"\n",
"#************** NOISE HISTOGRAM **************#\n",
"binwidth = noise_sigma/100\n",
"hn, cn = np.histogram(constant_maps['Noise'].data.flatten(),\n",
" bins=np.arange((min(constant_maps['Noise'].data.flatten())),\n",
" max(constant_maps['Noise'].data.flatten()) + binwidth,\n",
" binwidth))\n",
"dn = {'x': cn[:-1],\n",
" 'y': hn,\n",
" 'y_err': np.sqrt(hn[:]),\n",
" 'drawstyle': 'bars',\n",
" 'color': 'cornflowerblue'}\n",
"\n",
"fig4 = xana.simplePlot(dn,\n",
" aspect=1.5,\n",
" x_range=(max(0,noise_median - badpixel_threshold_sigma*noise_sigma), \n",
" noise_median + badpixel_threshold_sigma*noise_sigma),\n",
" y_range=(0,max(hn)*1.1),\n",
" x_label='Noise [ADU]',\n",
" y_label='Counts',\n",
" y_log=True)\n",
"\n",
"fig4.suptitle('Noise Distribution',x=.5,y=.92,fontsize=16);\n",
"\n",
"stats_str = 'mean : ' + \"{:.2f}\".format(np.round(noise_mean,2)) \\\n",
" + '\\nstd : ' + \"{:.2f}\".format(np.round(noise_sigma,2)) \\\n",
" + '\\nmedian: ' + \"{:.2f}\".format(np.round(noise_median,2)) \\\n",
" + '\\nmin : ' + \"{:.2f}\".format(np.min(constant_maps['Noise'].data.flatten())) \\\n",
" + '\\nmax : ' + \"{:.2f}\".format(np.max(constant_maps['Noise'].data.flatten()))\n",
"fig4.text(s=stats_str,x=.72,y=.7,fontsize=14, bbox=dict(facecolor='yellow', edgecolor='black', alpha=.1));"
]
},
{
"cell_type": "markdown",
"id": "02702ec6-0dbc-4cbf-9421-7cef2a6290c4",
"metadata": {},
"source": [
"## Bad Pixel Map ###\n",
"\n",
"The bad pixel map is deduced by comparing offset and noise of each pixel ($v_i$) against its median value:\n",
"\n",
"$$ \n",
"v_i > \\mathrm{median}(v_{k}) + n \\sigma_{v_{k}}\n",
"$$\n",
"or\n",
"$$\n",
"v_i < \\mathrm{median}(v_{k}) - n \\sigma_{v_{k}} \n",
"$$\n",
"\n",
"Values are encoded in a 32 bit mask, where for the dark image deduced bad pixels the following non-zero entries are relevant:"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0edad7cd-e719-4274-a1f0-9d0ed0937ef2",
"metadata": {},
"outputs": [],
"source": [
"def print_bp_entry(bp):\n",
" print(\"{:<30s} {:032b}\".format(bp.name, bp.value))\n",
"\n",
"print_bp_entry(BadPixels.OFFSET_OUT_OF_THRESHOLD)\n",
"print_bp_entry(BadPixels.NOISE_OUT_OF_THRESHOLD)\n",
"print_bp_entry(BadPixels.OFFSET_NOISE_EVAL_ERROR)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b71b82c2-206b-4862-a134-8ae3cc065780",
"metadata": {},
"outputs": [],
"source": [
"def eval_bpidx(d):\n",
"\n",
" mdn = np.nanmedian(d, axis=(0, 1))\n",
" std = np.nanstd(d, axis=(0, 1)) \n",
" idx = (d > mdn + badpixel_threshold_sigma*std) | (d < mdn - badpixel_threshold_sigma*std)\n",
"\n",
" return idx"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f23ae7ca-cf5b-457c-9457-57e38f12a484",
"metadata": {},
"outputs": [],
"source": [
"constant_maps['BadPixels'] = np.zeros(constant_maps['Offset'].shape, np.uint32)\n",
"\n",
"constant_maps['BadPixels'][eval_bpidx(constant_maps['Offset'])] = BadPixels.OFFSET_OUT_OF_THRESHOLD.value\n",
"constant_maps['BadPixels'][eval_bpidx(constant_maps['Noise'])] = BadPixels.NOISE_OUT_OF_THRESHOLD.value\n",
"constant_maps['BadPixels'][~np.isfinite(constant_maps['Offset'])] = BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
"constant_maps['BadPixels'][~np.isfinite(constant_maps['Noise'])] = BadPixels.OFFSET_NOISE_EVAL_ERROR.value\n",
"\n",
"#************** BAD PIXLES HEAT MAP **************#\n",
"fig5 = xana.heatmapPlot(constant_maps['BadPixels'][:, :, 0],\n",
" x_label='Columns', y_label='Rows',\n",
" lut_label='Noise (ADU)',\n",
" x_range=(0, pixels_y),\n",
" y_range=(0, pixels_x), vmax=2 * np.mean(constant_maps['Noise']))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Save constants to DB\n",
"dclass=\"ePix100\"\n",
"md = None\n",
"\n",
"for const_name in constant_maps.keys():\n",
" det = getattr(Constants, dclass)\n",
" const = getattr(det, const_name)()\n",
" const.data = constant_maps[const_name].data\n",
"\n",
" # set the operating condition\n",
" condition = getattr(Conditions.Dark, dclass)(bias_voltage=bias_voltage,\n",
" integration_time=integration_time,\n",
" temperature=temperature_k,\n",
" in_vacuum=in_vacuum)\n",
" for parm in condition.parameters:\n",
" if parm.name == \"Sensor Temperature\":\n",
" parm.lower_deviation = temp_limits\n",
" parm.upper_deviation = temp_limits\n",
"\n",
" db_module = get_pdu_from_db(karabo_id, karabo_da, const,\n",
" condition, cal_db_interface,\n",
" snapshot_at=creation_time)[0]\n",
"\n",
" if db_output:\n",
" md = send_to_db(db_module, karabo_id, const, condition,\n",
" file_loc=file_loc, report_path=report,\n",
" cal_db_interface=cal_db_interface,\n",
" creation_time=creation_time,\n",
" timeout=cal_db_timeout)\n",
" if local_output:\n",
" md = save_const_to_h5(db_module, karabo_id, const, condition,\n",
" const.data, file_loc, report,\n",
" creation_time, out_folder)\n",
" print(f\"Calibration constant {const_name} is stored locally at {out_folder} \\n\")\n",
"\n",
"print(\"Constants parameter conditions are:\\n\")\n",
"print(f\"• Bias voltage: {bias_voltage}\\n• Integration time: {integration_time}\\n\"\n",
" f\"• Temperature: {temperature_k}\\n• In Vacuum: {in_vacuum}\\n\"\n",
" f\"• Creation time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.11"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
"autocomplete": true,
"bibliofile": "biblio.bib",
"cite_by": "apalike",
"current_citInitial": 1,
"eqLabelWithNumbers": true,
"eqNumInitial": 1,
"hotkeys": {
"equation": "Ctrl-E",
"itemize": "Ctrl-I"
},
"labels_anchors": false,
"latex_user_defs": false,
"report_style_numbering": false,
"user_envs_cfg": false
}
},
"nbformat": 4,
"nbformat_minor": 1
}