{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# ePix10K Dark Characterization #\n",
"\n",
"Author: M. Karnevskiy, Version 1.0\n",
"\n",
"The following notebook provides dark image analysis of the ePix10K detector.\n",
"\n",
"Dark characterization evaluates offset and noise of the detector and gives information about bad pixels. Resulting maps are saved as .h5 files for a latter use and injected to the calibration DB."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # ipcluster profile to use\n",
"in_folder = '/gpfs/exfel/exp/HED/201922/p002550/raw' # input folder, required\n",
"out_folder = '/gpfs/exfel/data/scratch/ahmedk/test/epix10' # output folder, required\n",
"metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"sequence = 0 # sequence file to use\n",
"run = 55 # which run to read data from, required\n",
"\n",
"karabo_id = \"HED_IA1_EPIX10K-1\" # karabo karabo_id\n",
"karabo_da = [\"EPIX03\"] # data aggregators\n",
"receiver_id = \"RECEIVER\" # inset for receiver devices\n",
"path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
"h5path = '/INSTRUMENT/{}/DET/{}:daqOutput/data/image/pixels' # path in the HDF5 file to images\n",
"h5path_t = '/INSTRUMENT/{}/DET/{}:daqOutput/data/backTemp' # path to find temperature at\n",
"h5path_cntrl = '/CONTROL/{}/DET' # path to control data\n",
"\n",
"use_dir_creation_date = True\n",
"cal_db_interface = \"tcp://max-exfl016:8020\" # calibration DB interface to use\n",
"cal_db_timeout = 300000 # timeout on caldb requests\n",
"db_output = False # Output constants to the calibration database\n",
"local_output = True # output constants locally\n",
"\n",
"temp_limits = 5 # limit for parameter Operational temperature\n",
"number_dark_frames = 0 # number of images to be used, if set to 0 all available images are used\n",
"db_module = 'ePix10K_M43' # detector karabo_id\n",
"bias_voltage = 200 # bias voltage\n",
"in_vacuum = False # detector operated in vacuum\n",
"fix_temperature = 290. # fix temperature to this value\n",
"operation_mode = '' # Detector operation mode, optional"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import warnings\n",
"\n",
"warnings.filterwarnings('ignore')\n",
"import os\n",
"\n",
"import h5py\n",
"import matplotlib.pyplot as plt\n",
"\n",
"%matplotlib inline\n",
"import numpy as np\n",
"from cal_tools.tools import (\n",
" get_dir_creation_date,\n",
" get_pdu_from_db,\n",
" get_random_db_interface,\n",
" get_report,\n",
" save_const_to_h5,\n",
" send_to_db,\n",
")\n",
"from iCalibrationDB import Conditions, Constants, Detectors, Versions\n",
"from iCalibrationDB.detectors import DetectorTypes\n",
"from XFELDetAna import xfelpyanatools as xana\n",
"from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna.plotting.util import prettyPlotting\n",
"\n",
"prettyPlotting = True\n",
"from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
"from XFELDetAna.util import env\n",
"\n",
"env.iprofile = cluster_profile\n",
"import XFELDetAna.xfelprofiler as xprof\n",
"\n",
"profiler = xprof.Profiler()\n",
"profiler.disable()\n",
"from XFELDetAna.xfelreaders import ChunkReader\n",
"\n",
"h5path = h5path.format(karabo_id, receiver_id)\n",
"h5path_t = h5path_t.format(karabo_id, receiver_id)\n",
"h5path_cntrl = h5path_cntrl.format(karabo_id)\n",
"\n",
"def nImagesOrLimit(nImages, limit):\n",
" if limit == 0:\n",
" return nImages\n",
" else:\n",
" return min(nImages, limit)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Read report path and create file location tuple to add with the injection\n",
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
"file_loc = 'proposal:{} runs:{}'.format(proposal, run)\n",
"\n",
"report = get_report(metadata_folder)\n",
"\n",
"x = 356 # rows of the ePix10K\n",
"y = 384 # columns of the ePix10K\n",
"\n",
"ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
"fp_name = path_template.format(run, karabo_da[0]).format(sequence)\n",
"filename = '{}/{}'.format(ped_dir, fp_name)\n",
"\n",
"print(\"Reading data from: {}\\n\".format(filename))\n",
"print(\"Run number: {}\".format(run))\n",
"print(\"HDF5 path: {}\".format(h5path))\n",
"if use_dir_creation_date:\n",
" creation_time = get_dir_creation_date(in_folder, run)\n",
" print(\"Using {} as creation time\".format(creation_time.isoformat()))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"sensorSize = [x, y]\n",
"chunkSize = 100 #Number of images to read per chunk\n",
"\n",
"#Sensor area will be analysed according to blocksize\n",
"blockSize = [sensorSize[0] // 2, sensorSize[1] // 2]\n",
"xcal.defaultBlockSize = blockSize\n",
"cpuCores = 4 #Specifies the number of running cpu cores\n",
"memoryCells = 1 #No mamery cells\n",
"\n",
"#Specifies total number of images to proceed\n",
"nImages = fastccdreaderh5.getDataSize(filename, h5path)[0]\n",
"nImages = nImagesOrLimit(nImages, number_dark_frames)\n",
"print(\"\\nNumber of dark images to analyze: \", nImages)\n",
"run_parallel = False\n",
"\n",
"with h5py.File(filename, 'r') as f:\n",
" integration_time = int(f['{}/CONTROL/expTime/value'.format(h5path_cntrl)][0])\n",
" gain_setting = int(f['{}/CONTROL/encodedGainSetting/value'.format(h5path_cntrl)][0])\n",
" temperature = np.mean(f[h5path_t])/100.\n",
" temperature_k = temperature + 273.15\n",
" if fix_temperature != 0:\n",
" temperature_k = fix_temperature\n",
" print(\"Temperature is fixed!\")\n",
" print(\"Bias voltage is {} V\".format(bias_voltage))\n",
" print(\"Gain setting {}\".format(gain_setting))\n",
" print(\"Detector integration time is set to {}\".format(integration_time))\n",
" print(\"Mean temperature was {:0.2f} °C / {:0.2f} K\".format(temperature,\n",
" temperature_k))\n",
" print(\"Operated in vacuum: {} \".format(in_vacuum))\n",
" \n",
"os.makedirs(out_folder, exist_ok=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"reader = ChunkReader(filename, fastccdreaderh5.readData,\n",
" nImages, chunkSize,\n",
" path=h5path,\n",
" pixels_x=sensorSize[0],\n",
" pixels_y=sensorSize[1], )"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"noiseCal = xcal.NoiseCalculator(sensorSize, memoryCells,\n",
" cores=cpuCores, blockSize=blockSize,\n",
" parallel=run_parallel)\n",
"histCalRaw = xcal.HistogramCalculator(sensorSize, bins=1000,\n",
" range=[0, 10000], parallel=False,\n",
" memoryCells=memoryCells,\n",
" cores=cpuCores, blockSize=blockSize)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"for data in reader.readChunks():\n",
" data = np.bitwise_and(data.astype(np.uint16), 0b0011111111111111).astype(np.float32)\n",
" dx = np.count_nonzero(data, axis=(0, 1))\n",
" data = data[:, :, dx != 0]\n",
" histCalRaw.fill(data)\n",
" noiseCal.fill(data) #Fill calculators with data\n",
" \n",
"constant_maps = {}\n",
"constant_maps['Offset'] = noiseCal.getOffset() #Produce offset map\n",
"constant_maps['Noise'] = noiseCal.get() #Produce noise map\n",
"\n",
"noiseCal.reset() #Reset noise calculator\n",
"print(\"Initial maps were created\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#**************OFFSET MAP HISTOGRAM***********#\n",
"ho, co = np.histogram(constant_maps['Offset'].flatten(), bins=700)\n",
"\n",
"do = {'x': co[:-1],\n",
" 'y': ho,\n",
" 'y_err': np.sqrt(ho[:]),\n",
" 'drawstyle': 'bars',\n",
" 'color': 'cornflowerblue',\n",
" }\n",
"\n",
"fig = xana.simplePlot(do, figsize='1col', aspect=2,\n",
" x_label='Offset (ADU)',\n",
" y_label=\"Counts\", y_log=True,\n",
" )\n",
"\n",
"#*****NOISE MAP HISTOGRAM FROM THE OFFSET CORRECTED DATA*******#\n",
"hn, cn = np.histogram(constant_maps['Noise'].flatten(), bins=200)\n",
"\n",
"dn = {'x': cn[:-1],\n",
" 'y': hn,\n",
" 'y_err': np.sqrt(hn[:]),\n",
" 'drawstyle': 'bars',\n",
" 'color': 'cornflowerblue',\n",
" }\n",
"\n",
"fig = xana.simplePlot(dn, figsize='1col', aspect=2,\n",
" x_label='Noise (ADU)',\n",
" y_label=\"Counts\",\n",
" y_log=True)\n",
"\n",
"#**************HEAT MAPS*******************#\n",
"fig = xana.heatmapPlot(constant_maps['Offset'][:, :, 0],\n",
" x_label='Columns', y_label='Rows',\n",
" lut_label='Offset (ADU)',\n",
" x_range=(0, y),\n",
" y_range=(0, x), vmin=1000, vmax=4000)\n",
"\n",
"fig = xana.heatmapPlot(constant_maps['Noise'][:, :, 0],\n",
" x_label='Columns', y_label='Rows',\n",
" lut_label='Noise (ADU)',\n",
" x_range=(0, y),\n",
" y_range=(0, x), vmax=2 * np.mean(constant_maps['Noise']))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": false
},
"outputs": [],
"source": [
"# Save constants to DB\n",
"dclass=\"ePix10K\"\n",
"md = None\n",
"\n",
"for const_name in constant_maps.keys():\n",
" det = getattr(Constants, dclass)\n",
" const = getattr(det, const_name)()\n",
" const.data = constant_maps[const_name].data\n",
"\n",
" # set the operating condition\n",
" dcond = Conditions.Dark\n",
" condition = getattr(dcond, dclass)(bias_voltage=bias_voltage,\n",
" integration_time=integration_time,\n",
" temperature=temperature_k,\n",
" in_vacuum=in_vacuum,\n",
" gain_setting=gain_setting)\n",
"\n",
" for parm in condition.parameters:\n",
" if parm.name == \"Sensor Temperature\":\n",
" parm.lower_deviation = temp_limits\n",
" parm.upper_deviation = temp_limits\n",
"\n",
" # This should be used in case of running notebook \n",
" # by a different method other than myMDC which already\n",
" # sends CalCat info.\n",
" # TODO: Set db_module to \"\" by default in the first cell\n",
" if not db_module:\n",
" db_module = get_pdu_from_db(karabo_id, karabo_da, const,\n",
" condition, cal_db_interface,\n",
" snapshot_at=creation_time)[0]\n",
"\n",
" if db_output:\n",
" md = send_to_db(db_module, karabo_id, const, condition,\n",
" file_loc=file_loc, report_path=report,\n",
" cal_db_interface=cal_db_interface,\n",
" creation_time=creation_time,\n",
" timeout=cal_db_timeout)\n",
" if local_output:\n",
" md = save_const_to_h5(db_module, karabo_id, const, condition,\n",
" const.data, file_loc, report,\n",
" creation_time, out_folder)\n",
" print(f\"Calibration constant {const_name} is stored locally.\")\n",
"\n",
"print(\"Constants parameter conditions are:\\n\")\n",
"print(f\"• bias_voltage: {bias_voltage}\\n• integration_time: {integration_time}\\n\"\n",
" f\"• temperature: {temperature_k}\\n• gain_setting: {gain_setting}\\n\"\n",
" f\"• in_vacuum: {in_vacuum}\\n\"\n",
" f\"• creation_time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
"autocomplete": true,
"bibliofile": "biblio.bib",
"cite_by": "apalike",
"current_citInitial": 1,
"eqLabelWithNumbers": true,
"eqNumInitial": 1,
"hotkeys": {
"equation": "Ctrl-E",
"itemize": "Ctrl-I"
},
"labels_anchors": false,
"latex_user_defs": false,
"report_style_numbering": false,
"user_envs_cfg": false
}
},
"nbformat": 4,
"nbformat_minor": 1
}