{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# ePix100 Dark Characterization\n",
"\n",
"Author: M. Karnevskiy, Version 1.0\n",
"\n",
"The following notebook provides dark image analysis of the ePix100 detector.\n",
"\n",
"Dark characterization evaluates offset and noise of the detector and gives information about bad pixels. Resulting maps are saved as .h5 files for a latter use and injected to calibration DB."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # ipcluster profile to use\n",
"in_folder = '/gpfs/exfel/exp/HED/202030/p900136/raw' # input folder, required\n",
"out_folder = '/gpfs/exfel/data/scratch/ahmedk/test/HED_dark/' # output folder, required\n",
"sequence = 0 # sequence file to use\n",
"run = 182 # which run to read data from, required\n",
"\n",
"karabo_id = \"HED_IA1_EPX100-2\" # karabo karabo_id\n",
"karabo_da = [\"EPIX02\"] # data aggregators\n",
"receiver_id = \"RECEIVER\" # inset for receiver devices\n",
"path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
"h5path = '/INSTRUMENT/{}/DET/{}:daqOutput/data/image/pixels' # path in the HDF5 file to images\n",
"h5path_t = '/INSTRUMENT/{}/DET/{}:daqOutput/data/backTemp' # path to find temperature at\n",
"h5path_cntrl = '/CONTROL/{}/DET' # path to control data\n",
"\n",
"use_dir_creation_date = True\n",
"cal_db_interface = \"tcp://max-exfl016:8020\" # calibration DB interface to use\n",
"cal_db_timeout = 300000 # timeout on caldb requests\n",
"db_output = False # Output constants to the calibration database\n",
"local_output = True # output constants locally\n",
"\n",
"number_dark_frames = 0 # number of images to be used, if set to 0 all available images are used\n",
"temp_limits = 5 # limit for parameter Operational temperature\n",
"db_module = 'ePix100_M17' # detector karabo_id\n",
"bias_voltage = 200 # bias voltage\n",
"in_vacuum = False # detector operated in vacuum\n",
"fix_temperature = 290. # fix temperature to this value"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from IPython.display import display, Markdown, Latex\n",
"import os\n",
"import textwrap\n",
"\n",
"import numpy as np\n",
"import h5py\n",
"import matplotlib.pyplot as plt\n",
"%matplotlib inline\n",
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"from cal_tools.tools import get_dir_creation_date, save_const_to_h5, get_random_db_interface\n",
"from iCalibrationDB import (ConstantMetaData, Constants, Conditions, Detectors,\n",
" Versions)\n",
"from iCalibrationDB.detectors import DetectorTypes\n",
"import XFELDetAna.xfelprofiler as xprof\n",
"profiler = xprof.Profiler()\n",
"profiler.disable()\n",
"from XFELDetAna.util import env\n",
"env.iprofile = cluster_profile\n",
"from XFELDetAna import xfelpycaltools as xcal\n",
"from XFELDetAna import xfelpyanatools as xana\n",
"from XFELDetAna.plotting.util import prettyPlotting\n",
"prettyPlotting = True\n",
"from XFELDetAna.xfelreaders import ChunkReader\n",
"from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
"\n",
"h5path = h5path.format(karabo_id, receiver_id)\n",
"h5path_t = h5path_t.format(karabo_id, receiver_id)\n",
"h5path_cntrl = h5path_cntrl.format(karabo_id)\n",
"\n",
"def nImagesOrLimit(nImages, limit):\n",
" if limit == 0:\n",
" return nImages\n",
" else:\n",
" return min(nImages, limit)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
"file_loc = f'proposal:{proposal} runs:{run}'\n",
"\n",
"x = 708 # rows of the xPix100\n",
"y = 768 # columns of the xPix100\n",
"\n",
"ped_dir = os.path.join(in_folder, f\"r{run:04d}\")\n",
"fp_name = path_template.format(run, karabo_da[0]).format(sequence)\n",
"filename = os.path.join(ped_dir, fp_name)\n",
"\n",
"print(f\"Reading data from: {filename}\\n\")\n",
"print(f\"Run number: {run}\")\n",
"print(f\"HDF5 path: {h5path}\")\n",
"if use_dir_creation_date:\n",
" creation_time = get_dir_creation_date(in_folder, run)\n",
" print(f\"Using {creation_time.isoformat()} as creation time\")\n",
"os.makedirs(out_folder, exist_ok=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"sensorSize = [x, y]\n",
"chunkSize = 100 #Number of images to read per chunk\n",
"\n",
"#Sensor area will be analysed according to blocksize\n",
"blockSize = [sensorSize[0] // 2, sensorSize[1] // 2]\n",
"xcal.defaultBlockSize = blockSize\n",
"cpuCores = 4 #Specifies the number of running cpu cores\n",
"memoryCells = 1 #No mamery cells\n",
"\n",
"#Specifies total number of images to proceed\n",
"nImages = fastccdreaderh5.getDataSize(filename, h5path)[0]\n",
"nImages = nImagesOrLimit(nImages, number_dark_frames)\n",
"print(\"\\nNumber of dark images to analyze: \", nImages)\n",
"run_parallel = False\n",
"\n",
"with h5py.File(filename, 'r') as f:\n",
" integration_time = int(f[os.path.join(h5path_cntrl, \"CONTROL\",\"expTime\", \"value\")][0])\n",
" temperature = np.mean(f[h5path_t])/100.\n",
" temperature_k = temperature + 273.15\n",
" if fix_temperature != 0:\n",
" temperature_k = fix_temperature\n",
" print(\"Temperature is fixed!\")\n",
" print(f\"Bias voltage is {bias_voltage} V\")\n",
" print(f\"Detector integration time is set to {integration_time}\")\n",
" print(f\"Mean temperature was {temperature:0.2f} °C / {temperature_k:0.2f} K\")\n",
" print(f\"Operated in vacuum: {in_vacuum} \")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"reader = ChunkReader(filename, fastccdreaderh5.readData,\n",
" nImages, chunkSize,\n",
" path=h5path,\n",
" pixels_x=sensorSize[0],\n",
" pixels_y=sensorSize[1], )\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"noiseCal = xcal.NoiseCalculator(sensorSize, memoryCells,\n",
" cores=cpuCores, blockSize=blockSize,\n",
" parallel=run_parallel)\n",
"histCalRaw = xcal.HistogramCalculator(sensorSize, bins=1000,\n",
" range=[0, 10000], parallel=False,\n",
" memoryCells=memoryCells,\n",
" cores=cpuCores, blockSize=blockSize)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"for data in reader.readChunks():\n",
" dx = np.count_nonzero(data, axis=(0, 1))\n",
" data = data[:, :, dx != 0]\n",
" histCalRaw.fill(data)\n",
" noiseCal.fill(data) #Fill calculators with data\n",
" \n",
"constant_maps = {}\n",
"constant_maps['Offset'] = noiseCal.getOffset() #Produce offset map\n",
"constant_maps['Noise'] = noiseCal.get() #Produce noise map\n",
"\n",
"noiseCal.reset() #Reset noise calculator\n",
"print(\"Initial maps were created\")\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#**************OFFSET MAP HISTOGRAM***********#\n",
"ho, co = np.histogram(constant_maps['Offset'].flatten(), bins=700)\n",
"\n",
"do = {'x': co[:-1],\n",
" 'y': ho,\n",
" 'y_err': np.sqrt(ho[:]),\n",
" 'drawstyle': 'bars',\n",
" 'color': 'cornflowerblue',\n",
" }\n",
"\n",
"fig = xana.simplePlot(do, figsize='1col', aspect=2,\n",
" x_label='Offset (ADU)',\n",
" y_label=\"Counts\", y_log=True,\n",
" )\n",
"\n",
"#*****NOISE MAP HISTOGRAM FROM THE OFFSET CORRECTED DATA*******#\n",
"hn, cn = np.histogram(constant_maps['Noise'].flatten(), bins=200)\n",
"\n",
"dn = {'x': cn[:-1],\n",
" 'y': hn,\n",
" 'y_err': np.sqrt(hn[:]),\n",
" 'drawstyle': 'bars',\n",
" 'color': 'cornflowerblue',\n",
" }\n",
"\n",
"fig = xana.simplePlot(dn, figsize='1col', aspect=2,\n",
" x_label='Noise (ADU)',\n",
" y_label=\"Counts\",\n",
" y_log=True)\n",
"\n",
"#**************HEAT MAPS*******************#\n",
"fig = xana.heatmapPlot(constant_maps['Offset'][:, :, 0],\n",
" x_label='Columns', y_label='Rows',\n",
" lut_label='Offset (ADU)',\n",
" x_range=(0, y),\n",
" y_range=(0, x), vmin=1000, vmax=4000)\n",
"\n",
"fig = xana.heatmapPlot(constant_maps['Noise'][:, :, 0],\n",
" x_label='Columns', y_label='Rows',\n",
" lut_label='Noise (ADU)',\n",
" x_range=(0, y),\n",
" y_range=(0, x), vmax=2 * np.mean(constant_maps['Noise']))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": false
},
"outputs": [],
"source": [
"# Save constants to DB\n",
"dclass=\"ePix100\"\n",
"cal_db_interface = get_random_db_interface(cal_db_interface)\n",
"for const_name in constant_maps.keys():\n",
" metadata = ConstantMetaData()\n",
" det = getattr(Constants, dclass)\n",
" const = getattr(det, const_name)()\n",
" const.data = constant_maps[const_name].data\n",
"\n",
" metadata.calibration_constant = const\n",
"\n",
" # set the operating condition\n",
" dcond = Conditions.Dark\n",
" condition = getattr(dcond, dclass)(bias_voltage=bias_voltage,\n",
" integration_time=integration_time,\n",
" temperature=temperature_k,\n",
" in_vacuum=in_vacuum)\n",
"\n",
" for parm in condition.parameters:\n",
" if parm.name == \"Sensor Temperature\":\n",
" parm.lower_deviation = temp_limits\n",
" parm.upper_deviation = temp_limits\n",
"\n",
" device = getattr(Detectors, db_module)\n",
" metadata.detector_condition = condition\n",
"\n",
" # specify the a version for this constant\n",
" if creation_time is None:\n",
" metadata.calibration_constant_version = Versions.Now(device=device)\n",
" else:\n",
" metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
" start=creation_time)\n",
" \n",
" metadata.calibration_constant_version.raw_data_location = file_loc\n",
"\n",
" if db_output:\n",
" try:\n",
" metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
" print(f\"Inject {const_name} constants from {metadata.calibration_constant_version.begin_at}\\n\")\n",
" except Exception as e: \n",
" if \"has already been take\" in str(e):\n",
" print(f\"{const_name} has already been injected with the same parameter conditions\\n\")\n",
" else:\n",
" # To prevent having big error message in the pdf report.\n",
" print(\"\\n\".join(textwrap.wrap(str(e),100)))\n",
"\n",
" if local_output:\n",
" save_const_to_h5(metadata, out_folder)\n",
" print(f\"Calibration constant {const_name} is stored locally at {out_folder}.\")\n",
"\n",
"print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Integration time: {integration_time}\\n\"\n",
" f\"• Temperature: {temperature_k}\\n• In Vacuum: {in_vacuum}\\n\"\n",
" f\"• Creation time: {metadata.calibration_constant_version.begin_at}\\n\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
"autocomplete": true,
"bibliofile": "biblio.bib",
"cite_by": "apalike",
"current_citInitial": 1,
"eqLabelWithNumbers": true,
"eqNumInitial": 1,
"hotkeys": {
"equation": "Ctrl-E",
"itemize": "Ctrl-I"
},
"labels_anchors": false,
"latex_user_defs": false,
"report_style_numbering": false,
"user_envs_cfg": false
}
},
"nbformat": 4,
"nbformat_minor": 1
}