Newer
Older
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Statistical analysis of calibration factors#\n",
"\n",
"Author: Mikhail Karnevskiy, Version 0.2\n",
"Plot calibration constants retrieved from the cal. DB.\n",
"\n",
"To be visualized, calibration constants are averaged per group of pixels. Plots shows calibration constant over time for each constant.\n",
"\n",
"Values shown in plots are saved in h5 files."
]
},
{
"cell_type": "code",
"execution_count": null,
"out_folder = \"/gpfs/exfel/data/scratch/karnem/test_bla4/\" # Output folder, required\n",
"use_existing = \"/home/karnem/myscratch/PlotCalDB/SPB/AGIPD1M1/\" # Input folder\n",
"dclass = \"AGIPD\" # Detector class\n",
"nMemToShow = 32 # Number of memory cells to be shown in plots over ASICs\n",
"range_offset = [4000., 5500, 6500, 8500] # plotting range for offset: high gain l, r, medium gain l, r \n",
"range_noise = [2.5, 15, 7.5, 17.0] # plotting range for noise: high gain l, r, medium gain l, r \n",
"range_gain = [0.8, 1.2, 0.8, 1.2] # plotting range for gain: high gain l, r, medium gain l, r \n",
"range_noise_e = [85., 500., 85., 500.] # plotting range for noise in [e-]: high gain l, r, medium gain l, r \n",
"range_slopesPC = [22.0, 27.0, -0.5, 1.5] # plotting range for slope PC: high gain l, r, medium gain l, r \n",
"range_slopesCI = [22.0, 27.0, -0.5, 1.5] # plotting range for slope CI: high gain l, r, medium gain l, r \n",
"range_slopesFF = [0.8, 1.2, 0.6, 1.2] # plotting range for slope FF: high gain l, r, medium gain l, r \n",
"plot_range = 3 # range for plotting in units of median absolute deviations\n",
"x_labels = ['Sensor Bias Voltage', 'Memory cells'] # parameters to be shown on X axis\n",
"spShape = [64, 64] # Shape of superpixel\n",
"gain_titles = ['High gain', 'Medium gain', 'Low gain'] # Title inset related to gain"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"warnings.filterwarnings('ignore')\n",
"from cal_tools.ana_tools import (\n",
" HMType,\n",
" IMType,\n",
" get_range,\n",
" hm_combine,\n",
" load_data_from_hdf5,\n",
" multi_union,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [
"if use_existing == '':\n",
" use_existing = out_folder"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"print('Load data from {}/CalDBAna_{}_*.h5'.format(use_existing, dclass))\n",
"ret_constants = load_data_from_hdf5(\n",
" '{}/CalDBAna_{}_*.h5'.format(use_existing, dclass))\n",
"\n",
"print('Evaluated data from {}/CalDBAna_{}_Gain*.h5'.format(out_folder, dclass))\n",
"ret_constants.update(load_data_from_hdf5(\n",
" '{}/CalDBAna_{}_Gain*.h5'.format(out_folder, dclass)))"
]
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [
"# Parameters for plotting\n",
"\n",
"# Define range for plotting\n",
"rangevals = {\n",
" \"Offset\": [range_offset[0:2], range_offset[2:4]],\n",
" \"Noise\": [range_noise[0:2], range_noise[2:4]],\n",
" \"Gain\": [range_gain[0:2], range_gain[2:4]],\n",
" \"Noise-e\": [range_noise_e[0:2], range_noise_e[2:4]],\n",
" \"SlopesPC\": [range_slopesPC[0:2], range_slopesPC[2:4]],\n",
" \"SlopesCI\": [range_slopesCI[0:2], range_slopesCI[2:4]],\n",
" \"SlopesFF\": [range_slopesFF[0:2], range_slopesFF[2:4]]\n",
"}\n",
"\n",
"keys = {\n",
" 'Mean': ['data', '', 'Mean over pixels'],\n",
" 'std': ['dataStd', '', '$\\sigma$ over pixels'],\n",
" 'MeanBP': ['dataBP', 'Good pixels only', 'Mean over pixels'],\n",
" 'NBP': ['nBP', '', 'Fraction of BP'],\n",
" 'stdBP': ['dataBPStd', 'Good pixels only', '$\\sigma$ over pixels'],\n",
" 'stdASIC': ['', '', '$\\sigma$ over ASICs'],\n",
" 'stdCell': ['', '', '$\\sigma$ over Cells'],\n",
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"print('Plot calibration constants')\n",
"\n",
"# loop over constat type\n",
"for const, modules in ret_constants.items():\n",
"\n",
" const = const.split(\"_\")\n",
" gain = [int(x[1]) for x in const if 'g' in x]\n",
" gain = gain[0] if len(gain)>0 else None\n",
" print('Const: {}, gain {}'.format(const, gain))\n",
" # loop over modules\n",
" mod_data = {}\n",
" mod_data['stdASIC'] = []\n",
" mod_data['stdCell'] = []\n",
" mod_names = []\n",
" mod_times = []\n",
" # Loop over modules\n",
" for mod, data in modules.items():\n",
" ctimes = np.array(data[\"ctime\"])\n",
" ctimes_ticks = [x.strftime('%y-%m-%d') for x in ctimes]\n",
" if (\"mdata\" in data):\n",
" cmdata = np.array(data[\"mdata\"])\n",
" for i, tick in enumerate(ctimes_ticks):\n",
" for entr in x_labels:\n",
" key = entr[0].upper()\n",
" val = cmdata[i].get(entr, None)\n",
" if val is not None:\n",
" ctimes_ticks[i] += ', {}={:.1f}'.format(key, val)\n",
" sort_ind = np.argsort(ctimes_ticks)\n",
" ctimes_ticks = list(np.array(ctimes_ticks)[sort_ind])\n",
" # Create sorted by data dataset\n",
" rdata = {}\n",
" for key, item in keys.items():\n",
" if item[0] in data:\n",
" rdata[key] = np.array(data[item[0]])[sort_ind]\n",
" nTimes = rdata['Mean'].shape[0]\n",
" nPixels = rdata['Mean'].shape[1] * rdata['Mean'].shape[2]\n",
" nBins = nMemToShow * nPixels\n",
" if 'NBP' in rdata:\n",
" rdata[\"NBP\"] = rdata[\"NBP\"] / (spShape[0] * spShape[1]) * 100\n",
" # Reshape: ASICs over cells for plotting\n",
" pdata = {}\n",
" for key in rdata:\n",
" pdata[key] = rdata[key][:, :, :, :nMemToShow].reshape(\n",
" nTimes, nBins).swapaxes(0, 1)\n",
" # Summary information over modules\n",
" for key in pdata:\n",
" if key not in mod_data:\n",
" mod_data[key] = []\n",
" mod_data[key].append(np.nanmean(pdata[key], axis=0))\n",
" # Avoid too low values\n",
" if const[0] in [\"Noise\", \"Offset\", \"Noise-e\"] and key in ['Mean', 'MeanBP']:\n",
" mod_data[key][-1][mod_data[key][-1] == 0.0] = IMType.STRANGE_VAL.value\n",
" if key=='NBP':\n",
" if 'Mean' in mod_data:\n",
" mod_data['Mean'][-1][mod_data[key][-1] == 100] = IMType.ALL_BAD.value\n",
" if 'MeanBP' in mod_data:\n",
" mod_data['MeanBP'][-1][mod_data[key][-1] == 100] = IMType.ALL_BAD.value\n",
" mod_data[key][-1][mod_data[key][-1] == 100] = IMType.ALL_BAD.value\n",
" mod_data['stdASIC'].append(np.nanstd(\n",
" np.nanmean(rdata['Mean'][:, :, :, :nMemToShow], axis=(1, 2)), axis=1))\n",
" mod_data['stdCell'].append(np.nanstd(\n",
" np.nanmean(rdata['Mean'][:, :, :, :nMemToShow], axis=3), axis=(1, 2)))\n",
" mod_names.append(mod)\n",
" mod_times.append(ctimes_ticks)\n",
" # Insert nans to get array-like list of data\n",
" uTime = mod_times[0]\n",
" for tlist in mod_times:\n",
" uTime = sorted(multi_union(uTime, tlist))\n",
" for i, tlist in enumerate(mod_times):\n",
" for t, time in enumerate(uTime):\n",
" if t == len(tlist) or time != tlist[t]:\n",
" tlist.insert(t, time)\n",
" for key in mod_data:\n",
" mod_data[key][i] = np.insert(\n",
" mod_data[key][i], t, IMType.NO_CONST.value )\n",
" # Plotting\n",
" nModules = len(mod_names)\n",
" mod_idx = np.argsort(mod_names)\n",
" for key in mod_data:\n",
" vmin,vmax = get_range(np.array(mod_data[key])[mod_idx][::-1].flatten(), plot_range)\n",
" htype = None\n",
" if const[0] in ['SlopesFF', 'SlopesPC', 'SlopesCI']:\n",
" htype = HMType.INSET_1D\n",
" \n",
" if key == 'NBP':\n",
" unit = '[%]'\n",
" title = 'BadPixelsDark'\n",
" else:\n",
" unit = '[ADU]'\n",
" title = const[0]\n",
" title += ', All modules'\n",
" if keys[key][1] != '':\n",
" title += ', {}'.format(keys[key][1])\n",
" if gain is not None:\n",
" title += ', {}'.format(gain_titles[gain])\n",
" \n",
" cb_label = '{}, {} {}'.format(const[0], keys[key][2], unit)\n",
" hm_combine(np.array(mod_data[key])[mod_idx][::-1],\n",
" y_ticks=np.arange(nModules)[::-1]+0.8,\n",
" y_ticklabels=np.array(mod_names)[mod_idx],\n",
" x_label='Creation Time', y_label='Module ID',\n",
" x_ticklabels=ctimes_ticks, x_ticks=np.arange(\n",
" len(ctimes_ticks))+0.3,\n",
" title=title, cb_label=cb_label,\n",
" fname='{}/{}_all_g{}_{}.png'.format(\n",
" out_folder, const[0], gain, key),\n",
" vmin=vmin, vmax=vmax,\n",
" pad=[0.125, 0.151, 0.12, 0.17], htype=htype)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
}
},
"nbformat": 4,
"nbformat_minor": 2
}