{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Statistical analysis of calibration factors#\n",
"\n",
"Author: Mikhail Karnevskiy, Steffen Hauf, Version 0.2\n",
"\n",
"Plot calibration constants for AGIPD1M1 detector aggregated within detector modules. Input information is taken from folder `use_existing`. Corresponding files are prepared by `PlotFromCalDB` notebook."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
"out_folder = \"/gpfs/exfel/data/scratch/karnem/testLPD_11/\" # Output folder, required\n",
"use_existing = \"/gpfs/exfel/data/scratch/karnem/testLPD_10/\" # Input folder\n",
"dclass = \"LPD\" # Detector class\n",
"nMemToShow = 32 # Number of memory cells to be shown in plots over ASICs\n",
"range_offset = [4000., 5500, 6500, 8500] # plotting range for offset: high gain l, r, medium gain l, r \n",
"range_noise = [2.5, 15, 7.5, 17.0] # plotting range for noise: high gain l, r, medium gain l, r \n",
"range_gain = [0.8, 1.2, 0.8, 1.2] # plotting range for gain: high gain l, r, medium gain l, r \n",
"range_noise_e = [85., 500., 85., 500.] # plotting range for noise in [e-]: high gain l, r, medium gain l, r \n",
"range_slopesPC = [22.0, 27.0, -0.5, 1.5] # plotting range for slope PC: high gain l, r, medium gain l, r \n",
"range_slopesCI = [22.0, 27.0, -0.5, 1.5] # plotting range for slope CI: high gain l, r, medium gain l, r \n",
"range_slopesFF = [0.8, 1.2, 0.6, 1.2] # plotting range for slope FF: high gain l, r, medium gain l, r "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"import numpy as np\n",
"import matplotlib\n",
"# %matplotlib inline\n",
"\n",
"from cal_tools.ana_tools import (load_data_from_hdf5, \n",
" HMType, multi_union,\n",
" hm_combine)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"if use_existing == '':\n",
" use_existing = out_folder"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"print('Load data from {}/CalDBAna_{}_*.h5'.format(use_existing, dclass))\n",
"ret_constants = load_data_from_hdf5(\n",
" '{}/CalDBAna_{}_*.h5'.format(use_existing, dclass))\n",
"\n",
"print('Evaluated data from {}/CalDBAna_{}_Gain*.h5'.format(out_folder, dclass))\n",
"ret_constants.update(load_data_from_hdf5(\n",
" '{}/CalDBAna_{}_Gain*.h5'.format(out_folder, dclass)))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": true
},
"outputs": [],
"source": [
"# Parameters for plotting\n",
"\n",
"# Define range for plotting\n",
"rangevals = {\n",
" \"Offset\": [range_offset[0:2], range_offset[2:4]],\n",
" \"Noise\": [range_noise[0:2], range_noise[2:4]],\n",
" \"Gain\": [range_gain[0:2], range_gain[2:4]],\n",
" \"Noise-e\": [range_noise_e[0:2], range_noise_e[2:4]],\n",
" \"SlopesPC\": [range_slopesPC[0:2], range_slopesPC[2:4]],\n",
" \"SlopesCI\": [range_slopesCI[0:2], range_slopesCI[2:4]],\n",
" \"SlopesFF\": [range_slopesFF[0:2], range_slopesFF[2:4]]\n",
"}\n",
"\n",
"keys = {\n",
" 'Mean': ['data', '', 'Mean over pixels'],\n",
" 'std': ['dataStd', '', '$\\sigma$ over pixels'],\n",
" 'MeanBP': ['dataBP', 'Good pixels only', 'Mean over pixels'],\n",
" 'NBP': ['nBP', 'Fraction of BP', 'Fraction of BP'],\n",
" 'stdBP': ['dataBPStd', 'Good pixels only', '$\\sigma$ over pixels'],\n",
" 'stdASIC': ['', '', '$\\sigma$ over ASICs'],\n",
" 'stdCell': ['', '', '$\\sigma$ over Cells'],\n",
"}\n",
"\n",
"gain_name = ['High', 'Medium', 'Low']"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"print('Plot calibration constants')\n",
"\n",
"# loop over constat type\n",
"for const, modules in ret_constants.items():\n",
"\n",
" # Loop over gain\n",
" for gain in range(2):\n",
" print('Const: {}, gain : {}'.format(const, gain))\n",
"\n",
" if const in [\"Gain\", \"Noise-e\"] and gain == 1:\n",
" continue\n",
"\n",
" # loop over modules\n",
" mod_data = {}\n",
" mod_data['stdASIC'] = []\n",
" mod_data['stdCell'] = []\n",
" mod_names = []\n",
" mod_times = []\n",
"\n",
" # Loop over modules\n",
" for mod, data in modules.items():\n",
" ctimes = np.array(data[\"ctime\"])\n",
" ctimes_ticks = [x.strftime('%y-%m-%d') for x in ctimes]\n",
"\n",
" if (\"mdata\" in data):\n",
" cmdata = np.array(data[\"mdata\"])\n",
" for i, tick in enumerate(ctimes_ticks):\n",
" ctimes_ticks[i] = ctimes_ticks[i] + \\\n",
" ', V={:1.0f}'.format(cmdata[i]['Sensor Bias Voltage']) + \\\n",
" ', M={:1.0f}'.format(\n",
" cmdata[i]['Memory cells'])\n",
"\n",
" sort_ind = np.argsort(ctimes_ticks)\n",
" ctimes_ticks = list(np.array(ctimes_ticks)[sort_ind])\n",
"\n",
" # Create sorted by data dataset\n",
" rdata = {}\n",
" for key, item in keys.items():\n",
" if item[0] in data:\n",
" rdata[key] = np.array(data[item[0]])[sort_ind]\n",
"\n",
" nTimes = rdata['Mean'].shape[0]\n",
" nPixels = rdata['Mean'].shape[1] * rdata['Mean'].shape[2]\n",
" nBins = nMemToShow * nPixels\n",
"\n",
" # Select gain\n",
" if const not in [\"Gain\", \"Noise-e\"]:\n",
" for key in rdata:\n",
" rdata[key] = rdata[key][..., gain]\n",
"\n",
" # Avoid to low values\n",
" if const in [\"Noise\", \"Offset\", \"Noise-e\"]:\n",
" rdata['Mean'][rdata['Mean'] < 0.1] = np.nan\n",
" if 'MeanBP' in rdata:\n",
" rdata['MeanBP'][rdata['MeanBP'] < 0.1] = np.nan\n",
"\n",
" if 'NBP' in rdata:\n",
" rdata[\"NBP\"][rdata[\"NBP\"] == 4096] = np.nan\n",
" rdata[\"NBP\"] = rdata[\"NBP\"] / (64 * 64) * 100\n",
"\n",
" # Reshape: ASICs over cells for plotting\n",
" pdata = {}\n",
" for key in rdata:\n",
" pdata[key] = rdata[key][:, :, :, :nMemToShow].reshape(\n",
" nTimes, nBins).swapaxes(0, 1)\n",
"\n",
" # Summary information over modules\n",
" for key in pdata:\n",
" if key not in mod_data:\n",
" mod_data[key] = []\n",
" mod_data[key].append(np.nanmean(pdata[key], axis=0))\n",
"\n",
" mod_data['stdASIC'].append(np.nanstd(\n",
" np.nanmean(rdata['Mean'][:, :, :, :nMemToShow], axis=(1, 2)), axis=1))\n",
" mod_data['stdCell'].append(np.nanstd(\n",
" np.nanmean(rdata['Mean'][:, :, :, :nMemToShow], axis=3), axis=(1, 2)))\n",
"\n",
" mod_names.append(mod)\n",
" mod_times.append(ctimes_ticks)\n",
"\n",
" # Incert nans to get array-like list of data\n",
" uTime = mod_times[0]\n",
" for tlist in mod_times:\n",
" uTime = sorted(multi_union(uTime, tlist))\n",
"\n",
" for i, tlist in enumerate(mod_times):\n",
" for t, time in enumerate(uTime):\n",
" if t == len(tlist) or time != tlist[t]:\n",
" tlist.insert(t, time)\n",
" for key in mod_data:\n",
" mod_data[key][i] = np.insert(\n",
" mod_data[key][i], t, np.nan)\n",
"\n",
" # Plotting\n",
" nModules = len(mod_names)\n",
" mod_idx = np.argsort(mod_names)\n",
" for key in mod_data:\n",
" vmin = None\n",
" vmax = None\n",
" if const in rangevals and key in ['Mean', 'MeanBP']:\n",
" vmin = rangevals[const][gain][0]\n",
" vmax = rangevals[const][gain][1]\n",
" else:\n",
" vmin = np.nanmin(np.array(mod_data[key]))\n",
" vmax = np.nanmean(\n",
" np.array(mod_data[key])) + 2*np.nanstd(np.array(mod_data[key]))\n",
"\n",
" htype = None\n",
" if const in ['SlopesFF', 'SlopesPC', 'SlopesCI']:\n",
" htype = HMType.INSET_1D\n",
"\n",
" if key == 'NBP':\n",
" unit = '[%]'\n",
" else:\n",
" unit = '[ADU]'\n",
" if const == 'Noise-e':\n",
" unit = '[$e^-$]'\n",
"\n",
" title = '{}, All modules, {} gain, {}'.format(\n",
" const, gain_name[gain], keys[key][1])\n",
" cb_label = '{}, {} {}'.format(const, keys[key][2], unit)\n",
"\n",
" hm_combine(np.array(mod_data[key])[mod_idx][::-1],\n",
" y_ticks=np.arange(nModules)[::-1]+0.8,\n",
" y_ticklabels=np.array(mod_names)[mod_idx],\n",
" x_label='Creation Time', y_label='Module ID',\n",
" x_ticklabels=ctimes_ticks, x_ticks=np.arange(\n",
" len(ctimes_ticks))+0.3,\n",
" title=title, cb_label=cb_label,\n",
" fname='{}/{}_all_g{}_{}.png'.format(\n",
" out_folder, const, gain, key),\n",
" vmin=vmin, vmax=vmax,\n",
" pad=[0.125, 0.151, 0.12, 0.17], htype=htype)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
}
},
"nbformat": 4,
"nbformat_minor": 2
}