Newer
Older
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Summary of AGIPD dark characterization #\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/SPB/DARK/AGIPD3/\" # path to output to, required\n",
"dinstance = \"AGIPD1M1\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from collections import OrderedDict\n",
"import copy\n",
"from datetime import datetime\n",
"import os\n",
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"import glob\n",
"import h5py\n",
"from IPython.display import display, Markdown, Latex\n",
"import numpy as np\n",
"import matplotlib\n",
"matplotlib.use(\"agg\")\n",
"import matplotlib.patches as patches\n",
"import matplotlib.pyplot as plt\n",
"%matplotlib inline\n",
"import tabulate\n",
"from cal_tools.ana_tools import get_range\n",
"from extra_geom import AGIPD_1MGeometry\n",
"from iCalibrationDB import Detectors\n",
"from XFELDetAna.plotting.heatmap import heatmapPlot\n",
"from XFELDetAna.plotting.simpleplot import simplePlot"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# TODO: After changes in the Cal DB interface read files from cal repository\n",
"\n",
"# Load constants from local files\n",
"data = OrderedDict()\n",
"mod_names = []\n",
"# Loop over modules\n",
"for i in range(16):\n",
" qm = \"Q{}M{}\".format(i//4 + 1, i % 4 + 1)\n",
" # loop over constants\n",
" detinst = getattr(Detectors, dinstance)\n",
" for const in ['Offset', 'Noise', 'ThresholdsDark', 'BadPixelsDark']:\n",
" det = getattr(detinst, qm)\n",
" if det is None:\n",
" continue\n",
" det_name = det.device_name\n",
"\n",
" fpath = '{}/const_{}_{}.h5'.format(out_folder, const, det_name)\n",
" oldfpath = '{}/old/const_{}_{}.h5'.format(out_folder, const, det_name)\n",
" if not os.path.isfile(fpath):\n",
" continue\n",
" with h5py.File(fpath, 'r') as f:\n",
" if qm not in data:\n",
" mod_names.append(qm)\n",
" data[qm] = OrderedDict()\n",
"\n",
" data[qm][const] = f[\"data\"][()]\n",
"\n",
" if not os.path.isfile(oldfpath):\n",
" continue\n",
"\n",
" with h5py.File(oldfpath, 'r') as oldf:\n",
" if qm not in old_cons:\n",
" old_cons[qm] = OrderedDict()\n",
" old_cons[qm][const] = oldf[\"data\"][()]\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"modules = np.argsort(mod_names)\n",
"# extracting constant shape.\n",
"for qm, constant in data.items():\n",
" for cname, cons in constant.items():\n",
" shape = data[qm][cname].shape\n",
" if cname not in cons_shape:\n",
" cons_shape[cname] = shape\n",
"constants = {}\n",
"prev_const = {}\n",
"for cname, sh in cons_shape.items():\n",
" constants[cname]= np.zeros((len(mod_names),) + sh[:])\n",
" prev_const[cname]= np.zeros((len(mod_names),) + sh[:])\n",
"for i, mod in enumerate(modules):\n",
" for cname, cval in constants.items():\n",
" # move 3 gain maps from last indices to first three\n",
" if cname == 'ThresholdsDark':\n",
" cval[i][..., :3] = data[mod_names[mod]][cname][...,2:]\n",
" cval[i][..., 3:] = data[mod_names[mod]][cname][...,:2]\n",
" else:\n",
" cval[i] = data[mod_names[mod]][cname]\n",
"\n",
" for cname, cval in prev_const.items():\n",
" if cname == 'ThresholdsDark':\n",
" cval[i][..., :3] = old_cons[mod_names[mod]][cname][..., 2:]\n",
" cval[i][..., 3:] = old_cons[mod_names[mod]][cname][..., :2]\n",
" else:\n",
" cval[i] = old_cons[mod_names[mod]][cname]\n",
"mod_names = np.array(mod_names)[modules]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"display(Markdown('## Processed modules ##'))\n",
"\n",
"fig, ax = plt.subplots(1, figsize=(10, 10))\n",
"ax.set_axis_off()\n",
"\n",
"ax.set_xlim(0, 90)\n",
"ax.set_ylim(0, 75)\n",
"asic_pos = 5\n",
"for iq, q_x in enumerate([[43,66],[45,34],[4,32],[2,64]]):\n",
" for im in range(4):\n",
" color = 'gray'\n",
" if 'Q{}M{}'.format(iq+1, im+1) in mod_names:\n",
" color = 'green'\n",
" if np.nanmean(constants['Noise'][counter, :, :, :, 0]) == 0:\n",
" color = 'red'\n",
" counter += 1\n",
" x = q_x[0]\n",
" for i_as in range(8):\n",
" ax.add_patch(matplotlib.patches.Rectangle((x,q_x[1]-q_st*im), l_x, l_y, linewidth=2,edgecolor='darkgreen',\n",
" facecolor=color, fill=True))\n",
" x += asic_pos\n",
" ax.text(q_x[0]+14.5, q_x[1]-q_st*im+1.5, 'Q{}M{}'.format(\n",
" iq+1, im+1), fontsize=24, color='gold')\n",
"\n",
"_ = ax.legend(handles=[patches.Patch(facecolor='red', label='No data'),\n",
" patches.Patch(facecolor='gray', label='Not processed'),\n",
" patches.Patch(facecolor='green', label='Processed')],\n",
" loc='outside-top', ncol=3, bbox_to_anchor=(0.1, 0.25, 0.7, 0.8))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Summary figures across Modules ##\n",
"\n",
"The following plots give an overview of calibration constants averaged across pixels and memory cells. A bad pixel mask is applied."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": false
},
"outputs": [],
"source": [
"#Initializing AGIPD geometry\n",
"geom = AGIPD_1MGeometry.from_quad_positions(quad_pos=[\n",
" (-525, 625),\n",
" (-550, -10),\n",
" (520, -160),\n",
" (542.5, 475),\n",
" ])\n",
"gain_names = ['High-gain', 'Medium-gain', 'Low-gain', 'High-Threshold', 'Medium-Threshold']\n",
"Mod_data=OrderedDict()\n",
"for const_name, const in constants.items():\n",
" \n",
" if const_name == 'BadPixelsDark':\n",
" continue\n",
" Mod_data[const_name] = OrderedDict()\n",
" Mod_data['prev-{}'.format(const_name)] = OrderedDict()\n",
" display(Markdown('<h3 align=\"center\">{}</h3>'.format(const_name)))\n",
" display(Markdown('<h3 align=\"center\">-------</h3>'.format(const_name)))\n",
" \n",
" for gain in range(5):\n",
"\n",
" # For high gain than 3, don't pass except for Threshold.\n",
" if const.shape[-1] <= gain:\n",
" continue\n",
"\n",
" display(Markdown('<h3 align=\"center\">{}</h3>'.format(gain_names[gain])))\n",
" fig, (ax0, ax1) = plt.subplots(ncols=2, figsize=(20, 17.5))\n",
" ax_cbar = fig.add_axes([0.15, 0.2, 0.7, 0.02]) # Create extra axes for the colorbar\n",
" \n",
" for i in range(16):\n",
" qm = \"Q{}M{}\".format(i//4 + 1, i % 4 + 1)\n",
"\n",
" values = np.nanmean(const[i, :, :, :, gain], axis=2)\n",
" prev_val = np.nanmean(prev_const[const_name][i, :, :, :, gain], axis=2)\n",
" values[values == 0] = np.nan\n",
" prev_val[prev_val == 0] = np.nan\n",
" values = np.moveaxis(values, 0, -1).reshape(1, values.shape[1], values.shape[0])\n",
" prev_val = np.moveaxis(prev_val, 0, -1).reshape(1, prev_val.shape[1], prev_val.shape[0])\n",
" try:\n",
" \n",
" Mod_data[const_name][gain_names[gain]] = np.concatenate((Mod_data[const_name][gain_names[gain]],\n",
" values), axis=0)\n",
" \n",
" Mod_data['prev-{}'.format(const_name)][gain_names[gain]] = \\\n",
" np.concatenate((Mod_data['prev-{}'.format(const_name)][gain_names[gain]],\n",
" prev_val), axis=0)\n",
" Mod_data[const_name][gain_names[gain]] = values\n",
" Mod_data['prev-{}'.format(const_name)][gain_names[gain]] = prev_val\n",
"\n",
" vmin, vmax = get_range(Mod_data[const_name][gain_names[gain]], 2)\n",
" prev_vmin, prev_vmax = get_range(Mod_data['prev-{}'.format(const_name)][gain_names[gain]], 2)\n",
"\n",
" geom.plot_data_fast(Mod_data[const_name][gain_names[gain]], vmin=vmin, vmax=vmax,\n",
" ax=ax0,colorbar={\n",
" 'cax': ax_cbar,\n",
" 'shrink': 0.6,\n",
" 'pad': 0.1,\n",
" 'orientation': 'horizontal'\n",
" })\n",
" colorbar = ax0.images[0].colorbar\n",
" colorbar.set_label('ADUs')\n",
" geom.plot_data_fast(Mod_data['prev-{}'.format(const_name)][gain_names[gain]], \n",
" vmin=prev_vmin, vmax=prev_vmax, ax=ax1,colorbar=False)\n",
" ax0.set_title('{} {}'.format(const_name, gain_names[gain]), fontsize=20)\n",
" ax0.set_xlabel('Columns', fontsize=15)\n",
" ax0.set_ylabel('Rows', fontsize=15)\n",
" ax1.set_title('Previous {} {}'.format(const_name, gain_names[gain]), fontsize=20)\n",
" ax1.set_xlabel('Columns', fontsize=15)\n",
" ax1.set_ylabel('Rows', fontsize=15)\n",
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"# Loop over capacitor settings, modules, constants\n",
"for const_name, const in constants.items():\n",
"\n",
" display(Markdown('### Summary across Modules - {}'.format(const_name)))\n",
" for gain in range(3):\n",
" data = np.copy(const[:, :, :, :, gain])\n",
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
" if const_name != 'BadPixelsDark':\n",
" label = '{} value [ADU], good pixels only'.format(const_name)\n",
" data[constants['BadPixelsDark'][:, :, :, :, gain] > 0] = np.nan\n",
" datamean = np.nanmean(data, axis=(1, 2))\n",
"\n",
" fig = plt.figure(figsize=(15, 6), tight_layout={\n",
" 'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})\n",
" ax = fig.add_subplot(121)\n",
" else:\n",
" label = 'Fraction of bad pixels'\n",
" data[data > 0] = 1.0\n",
" datamean = np.nanmean(data, axis=(1, 2))\n",
" datamean[datamean == 1.0] = np.nan\n",
"\n",
" fig = plt.figure(figsize=(15, 6), tight_layout={\n",
" 'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})\n",
" ax = fig.add_subplot(111)\n",
"\n",
" d = []\n",
" for im, mod in enumerate(datamean):\n",
" d.append({'x': np.arange(mod.shape[0]),\n",
" 'y': mod,\n",
" 'drawstyle': 'steps-pre',\n",
" 'label': mod_names[im],\n",
" })\n",
"\n",
" _ = simplePlot(d, figsize=(10, 10), xrange=(-12, 510),\n",
" x_label='Memory Cell ID',\n",
" y_label=label,\n",
" use_axis=ax,\n",
" title='{} gain'.format(gain_names[gain]),\n",
" title_position=[0.5, 1.18],\n",
" legend='outside-top-ncol6-frame', legend_size='18%',\n",
" legend_pad=0.00)\n",
"\n",
" if const_name != 'BadPixelsDark':\n",
" ax = fig.add_subplot(122)\n",
" label = '$\\sigma$ {} [ADU], good pixels only'.format(const_name)\n",
" d = []\n",
" for im, mod in enumerate(np.nanstd(data, axis=(1, 2))):\n",
" d.append({'x': np.arange(mod.shape[0]),\n",
" 'y': mod,\n",
" 'drawstyle': 'steps-pre',\n",
" 'label': mod_names[im],\n",
" })\n",
"\n",
" _ = simplePlot(d, figsize=(10, 10), xrange=(-12, 510),\n",
" x_label='Memory Cell ID',\n",
" y_label=label,\n",
" use_axis=ax,\n",
" title='{} gain'.format(gain_names[gain]),\n",
" title_position=[0.5, 1.18],\n",
" legend='outside-top-ncol6-frame', legend_size='18%',\n",
" legend_pad=0.00)\n",
"\n",
" plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Summary tables across Modules ##\n",
"\n",
"Tables show values averaged across all pixels and memory cells of a given detector module."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"if u'$' in tabulate.LATEX_ESCAPE_RULES:\n",
" del(tabulate.LATEX_ESCAPE_RULES[u'$'])\n",
" \n",
"if u'\\\\' in tabulate.LATEX_ESCAPE_RULES:\n",
" del(tabulate.LATEX_ESCAPE_RULES[u'\\\\'])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"header = ['Module', 'High gain', 'Medium gain', 'Low gain']\n",
"\n",
"for const_name, const in constants.items():\n",
" table = []\n",
"\n",
" for i_mod, mod in enumerate(mod_names):\n",
"\n",
" t_line = [mod]\n",
" for gain in range(3):\n",
" data = np.copy(const[i_mod, :, :, :, gain])\n",
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
" if const_name == 'BadPixelsDark':\n",
" data[data > 0] = 1.0\n",
" datasum = np.nansum(data)\n",
" datamean = np.nanmean(data)\n",
" if datamean == 1.0:\n",
" datamean = np.nan\n",
" datasum = np.nan\n",
"\n",
" t_line.append('{:6.0f} ({:6.3f}) '.format(\n",
" datasum, datamean))\n",
"\n",
" label = '## Number (fraction) of bad pixels'\n",
" else:\n",
"\n",
" data[constants['BadPixelsDark']\n",
" [i_mod, :, :, :, gain] > 0] = np.nan\n",
"\n",
" t_line.append('{:6.1f} $\\\\pm$ {:6.1f}'.format(\n",
" np.nanmean(data), np.nanstd(data)))\n",
"\n",
" label = '## Average {} [ADU], good pixels only ##'.format(const_name)\n",
"\n",
" table.append(t_line)\n",
"\n",
" display(Markdown(label))\n",
" md = display(Latex(tabulate.tabulate(\n",
" table, tablefmt='latex', headers=header)))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
}
},
"nbformat": 4,
"nbformat_minor": 2
}