Skip to content
Snippets Groups Projects
overallmodules_Darks_Summary_NBC.ipynb 24.5 KiB
Newer Older
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Author: K. Ahmed, M. Karnevsky, Version: 0.1\n",
    "#The following is a summary for the processing of dark images and calibration constants production.\n",
    "\n",
    "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/miniHalfAGIPD\" # path to output to, required\n",
    "karabo_id = \"DETLAB_DET_AGIPD500K2G\" # detector instance\n",
    "gain_names = ['High gain', 'Medium gain', 'Low gain'] # a list of gain names to be used in plotting\n",
    "threshold_names = ['HG-MG threshold', 'MG_LG threshold'] # a list of gain names to be used in plotting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
    "from collections import OrderedDict\n",
    "import copy\n",
    "from datetime import datetime\n",
    "import os\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import glob\n",
    "import h5py\n",
    "from IPython.display import display, Markdown, Latex\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "matplotlib.use(\"agg\")\n",
    "import matplotlib.gridspec as gridspec\n",
    "import matplotlib.patches as patches\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import tabulate\n",
    "from cal_tools.ana_tools import get_range\n",
    "from cal_tools.plotting import show_processed_modules\n",
    "import extra_geom\n",
    "from iCalibrationDB import Detectors\n",
    "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
    "from XFELDetAna.plotting.simpleplot import simplePlot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if \"AGIPD\" in karabo_id:\n",
    "    if \"SPB\" in karabo_id:\n",
    "        dinstance = \"AGIPD1M1\"\n",
    "    elif \"MID\" in karabo_id:\n",
    "        dinstance = \"AGIPD1M2\"\n",
Cyril Danilevski's avatar
Cyril Danilevski committed
    "    elif \"HED\" in karabo_id or \"DETLAB\" in karabo_id:\n",
    "        dinstance = \"AGIPD500K\"\n",
    "    display(Markdown(\"\"\"\n",
    "    \n",
Karim Ahmed's avatar
Karim Ahmed committed
    "# Summary of AGIPD dark characterization #\n",
    "\n",
    "The following report shows a set of dark images taken with the AGIPD detector to deduce detector offsets, noise, bad-pixel maps and thresholding. All four types of constants are evaluated per-pixel and per-memory cell.\n",
    "\n",
    "\n",
    "**The offset** ($O$) is defined as the median ($M$) of the dark signal ($Ds$) over trains ($t$) for a given pixel ($x,y$) and memory cell ($c$). \n",
    "\n",
    "**The noise** $N$ is the standard deviation $\\sigma$ of the dark signal.\n",
    "\n",
    "$$ O_{x,y,c} = M(Ds)_{t} ,\\,\\,\\,\\,\\,\\, N_{x,y,c} = \\sigma(Ds)_{t}$$\n",
    "\n",
    "**The bad pixel** mask is encoded as a bit mask.\n",
    "\n",
    "**\"OFFSET_OUT_OF_THRESHOLD\":**\n",
    "\n",
    "Offset outside of bounds:\n",
    "\n",
    "$$M(O)_{x,y} - \\sigma(O)_{x,y} * \\mathrm{thresholds\\_offset\\_sigma} < O < M(O)_{x,y} + \\sigma(O)_{x,y} * \\mathrm{thresholds\\_offset\\_sigma} $$\n",
    "\n",
    "or offset outside of hard limits\n",
    "\n",
    "$$ \\mathrm{thresholds\\_offset\\_hard}_\\mathrm{low} < O < \\mathrm{thresholds\\_offset\\_hard}_\\mathrm{high} $$\n",
    "\n",
    "**\"NOISE_OUT_OF_THRESHOLD\":**\n",
    "\n",
    "Noise outside of bounds:\n",
    "\n",
    "$$M(N)_{x,y} - \\sigma(N)_{x,y} * \\mathrm{thresholds\\_noise\\_sigma} < N < M(N)_{x,y} + \\sigma(N)_{x,y} * \\mathrm{thresholds\\_noise\\_sigma} $$\n",
    "\n",
    "or noise outside of hard limits\n",
    "\n",
    "$$\\mathrm{thresholds\\_noise\\_hard}_\\mathrm{low} < N < \\mathrm{thresholds\\_noise\\_hard}_\\mathrm{high} $$\n",
    "\n",
    "**\"OFFSET_NOISE_EVAL_ERROR\":**\n",
    "\n",
    "Offset and Noise both not $nan$ values\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "\n",
    "Values: $\\mathrm{thresholds\\_offset\\_sigma}$, $\\mathrm{thresholds\\_offset\\_hard}$, $\\mathrm{thresholds\\_noise\\_sigma}$, $\\mathrm{thresholds\\_noise\\_hard}$ are given as parameters.\n",
    "\n",
    "\"**\\\"GAIN_THRESHOLDING_ERROR\\\":**\n",
    "Bad gain separated pixels with sigma separation less than gain_separation_sigma_threshold\n",
    "$$ sigma\\_separation = \\\\frac{\\mathrm{gain\\_offset} - \\mathrm{previous\\_gain\\_offset}}{\\sqrt{\\mathrm{gain\\_offset_{std}}^\\mathrm{2} + \\mathrm{previuos\\_gain\\_offset_{std}}^\\mathrm{2}}}$$ \n",
    "$$ Bad\\_separation = sigma\\_separation < \\mathrm{gain\\_separation\\_sigma\\_threshold} $$\n",
    "\n",
    "\"\"\"))\n",
    "    \n",
    "elif \"LPD\" in karabo_id:\n",
    "    dinstance = \"LPD1M1\"\n",
    "    display(Markdown(\"\"\"\n",
    "    \n",
    "# Summary of LPD dark characterization #\n",
    "\n",
    "The following report shows a set of dark images taken with the LPD detector to deduce detector offsets, noise, bad-pixel maps. All three types of constants are evaluated per-pixel and per-memory cell.\n",
    "\n",
    "**The offset** ($O$) is defined as the median ($M$) of the dark signal ($Ds$) over trains ($t$) for a given pixel ($x,y$) and memory cell ($c$). \n",
    "\n",
    "**The noise** $N$ is the standard deviation $\\sigma$ of the dark signal.\n",
    "\n",
    "$$ O_{x,y,c} = M(Ds)_{t} ,\\,\\,\\,\\,\\,\\, N_{x,y,c} = \\sigma(Ds)_{t}$$\n",
    "\n",
    "**The bad pixel** mask is encoded as a bit mask.\n",
    "\n",
    "**\"OFFSET_OUT_OF_THRESHOLD\":**\n",
    "\n",
    "Offset outside of bounds:\n",
    "\n",
    "$$M(O)_{x,y} - \\sigma(O)_{x,y} * \\mathrm{thresholds\\_offset\\_sigma} < O < M(O)_{x,y} + \\sigma(O)_{x,y} * \\mathrm{thresholds\\_offset\\_sigma} $$\n",
    "\n",
    "or offset outside of hard limits\n",
    "\n",
    "$$ \\mathrm{thresholds\\_offset\\_hard}_\\mathrm{low} < O < \\mathrm{thresholds\\_offset\\_hard}_\\mathrm{high} $$\n",
    "\n",
    "**\"NOISE_OUT_OF_THRESHOLD\":**\n",
    "\n",
    "Noise outside of bounds:\n",
    "\n",
    "$$M(N)_{x,y} - \\sigma(N)_{x,y} * \\mathrm{thresholds\\_noise\\_sigma} < N < M(N)_{x,y} + \\sigma(N)_{x,y} * \\mathrm{thresholds\\_noise\\_sigma} $$\n",
    "\n",
    "or noise outside of hard limits\n",
    "\n",
    "$$\\mathrm{thresholds\\_noise\\_hard}_\\mathrm{low} < N < \\mathrm{thresholds\\_noise\\_hard}_\\mathrm{high} $$\n",
    "\n",
    "**\"OFFSET_NOISE_EVAL_ERROR\":**\n",
    "\n",
    "Offset and Noise both not $nan$ values \n",
    "\n",
    "\"Values: $\\\\mathrm{thresholds\\\\_offset\\\\_sigma}$, $\\\\mathrm{thresholds\\\\_offset\\\\_hard}$, $\\\\mathrm{thresholds\\\\_noise\\\\_sigma}$, $\\\\mathrm{thresholds\\\\_noise\\\\_hard}$ are given as parameters.\\n\",\n",
    "\"\"\"))\n",
    "elif \"DSSC\" in karabo_id:\n",
    "    dinstance = \"DSSC1M1\"\n",
    "    display(Markdown(\"\"\"\n",
    "    \n",
    "# Summary of DSSC dark characterization #\n",
    "    \n",
    "    \"\"\"))"
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preparing newly injected and previous constants from produced local folder in out_folder."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: After changes in the Cal DB interface read files from cal repository\n",
    "# Load constants from local files\n",
    "data = OrderedDict()\n",
    "old_cons = OrderedDict()\n",
    "mod_names = []\n",
    "# Loop over modules\n",
    "for i in range(nmods):\n",
    "    qm = f\"Q{i//4+1}M{i%4+1}\"\n",
    "    # loop over constants\n",
    "    detinst = getattr(Detectors, dinstance)\n",
    "    for const in ['Offset', 'Noise', 'ThresholdsDark', 'BadPixelsDark']:\n",
    "        det = getattr(detinst, qm)\n",
    "        if det is None:\n",
    "            continue\n",
    "        det_name = det.device_name\n",
    "        fpath = '{}/const_{}_{}.h5'.format(out_folder, const, det_name)\n",
    "        oldfpath = '{}/old/const_{}_{}.h5'.format(out_folder, const, det_name)\n",
    "        if not os.path.isfile(fpath):\n",
    "            continue\n",
    "        with h5py.File(fpath, 'r') as f:\n",
    "            if qm not in data:\n",
    "                mod_names.append(qm)\n",
    "                data[qm] = OrderedDict()\n",
    "\n",
    "            data[qm][const] = f[\"data\"][()]\n",
    "\n",
    "        if not os.path.isfile(oldfpath):\n",
    "            continue\n",
    "\n",
    "        with h5py.File(oldfpath, 'r') as oldf:\n",
    "            if qm not in old_cons:\n",
    "                old_cons[qm] = OrderedDict()\n",
    "            old_cons[qm][const] = oldf[\"data\"][()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cons_shape = {}\n",
    "# extracting constant shape.\n",
    "for qm, constant in data.items():\n",
    "    for cname, cons in constant.items():\n",
    "        shape = data[qm][cname].shape\n",
    "        if cname not in cons_shape:\n",
    "            cons_shape[cname] = shape\n",
    "constants = {}\n",
    "prev_const = {}\n",
    "for cname, sh in cons_shape.items():\n",
    "    constants[cname]= np.zeros((len(mod_names),) + sh[:])\n",
    "    prev_const[cname]= np.zeros((len(mod_names),) + sh[:])\n",
    "for i in range(len(mod_names)):\n",
    "    for cname, cval in constants.items():\n",
    "        cval[i] = data[mod_names[i]][cname]\n",
    "        if mod_names[i] in old_cons.keys():\n",
    "            prev_const[cname][i] = old_cons[mod_names[i]][cname]\n",
    "        else:\n",
    "            print(f\"No previous {cname} found for {mod_names[i]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "display(Markdown('## Processed modules ##'))\n",
    "show_processed_modules(dinstance, constants, mod_names, mode=\"processed\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary figures across Modules ##\n",
    "\n",
    "The following plots give an overview of calibration constants averaged across pixels and memory cells. A bad pixel mask is applied."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "if \"LPD\" in dinstance:\n",
    "    geom = extra_geom.LPD_1MGeometry.from_quad_positions(quad_pos=[(11.4, 299),\n",
    "                                                                   (-11.5, 8),\n",
    "                                                                   (254.5, -16),\n",
    "                                                                   (278.5, 275)])\n",
    "    pixels_y = 256\n",
    "    pixels_x = 256\n",
    "\n",
    "elif \"AGIPD\" in dinstance:\n",
    "    geom = extra_geom.AGIPD_1MGeometry.from_quad_positions(quad_pos=[(-525, 625),\n",
    "                                                                     (-550, -10),\n",
    "                                                                     (520, -160),\n",
    "                                                                     (542.5, 475)])\n",
    "    pixels_y = 128\n",
    "    pixels_x = 512\n",
    "\n",
    "elif \"DSSC\" in dinstance:\n",
    "    pixels_y = 512\n",
    "    pixels_x = 128\n",
    "    quadpos = [(-130, 5), (-130, -125), (5, -125), (5, 5)]\n",
    "\n",
    "    extrageom_pth = os.path.dirname(extra_geom.__file__)\n",
    "    geom = extra_geom.DSSC_1MGeometry.from_h5_file_and_quad_positions(\n",
    "                f\"{extrageom_pth}/tests/dssc_geo_june19.h5\", positions=quadpos)\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "\n",
    "Mod_data=OrderedDict()\n",
    "gainstages = 1\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "\n",
    "for const_name, const in constants.items():\n",
    "    \n",
    "    \n",
    "    #TODO: add a Summary figure across modules\n",
    "    if dinstance == \"AGIPD500K\":\n",
    "        display(Markdown(f'WARNING: No summary figures are available for AGIPD mini-half at the moment.'))\n",
    "        break\n",
    "    \n",
    "    \n",
    "    if const_name == 'BadPixelsDark':\n",
    "        continue\n",
    "    # Check if constant gain available in constant e.g. AGIPD, LPD\n",
    "    if len(const.shape) == 5:\n",
    "        gainstages = 3\n",
    "    else:\n",
    "        gainstages = 1\n",
    "\n",
    "    for dname in ['{}', 'd-{}', 'dpct-{}']:\n",
    "        Mod_data[dname.format(const_name)] = OrderedDict()\n",
    "    \n",
    "    display(Markdown(f'##### {const_name}'))\n",
    "    print_once = True\n",
    "    for gain in range(gainstages):\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "        if const_name == 'ThresholdsDark':\n",
    "            if gain > 1:\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "                continue\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "            glabel = threshold_names\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "        else:\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "            glabel = gain_names\n",
    "        for i in range(nmods):\n",
    "            qm = f\"Q{i//4+1}M{i%4+1}\"\n",
    "            if qm in mod_names:\n",
    "                m_idx = mod_names.index(qm)\n",
    "                # Check if constant shape of 5 indices e.g. AGIPD, LPD\n",
    "                if len(const.shape) == 5:\n",
    "                    values = np.nanmean(const[m_idx, :, :, :, gain], axis=2)\n",
    "                    dval = np.nanmean(prev_const[const_name][m_idx, :, :, :, gain], axis=2)\n",
    "                else:\n",
    "                    values = np.nanmean(const[m_idx, :, :, :], axis=2)\n",
    "                    dval = np.nanmean(prev_const[const_name][m_idx, :, :, :], axis=2)\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "                values[values == 0] = np.nan\n",
    "                dval[dval == 0] = np.nan\n",
    "                dval = values - dval\n",
    "                dval_pct = dval/values * 100\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "                values = np.moveaxis(values, 0, -1).reshape(1, values.shape[1], values.shape[0])\n",
    "                dval = np.moveaxis(dval, 0, -1).reshape(1, dval.shape[1], dval.shape[0])\n",
    "                dval_pct = np.moveaxis(dval_pct, 0, -1).reshape(1, dval_pct.shape[1], dval_pct.shape[0])\n",
    "            else:\n",
    "                # if module not available fill arrays with nan\n",
    "                values = np.zeros((1, pixels_x, pixels_y),dtype=np.float64)\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "                values[values == 0] = np.nan\n",
    "                dval = values \n",
    "                dval_pct = dval\n",
    "            for k, v in {'{}': values, 'd-{}': dval , 'dpct-{}': dval_pct}.items():\n",
    "                try:\n",
    "                    Mod_data[k.format(const_name)][gain_names[gain]] = \\\n",
    "                            np.concatenate((Mod_data[k.format(const_name)][gain_names[gain]],\n",
    "                                                                         v), axis=0)\n",
    "                except:\n",
    "                    Mod_data[k.format(const_name)][gain_names[gain]] = v\n",
    "        if np.count_nonzero(dval) == 0 and print_once:\n",
    "            display(Markdown(f'New and previous {const_name} are the same, hence there is no difference.'))\n",
    "            print_once = False\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "        # Plotting constant overall modules.\n",
    "        display(Markdown(f'###### {glabel[gain]} ######'))\n",
    "\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "        gs = gridspec.GridSpec(2, 2)\n",
    "        fig = plt.figure(figsize=(24, 32))\n",
    "\n",
    "        axis = OrderedDict()\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "        axis = {\"ax0\": {\"cname\": \"{}\" ,\"gs\": gs[0, :], \"shrink\": 0.9, \"pad\": 0.05, \"label\": \"ADUs\", \"title\": '{}'},\n",
    "                \"ax1\": {\"cname\": \"d-{}\",\"gs\": gs[1, 0], \"shrink\": 0.6, \"pad\": 0.1, \"label\": \"ADUs\", \"title\": 'Difference with previous {}'},\n",
    "                \"ax2\": {\"cname\": \"dpct-{}\", \"gs\": gs[1, 1], \"shrink\": 0.6, \"pad\": 0.1, \"label\": \"%\", \"title\": 'Difference with previous {} %'}}\n",
    "\n",
    "        for ax, axv in axis.items():\n",
    "            # Add the min and max plot values for each axis.\n",
    "            vmin, vmax =  get_range(Mod_data[axv[\"cname\"].format(const_name)][gain_names[gain]], 2)\n",
    "            ax = fig.add_subplot(axv[\"gs\"])\n",
    "            geom.plot_data_fast(Mod_data[axv[\"cname\"].format(const_name)][gain_names[gain]],\n",
    "                                vmin=vmin, vmax=vmax, ax=ax, \n",
    "                                colorbar={'shrink': axv[\"shrink\"],\n",
    "                                          'pad': axv[\"pad\"]\n",
    "                                         }\n",
    "                               )\n",
    "\n",
    "            colorbar = ax.images[0].colorbar\n",
    "            colorbar.set_label(axv[\"label\"])\n",
    "\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "            ax.set_title(axv[\"title\"].format(f\"{const_name} {glabel[gain]}\"), fontsize=15)\n",
    "            ax.set_xlabel('Columns', fontsize=15)\n",
    "            ax.set_ylabel('Rows', fontsize=15)\n",
    "\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
Karim Ahmed's avatar
Karim Ahmed committed
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Loop over modules and constants\n",
    "for const_name, const in constants.items():\n",
    "    display(Markdown(f'### Summary across Modules - {const_name}'))\n",
    "    for gain in range(gainstages):\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "        if const_name == 'ThresholdsDark':\n",
    "            if gain == 2:\n",
    "                continue\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "            glabel = threshold_names\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "        else:\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "            glabel = gain_names\n",
    "\n",
    "        if len(const.shape) == 5:\n",
    "            data = np.copy(const[:, :, :, :, gain])\n",
    "        else:\n",
    "            data = np.copy(const[:, :, :, :])\n",
    "        if const_name != 'BadPixelsDark':\n",
    "            if \"BadPixelsDark\" in constants.keys():\n",
    "                label = f'{const_name} value [ADU], good pixels only'\n",
    "                if len(const.shape) == 5:\n",
    "                    data[constants['BadPixelsDark'][:, :, :, :, gain] > 0] = np.nan\n",
    "                else:\n",
    "                    data[constants['BadPixelsDark'][:, :, :, :] > 0] = np.nan\n",
    "            else:\n",
    "                label = f'{const_name} value [ADU], good and bad pixels'\n",
    "            datamean = np.nanmean(data, axis=(1, 2))\n",
    "\n",
    "            fig = plt.figure(figsize=(15, 6), tight_layout={\n",
    "                             'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})\n",
    "            ax = fig.add_subplot(121)\n",
    "        else:\n",
    "            label = 'Fraction of bad pixels'\n",
    "            data[data > 0] = 1.0\n",
    "            datamean = np.nanmean(data, axis=(1, 2))\n",
    "            datamean[datamean == 1.0] = np.nan\n",
    "\n",
    "            fig = plt.figure(figsize=(15, 6), tight_layout={\n",
    "                             'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})\n",
    "            ax = fig.add_subplot(111)\n",
    "\n",
    "        d = []\n",
    "        for im, mod in enumerate(datamean):\n",
    "            d.append({'x': np.arange(mod.shape[0]),\n",
    "                      'y': mod,\n",
    "                      'drawstyle': 'steps-pre',\n",
    "                      'label': mod_names[im],\n",
    "                      })\n",
    "\n",
    "        _ = simplePlot(d, figsize=(10, 10), xrange=(-12, 510),\n",
    "                            x_label='Memory Cell ID',\n",
    "                            y_label=label,\n",
    "                            use_axis=ax,\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "                            title='{}'.format(glabel[gain]),\n",
    "                            title_position=[0.5, 1.18],\n",
    "                            legend='outside-top-ncol6-frame', legend_size='18%',\n",
    "                            legend_pad=0.00)\n",
    "        if const_name != 'BadPixelsDark':\n",
    "            ax = fig.add_subplot(122)\n",
    "            if \"BadPixelsDark\" in constants.keys():\n",
    "                label = f'$\\sigma$ {const_name} [ADU], good pixels only'\n",
    "            else:\n",
    "                label = f'$\\sigma$ {const_name} [ADU], good and bad pixels'\n",
    "            d = []\n",
    "            for im, mod in enumerate(np.nanstd(data, axis=(1, 2))):\n",
    "                d.append({'x': np.arange(mod.shape[0]),\n",
    "                          'y': mod,\n",
    "                          'drawstyle': 'steps-pre',\n",
    "                          'label': mod_names[im],\n",
    "                          })\n",
    "\n",
    "            _ = simplePlot(d, figsize=(10, 10), xrange=(-12, 510),\n",
    "                                x_label='Memory Cell ID',\n",
    "                                y_label=label,\n",
    "                                use_axis=ax,\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "                                title='{} $\\sigma$'.format(glabel[gain]),\n",
    "                                title_position=[0.5, 1.18],\n",
    "                                legend='outside-top-ncol6-frame', legend_size='18%',\n",
    "                                legend_pad=0.00)\n",
    "\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary tables across Modules ##\n",
    "\n",
    "Tables show values averaged across all pixels and memory cells of a given detector module."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if u'$' in tabulate.LATEX_ESCAPE_RULES:\n",
    "    del(tabulate.LATEX_ESCAPE_RULES[u'$'])\n",
    "    \n",
    "if u'\\\\' in tabulate.LATEX_ESCAPE_RULES:\n",
    "    del(tabulate.LATEX_ESCAPE_RULES[u'\\\\'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
Karim Ahmed's avatar
Karim Ahmed committed
    "head = ['Module', 'High gain', 'Medium gain', 'Low gain']\n",
    "head_th = ['Module', 'HG_MG threshold', 'MG_LG threshold']\n",
    "for const_name, const in constants.items():\n",
    "    table = []\n",
    "\n",
    "    for i_mod, mod in enumerate(mod_names):\n",
    "\n",
    "        t_line = [mod]\n",
    "        for gain in range(gainstages):\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "            \n",
    "            if const_name == 'ThresholdsDark': \n",
    "                if gain == 2:\n",
    "                    continue\n",
    "                header = head_th\n",
    "            else:\n",
    "                header = head\n",
    "            if len(const.shape) == 5:    \n",
    "                data = np.copy(const[i_mod, :, :, :, gain])\n",
    "            else:\n",
    "                data = np.copy(const[i_mod, :, :, :])\n",
    "            if const_name == 'BadPixelsDark':\n",
    "                data[data > 0] = 1.0\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "\n",
    "                datasum = np.nansum(data)\n",
    "                datamean = np.nanmean(data)\n",
    "                if datamean == 1.0:\n",
    "                    datamean = np.nan\n",
    "                    datasum = np.nan\n",
    "\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "                t_line.append(f'{datasum:6.0f} ({datamean:6.3f}) ')\n",
    "                label = '## Number(fraction) of bad pixels'\n",
    "                if \"BadPixelsDark\" in constants.keys():\n",
    "                    data[constants['BadPixelsDark']\n",
    "                         [i_mod, :, :, :, gain] > 0] = np.nan\n",
    "                    label = f'### Average {const_name} [ADU], good pixels only'\n",
    "                else:\n",
    "                    label = f'### Average {const_name} [ADU], good and bad pixels'\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "                t_line.append(f'{np.nanmean(data):6.1f} $\\\\pm$ {np.nanstd(data):6.1f}')\n",
    "                label = f'## Average {const_name} [ADU], good pixels only'\n",
    "\n",
    "        table.append(t_line)\n",
    "\n",
    "    display(Markdown(label))\n",
    "    md = display(Latex(tabulate.tabulate(\n",
    "        table, tablefmt='latex', headers=header)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}