Skip to content
Snippets Groups Projects
overallmodules_Darks_Summary_NBC.ipynb 24.2 KiB
Newer Older
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Author: K. Ahmed, M. Karnevsky, Version: 0.1\n",
    "#The following is a summary for the processing of dark images and calibration constants production.\n",
    "\n",
    "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/miniHalfAGIPD\" # path to output to, required\n",
    "karabo_id = \"HED_DET_AGIPD500K2G\" # detector instance\n",
    "gain_names = ['High gain', 'Medium gain', 'Low gain'] # a list of gain names to be used in plotting\n",
    "threshold_names = ['HG-MG threshold', 'MG_LG threshold'] # a list of gain names to be used in plotting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
    "import copy\n",
    "import os\n",
    "import warnings\n",
    "from collections import OrderedDict\n",
    "from datetime import datetime\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import glob\n",
    "import h5py\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "from IPython.display import Latex, Markdown, display\n",
    "\n",
    "matplotlib.use(\"agg\")\n",
    "import matplotlib.gridspec as gridspec\n",
    "import matplotlib.patches as patches\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import extra_geom\n",
    "import tabulate\n",
    "from cal_tools.ana_tools import get_range\n",
    "from cal_tools.plotting import show_processed_modules\n",
    "from cal_tools.tools import CalibrationMetadata\n",
    "from iCalibrationDB import Detectors\n",
    "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
    "from XFELDetAna.plotting.simpleplot import simplePlot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if \"AGIPD\" in karabo_id:\n",
    "    if \"SPB\" in karabo_id:\n",
    "        dinstance = \"AGIPD1M1\"\n",
    "    elif \"MID\" in karabo_id:\n",
    "        dinstance = \"AGIPD1M2\"\n",
    "    elif \"HED\" in karabo_id:\n",
    "        dinstance = \"AGIPD500K\"\n",
    "    display(Markdown(\"\"\"\n",
    "    \n",
Karim Ahmed's avatar
Karim Ahmed committed
    "# Summary of AGIPD dark characterization #\n",
    "\n",
    "The following report shows a set of dark images taken with the AGIPD detector to deduce detector offsets, noise, bad-pixel maps and thresholding. All four types of constants are evaluated per-pixel and per-memory cell.\n",
    "\n",
    "\n",
    "**The offset** ($O$) is defined as the median ($M$) of the dark signal ($Ds$) over trains ($t$) for a given pixel ($x,y$) and memory cell ($c$). \n",
    "\n",
    "**The noise** $N$ is the standard deviation $\\sigma$ of the dark signal.\n",
    "\n",
    "$$ O_{x,y,c} = M(Ds)_{t} ,\\,\\,\\,\\,\\,\\, N_{x,y,c} = \\sigma(Ds)_{t}$$\n",
    "\n",
    "**The bad pixel** mask is encoded as a bit mask.\n",
    "\n",
    "**\"OFFSET_OUT_OF_THRESHOLD\":**\n",
    "\n",
    "Offset outside of bounds:\n",
    "\n",
    "$$M(O)_{x,y} - \\sigma(O)_{x,y} * \\mathrm{thresholds\\_offset\\_sigma} < O < M(O)_{x,y} + \\sigma(O)_{x,y} * \\mathrm{thresholds\\_offset\\_sigma} $$\n",
    "\n",
    "or offset outside of hard limits\n",
    "\n",
    "$$ \\mathrm{thresholds\\_offset\\_hard}_\\mathrm{low} < O < \\mathrm{thresholds\\_offset\\_hard}_\\mathrm{high} $$\n",
    "\n",
    "**\"NOISE_OUT_OF_THRESHOLD\":**\n",
    "\n",
    "Noise outside of bounds:\n",
    "\n",
    "$$M(N)_{x,y} - \\sigma(N)_{x,y} * \\mathrm{thresholds\\_noise\\_sigma} < N < M(N)_{x,y} + \\sigma(N)_{x,y} * \\mathrm{thresholds\\_noise\\_sigma} $$\n",
    "\n",
    "or noise outside of hard limits\n",
    "\n",
    "$$\\mathrm{thresholds\\_noise\\_hard}_\\mathrm{low} < N < \\mathrm{thresholds\\_noise\\_hard}_\\mathrm{high} $$\n",
    "\n",
    "**\"OFFSET_NOISE_EVAL_ERROR\":**\n",
    "\n",
    "Offset and Noise both not $nan$ values\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "\n",
    "Values: $\\mathrm{thresholds\\_offset\\_sigma}$, $\\mathrm{thresholds\\_offset\\_hard}$, $\\mathrm{thresholds\\_noise\\_sigma}$, $\\mathrm{thresholds\\_noise\\_hard}$ are given as parameters.\n",
    "\n",
    "\"**\\\"GAIN_THRESHOLDING_ERROR\\\":**\n",
    "Bad gain separated pixels with sigma separation less than gain_separation_sigma_threshold\n",
    "$$ sigma\\_separation = \\\\frac{\\mathrm{gain\\_offset} - \\mathrm{previous\\_gain\\_offset}}{\\sqrt{\\mathrm{gain\\_offset_{std}}^\\mathrm{2} + \\mathrm{previuos\\_gain\\_offset_{std}}^\\mathrm{2}}}$$ \n",
    "$$ Bad\\_separation = sigma\\_separation < \\mathrm{gain\\_separation\\_sigma\\_threshold} $$\n",
    "\n",
    "\"\"\"))\n",
    "    \n",
    "elif \"LPD\" in karabo_id:\n",
    "    dinstance = \"LPD1M1\"\n",
    "    display(Markdown(\"\"\"\n",
    "    \n",
    "# Summary of LPD dark characterization #\n",
    "\n",
    "The following report shows a set of dark images taken with the LPD detector to deduce detector offsets, noise, bad-pixel maps. All three types of constants are evaluated per-pixel and per-memory cell.\n",
    "\n",
    "**The offset** ($O$) is defined as the median ($M$) of the dark signal ($Ds$) over trains ($t$) for a given pixel ($x,y$) and memory cell ($c$). \n",
    "\n",
    "**The noise** $N$ is the standard deviation $\\sigma$ of the dark signal.\n",
    "\n",
    "$$ O_{x,y,c} = M(Ds)_{t} ,\\,\\,\\,\\,\\,\\, N_{x,y,c} = \\sigma(Ds)_{t}$$\n",
    "\n",
    "**The bad pixel** mask is encoded as a bit mask.\n",
    "\n",
    "**\"OFFSET_OUT_OF_THRESHOLD\":**\n",
    "\n",
    "Offset outside of bounds:\n",
    "\n",
    "$$M(O)_{x,y} - \\sigma(O)_{x,y} * \\mathrm{thresholds\\_offset\\_sigma} < O < M(O)_{x,y} + \\sigma(O)_{x,y} * \\mathrm{thresholds\\_offset\\_sigma} $$\n",
    "\n",
    "or offset outside of hard limits\n",
    "\n",
    "$$ \\mathrm{thresholds\\_offset\\_hard}_\\mathrm{low} < O < \\mathrm{thresholds\\_offset\\_hard}_\\mathrm{high} $$\n",
    "\n",
    "**\"NOISE_OUT_OF_THRESHOLD\":**\n",
    "\n",
    "Noise outside of bounds:\n",
    "\n",
    "$$M(N)_{x,y} - \\sigma(N)_{x,y} * \\mathrm{thresholds\\_noise\\_sigma} < N < M(N)_{x,y} + \\sigma(N)_{x,y} * \\mathrm{thresholds\\_noise\\_sigma} $$\n",
    "\n",
    "or noise outside of hard limits\n",
    "\n",
    "$$\\mathrm{thresholds\\_noise\\_hard}_\\mathrm{low} < N < \\mathrm{thresholds\\_noise\\_hard}_\\mathrm{high} $$\n",
    "\n",
    "**\"OFFSET_NOISE_EVAL_ERROR\":**\n",
    "\n",
    "Offset and Noise both not $nan$ values \n",
    "\n",
    "\"Values: $\\\\mathrm{thresholds\\\\_offset\\\\_sigma}$, $\\\\mathrm{thresholds\\\\_offset\\\\_hard}$, $\\\\mathrm{thresholds\\\\_noise\\\\_sigma}$, $\\\\mathrm{thresholds\\\\_noise\\\\_hard}$ are given as parameters.\\n\",\n",
    "\"\"\"))\n",
    "elif \"DSSC\" in karabo_id:\n",
    "    dinstance = \"DSSC1M1\"\n",
    "    display(Markdown(\"\"\"\n",
    "    \n",
    "# Summary of DSSC dark characterization #\n",
    "    \n",
    "    \"\"\"))"
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Preparing newly injected and previous constants from produced local folder in out_folder."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: After changes in the Cal DB interface read files from cal repository\n",
    "# Load constants from local files\n",
    "data = OrderedDict()\n",
    "old_cons = OrderedDict()\n",
    "mod_names = []\n",
    "metadata = CalibrationMetadata(out_folder)\n",
    "\n",
    "# Loop over modules\n",
    "for i in range(nmods):\n",
    "    qm = f\"Q{i//4+1}M{i%4+1}\"\n",
    "    if qm not in metadata[\"modules_mapping\"].keys():\n",
    "    det_name = metadata[\"modules_mapping\"][qm]\n",
    "    # loop over constants\n",
    "    for const in ['Offset', 'Noise', 'ThresholdsDark', 'BadPixelsDark']:\n",
    "        fpath = '{}/const_{}_{}.h5'.format(out_folder, const, det_name)\n",
    "        oldfpath = '{}/old/const_{}_{}.h5'.format(out_folder, const, det_name)\n",
    "        if not os.path.isfile(fpath):\n",
    "            continue\n",
    "        with h5py.File(fpath, 'r') as f:\n",
    "            if qm not in data:\n",
    "                mod_names.append(qm)\n",
    "                data[qm] = OrderedDict()\n",
    "\n",
    "            data[qm][const] = f[\"data\"][()]\n",
    "\n",
    "        if not os.path.isfile(oldfpath):\n",
    "            continue\n",
    "\n",
    "        with h5py.File(oldfpath, 'r') as oldf:\n",
    "            if qm not in old_cons:\n",
    "                old_cons[qm] = OrderedDict()\n",
    "            old_cons[qm][const] = oldf[\"data\"][()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cons_shape = {}\n",
    "# extracting constant shape.\n",
    "for qm, constant in data.items():\n",
    "    for cname, cons in constant.items():\n",
    "        shape = data[qm][cname].shape\n",
    "        if cname not in cons_shape:\n",
    "            cons_shape[cname] = shape\n",
    "constants = {}\n",
    "prev_const = {}\n",
    "for cname, sh in cons_shape.items():\n",
    "    constants[cname]= np.zeros((len(mod_names),) + sh[:])\n",
    "    prev_const[cname]= np.zeros((len(mod_names),) + sh[:])\n",
    "for i in range(len(mod_names)):\n",
    "    for cname, cval in constants.items():\n",
    "        cval[i] = data[mod_names[i]][cname]\n",
    "        if mod_names[i] in old_cons.keys():\n",
    "            prev_const[cname][i] = old_cons[mod_names[i]][cname]\n",
    "        else:\n",
    "            print(f\"No previous {cname} found for {mod_names[i]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "display(Markdown('## Processed modules ##'))\n",
    "show_processed_modules(dinstance, constants, mod_names, mode=\"processed\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary figures across Modules ##\n",
    "\n",
    "The following plots give an overview of calibration constants averaged across pixels and memory cells. A bad pixel mask is applied."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "if \"LPD\" in dinstance:\n",
    "    geom = extra_geom.LPD_1MGeometry.from_quad_positions(quad_pos=[(11.4, 299),\n",
    "                                                                   (-11.5, 8),\n",
    "                                                                   (254.5, -16),\n",
    "                                                                   (278.5, 275)])\n",
    "    pixels_y = 256\n",
    "    pixels_x = 256\n",
    "\n",
    "elif dinstance in ('AGIPD1M1', 'AGIPD1M2'):\n",
    "    geom = extra_geom.AGIPD_1MGeometry.from_quad_positions(quad_pos=[(-525, 625),\n",
    "                                                                     (-550, -10),\n",
    "                                                                     (520, -160),\n",
    "                                                                     (542.5, 475)])\n",
    "    pixels_y = 128\n",
    "    pixels_x = 512\n",
    "\n",
    "elif dinstance == \"AGIPD500K\":\n",
    "    geom = extra_geom.AGIPD_500K2GGeometry.from_origin()\n",
    "    pixels_y = 128\n",
    "    pixels_x = 512\n",
    "    \n",
    "elif \"DSSC\" in dinstance:\n",
    "    pixels_y = 512\n",
    "    pixels_x = 128\n",
    "    quadpos = [(-130, 5), (-130, -125), (5, -125), (5, 5)]\n",
    "\n",
    "    extrageom_pth = os.path.dirname(extra_geom.__file__)\n",
    "    geom = extra_geom.DSSC_1MGeometry.from_h5_file_and_quad_positions(\n",
    "                f\"{extrageom_pth}/tests/dssc_geo_june19.h5\", positions=quadpos)\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "\n",
    "Mod_data=OrderedDict()\n",
    "gainstages = 1\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "\n",
    "for const_name, const in constants.items():\n",
    "    if const_name == 'BadPixelsDark':\n",
    "        continue\n",
    "    # Check if constant gain available in constant e.g. AGIPD, LPD\n",
    "    if len(const.shape) == 5:\n",
    "        gainstages = 3\n",
    "    else:\n",
    "        gainstages = 1\n",
    "\n",
    "    for dname in ['{}', 'd-{}', 'dpct-{}']:\n",
    "        Mod_data[dname.format(const_name)] = OrderedDict()\n",
    "    \n",
    "    display(Markdown(f'##### {const_name}'))\n",
    "    print_once = True\n",
    "    for gain in range(gainstages):\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "        if const_name == 'ThresholdsDark':\n",
    "            if gain > 1:\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "                continue\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "            glabel = threshold_names\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "        else:\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "            glabel = gain_names\n",
    "        for i in range(nmods):\n",
    "            qm = f\"Q{i//4+1}M{i%4+1}\"\n",
    "            if qm in mod_names:\n",
    "                m_idx = mod_names.index(qm)\n",
    "                # Check if constant shape of 5 indices e.g. AGIPD, LPD\n",
    "                if len(const.shape) == 5:\n",
    "                    values = np.nanmean(const[m_idx, :, :, :, gain], axis=2)\n",
    "                    dval = np.nanmean(prev_const[const_name][m_idx, :, :, :, gain], axis=2)\n",
    "                else:\n",
    "                    values = np.nanmean(const[m_idx, :, :, :], axis=2)\n",
    "                    dval = np.nanmean(prev_const[const_name][m_idx, :, :, :], axis=2)\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "                values[values == 0] = np.nan\n",
    "                dval[dval == 0] = np.nan\n",
    "                dval = values - dval\n",
    "                dval_pct = dval/values * 100\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "                values = np.moveaxis(values, 0, -1).reshape(1, values.shape[1], values.shape[0])\n",
    "                dval = np.moveaxis(dval, 0, -1).reshape(1, dval.shape[1], dval.shape[0])\n",
    "                dval_pct = np.moveaxis(dval_pct, 0, -1).reshape(1, dval_pct.shape[1], dval_pct.shape[0])\n",
    "            else:\n",
    "                # if module not available fill arrays with nan\n",
    "                values = np.zeros((1, pixels_x, pixels_y),dtype=np.float64)\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "                values[values == 0] = np.nan\n",
    "                dval = values \n",
    "                dval_pct = dval\n",
    "            for k, v in {'{}': values, 'd-{}': dval , 'dpct-{}': dval_pct}.items():\n",
    "                try:\n",
    "                    Mod_data[k.format(const_name)][gain_names[gain]] = \\\n",
    "                            np.concatenate((Mod_data[k.format(const_name)][gain_names[gain]],\n",
    "                                                                         v), axis=0)\n",
    "                except:\n",
    "                    Mod_data[k.format(const_name)][gain_names[gain]] = v\n",
    "        if np.count_nonzero(dval) == 0 and print_once:\n",
    "            display(Markdown(f'New and previous {const_name} are the same, hence there is no difference.'))\n",
    "            print_once = False\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "        # Plotting constant overall modules.\n",
    "        display(Markdown(f'###### {glabel[gain]} ######'))\n",
    "\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "        gs = gridspec.GridSpec(2, 2)\n",
    "        fig = plt.figure(figsize=(24, 32))\n",
    "\n",
    "        axis = OrderedDict()\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "        axis = {\"ax0\": {\"cname\": \"{}\" ,\"gs\": gs[0, :], \"shrink\": 0.9, \"pad\": 0.05, \"label\": \"ADUs\", \"title\": '{}'},\n",
    "                \"ax1\": {\"cname\": \"d-{}\",\"gs\": gs[1, 0], \"shrink\": 0.6, \"pad\": 0.1, \"label\": \"ADUs\", \"title\": 'Difference with previous {}'},\n",
    "                \"ax2\": {\"cname\": \"dpct-{}\", \"gs\": gs[1, 1], \"shrink\": 0.6, \"pad\": 0.1, \"label\": \"%\", \"title\": 'Difference with previous {} %'}}\n",
    "\n",
    "        for ax, axv in axis.items():\n",
    "            # Add the min and max plot values for each axis.\n",
    "            vmin, vmax =  get_range(Mod_data[axv[\"cname\"].format(const_name)][gain_names[gain]], 2)\n",
    "            ax = fig.add_subplot(axv[\"gs\"])\n",
    "            geom.plot_data_fast(Mod_data[axv[\"cname\"].format(const_name)][gain_names[gain]],\n",
    "                                vmin=vmin, vmax=vmax, ax=ax, \n",
    "                                colorbar={'shrink': axv[\"shrink\"],\n",
    "                                          'pad': axv[\"pad\"]\n",
    "                                         }\n",
    "                               )\n",
    "\n",
    "            colorbar = ax.images[0].colorbar\n",
    "            colorbar.set_label(axv[\"label\"])\n",
    "\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "            ax.set_title(axv[\"title\"].format(f\"{const_name} {glabel[gain]}\"), fontsize=15)\n",
    "            ax.set_xlabel('Columns', fontsize=15)\n",
    "            ax.set_ylabel('Rows', fontsize=15)\n",
    "\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
Karim Ahmed's avatar
Karim Ahmed committed
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Loop over modules and constants\n",
    "for const_name, const in constants.items():\n",
    "    display(Markdown(f'### Summary across Modules - {const_name}'))\n",
    "    for gain in range(gainstages):\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "        if const_name == 'ThresholdsDark':\n",
    "            if gain == 2:\n",
    "                continue\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "            glabel = threshold_names\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "        else:\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "            glabel = gain_names\n",
    "\n",
    "        if len(const.shape) == 5:\n",
    "            data = np.copy(const[:, :, :, :, gain])\n",
    "        else:\n",
    "            data = np.copy(const[:, :, :, :])\n",
    "        if const_name != 'BadPixelsDark':\n",
    "            if \"BadPixelsDark\" in constants.keys():\n",
    "                label = f'{const_name} value [ADU], good pixels only'\n",
    "                if len(const.shape) == 5:\n",
    "                    data[constants['BadPixelsDark'][:, :, :, :, gain] > 0] = np.nan\n",
    "                else:\n",
    "                    data[constants['BadPixelsDark'][:, :, :, :] > 0] = np.nan\n",
    "            else:\n",
    "                label = f'{const_name} value [ADU], good and bad pixels'\n",
    "            datamean = np.nanmean(data, axis=(1, 2))\n",
    "\n",
    "            fig = plt.figure(figsize=(15, 6), tight_layout={\n",
    "                             'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})\n",
    "            ax = fig.add_subplot(121)\n",
    "        else:\n",
    "            label = 'Fraction of bad pixels'\n",
    "            data[data > 0] = 1.0\n",
    "            datamean = np.nanmean(data, axis=(1, 2))\n",
    "            datamean[datamean == 1.0] = np.nan\n",
    "\n",
    "            fig = plt.figure(figsize=(15, 6), tight_layout={\n",
    "                             'pad': 0.2, 'w_pad': 1.3, 'h_pad': 1.3})\n",
    "            ax = fig.add_subplot(111)\n",
    "\n",
    "        d = []\n",
    "        for im, mod in enumerate(datamean):\n",
    "            d.append({'x': np.arange(mod.shape[0]),\n",
    "                      'y': mod,\n",
    "                      'drawstyle': 'steps-pre',\n",
    "                      'label': mod_names[im],\n",
    "                      })\n",
    "\n",
    "        _ = simplePlot(d, figsize=(10, 10), xrange=(-12, 510),\n",
    "                            x_label='Memory Cell ID',\n",
    "                            y_label=label,\n",
    "                            use_axis=ax,\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "                            title='{}'.format(glabel[gain]),\n",
    "                            title_position=[0.5, 1.18],\n",
    "                            legend='outside-top-ncol6-frame', legend_size='18%',\n",
    "                            legend_pad=0.00)\n",
    "        if const_name != 'BadPixelsDark':\n",
    "            ax = fig.add_subplot(122)\n",
    "            if \"BadPixelsDark\" in constants.keys():\n",
    "                label = f'$\\sigma$ {const_name} [ADU], good pixels only'\n",
    "            else:\n",
    "                label = f'$\\sigma$ {const_name} [ADU], good and bad pixels'\n",
    "            d = []\n",
    "            for im, mod in enumerate(np.nanstd(data, axis=(1, 2))):\n",
    "                d.append({'x': np.arange(mod.shape[0]),\n",
    "                          'y': mod,\n",
    "                          'drawstyle': 'steps-pre',\n",
    "                          'label': mod_names[im],\n",
    "                          })\n",
    "\n",
    "            _ = simplePlot(d, figsize=(10, 10), xrange=(-12, 510),\n",
    "                                x_label='Memory Cell ID',\n",
    "                                y_label=label,\n",
    "                                use_axis=ax,\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "                                title='{} $\\sigma$'.format(glabel[gain]),\n",
    "                                title_position=[0.5, 1.18],\n",
    "                                legend='outside-top-ncol6-frame', legend_size='18%',\n",
    "                                legend_pad=0.00)\n",
    "\n",
    "        plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary tables across Modules ##\n",
    "\n",
    "Tables show values averaged across all pixels and memory cells of a given detector module."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if u'$' in tabulate.LATEX_ESCAPE_RULES:\n",
    "    del(tabulate.LATEX_ESCAPE_RULES[u'$'])\n",
    "    \n",
    "if u'\\\\' in tabulate.LATEX_ESCAPE_RULES:\n",
    "    del(tabulate.LATEX_ESCAPE_RULES[u'\\\\'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
Karim Ahmed's avatar
Karim Ahmed committed
    "head = ['Module', 'High gain', 'Medium gain', 'Low gain']\n",
    "head_th = ['Module', 'HG_MG threshold', 'MG_LG threshold']\n",
    "for const_name, const in constants.items():\n",
    "    table = []\n",
    "\n",
    "    for i_mod, mod in enumerate(mod_names):\n",
    "\n",
    "        t_line = [mod]\n",
    "        for gain in range(gainstages):\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "            \n",
    "            if const_name == 'ThresholdsDark': \n",
    "                if gain == 2:\n",
    "                    continue\n",
    "                header = head_th\n",
    "            else:\n",
    "                header = head\n",
    "            if len(const.shape) == 5:    \n",
    "                data = np.copy(const[i_mod, :, :, :, gain])\n",
    "            else:\n",
    "                data = np.copy(const[i_mod, :, :, :])\n",
    "            if const_name == 'BadPixelsDark':\n",
    "                data[data > 0] = 1.0\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "\n",
    "                datasum = np.nansum(data)\n",
    "                datamean = np.nanmean(data)\n",
    "                if datamean == 1.0:\n",
    "                    datamean = np.nan\n",
    "                    datasum = np.nan\n",
    "\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "                t_line.append(f'{datasum:6.0f} ({datamean:6.3f}) ')\n",
    "                label = '## Number(fraction) of bad pixels'\n",
    "                if \"BadPixelsDark\" in constants.keys():\n",
    "                    data[constants['BadPixelsDark']\n",
    "                         [i_mod, :, :, :, gain] > 0] = np.nan\n",
    "                    label = f'### Average {const_name} [ADU], good pixels only'\n",
    "                else:\n",
    "                    label = f'### Average {const_name} [ADU], good and bad pixels'\n",
Karim Ahmed's avatar
Karim Ahmed committed
    "                t_line.append(f'{np.nanmean(data):6.1f} $\\\\pm$ {np.nanstd(data):6.1f}')\n",
    "                label = f'## Average {const_name} [ADU], good pixels only'\n",
    "\n",
    "        table.append(t_line)\n",
    "\n",
    "    display(Markdown(label))\n",
    "    md = display(Latex(tabulate.tabulate(\n",
    "        table, tablefmt='latex', headers=header)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}