From 6d1aa45d127f95d4bfe8e23f1ef832b2413b35d4 Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Fri, 30 Aug 2019 15:55:30 +0200
Subject: [PATCH 1/6] Add plotting constants of FCCD

---
 cal_tools/cal_tools/ana_tools.py              |  14 +
 .../FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb   | 531 ++++++++++++++++++
 xfel_calibrate/notebooks.py                   |   6 +
 3 files changed, 551 insertions(+)
 create mode 100644 notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb

diff --git a/cal_tools/cal_tools/ana_tools.py b/cal_tools/cal_tools/ana_tools.py
index f58ace59b..dce75cf70 100644
--- a/cal_tools/cal_tools/ana_tools.py
+++ b/cal_tools/cal_tools/ana_tools.py
@@ -352,6 +352,20 @@ class HMType(Enum):
     INSET_AXIS = 2
 
 
+def get_range(data, scale):
+    """
+    Get range, which includes most of the data points.
+    Range is calculated in units of median absolute deviations
+
+    :param data: numpy.array of data points
+    :param scale: range in units of median absolute deviations
+    :return:
+    """
+    med = np.nanmedian(data)
+    mad = np.nanmedian(np.abs(data.flatten() - med))
+    return med - scale * mad, med + scale * mad
+
+
 def hm_combine(data, fname=None, htype=None, **kwargs):
     """
     Plot heatmap for calibration report
diff --git a/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb b/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
new file mode 100644
index 000000000..406e88e09
--- /dev/null
+++ b/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
@@ -0,0 +1,531 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Statistical analysis of calibration factors#\n",
+    "\n",
+    "Author: Mikhail Karnevskiy, Steffen Hauf, Version 0.1\n",
+    "\n",
+    "A description of the notebook."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cluster_profile = \"noDB\" # The ipcluster profile to use\n",
+    "start_date = \"2019-01-30\" # date to start investigation interval from\n",
+    "end_date = \"2019-08-30\" # date to end investigation interval at, can be \"now\"\n",
+    "nconstants = 10 # Number of time stamps to plot. If not 0, overcome start_date.\n",
+    "dclass=\"CCD\" # Detector class\n",
+    "db_module = \"fastCCD1\" # detector entry in the DB to investigate\n",
+    "constants = [\"Noise\"]#, \"Offset\"] # constants to plot\n",
+    "\n",
+    "gain_setting = [0,1,2,8] # gain stages\n",
+    "bias_voltage = [79] # Bias voltage\n",
+    "temperature = [235]#, 216, 245] # Operation temperature\n",
+    "integration_time = [1, 50] # Integration time\n",
+    "pixels_x=[1934]\n",
+    "pixels_y=[960]\n",
+    "max_time = 15\n",
+    "parameter_names = ['bias_voltage', 'integration_time', 'temperature', \n",
+    "                   'gain_setting', 'pixels_x', 'pixels_y'] # names of parameters\n",
+    "photon_energy = 9.2 # Photon energy of the beam\n",
+    "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_FCCD5/\" # output folder\n",
+    "use_existing = \"\" # If not empty, constants stored in given folder will be used\n",
+    "cal_db_interface = \"tcp://max-exfl016:8015#8025\" # the database interface to use\n",
+    "cal_db_timeout = 180000 # timeout on caldb requests\",\n",
+    "plot_range = 3 # range for plotting in units of median absolute deviations"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "import copy\n",
+    "import datetime\n",
+    "import dateutil.parser\n",
+    "import numpy as np\n",
+    "from operator import itemgetter\n",
+    "import os\n",
+    "import sys\n",
+    "import warnings\n",
+    "warnings.filterwarnings('ignore')\n",
+    "\n",
+    "import h5py\n",
+    "import matplotlib\n",
+    "%matplotlib inline\n",
+    "\n",
+    "from iCalibrationDB import Constants, Conditions, Detectors, ConstantMetaData\n",
+    "from cal_tools.tools import get_from_db, get_random_db_interface\n",
+    "from cal_tools.ana_tools import (save_dict_to_hdf5, load_data_from_hdf5, \n",
+    "                                 combine_constants, HMType,\n",
+    "                                 hm_combine, combine_lists, get_range)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Prepare variables\n",
+    "spShape = (967, 10) # Shape of superpixel\n",
+    "\n",
+    "parameters = [globals()[x] for x in parameter_names]\n",
+    "\n",
+    "constantsDark = {'Noise': 'BadPixelsDark',\n",
+    "                 'Offset': 'BadPixelsDark'}\n",
+    "print('Bad pixels data: ', constantsDark)\n",
+    "\n",
+    "# Define parameters in order to perform loop over time stamps\n",
+    "start = datetime.datetime.now() if start_date.upper() == \"NOW\" else dateutil.parser.parse(\n",
+    "    start_date)\n",
+    "end = datetime.datetime.now() if end_date.upper() == \"NOW\" else dateutil.parser.parse(\n",
+    "    end_date)\n",
+    "\n",
+    "# Create output folder\n",
+    "os.makedirs(out_folder, exist_ok=True)\n",
+    "\n",
+    "# Get getector conditions\n",
+    "det = getattr(Detectors, db_module)\n",
+    "dconstants = getattr(Constants, dclass)(det.detector_type)\n",
+    "\n",
+    "print('CalDB Interface: {}'.format(cal_db_interface))\n",
+    "print('Start time at: ', start)\n",
+    "print('End time at: ', end)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "parameter_list = combine_lists(*parameters, names = parameter_names)\n",
+    "print(parameter_list)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [],
+   "source": [
+    "# Retrieve list of meta-data\n",
+    "constant_versions = []\n",
+    "constant_parameters = []\n",
+    "constantBP_versions = []\n",
+    "\n",
+    "# Loop over constants\n",
+    "for c, const in enumerate(constants):\n",
+    "    \n",
+    "    if use_existing != \"\":\n",
+    "        break\n",
+    "    \n",
+    "    # Loop over parameters\n",
+    "    for pars in parameter_list:\n",
+    "    \n",
+    "        if (const in [\"Offset\", \"Noise\", \"SlopesPC\"] or \"DARK\" in const.upper()):\n",
+    "            dcond = Conditions.Dark\n",
+    "            mcond = getattr(dcond, dclass)(**pars)\n",
+    "        else:\n",
+    "            dcond = Conditions.Illuminated\n",
+    "            mcond = getattr(dcond, dclass)(**pars,\n",
+    "                                photon_energy=photon_energy)\n",
+    "\n",
+    "            \n",
+    "            \n",
+    "        print('Request: ', const, 'with paramters:', pars)\n",
+    "        # Request Constant versions for given parameters and module\n",
+    "        data = get_from_db(det,\n",
+    "                           getattr(dconstants,\n",
+    "                                   const)(),\n",
+    "                           copy.deepcopy(mcond), None,\n",
+    "                           cal_db_interface,\n",
+    "                           creation_time=start,\n",
+    "                           verbosity=0,\n",
+    "                           timeout=cal_db_timeout,\n",
+    "                           meta_only=True,\n",
+    "                           version_info=True)\n",
+    "        \n",
+    "        if not isinstance(data, list):\n",
+    "                continue\n",
+    "            \n",
+    "        data = sorted(data, key=itemgetter('begin_at'))\n",
+    "        print('Number of retrieved constants: {}'.format(len(data)) )\n",
+    "                \n",
+    "        if const in constantsDark:\n",
+    "            # Request BP constant versions\n",
+    "            dataBP = get_from_db(det,\n",
+    "                                 getattr(dconstants, \n",
+    "                                         constantsDark[const])(),\n",
+    "                                 copy.deepcopy(mcond), None,\n",
+    "                                 cal_db_interface,\n",
+    "                                 creation_time=start,\n",
+    "                                 verbosity=0,\n",
+    "                                 timeout=cal_db_timeout,\n",
+    "                                 meta_only=True,\n",
+    "                                 version_info=True)\n",
+    "        \n",
+    "            if not isinstance(data, list) or not isinstance(dataBP, list):\n",
+    "                continue\n",
+    "            print('Number of retrieved darks: {}'.format(len(dataBP)) )\n",
+    "            found_BPmatch = False\n",
+    "            for d in data:\n",
+    "                # Match proper BP constant version\n",
+    "                # and get constant version within\n",
+    "                # requested time range\n",
+    "                if d is None:\n",
+    "                    print('Time or data is not found!')\n",
+    "                    continue\n",
+    "\n",
+    "                dt = dateutil.parser.parse(d['begin_at'])\n",
+    "\n",
+    "                if (dt.replace(tzinfo=None) > end or \n",
+    "                    (nconstants==0 and dt.replace(tzinfo=None) < start)):\n",
+    "                    continue\n",
+    "                    \n",
+    "                if nconstants>0 and constant_parameters.count(pars)>nconstants-1:\n",
+    "                    break\n",
+    "\n",
+    "                closest_BP = None\n",
+    "                closest_BPtime = None\n",
+    "\n",
+    "                for dBP in dataBP:\n",
+    "                    if dBP is None:\n",
+    "                        print(\"Bad pixels are not found!\")\n",
+    "                        continue\n",
+    "\n",
+    "                    dt = dateutil.parser.parse(d['begin_at'])\n",
+    "                    dBPt = dateutil.parser.parse(dBP['begin_at'])\n",
+    "\n",
+    "                    if dt == dBPt:\n",
+    "                        found_BPmatch = True\n",
+    "                    else:\n",
+    "\n",
+    "                        if np.abs(dBPt-dt).seconds < (max_time*60):\n",
+    "                            if closest_BP is None:\n",
+    "                                closest_BP = dBP\n",
+    "                                closest_BPtime = dBPt\n",
+    "                            else:\n",
+    "                                if np.abs(dBPt-dt) < np.abs(closest_BPtime-dt):\n",
+    "                                    closest_BP = dBP\n",
+    "                                    closest_BPtime = dBPt\n",
+    "\n",
+    "                        if dataBP.index(dBP) ==  len(dataBP)-1:\n",
+    "                            if closest_BP:\n",
+    "                                dBP = closest_BP\n",
+    "                                dBPt = closest_BPtime\n",
+    "                                found_BPmatch = True\n",
+    "                            else:\n",
+    "                                print('Bad pixels are not found!')\n",
+    "\n",
+    "                    if found_BPmatch:\n",
+    "                        print(\"Found constant {}: begin at {}\".format(const, dt))\n",
+    "                        print(\"Found bad pixels at {}\".format(dBPt))\n",
+    "                        constantBP_versions.append(dBP)\n",
+    "                        constant_versions.append(d)\n",
+    "                        constant_parameters.append(copy.deepcopy(pars))\n",
+    "                        found_BPmatch = False\n",
+    "                        break\n",
+    "        else:\n",
+    "            constant_versions += data\n",
+    "            constant_parameters += [copy.deepcopy(pars)]*len(data)\n",
+    "\n",
+    "# Remove dublications\n",
+    "constant_versions_tmp = []\n",
+    "constant_parameters_tmp = []\n",
+    "for i, x in enumerate(constant_versions):\n",
+    "    if x not in constant_versions_tmp:\n",
+    "        constant_versions_tmp.append(x)\n",
+    "        constant_parameters_tmp.append(constant_parameters[i])\n",
+    "        \n",
+    "constant_versions=constant_versions_tmp\n",
+    "constant_parameters=constant_parameters_tmp\n",
+    "        \n",
+    "print('Number of stored constant versions is {}'.format(len(constant_versions)))\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def get_rebined(a, rebin):\n",
+    "    return a[:,:,0].reshape(\n",
+    "                int(a.shape[0] / rebin[0]),\n",
+    "                rebin[0],\n",
+    "                int(a.shape[1] / rebin[1]),\n",
+    "                rebin[1])\n",
+    "    \n",
+    "def modify_const(const, data, isBP = False):\n",
+    "    return data\n",
+    "\n",
+    "ret_constants = {}\n",
+    "constand_data = ConstantMetaData()\n",
+    "constant_BP = ConstantMetaData()\n",
+    "for i, constant_version in enumerate(constant_versions):\n",
+    "\n",
+    "    const = constant_version['data_set_name'].split('/')[-2]\n",
+    "    qm = db_module\n",
+    "    \n",
+    "    print(\"constant: {}, module {}\".format(const,qm))\n",
+    "    \n",
+    "    constand_data.retrieve_from_version_info(constant_version)\n",
+    "    \n",
+    "    # Convert parameters to dict\n",
+    "    dpar = {p.name: p.value for p in constand_data.detector_condition.parameters}\n",
+    "    \n",
+    "    const = \"{}_{}_{}_{}\".format(const, \n",
+    "                              constant_parameters[i]['gain_setting'],\n",
+    "                              constant_parameters[i]['temperature'],\n",
+    "                              constant_parameters[i]['integration_time'])\n",
+    "    \n",
+    "    if not const in ret_constants:\n",
+    "        ret_constants[const] = {}\n",
+    "    if not qm in ret_constants[const]:\n",
+    "            ret_constants[const][qm] = []\n",
+    "    \n",
+    "    cdata = constand_data.calibration_constant.data\n",
+    "    ctime = constand_data.calibration_constant_version.begin_at\n",
+    "    \n",
+    "    cdata = modify_const(const, cdata)\n",
+    "    \n",
+    "    if len(constantBP_versions)>0:\n",
+    "        constant_BP.retrieve_from_version_info(constantBP_versions[i])\n",
+    "        cdataBP = constant_BP.calibration_constant.data\n",
+    "        cdataBP = modify_const(const, cdataBP, True)\n",
+    "        \n",
+    "        if cdataBP.shape != cdata.shape:\n",
+    "            print('Wrong bad pixel shape! {}, expected {}'.format(cdataBP.shape, cdata.shape))\n",
+    "            continue\n",
+    "        \n",
+    "        # Apply bad pixel mask\n",
+    "        cdataABP = np.copy(cdata)\n",
+    "        cdataABP[cdataBP > 0] = np.nan\n",
+    "    \n",
+    "        # Create superpixels for constants with BP applied\n",
+    "        cdataABP = get_rebined(cdataABP, spShape)\n",
+    "        toStoreBP = np.nanmean(cdataABP, axis=(1, 3))\n",
+    "        toStoreBPStd = np.nanstd(cdataABP, axis=(1, 3))\n",
+    "\n",
+    "        # Prepare number of bad pixels per superpixels\n",
+    "        cdataBP = get_rebined(cdataBP, spShape)\n",
+    "        cdataNBP = np.nansum(cdataBP > 0, axis=(1, 3))\n",
+    "    else:\n",
+    "        toStoreBP = 0\n",
+    "        toStoreBPStd = 0\n",
+    "        cdataNBP = 0\n",
+    "\n",
+    "    # Create superpixels for constants without BP applied\n",
+    "    cdata = get_rebined(cdata, spShape)\n",
+    "    toStoreStd = np.nanstd(cdata, axis=(1, 3))\n",
+    "    toStore = np.nanmean(cdata, axis=(1, 3))\n",
+    "    \n",
+    "    # Convert parameters to dict\n",
+    "    dpar = {p.name: p.value for p in constand_data.detector_condition.parameters}\n",
+    "    \n",
+    "    print(\"Store values in dict\", const, qm, ctime)\n",
+    "    ret_constants[const][qm].append({'ctime': ctime,\n",
+    "                                     'nBP': cdataNBP,\n",
+    "                                     'dataBP': toStoreBP,\n",
+    "                                     'dataBPStd': toStoreBPStd,\n",
+    "                                     'data': toStore,\n",
+    "                                     'dataStd': toStoreStd,\n",
+    "                                     'mdata': dpar})  \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "if use_existing == \"\":\n",
+    "    print('Save data to /CalDBAna_{}_{}.h5'.format(dclass, db_module))\n",
+    "    save_dict_to_hdf5(ret_constants,\n",
+    "                      '{}/CalDBAna_{}_{}.h5'.format(out_folder, dclass, db_module))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if use_existing == \"\":\n",
+    "    fpath = '{}/CalDBAna_{}_*.h5'.format(out_folder, dclass)\n",
+    "else:\n",
+    "    fpath = '{}/CalDBAna_{}_*.h5'.format(use_existing, dclass)\n",
+    "\n",
+    "print('Load data from {}'.format(fpath))\n",
+    "ret_constants = load_data_from_hdf5(fpath)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Parameters for plotting\n",
+    "\n",
+    "keys = {\n",
+    "    'Mean': ['data', '', 'Mean over pixels'],\n",
+    "    'std': ['dataStd', '', '$\\sigma$ over pixels'],\n",
+    "    'MeanBP': ['dataBP', 'Good pixels only', 'Mean over pixels'],\n",
+    "    'NBP': ['nBP', 'Fraction of BP', 'Fraction of BP'],\n",
+    "    'stdBP': ['dataBPStd', 'Good pixels only', '$\\sigma$ over pixels'],\n",
+    "    'stdASIC': ['', '', '$\\sigma$ over ASICs'],\n",
+    "    'stdCell': ['', '', '$\\sigma$ over Cells'],\n",
+    "}\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print('Plot calibration constants')\n",
+    "\n",
+    "# loop over constat type\n",
+    "for const, modules in ret_constants.items():\n",
+    "\n",
+    "        const, gain, temp, int_time = const.split(\"_\")\n",
+    "        print('Const: {}'.format(const))\n",
+    "\n",
+    "        # loop over modules\n",
+    "        mod_data = {}\n",
+    "        mod_data['stdASIC'] = []\n",
+    "        mod_data['stdCell'] = []\n",
+    "        mod_names = []\n",
+    "        mod_times = []\n",
+    "\n",
+    "        # Loop over modules\n",
+    "        for mod, data in modules.items():\n",
+    "            print(mod)\n",
+    "\n",
+    "            ctimes = np.array(data[\"ctime\"])\n",
+    "            ctimes_ticks = [x.strftime('%y-%m-%d') for x in ctimes]\n",
+    "\n",
+    "            if (\"mdata\" in data):\n",
+    "                cmdata = np.array(data[\"mdata\"])\n",
+    "                for i, tick in enumerate(ctimes_ticks):\n",
+    "                    ctimes_ticks[i] = ctimes_ticks[i] + \\\n",
+    "                        ', V={:1.0f}'.format(cmdata[i]['Sensor Temperature']) + \\\n",
+    "                        ', T={:1.0f}'.format(\n",
+    "                        cmdata[i]['Integration Time'])\n",
+    "\n",
+    "            sort_ind = np.argsort(ctimes_ticks)\n",
+    "            ctimes_ticks = list(np.array(ctimes_ticks)[sort_ind])\n",
+    "\n",
+    "            # Create sorted by data dataset\n",
+    "            rdata = {}\n",
+    "            for key, item in keys.items():\n",
+    "                if item[0] in data:\n",
+    "                    rdata[key] = np.array(data[item[0]])[sort_ind]\n",
+    "\n",
+    "            nTimes = rdata['Mean'].shape[0]\n",
+    "            nPixels = rdata['Mean'].shape[1] * rdata['Mean'].shape[2]\n",
+    "            nBins = nPixels\n",
+    "\n",
+    "            # Avoid to low values\n",
+    "            if const in [\"Noise\", \"Offset\", \"Noise-e\"]:\n",
+    "                rdata['Mean'][rdata['Mean'] < 0.1] = np.nan\n",
+    "                if 'MeanBP' in rdata:\n",
+    "                    rdata['MeanBP'][rdata['MeanBP'] < 0.1] = np.nan\n",
+    "                    \n",
+    "            if 'NBP' in rdata:\n",
+    "                rdata[\"NBP\"] = rdata[\"NBP\"] / spShape[0] / spShape[1] * 100\n",
+    "\n",
+    "            # Reshape: ASICs over cells for plotting\n",
+    "            pdata = {}\n",
+    "            for key in rdata:\n",
+    "                if len(rdata[key].shape)<3:\n",
+    "                    continue\n",
+    "                pdata[key] = rdata[key][:, :, :].reshape(nTimes, nBins).swapaxes(0, 1)\n",
+    "\n",
+    "            # Summary over ASICs\n",
+    "            adata = {}\n",
+    "            for key in rdata:\n",
+    "                if len(rdata[key].shape)<3:\n",
+    "                    continue\n",
+    "                adata[key] = np.nanmean(rdata[key], axis=(1, 2))\n",
+    "\n",
+    "            # Summary information over modules\n",
+    "            for key in pdata:\n",
+    "                if key not in mod_data:\n",
+    "                    mod_data[key] = []\n",
+    "                mod_data[key].append(np.nanmean(pdata[key], axis=0))\n",
+    "\n",
+    "            mod_data['stdASIC'].append(np.nanstd(rdata['Mean'], axis=(1, 2)))\n",
+    "\n",
+    "            mod_names.append(mod)\n",
+    "            mod_times.append(ctimes_ticks)\n",
+    "\n",
+    "            # Plotting\n",
+    "            for key in pdata:\n",
+    "                if len(pdata[key].shape)<2:\n",
+    "                    continue\n",
+    "\n",
+    "                if key == 'NBP':\n",
+    "                    unit = '[%]'\n",
+    "                else:\n",
+    "                    unit = '[ADU]'\n",
+    "                    if const == 'Noise-e':\n",
+    "                        unit = '[$e^-$]'\n",
+    "\n",
+    "                title = '{}, module {}, gain {} {}'.format(\n",
+    "                    const, mod, gain, keys[key][1])\n",
+    "                cb_label = '{}, {} {}'.format(const, keys[key][2], unit)\n",
+    "\n",
+    "                vmin,vmax = get_range(pdata[key][::-1].flatten(), plot_range)\n",
+    "                hm_combine(pdata[key][::-1], htype=HMType.mro,\n",
+    "                          x_label='Creation Time', y_label='ASIC ID',\n",
+    "                          x_ticklabels=ctimes_ticks,\n",
+    "                          x_ticks=np.arange(len(ctimes_ticks))+0.3,\n",
+    "                          title=title, cb_label=cb_label,\n",
+    "                          vmin=vmin, vmax=vmax,\n",
+    "                          fname='{}/{}_{}_g{}_t{}_t{}_ASIC_{}.png'.format(\n",
+    "                                  out_folder, const, mod.replace('_', ''), gain, temp, int_time, key),\n",
+    "                          pad=[0.125, 0.125, 0.12, 0.185])\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.7"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py
index 71da69695..fe9df1c8a 100644
--- a/xfel_calibrate/notebooks.py
+++ b/xfel_calibrate/notebooks.py
@@ -129,6 +129,12 @@ notebooks = {
                                                "use function": "balance_sequences",
                                                "cluster cores": 4},
                                 },
+                       "STATS_FROM_DB":   {
+                               "notebook": "notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb",
+                               "concurrency": {"parameter": None,
+                                                "default concurrency": None,
+                                                "cluster cores": 1},
+                               },
                        },
              "JUNGFRAU": {
                         "DARK": {
-- 
GitLab


From 273145be398dabfa495f6122f00194d52924b609 Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Mon, 2 Sep 2019 10:15:50 +0200
Subject: [PATCH 2/6] Cleanup code

---
 .../FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb   | 66 ++++++-------------
 1 file changed, 20 insertions(+), 46 deletions(-)

diff --git a/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb b/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
index 406e88e09..029a487d1 100644
--- a/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
+++ b/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
@@ -23,11 +23,11 @@
     "nconstants = 10 # Number of time stamps to plot. If not 0, overcome start_date.\n",
     "dclass=\"CCD\" # Detector class\n",
     "db_module = \"fastCCD1\" # detector entry in the DB to investigate\n",
-    "constants = [\"Noise\"]#, \"Offset\"] # constants to plot\n",
+    "constants = [\"Noise\", \"Offset\"] # constants to plot\n",
     "\n",
     "gain_setting = [0,1,2,8] # gain stages\n",
     "bias_voltage = [79] # Bias voltage\n",
-    "temperature = [235]#, 216, 245] # Operation temperature\n",
+    "temperature = [235, 216, 245] # Operation temperature\n",
     "integration_time = [1, 50] # Integration time\n",
     "pixels_x=[1934]\n",
     "pixels_y=[960]\n",
@@ -35,7 +35,7 @@
     "parameter_names = ['bias_voltage', 'integration_time', 'temperature', \n",
     "                   'gain_setting', 'pixels_x', 'pixels_y'] # names of parameters\n",
     "photon_energy = 9.2 # Photon energy of the beam\n",
-    "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_FCCD5/\" # output folder\n",
+    "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_FCCD6/\" # output folder\n",
     "use_existing = \"\" # If not empty, constants stored in given folder will be used\n",
     "cal_db_interface = \"tcp://max-exfl016:8015#8025\" # the database interface to use\n",
     "cal_db_timeout = 180000 # timeout on caldb requests\",\n",
@@ -60,15 +60,11 @@
     "import warnings\n",
     "warnings.filterwarnings('ignore')\n",
     "\n",
-    "import h5py\n",
-    "import matplotlib\n",
-    "%matplotlib inline\n",
-    "\n",
     "from iCalibrationDB import Constants, Conditions, Detectors, ConstantMetaData\n",
-    "from cal_tools.tools import get_from_db, get_random_db_interface\n",
+    "from cal_tools.tools import get_from_db\n",
     "from cal_tools.ana_tools import (save_dict_to_hdf5, load_data_from_hdf5, \n",
-    "                                 combine_constants, HMType,\n",
-    "                                 hm_combine, combine_lists, get_range)"
+    "                                 HMType, hm_combine,\n",
+    "                                 combine_lists, get_range)"
    ]
   },
   {
@@ -246,15 +242,18 @@
     "# Remove dublications\n",
     "constant_versions_tmp = []\n",
     "constant_parameters_tmp = []\n",
+    "constantBP_versions_tmp = []\n",
     "for i, x in enumerate(constant_versions):\n",
     "    if x not in constant_versions_tmp:\n",
     "        constant_versions_tmp.append(x)\n",
     "        constant_parameters_tmp.append(constant_parameters[i])\n",
-    "        \n",
+    "        if i<len(constantBP_versions)-1:\n",
+    "            constantBP_versions_tmp.append(constantBP_versions[i])\n",
     "constant_versions=constant_versions_tmp\n",
+    "constantBP_versions=constantBP_versions_tmp\n",
     "constant_parameters=constant_parameters_tmp\n",
-    "        \n",
-    "print('Number of stored constant versions is {}'.format(len(constant_versions)))\n"
+    "\n",
+    "print('Number of stored constant versions is {}'.format(len(constant_versions)))"
    ]
   },
   {
@@ -274,7 +273,7 @@
     "    return data\n",
     "\n",
     "ret_constants = {}\n",
-    "constand_data = ConstantMetaData()\n",
+    "constant_data = ConstantMetaData()\n",
     "constant_BP = ConstantMetaData()\n",
     "for i, constant_version in enumerate(constant_versions):\n",
     "\n",
@@ -283,10 +282,10 @@
     "    \n",
     "    print(\"constant: {}, module {}\".format(const,qm))\n",
     "    \n",
-    "    constand_data.retrieve_from_version_info(constant_version)\n",
+    "    constant_data.retrieve_from_version_info(constant_version)\n",
     "    \n",
     "    # Convert parameters to dict\n",
-    "    dpar = {p.name: p.value for p in constand_data.detector_condition.parameters}\n",
+    "    dpar = {p.name: p.value for p in constant_data.detector_condition.parameters}\n",
     "    \n",
     "    const = \"{}_{}_{}_{}\".format(const, \n",
     "                              constant_parameters[i]['gain_setting'],\n",
@@ -298,8 +297,8 @@
     "    if not qm in ret_constants[const]:\n",
     "            ret_constants[const][qm] = []\n",
     "    \n",
-    "    cdata = constand_data.calibration_constant.data\n",
-    "    ctime = constand_data.calibration_constant_version.begin_at\n",
+    "    cdata = constant_data.calibration_constant.data\n",
+    "    ctime = constant_data.calibration_constant_version.begin_at\n",
     "    \n",
     "    cdata = modify_const(const, cdata)\n",
     "    \n",
@@ -335,7 +334,7 @@
     "    toStore = np.nanmean(cdata, axis=(1, 3))\n",
     "    \n",
     "    # Convert parameters to dict\n",
-    "    dpar = {p.name: p.value for p in constand_data.detector_condition.parameters}\n",
+    "    dpar = {p.name: p.value for p in constant_data.detector_condition.parameters}\n",
     "    \n",
     "    print(\"Store values in dict\", const, qm, ctime)\n",
     "    ret_constants[const][qm].append({'ctime': ctime,\n",
@@ -356,7 +355,7 @@
    "outputs": [],
    "source": [
     "if use_existing == \"\":\n",
-    "    print('Save data to /CalDBAna_{}_{}.h5'.format(dclass, db_module))\n",
+    "    print('Save data to {}/CalDBAna_{}_{}.h5'.format(out_folder, dclass, db_module))\n",
     "    save_dict_to_hdf5(ret_constants,\n",
     "                      '{}/CalDBAna_{}_{}.h5'.format(out_folder, dclass, db_module))"
    ]
@@ -409,13 +408,6 @@
     "        const, gain, temp, int_time = const.split(\"_\")\n",
     "        print('Const: {}'.format(const))\n",
     "\n",
-    "        # loop over modules\n",
-    "        mod_data = {}\n",
-    "        mod_data['stdASIC'] = []\n",
-    "        mod_data['stdCell'] = []\n",
-    "        mod_names = []\n",
-    "        mod_times = []\n",
-    "\n",
     "        # Loop over modules\n",
     "        for mod, data in modules.items():\n",
     "            print(mod)\n",
@@ -444,7 +436,7 @@
     "            nPixels = rdata['Mean'].shape[1] * rdata['Mean'].shape[2]\n",
     "            nBins = nPixels\n",
     "\n",
-    "            # Avoid to low values\n",
+    "            # Avoid too low values\n",
     "            if const in [\"Noise\", \"Offset\", \"Noise-e\"]:\n",
     "                rdata['Mean'][rdata['Mean'] < 0.1] = np.nan\n",
     "                if 'MeanBP' in rdata:\n",
@@ -460,24 +452,6 @@
     "                    continue\n",
     "                pdata[key] = rdata[key][:, :, :].reshape(nTimes, nBins).swapaxes(0, 1)\n",
     "\n",
-    "            # Summary over ASICs\n",
-    "            adata = {}\n",
-    "            for key in rdata:\n",
-    "                if len(rdata[key].shape)<3:\n",
-    "                    continue\n",
-    "                adata[key] = np.nanmean(rdata[key], axis=(1, 2))\n",
-    "\n",
-    "            # Summary information over modules\n",
-    "            for key in pdata:\n",
-    "                if key not in mod_data:\n",
-    "                    mod_data[key] = []\n",
-    "                mod_data[key].append(np.nanmean(pdata[key], axis=0))\n",
-    "\n",
-    "            mod_data['stdASIC'].append(np.nanstd(rdata['Mean'], axis=(1, 2)))\n",
-    "\n",
-    "            mod_names.append(mod)\n",
-    "            mod_times.append(ctimes_ticks)\n",
-    "\n",
     "            # Plotting\n",
     "            for key in pdata:\n",
     "                if len(pdata[key].shape)<2:\n",
-- 
GitLab


From c8809a80f1451d49d054013ecdd789787e378a72 Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Mon, 2 Sep 2019 11:42:42 +0200
Subject: [PATCH 3/6] simplify code. Include nconstants.

---
 notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb | 91 +++++++------------
 .../FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb   |  2 +-
 notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb     | 74 ++++++---------
 .../ePix/PlotFromCalDB_ePix100_NBC.ipynb      | 70 ++++++--------
 4 files changed, 90 insertions(+), 147 deletions(-)

diff --git a/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb b/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
index 545a9a475..e3efb528a 100644
--- a/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
+++ b/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
@@ -28,12 +28,14 @@
     "cluster_profile = \"noDB\"  # The ipcluster profile to use\n",
     "start_date = \"2019-01-01\"  # Date to start investigation interval from\n",
     "end_date = \"2019-12-12\"  # Date to end investigation interval at, can be \"now\"\n",
+    "nconstants = 20 # Number of time stamps to plot. If not 0, overcome start_date.\n",
     "constants = [\"Noise\", \"SlopesFF\", \"SlopesPC\", \"Offset\"]  # Constants to plot\n",
     "modules = [1]  # Modules, set to -1 for all, range allowed\n",
     "bias_voltages = [300, 500]  # Bias voltage\n",
-    "mem_cells = [128, 176]  # Number of used memory cells. Typically: 4,32,64,128,176.\n",
+    "mem_cells = [128, 176, 250]  # Number of used memory cells. Typically: 4,32,64,128,176.\n",
+    "acquisition_rate = [None]\n",
     "photon_energy = 9.2  # Photon energy of the beam\n",
-    "out_folder = \"/gpfs/exfel/data/scratch/karnem/testAGIPD16_21/\"  # Output folder, required\n",
+    "out_folder = \"/gpfs/exfel/data/scratch/karnem/testAGIPD16_25/\"  # Output folder, required\n",
     "use_existing = \"\" # If not empty, constants stored in given folder will be used\n",
     "cal_db_timeout = 120000 # timeout on caldb requests\",\n",
     "adu_to_photon = 33.17 # ADU to photon conversion factor (8000 / 3.6 / 67.)\n",
@@ -47,7 +49,8 @@
     "range_gain = [0.8, 1.2, 0.8, 1.2] # plotting range for gain: high gain l, r, medium gain l, r \n",
     "range_noise_e = [85., 500., 85., 500.] # plotting range for noise in [e-]: high gain l, r, medium gain l, r \n",
     "range_slopesPC = [22.0, 27.0, -0.5, 1.5] # plotting range for slope PC: high gain l, r, medium gain l, r \n",
-    "range_slopesFF = [0.8, 1.2, 0.6, 1.2] # plotting range for slope FF: high gain l, r, medium gain l, r "
+    "range_slopesFF = [0.8, 1.2, 0.6, 1.2] # plotting range for slope FF: high gain l, r, medium gain l, r \n",
+    "plot_range = 3 # range for plotting in units of median absolute deviations"
    ]
   },
   {
@@ -67,13 +70,11 @@
     "import warnings\n",
     "warnings.filterwarnings('ignore')\n",
     "\n",
-    "import h5py\n",
-    "\n",
     "from iCalibrationDB import Constants, Conditions, Detectors, ConstantMetaData\n",
     "from cal_tools.tools import get_from_db, get_random_db_interface\n",
     "from cal_tools.ana_tools import (save_dict_to_hdf5, load_data_from_hdf5, \n",
     "                                 combine_constants, HMType,\n",
-    "                                 hm_combine, combine_lists)"
+    "                                 hm_combine, combine_lists, get_range)"
    ]
   },
   {
@@ -122,7 +123,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "parameter_list = combine_lists(bias_voltages, modules, mem_cells, names = ['bias_voltage', 'module', 'mem_cells'])\n",
+    "parameter_list = combine_lists(bias_voltages, modules, mem_cells, acquisition_rate,\n",
+    "                               names = ['bias_voltage', 'module', 'mem_cells', 'acquisition_rate'])\n",
     "print(parameter_list)"
    ]
   },
@@ -152,13 +154,15 @@
     "            dcond = Conditions.Dark\n",
     "            mcond = getattr(dcond, dclass)(\n",
     "                        memory_cells=pars['mem_cells'],\n",
-    "                        bias_voltage=pars['bias_voltage'])\n",
+    "                        bias_voltage=pars['bias_voltage'],\n",
+    "                        acquisition_rate=pars['acquisition_rate'])\n",
     "        else:\n",
     "            dcond = Conditions.Illuminated\n",
     "            mcond = getattr(dcond, dclass)(\n",
-    "                                memory_cells=pars['mem_cells'],\n",
-    "                                bias_voltage=pars['bias_voltage'],\n",
-    "                                photon_energy=photon_energy)\n",
+    "                        memory_cells=pars['mem_cells'],\n",
+    "                        bias_voltage=pars['bias_voltage'],\n",
+    "                        acquisition_rate=pars['acquisition_rate'],\n",
+    "                        photon_energy=photon_energy)\n",
     "\n",
     "        print('Request: ', const, 'with paramters:', pars)\n",
     "        # Request Constant versions for given parameters and module\n",
@@ -173,7 +177,6 @@
     "                           meta_only=True,\n",
     "                           version_info=True)\n",
     "\n",
-    "        print(data)\n",
     "        # Request BP constant versions\n",
     "        print('constantDark:', constantsDark[const], )        \n",
     "        dataBP = get_from_db(getattr(det, pars['module']),\n",
@@ -187,13 +190,12 @@
     "                             meta_only=True,\n",
     "                             version_info=True)\n",
     "        \n",
-    "        print('BP!!!!!', dataBP)\n",
-    "        \n",
     "        if not isinstance(data, list) or not isinstance(dataBP, list):\n",
     "            continue\n",
     "            \n",
     "        found_BPmatch = False\n",
     "        for d in data:\n",
+    "            print('Item: ', d)\n",
     "            # Match proper BP constant version\n",
     "            # and get constant version within\n",
     "            # requested time range\n",
@@ -203,7 +205,8 @@
     "\n",
     "            dt = dateutil.parser.parse(d['begin_at'])\n",
     "\n",
-    "            if dt.replace(tzinfo=None) > end or dt.replace(tzinfo=None) < start:\n",
+    "            if (dt.replace(tzinfo=None) > end or \n",
+    "                (nconstants==0 and dt.replace(tzinfo=None) < start)):\n",
     "                continue\n",
     "                \n",
     "            closest_BP = None\n",
@@ -302,25 +305,31 @@
     "ret_constants = {}\n",
     "constand_data = ConstantMetaData()\n",
     "constant_BP = ConstantMetaData()\n",
-    "for i, constant_version in enumerate(constant_versions):\n",
+    "# sort over begin_at\n",
+    "idxs, _ = zip(*sorted(enumerate(constant_versions), \n",
+    "                     key=lambda x: x[1]['begin_at'], reverse=True))\n",
     "\n",
-    "    const = constant_version['data_set_name'].split('/')[-2]\n",
+    "for i in idxs:\n",
+    "    const = constant_versions[i]['data_set_name'].split('/')[-2]\n",
     "    qm = constant_parameters[i]['module']\n",
     "    \n",
-    "    constand_data.retrieve_from_version_info(constant_version)\n",
+    "    if not const in ret_constants:\n",
+    "        ret_constants[const] = {}\n",
+    "    if not qm in ret_constants[const]:\n",
+    "            ret_constants[const][qm] = []\n",
+    "            \n",
+    "    if nconstants>0 and len(ret_constants[const][qm])>=nconstants:\n",
+    "        continue\n",
+    "            \n",
+    "    constand_data.retrieve_from_version_info(constant_versions[i])\n",
     "    constant_BP.retrieve_from_version_info(constantBP_versions[i])\n",
     "    \n",
     "    cdata = constand_data.calibration_constant.data\n",
     "    cdataBP = constant_BP.calibration_constant.data\n",
     "    ctime = constand_data.calibration_constant_version.begin_at  \n",
-    "    \n",
+    "   \n",
     "    print(\"constant: {}, module {}, begin_at {}\".format(const, qm, ctime))\n",
-    "    \n",
-    "    if not const in ret_constants:\n",
-    "        ret_constants[const] = {}\n",
-    "    if not qm in ret_constants[const]:\n",
-    "            ret_constants[const][qm] = []\n",
-    "    \n",
+    "\n",
     "    cdata = modify_const(const, cdata)\n",
     "    cdataBP = modify_const(const, cdataBP, True)\n",
     "\n",
@@ -540,20 +549,12 @@
     "        else:\n",
     "            pass\n",
     "\n",
-    "        # loop over modules\n",
-    "        mod_data = {}\n",
-    "        mod_data['stdASIC'] = []\n",
-    "        mod_data['stdCell'] = []\n",
-    "        mod_names = []\n",
-    "        mod_times = []\n",
-    "\n",
     "        # Loop over modules\n",
     "        for mod, data in mods.items():\n",
     "            if mod not in modules:\n",
     "                continue\n",
     "\n",
     "            print(mod)\n",
-    "\n",
     "            ctimes = np.array(data[\"ctime\"])\n",
     "            ctimes_ticks = [x.strftime('%y-%m-%d') for x in ctimes]\n",
     "\n",
@@ -604,24 +605,9 @@
     "            for key in rdata:\n",
     "                adata[key] = np.nanmean(rdata[key], axis=(1, 2)).swapaxes(0, 1)\n",
     "\n",
-    "            # Summary information over modules\n",
-    "            for key in pdata:\n",
-    "                if key not in mod_data:\n",
-    "                    mod_data[key] = []\n",
-    "                mod_data[key].append(np.nanmean(pdata[key], axis=0))\n",
-    "\n",
-    "            mod_data['stdASIC'].append(np.nanstd(\n",
-    "                np.nanmean(rdata['Mean'][:, :, :, :nMemToShow], axis=(1, 2)), axis=1))\n",
-    "            mod_data['stdCell'].append(np.nanstd(\n",
-    "                np.nanmean(rdata['Mean'][:, :, :, :nMemToShow], axis=3), axis=(1, 2)))\n",
-    "\n",
-    "            mod_names.append(mod)\n",
-    "            mod_times.append(ctimes_ticks)\n",
-    "\n",
     "            # Plotting\n",
     "            for key in pdata:\n",
-    "                vmin = None\n",
-    "                vmax = None\n",
+    "                vmin,vmax = get_range(pdata[key][::-1], plot_range)\n",
     "                if const in rangevals and key in ['Mean', 'MeanBP']:\n",
     "                    vmin = rangevals[const][gain][0]\n",
     "                    vmax = rangevals[const][gain][1]\n",
@@ -658,13 +644,6 @@
     "                                  out_folder, const, mod, gain, key),\n",
     "                          vmin=vmin, vmax=vmax)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
diff --git a/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb b/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
index 029a487d1..1bd8c1792 100644
--- a/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
+++ b/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
@@ -158,7 +158,7 @@
     "        if not isinstance(data, list):\n",
     "                continue\n",
     "            \n",
-    "        data = sorted(data, key=itemgetter('begin_at'))\n",
+    "        data = sorted(data, key=itemgetter('begin_at'), reverse=True)\n",
     "        print('Number of retrieved constants: {}'.format(len(data)) )\n",
     "                \n",
     "        if const in constantsDark:\n",
diff --git a/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb b/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
index 8ecc446be..b3f50de67 100644
--- a/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
+++ b/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
@@ -22,18 +22,17 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "cluster_profile = \"noDB\"  # The ipcluster profile to use\n",
     "start_date = \"2018-01-30\"  # Date to start investigation interval from\n",
     "end_date = \"2018-12-12\"  # Date to end investigation interval at, can be \"now\"\n",
-    "constants = [\"Offset\", \"Noise\", \"SlopesFF\", \"SlopesCI\"] # constants to plot\n",
+    "nconstants = 10 # Number of time stamps to plot. If not 0, overcome start_date.\n",
+    "constants = [\"Offset\"]#, \"Noise\", \"SlopesFF\", \"SlopesCI\"] # constants to plot\n",
     "modules = [2]  # Modules, set to -1 for all, range allowed\n",
     "bias_voltages = [250, 500]  # Bias voltage\n",
-    "mem_cells = [1, 128, 256, 512]  # Number of used memory cells. Typically: 4,32,64,128,176.\n",
+    "mem_cells = [128, 256, 512]  # Number of used memory cells.\n",
     "photon_energy = 9.2  # Photon energy of the beam\n",
     "out_folder = \"/gpfs/exfel/data/scratch/karnem/testLPD_11/\"  # Output folder, required\n",
     "use_existing = \"\" # If not empty, constants stored in given folder will be used\n",
@@ -49,7 +48,8 @@
     "range_gain = [20, 30, 20, 30] # plotting range for gain: high gain l, r, medium gain l, r \n",
     "range_noise_e = [100., 600., 100., 600.] # plotting range for noise in [e-]: high gain l, r, medium gain l, r \n",
     "range_slopesCI = [0.95, 1.05, 0.0, 0.5] # plotting range for slope CI: high gain l, r, medium gain l, r \n",
-    "range_slopesFF = [0.8, 1.2, 0.8, 1.2] # plotting range for slope FF: high gain l, r, medium gain l, r "
+    "range_slopesFF = [0.8, 1.2, 0.8, 1.2] # plotting range for slope FF: high gain l, r, medium gain l, r \n",
+    "plot_range = 3 # range for plotting in units of median absolute deviations"
    ]
   },
   {
@@ -69,13 +69,11 @@
     "import warnings\n",
     "warnings.filterwarnings('ignore')\n",
     "\n",
-    "import h5py\n",
-    "\n",
     "from iCalibrationDB import Constants, Conditions, Detectors, ConstantMetaData\n",
     "from cal_tools.tools import get_from_db, get_random_db_interface\n",
     "from cal_tools.ana_tools import (save_dict_to_hdf5, load_data_from_hdf5, \n",
     "                                 combine_constants, HMType,\n",
-    "                                 hm_combine, combine_lists)"
+    "                                 hm_combine, combine_lists, get_range)"
    ]
   },
   {
@@ -203,7 +201,8 @@
     "\n",
     "            dt = dateutil.parser.parse(d['begin_at'])\n",
     "\n",
-    "            if dt.replace(tzinfo=None) > end or dt.replace(tzinfo=None) < start:\n",
+    "            if (dt.replace(tzinfo=None) > end or \n",
+    "                (nconstants==0 and dt.replace(tzinfo=None) < start)):\n",
     "                continue\n",
     "                \n",
     "            closest_BP = None\n",
@@ -293,12 +292,24 @@
     "ret_constants = {}\n",
     "constand_data = ConstantMetaData()\n",
     "constant_BP = ConstantMetaData()\n",
-    "for i, constant_version in enumerate(constant_versions):\n",
     "\n",
-    "    const = constant_version['data_set_name'].split('/')[-2]\n",
+    "# sort over begin_at\n",
+    "idxs, _ = zip(*sorted(enumerate(constant_versions), \n",
+    "                     key=lambda x: x[1]['begin_at'], reverse=True))\n",
+    "\n",
+    "for i in idxs:\n",
+    "    const = constant_versions[i]['data_set_name'].split('/')[-2]\n",
     "    qm = constant_parameters[i]['module']\n",
     "    \n",
-    "    constand_data.retrieve_from_version_info(constant_version)\n",
+    "    if not const in ret_constants:\n",
+    "        ret_constants[const] = {}\n",
+    "    if not qm in ret_constants[const]:\n",
+    "            ret_constants[const][qm] = []\n",
+    "    \n",
+    "    if nconstants>0 and len(ret_constants[const][qm])>=nconstants:\n",
+    "        continue\n",
+    "        \n",
+    "    constand_data.retrieve_from_version_info(constant_versions[i])\n",
     "    constant_BP.retrieve_from_version_info(constantBP_versions[i])\n",
     "    \n",
     "    cdata = constand_data.calibration_constant.data\n",
@@ -306,12 +317,7 @@
     "    ctime = constand_data.calibration_constant_version.begin_at  \n",
     "    \n",
     "    print(\"constant: {}, module {}, begin_at {}\".format(const, qm, ctime))\n",
-    "    \n",
-    "    if not const in ret_constants:\n",
-    "        ret_constants[const] = {}\n",
-    "    if not qm in ret_constants[const]:\n",
-    "            ret_constants[const][qm] = []\n",
-    "    \n",
+    "\n",
     "    cdata = modify_const(const, cdata)\n",
     "    cdataBP = modify_const(const, cdataBP)\n",
     "\n",
@@ -481,9 +487,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "collapsed": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "# Parameters for plotting\n",
@@ -515,7 +519,7 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "scrolled": true
+    "scrolled": false
    },
    "outputs": [],
    "source": [
@@ -533,13 +537,6 @@
     "        else:\n",
     "            pass\n",
     "\n",
-    "        # loop over modules\n",
-    "        mod_data = {}\n",
-    "        mod_data['stdASIC'] = []\n",
-    "        mod_data['stdCell'] = []\n",
-    "        mod_names = []\n",
-    "        mod_times = []\n",
-    "\n",
     "        # Loop over modules\n",
     "        for mod, data in mods.items():\n",
     "            \n",
@@ -598,24 +595,9 @@
     "            for key in rdata:\n",
     "                adata[key] = np.nanmean(rdata[key], axis=(1, 2)).swapaxes(0, 1)\n",
     "\n",
-    "            # Summary information over modules\n",
-    "            for key in pdata:\n",
-    "                if key not in mod_data:\n",
-    "                    mod_data[key] = []\n",
-    "                mod_data[key].append(np.nanmean(pdata[key], axis=0))\n",
-    "\n",
-    "            mod_data['stdASIC'].append(np.nanstd(\n",
-    "                np.nanmean(rdata['Mean'][:, :, :, :nMemToShow], axis=(1, 2)), axis=1))\n",
-    "            mod_data['stdCell'].append(np.nanstd(\n",
-    "                np.nanmean(rdata['Mean'][:, :, :, :nMemToShow], axis=3), axis=(1, 2)))\n",
-    "\n",
-    "            mod_names.append(mod)\n",
-    "            mod_times.append(ctimes_ticks)\n",
-    "\n",
     "            # Plotting\n",
     "            for key in pdata:\n",
-    "                vmin = None\n",
-    "                vmax = None\n",
+    "                vmin,vmax = get_range(pdata[key][::-1].flatten(), plot_range)\n",
     "                if const in rangevals and key in ['Mean', 'MeanBP']:\n",
     "                    vmin = rangevals[const][gain][0]\n",
     "                    vmax = rangevals[const][gain][1]\n",
diff --git a/notebooks/ePix/PlotFromCalDB_ePix100_NBC.ipynb b/notebooks/ePix/PlotFromCalDB_ePix100_NBC.ipynb
index 22aecccb9..e6163d8d4 100644
--- a/notebooks/ePix/PlotFromCalDB_ePix100_NBC.ipynb
+++ b/notebooks/ePix/PlotFromCalDB_ePix100_NBC.ipynb
@@ -24,6 +24,7 @@
     "cluster_profile = \"noDB\" # The ipcluster profile to use\n",
     "start_date = \"2019-01-30\" # date to start investigation interval from\n",
     "end_date = \"2019-05-01\" # date to end investigation interval at, can be \"now\"\n",
+    "nconstants = 10 # Number of time stamps to plot. If not 0, overcome start_date.\n",
     "dclass=\"ePix100\" # Detector class\n",
     "db_module = \"ePix100_M15\" # detector entry in the DB to investigate\n",
     "constants = [\"Noise\", \"Offset\"] # constants to plot\n",
@@ -35,10 +36,11 @@
     "photon_energy = 9.2 # Photon energy of the beam\n",
     "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_ePix/\" # output folder\n",
     "use_existing = \"\" # If not empty, constants stored in given folder will be used\n",
-    "cal_db_interface = \"tcp://max-exfl016:8016\" # the database interface to use\n",
+    "cal_db_interface = \"tcp://max-exfl016:8015#8025\" # the database interface to use\n",
     "cal_db_timeout = 180000 # timeout on caldb requests\",\n",
-    "range_offset = [1000., 2200] # plotting range for offset: high gain l, r, medium gain l, r \n",
-    "range_noise = [1.5, 3.3] # plotting range for noise: high gain l, r, medium gain l, r "
+    "range_offset = [1000., 2200] # plotting range for offset\n",
+    "range_noise = [1.5, 3.3] # plotting range for noise\n",
+    "plot_range = 3 # range for plotting in units of median absolute deviations"
    ]
   },
   {
@@ -58,13 +60,11 @@
     "import warnings\n",
     "warnings.filterwarnings('ignore')\n",
     "\n",
-    "import h5py\n",
-    "\n",
     "from iCalibrationDB import Constants, Conditions, Detectors, ConstantMetaData\n",
     "from cal_tools.tools import get_from_db, get_random_db_interface\n",
     "from cal_tools.ana_tools import (save_dict_to_hdf5, load_data_from_hdf5, \n",
-    "                                 combine_constants, HMType,\n",
-    "                                 hm_combine, combine_lists)"
+    "                                 HMType, hm_combine, \n",
+    "                                 combine_lists, get_range)"
    ]
   },
   {
@@ -170,8 +170,6 @@
     "                                 meta_only=True,\n",
     "                                 version_info=True)\n",
     "            \n",
-    "            print(dataBP)\n",
-    "        \n",
     "            if not isinstance(data, list) or not isinstance(dataBP, list):\n",
     "                continue\n",
     "            \n",
@@ -186,7 +184,8 @@
     "\n",
     "                dt = dateutil.parser.parse(d['begin_at'])\n",
     "\n",
-    "                if dt.replace(tzinfo=None) > end or dt.replace(tzinfo=None) < start:\n",
+    "                if (dt.replace(tzinfo=None) > end or \n",
+    "                    (nconstants==0 and dt.replace(tzinfo=None) < start)):\n",
     "                    continue\n",
     "\n",
     "                closest_BP = None\n",
@@ -236,12 +235,15 @@
     "# Remove dublications\n",
     "constant_versions_tmp = []\n",
     "constant_parameters_tmp = []\n",
+    "constantBP_versions_tmp = []\n",
     "for i, x in enumerate(constant_versions):\n",
     "    if x not in constant_versions_tmp:\n",
     "        constant_versions_tmp.append(x)\n",
     "        constant_parameters_tmp.append(constant_parameters[i])\n",
-    "        \n",
+    "        if i<len(constantBP_versions)-1:\n",
+    "            constantBP_versions_tmp.append(constantBP_versions[i])\n",
     "constant_versions=constant_versions_tmp\n",
+    "constantBP_versions=constantBP_versions_tmp\n",
     "constant_parameters=constant_parameters_tmp\n",
     "\n",
     "print('Number of stored constant versions is {}'.format(len(constant_versions)))\n"
@@ -266,19 +268,25 @@
     "ret_constants = {}\n",
     "constand_data = ConstantMetaData()\n",
     "constant_BP = ConstantMetaData()\n",
-    "for i, constant_version in enumerate(constant_versions):\n",
     "\n",
-    "    const = constant_version['data_set_name'].split('/')[-2]\n",
+    "# sort over begin_at\n",
+    "idxs, _ = zip(*sorted(enumerate(constant_versions), \n",
+    "                     key=lambda x: x[1]['begin_at'], reverse=True))\n",
+    "\n",
+    "for i in idxs:\n",
+    "    const = constant_versions[i]['data_set_name'].split('/')[-2]\n",
     "    qm = db_module\n",
     "    \n",
-    "    print(\"constant: {}, module {}\".format(const,qm))\n",
-    "    \n",
-    "    constand_data.retrieve_from_version_info(constant_version)\n",
-    "    \n",
     "    if not const in ret_constants:\n",
     "        ret_constants[const] = {}\n",
     "    if not qm in ret_constants[const]:\n",
     "            ret_constants[const][qm] = []\n",
+    "            \n",
+    "    if nconstants>0 and len(ret_constants[const][qm])>=nconstants:\n",
+    "        continue\n",
+    "        \n",
+    "    print(\"constant: {}, module {}\".format(const,qm))\n",
+    "    constand_data.retrieve_from_version_info(constant_versions[i])\n",
     "    \n",
     "    cdata = constand_data.calibration_constant.data\n",
     "    ctime = constand_data.calibration_constant_version.begin_at\n",
@@ -374,13 +382,6 @@
     "\n",
     "        print('Const: {}'.format(const))\n",
     "\n",
-    "        # loop over modules\n",
-    "        mod_data = {}\n",
-    "        mod_data['stdASIC'] = []\n",
-    "        mod_data['stdCell'] = []\n",
-    "        mod_names = []\n",
-    "        mod_times = []\n",
-    "\n",
     "        # Loop over modules\n",
     "        for mod, data in modules.items():\n",
     "            print(mod)\n",
@@ -422,32 +423,13 @@
     "                    continue\n",
     "                pdata[key] = rdata[key][:, :, :].reshape(nTimes, nBins).swapaxes(0, 1)\n",
     "\n",
-    "            # Summary over ASICs\n",
-    "            adata = {}\n",
-    "            for key in rdata:\n",
-    "                if key not in ['Mean', 'std']:\n",
-    "                    continue\n",
-    "                adata[key] = np.nanmean(rdata[key], axis=(1, 2))\n",
-    "\n",
-    "            # Summary information over modules\n",
-    "            for key in pdata:\n",
-    "                if key not in mod_data:\n",
-    "                    mod_data[key] = []\n",
-    "                mod_data[key].append(np.nanmean(pdata[key], axis=0))\n",
-    "\n",
-    "            mod_data['stdASIC'].append(np.nanstd(rdata['Mean'], axis=(1, 2)))\n",
-    "\n",
-    "            mod_names.append(mod)\n",
-    "            mod_times.append(ctimes_ticks)\n",
-    "\n",
     "            # Plotting\n",
     "            for key in pdata:\n",
     "                \n",
     "                if key not in ['Mean', 'std']:\n",
     "                    continue\n",
     "                    \n",
-    "                vmin = None\n",
-    "                vmax = None\n",
+    "                vmin,vmax = get_range(pdata[key][::-1].flatten(), plot_range)\n",
     "                if const in rangevals and key in ['Mean', 'MeanBP']:\n",
     "                    vmin = rangevals[const][0][0]\n",
     "                    vmax = rangevals[const][0][1]\n",
-- 
GitLab


From aec5abf416731d6ee0db1ba6774b5a5b29567479 Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Mon, 2 Sep 2019 11:59:38 +0200
Subject: [PATCH 4/6] simplify code. Include nconstants.

---
 notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb b/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
index b3f50de67..a9471e229 100644
--- a/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
+++ b/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
@@ -29,7 +29,7 @@
     "start_date = \"2018-01-30\"  # Date to start investigation interval from\n",
     "end_date = \"2018-12-12\"  # Date to end investigation interval at, can be \"now\"\n",
     "nconstants = 10 # Number of time stamps to plot. If not 0, overcome start_date.\n",
-    "constants = [\"Offset\"]#, \"Noise\", \"SlopesFF\", \"SlopesCI\"] # constants to plot\n",
+    "constants = [\"Offset\", \"Noise\", \"SlopesFF\", \"SlopesCI\"] # constants to plot\n",
     "modules = [2]  # Modules, set to -1 for all, range allowed\n",
     "bias_voltages = [250, 500]  # Bias voltage\n",
     "mem_cells = [128, 256, 512]  # Number of used memory cells.\n",
-- 
GitLab


From 50fe51c62ff349c11798c003eab25060791e6285 Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Wed, 4 Sep 2019 19:34:53 +0200
Subject: [PATCH 5/6] Use spShape for plotting

---
 cal_tools/cal_tools/ana_tools.py                  |  3 +--
 notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb     | 10 +++++-----
 notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb |  3 ++-
 notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb         |  9 +++------
 4 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/cal_tools/cal_tools/ana_tools.py b/cal_tools/cal_tools/ana_tools.py
index dce75cf70..83ee3a30f 100644
--- a/cal_tools/cal_tools/ana_tools.py
+++ b/cal_tools/cal_tools/ana_tools.py
@@ -354,8 +354,7 @@ class HMType(Enum):
 
 def get_range(data, scale):
     """
-    Get range, which includes most of the data points.
-    Range is calculated in units of median absolute deviations
+    Return a range calculated by median absolute deviations
 
     :param data: numpy.array of data points
     :param scale: range in units of median absolute deviations
diff --git a/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb b/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
index e3efb528a..09e8c0160 100644
--- a/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
+++ b/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
@@ -32,10 +32,10 @@
     "constants = [\"Noise\", \"SlopesFF\", \"SlopesPC\", \"Offset\"]  # Constants to plot\n",
     "modules = [1]  # Modules, set to -1 for all, range allowed\n",
     "bias_voltages = [300, 500]  # Bias voltage\n",
-    "mem_cells = [128, 176, 250]  # Number of used memory cells. Typically: 4,32,64,128,176.\n",
-    "acquisition_rate = [None]\n",
+    "mem_cells = [128, 176, 202, 250]  # Number of used memory cells. Typically: 4,32,64,128,176.\n",
+    "acquisition_rate = [None, 1.1, 2.2, 4.5]\n",
     "photon_energy = 9.2  # Photon energy of the beam\n",
-    "out_folder = \"/gpfs/exfel/data/scratch/karnem/testAGIPD16_25/\"  # Output folder, required\n",
+    "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_AGIPD/\"  # Output folder, required\n",
     "use_existing = \"\" # If not empty, constants stored in given folder will be used\n",
     "cal_db_timeout = 120000 # timeout on caldb requests\",\n",
     "adu_to_photon = 33.17 # ADU to photon conversion factor (8000 / 3.6 / 67.)\n",
@@ -591,8 +591,8 @@
     "                    rdata['MeanBP'][rdata['MeanBP'] < 0.1] = np.nan\n",
     "\n",
     "            if 'NBP' in rdata:\n",
-    "                rdata[\"NBP\"][rdata[\"NBP\"] == 4096] = np.nan\n",
-    "                rdata[\"NBP\"] = rdata[\"NBP\"] / (64 * 64) * 100\n",
+    "                rdata[\"NBP\"][rdata[\"NBP\"] == (spShape[0] * spShape[1])] = np.nan\n",
+    "                rdata[\"NBP\"] = rdata[\"NBP\"] / (spShape[0] * spShape[1]) * 100\n",
     "\n",
     "            # Reshape: ASICs over cells for plotting\n",
     "            pdata = {}\n",
diff --git a/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb b/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
index 1bd8c1792..b0b88ecd6 100644
--- a/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
+++ b/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
@@ -35,7 +35,7 @@
     "parameter_names = ['bias_voltage', 'integration_time', 'temperature', \n",
     "                   'gain_setting', 'pixels_x', 'pixels_y'] # names of parameters\n",
     "photon_energy = 9.2 # Photon energy of the beam\n",
-    "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_FCCD6/\" # output folder\n",
+    "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_FCCD/\" # output folder\n",
     "use_existing = \"\" # If not empty, constants stored in given folder will be used\n",
     "cal_db_interface = \"tcp://max-exfl016:8015#8025\" # the database interface to use\n",
     "cal_db_timeout = 180000 # timeout on caldb requests\",\n",
@@ -443,6 +443,7 @@
     "                    rdata['MeanBP'][rdata['MeanBP'] < 0.1] = np.nan\n",
     "                    \n",
     "            if 'NBP' in rdata:\n",
+    "                rdata[\"NBP\"][rdata[\"NBP\"] == (spShape[0] * spShape[1])] = np.nan\n",
     "                rdata[\"NBP\"] = rdata[\"NBP\"] / spShape[0] / spShape[1] * 100\n",
     "\n",
     "            # Reshape: ASICs over cells for plotting\n",
diff --git a/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb b/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
index a9471e229..d30fc84ba 100644
--- a/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
+++ b/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
@@ -34,7 +34,7 @@
     "bias_voltages = [250, 500]  # Bias voltage\n",
     "mem_cells = [128, 256, 512]  # Number of used memory cells.\n",
     "photon_energy = 9.2  # Photon energy of the beam\n",
-    "out_folder = \"/gpfs/exfel/data/scratch/karnem/testLPD_11/\"  # Output folder, required\n",
+    "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_LPD/\"  # Output folder, required\n",
     "use_existing = \"\" # If not empty, constants stored in given folder will be used\n",
     "cal_db_timeout = 180000 # timeout on caldb requests\",\n",
     "adu_to_photon = 33.17 # ADU to photon conversion factor (8000 / 3.6 / 67.)\n",
@@ -171,7 +171,6 @@
     "                           meta_only=True,\n",
     "                           version_info=True)\n",
     "\n",
-    "        print(data)\n",
     "        # Request BP constant versions\n",
     "        print('constantDark:', constantsDark[const], )        \n",
     "        dataBP = get_from_db(getattr(det, pars['module']),\n",
@@ -185,8 +184,6 @@
     "                             meta_only=True,\n",
     "                             version_info=True)\n",
     "        \n",
-    "        print('BP!!!!!', dataBP)\n",
-    "        \n",
     "        if not isinstance(data, list) or not isinstance(dataBP, list):\n",
     "            continue\n",
     "            \n",
@@ -581,8 +578,8 @@
     "                    rdata['MeanBP'][rdata['MeanBP'] < 0.1] = np.nan\n",
     "\n",
     "            if 'NBP' in rdata:\n",
-    "                rdata[\"NBP\"][rdata[\"NBP\"] == 4096] = np.nan\n",
-    "                rdata[\"NBP\"] = rdata[\"NBP\"] / (64 * 64) * 100\n",
+    "                rdata[\"NBP\"][rdata[\"NBP\"] == (spShape[0] * spShape[1])] = np.nan\n",
+    "                rdata[\"NBP\"] = rdata[\"NBP\"] / (spShape[0] * spShape[1]) * 100\n",
     "\n",
     "            # Reshape: ASICs over cells for plotting\n",
     "            pdata = {}\n",
-- 
GitLab


From 48439e07954d77d83fa6bc1a593f24f71787a03c Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Thu, 5 Sep 2019 10:55:42 +0200
Subject: [PATCH 6/6] Convert NBP to float

---
 notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb | 121 ++++++++++++++++--
 .../FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb   |   1 +
 notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb     |   1 +
 3 files changed, 114 insertions(+), 9 deletions(-)

diff --git a/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb b/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
index 09e8c0160..7f4b500b3 100644
--- a/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
+++ b/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
@@ -21,7 +21,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -33,7 +33,7 @@
     "modules = [1]  # Modules, set to -1 for all, range allowed\n",
     "bias_voltages = [300, 500]  # Bias voltage\n",
     "mem_cells = [128, 176, 202, 250]  # Number of used memory cells. Typically: 4,32,64,128,176.\n",
-    "acquisition_rate = [None, 1.1, 2.2, 4.5]\n",
+    "acquisition_rate = [0.0, 1.1, 2.2, 4.5]\n",
     "photon_energy = 9.2  # Photon energy of the beam\n",
     "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_AGIPD/\"  # Output folder, required\n",
     "use_existing = \"\" # If not empty, constants stored in given folder will be used\n",
@@ -55,7 +55,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {
     "scrolled": true
    },
@@ -79,9 +79,21 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Bad pixels data:  {'SlopesFF': 'BadPixelsFF', 'SlopesPC': 'BadPixelsPC', 'Noise': 'BadPixelsDark', 'Offset': 'BadPixelsDark'}\n",
+      "CalDB Interface: tcp://max-exfl016:8015#8025\n",
+      "Start time at:  2019-01-01 00:00:00\n",
+      "End time at:  2019-12-12 00:00:00\n",
+      "Modules:  ['Q1M2']\n"
+     ]
+    }
+   ],
    "source": [
     "# Prepare variables\n",
     "nMem = max(mem_cells) # Number of mem Cells to store\n",
@@ -92,6 +104,8 @@
     "    \n",
     "modules = [\"Q{}M{}\".format(x // 4 + 1, x % 4 + 1) for x in modules]\n",
     "\n",
+    "acquisition_rate[acquisition_rate==0] = None\n",
+    "\n",
     "constantsDark = {\"SlopesFF\": 'BadPixelsFF',\n",
     "                 'SlopesPC': 'BadPixelsPC',\n",
     "                 'Noise': 'BadPixelsDark',\n",
@@ -119,9 +133,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[{'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': None}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': 1.1}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': 2.2}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': 4.5}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 176, 'acquisition_rate': None}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 176, 'acquisition_rate': 1.1}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 176, 'acquisition_rate': 2.2}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 176, 'acquisition_rate': 4.5}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 202, 'acquisition_rate': None}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 202, 'acquisition_rate': 1.1}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 202, 'acquisition_rate': 2.2}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 202, 'acquisition_rate': 4.5}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 250, 'acquisition_rate': None}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 250, 'acquisition_rate': 1.1}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 250, 'acquisition_rate': 2.2}, {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 250, 'acquisition_rate': 4.5}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': None}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': 1.1}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': 2.2}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': 4.5}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 176, 'acquisition_rate': None}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 176, 'acquisition_rate': 1.1}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 176, 'acquisition_rate': 2.2}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 176, 'acquisition_rate': 4.5}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 202, 'acquisition_rate': None}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 202, 'acquisition_rate': 1.1}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 202, 'acquisition_rate': 2.2}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 202, 'acquisition_rate': 4.5}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 250, 'acquisition_rate': None}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 250, 'acquisition_rate': 1.1}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 250, 'acquisition_rate': 2.2}, {'bias_voltage': 500, 'module': 'Q1M2', 'mem_cells': 250, 'acquisition_rate': 4.5}]\n"
+     ]
+    }
+   ],
    "source": [
     "parameter_list = combine_lists(bias_voltages, modules, mem_cells, acquisition_rate,\n",
     "                               names = ['bias_voltage', 'module', 'mem_cells', 'acquisition_rate'])\n",
@@ -130,11 +152,91 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "metadata": {
     "scrolled": false
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Request:  Noise with paramters: {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': None}\n",
+      "constantDark: BadPixelsDark\n",
+      "Item:  {'id': 6990, 'name': '20190220_072419_sIdx=0', 'file_name': 'cal.1550647458.8201447.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2019-02-19T15:54:20.000+01:00', 'end_validity_at': None, 'begin_at': '2019-02-19T15:54:20.000+01:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
+      "Found constant Noise: begin at 2019-02-19 15:54:20+01:00\n",
+      "Found bad pixels at 2019-02-19 15:54:20+01:00\n",
+      "Item:  {'id': 5471, 'name': '20181027_075853_sIdx=0', 'file_name': 'cal.1540627132.6379695.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-10-27T09:20:58.000+02:00', 'end_validity_at': None, 'begin_at': '2018-10-27T09:20:58.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
+      "Found constant Noise: begin at 2018-10-27 09:20:58+02:00\n",
+      "Found bad pixels at 2018-10-27 09:20:58+02:00\n",
+      "Item:  {'id': 6791, 'name': '20181213_160158_sIdx=0', 'file_name': 'cal.1544716917.4122818.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-10-22T22:24:16.000+02:00', 'end_validity_at': None, 'begin_at': '2018-10-22T22:24:16.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
+      "Found constant Noise: begin at 2018-10-22 22:24:16+02:00\n",
+      "Found bad pixels at 2018-10-22 22:24:16+02:00\n",
+      "Item:  {'id': 6834, 'name': '20190207_135120_sIdx=0', 'file_name': 'cal.1549547479.2884257.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-10-21T20:55:02.000+02:00', 'end_validity_at': None, 'begin_at': '2018-10-21T20:55:02.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
+      "Found constant Noise: begin at 2018-10-21 20:55:02+02:00\n",
+      "Found bad pixels at 2018-10-21 20:55:02+02:00\n",
+      "Item:  {'id': 4670, 'name': '20180923_021047_sIdx=0', 'file_name': 'cal.1537668646.6458402.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-09-23T04:10:45.000+02:00', 'end_validity_at': None, 'begin_at': '2018-09-23T04:10:45.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
+      "Found constant Noise: begin at 2018-09-23 04:10:45+02:00\n",
+      "Found bad pixels at 2018-09-23 04:10:50+02:00\n",
+      "Item:  {'id': 4546, 'name': '20180923_000251_sIdx=0', 'file_name': 'cal.1537660970.398414.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-09-23T02:02:49.000+02:00', 'end_validity_at': None, 'begin_at': '2018-09-23T02:02:49.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
+      "Found constant Noise: begin at 2018-09-23 02:02:49+02:00\n",
+      "Found bad pixels at 2018-09-23 02:02:57+02:00\n",
+      "Item:  {'id': 4482, 'name': '20180922_235703_sIdx=0', 'file_name': 'cal.1537660622.794079.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-09-23T01:57:01.000+02:00', 'end_validity_at': None, 'begin_at': '2018-09-23T01:57:01.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
+      "Found constant Noise: begin at 2018-09-23 01:57:01+02:00\n",
+      "Found bad pixels at 2018-09-23 01:57:06+02:00\n",
+      "Item:  {'id': 5269, 'name': '20181025_061038_sIdx=0', 'file_name': 'cal.1540447837.275528.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-09-02T16:25:41.000+02:00', 'end_validity_at': None, 'begin_at': '2018-09-02T16:25:41.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
+      "Found constant Noise: begin at 2018-09-02 16:25:41+02:00\n",
+      "Found bad pixels at 2018-09-02 16:25:41+02:00\n",
+      "Item:  {'id': 3431, 'name': '20180901_173920_sIdx=0', 'file_name': 'cal.1535823559.5818477.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-09-01T19:39:13.000+02:00', 'end_validity_at': None, 'begin_at': '2018-09-01T19:39:13.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
+      "Found constant Noise: begin at 2018-09-01 19:39:13+02:00\n",
+      "Found bad pixels at 2018-09-01 19:39:29+02:00\n",
+      "Item:  {'id': 3229, 'name': '20180830_164924_sIdx=0', 'file_name': 'cal.1535647763.2838182.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-08-30T18:49:21.000+02:00', 'end_validity_at': None, 'begin_at': '2018-08-30T18:49:21.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
+      "Found constant Noise: begin at 2018-08-30 18:49:21+02:00\n",
+      "Found bad pixels at 2018-08-30 18:49:26+02:00\n",
+      "Item:  {'id': 6413, 'name': '20181210_082033_sIdx=0', 'file_name': 'cal.1544430032.7715538.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2018-01-29T17:10:23.000+01:00', 'end_validity_at': None, 'begin_at': '2018-01-29T17:10:23.000+01:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
+      "Found constant Noise: begin at 2018-01-29 17:10:23+01:00\n",
+      "Found bad pixels at 2018-01-29 17:10:23+01:00\n",
+      "Item:  {'id': 6348, 'name': '20181210_081739_sIdx=0', 'file_name': 'cal.1544429858.2109814.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2017-12-04T20:15:40.000+01:00', 'end_validity_at': None, 'begin_at': '2017-12-04T20:15:40.000+01:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
+      "Bad pixels are not found!\n",
+      "Item:  {'id': 6590, 'name': '20181210_092654_sIdx=0', 'file_name': 'cal.1544434013.2191806.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2017-10-10T19:00:34.000+02:00', 'end_validity_at': None, 'begin_at': '2017-10-10T19:00:34.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
+      "Found constant Noise: begin at 2017-10-10 19:00:34+02:00\n",
+      "Found bad pixels at 2017-10-10 19:00:34+02:00\n",
+      "Item:  {'id': 6509, 'name': '20181210_085802_sIdx=0', 'file_name': 'cal.1544432281.1941464.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2017-09-19T15:10:49.000+02:00', 'end_validity_at': None, 'begin_at': '2017-09-19T15:10:49.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
+      "Found constant Noise: begin at 2017-09-19 15:10:49+02:00\n",
+      "Found bad pixels at 2017-09-19 15:10:49+02:00\n",
+      "Item:  {'id': 6543, 'name': '20181210_085904_sIdx=0', 'file_name': 'cal.1544432342.903418.h5', 'path_to_file': 'xfel/cal/agipd-type/agipd_siv1_agipdv11_m315/', 'data_set_name': '/AGIPD_SIV1_AGIPDV11_M315/Noise/0', 'flg_deployed': True, 'flg_good_quality': True, 'begin_validity_at': '2017-09-15T23:33:29.000+02:00', 'end_validity_at': None, 'begin_at': '2017-09-15T23:33:29.000+02:00', 'start_idx': 0, 'end_idx': 0, 'raw_data_location': '', 'description': '', 'calibration_constant': {'id': 913, 'name': 'AGIPD-Type_Noise_AGIPD DefJAwEUE0Km+', 'flg_auto_approve': True, 'flg_available': True, 'description': 'Per-pixel (per-memory cell) noise', 'device_type_id': 2, 'calibration_id': 2, 'condition_id': 271}, 'physical_device': {'id': 58, 'name': 'AGIPD_SIV1_AGIPDV11_M315', 'device_type_id': 2, 'flg_available': True, 'parent_id': None, 'description': ''}}\n",
+      "Found constant Noise: begin at 2017-09-15 23:33:29+02:00\n",
+      "Found bad pixels at 2017-09-15 23:33:29+02:00\n",
+      "Request:  Noise with paramters: {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': 1.1}\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "constantDark: BadPixelsDark\n",
+      "Request:  Noise with paramters: {'bias_voltage': 300, 'module': 'Q1M2', 'mem_cells': 128, 'acquisition_rate': 2.2}\n"
+     ]
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-5-a602d73b9ab9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     38\u001b[0m                            \u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcal_db_timeout\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m                            \u001b[0mmeta_only\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m                            version_info=True)\n\u001b[0m\u001b[1;32m     41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m         \u001b[0;31m# Request BP constant versions\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/gpfs/exfel/data/scratch/karnem/calibration3/pycalibration/cal_tools/cal_tools/tools.py\u001b[0m in \u001b[0;36mget_from_db\u001b[0;34m(device, constant, condition, empty_constant, cal_db_interface, creation_time, verbosity, timeout, ntries, meta_only, version_info)\u001b[0m\n\u001b[1;32m    512\u001b[0m                 r = metadata.retrieve(this_interface, timeout=timeout,\n\u001b[1;32m    513\u001b[0m                                       \u001b[0mwhen\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mwhen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmeta_only\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmeta_only\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 514\u001b[0;31m                                       version_info=version_info)\n\u001b[0m\u001b[1;32m    515\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mversion_info\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    516\u001b[0m                     \u001b[0;32mreturn\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/.local/lib/python3.6/site-packages/iCalibrationDB/meta_data.py\u001b[0m in \u001b[0;36mretrieve\u001b[0;34m(self, receiver, when, silent, timeout, meta_only, version_info)\u001b[0m\n\u001b[1;32m    285\u001b[0m             \u001b[0msock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconnect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreceiver\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    286\u001b[0m             \u001b[0msock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_pyobj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 287\u001b[0;31m             \u001b[0mresp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msock\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv_pyobj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    288\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    289\u001b[0m             \u001b[0;32mdel\u001b[0m \u001b[0msock\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m/software/anaconda3/5.2/lib/python3.6/site-packages/zmq/sugar/socket.py\u001b[0m in \u001b[0;36mrecv_pyobj\u001b[0;34m(self, flags)\u001b[0m\n\u001b[1;32m    620\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0many\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mreasons\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m~\u001b[0m\u001b[0mSocket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mmight\u001b[0m \u001b[0mfail\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    621\u001b[0m         \"\"\"\n\u001b[0;32m--> 622\u001b[0;31m         \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mflags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    623\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_deserialize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    624\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket.Socket.recv\u001b[0;34m()\u001b[0m\n",
+      "\u001b[0;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket.Socket.recv\u001b[0;34m()\u001b[0m\n",
+      "\u001b[0;32mzmq/backend/cython/socket.pyx\u001b[0m in \u001b[0;36mzmq.backend.cython.socket._recv_copy\u001b[0;34m()\u001b[0m\n",
+      "\u001b[0;32m/software/anaconda3/5.2/lib/python3.6/site-packages/zmq/backend/cython/checkrc.pxd\u001b[0m in \u001b[0;36mzmq.backend.cython.checkrc._check_rc\u001b[0;34m()\u001b[0m\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
+     ]
+    }
+   ],
    "source": [
     "# Retrieve list of meta-data\n",
     "constant_versions = []\n",
@@ -591,6 +693,7 @@
     "                    rdata['MeanBP'][rdata['MeanBP'] < 0.1] = np.nan\n",
     "\n",
     "            if 'NBP' in rdata:\n",
+    "                rdata['NBP'] = rdata['NBP'].astype(float)\n",
     "                rdata[\"NBP\"][rdata[\"NBP\"] == (spShape[0] * spShape[1])] = np.nan\n",
     "                rdata[\"NBP\"] = rdata[\"NBP\"] / (spShape[0] * spShape[1]) * 100\n",
     "\n",
diff --git a/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb b/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
index b0b88ecd6..e2b9a436f 100644
--- a/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
+++ b/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
@@ -443,6 +443,7 @@
     "                    rdata['MeanBP'][rdata['MeanBP'] < 0.1] = np.nan\n",
     "                    \n",
     "            if 'NBP' in rdata:\n",
+    "                rdata['NBP'] = rdata['NBP'].astype(float)\n",
     "                rdata[\"NBP\"][rdata[\"NBP\"] == (spShape[0] * spShape[1])] = np.nan\n",
     "                rdata[\"NBP\"] = rdata[\"NBP\"] / spShape[0] / spShape[1] * 100\n",
     "\n",
diff --git a/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb b/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
index d30fc84ba..d6eba9461 100644
--- a/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
+++ b/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
@@ -578,6 +578,7 @@
     "                    rdata['MeanBP'][rdata['MeanBP'] < 0.1] = np.nan\n",
     "\n",
     "            if 'NBP' in rdata:\n",
+    "                rdata['NBP'] = rdata['NBP'].astype(float)\n",
     "                rdata[\"NBP\"][rdata[\"NBP\"] == (spShape[0] * spShape[1])] = np.nan\n",
     "                rdata[\"NBP\"] = rdata[\"NBP\"] / (spShape[0] * spShape[1]) * 100\n",
     "\n",
-- 
GitLab