From 6d1aa45d127f95d4bfe8e23f1ef832b2413b35d4 Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Fri, 30 Aug 2019 15:55:30 +0200
Subject: [PATCH] Add plotting constants of FCCD
---
cal_tools/cal_tools/ana_tools.py | 14 +
.../FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb | 531 ++++++++++++++++++
xfel_calibrate/notebooks.py | 6 +
3 files changed, 551 insertions(+)
create mode 100644 notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
diff --git a/cal_tools/cal_tools/ana_tools.py b/cal_tools/cal_tools/ana_tools.py
index f58ace59b..dce75cf70 100644
--- a/cal_tools/cal_tools/ana_tools.py
+++ b/cal_tools/cal_tools/ana_tools.py
@@ -352,6 +352,20 @@ class HMType(Enum):
INSET_AXIS = 2
+def get_range(data, scale):
+ """
+ Get range, which includes most of the data points.
+ Range is calculated in units of median absolute deviations
+
+ :param data: numpy.array of data points
+ :param scale: range in units of median absolute deviations
+ :return:
+ """
+ med = np.nanmedian(data)
+ mad = np.nanmedian(np.abs(data.flatten() - med))
+ return med - scale * mad, med + scale * mad
+
+
def hm_combine(data, fname=None, htype=None, **kwargs):
"""
Plot heatmap for calibration report
diff --git a/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb b/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
new file mode 100644
index 000000000..406e88e09
--- /dev/null
+++ b/notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb
@@ -0,0 +1,531 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Statistical analysis of calibration factors#\n",
+ "\n",
+ "Author: Mikhail Karnevskiy, Steffen Hauf, Version 0.1\n",
+ "\n",
+ "A description of the notebook."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cluster_profile = \"noDB\" # The ipcluster profile to use\n",
+ "start_date = \"2019-01-30\" # date to start investigation interval from\n",
+ "end_date = \"2019-08-30\" # date to end investigation interval at, can be \"now\"\n",
+ "nconstants = 10 # Number of time stamps to plot. If not 0, overcome start_date.\n",
+ "dclass=\"CCD\" # Detector class\n",
+ "db_module = \"fastCCD1\" # detector entry in the DB to investigate\n",
+ "constants = [\"Noise\"]#, \"Offset\"] # constants to plot\n",
+ "\n",
+ "gain_setting = [0,1,2,8] # gain stages\n",
+ "bias_voltage = [79] # Bias voltage\n",
+ "temperature = [235]#, 216, 245] # Operation temperature\n",
+ "integration_time = [1, 50] # Integration time\n",
+ "pixels_x=[1934]\n",
+ "pixels_y=[960]\n",
+ "max_time = 15\n",
+ "parameter_names = ['bias_voltage', 'integration_time', 'temperature', \n",
+ " 'gain_setting', 'pixels_x', 'pixels_y'] # names of parameters\n",
+ "photon_energy = 9.2 # Photon energy of the beam\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_FCCD5/\" # output folder\n",
+ "use_existing = \"\" # If not empty, constants stored in given folder will be used\n",
+ "cal_db_interface = \"tcp://max-exfl016:8015#8025\" # the database interface to use\n",
+ "cal_db_timeout = 180000 # timeout on caldb requests\",\n",
+ "plot_range = 3 # range for plotting in units of median absolute deviations"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "import copy\n",
+ "import datetime\n",
+ "import dateutil.parser\n",
+ "import numpy as np\n",
+ "from operator import itemgetter\n",
+ "import os\n",
+ "import sys\n",
+ "import warnings\n",
+ "warnings.filterwarnings('ignore')\n",
+ "\n",
+ "import h5py\n",
+ "import matplotlib\n",
+ "%matplotlib inline\n",
+ "\n",
+ "from iCalibrationDB import Constants, Conditions, Detectors, ConstantMetaData\n",
+ "from cal_tools.tools import get_from_db, get_random_db_interface\n",
+ "from cal_tools.ana_tools import (save_dict_to_hdf5, load_data_from_hdf5, \n",
+ " combine_constants, HMType,\n",
+ " hm_combine, combine_lists, get_range)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Prepare variables\n",
+ "spShape = (967, 10) # Shape of superpixel\n",
+ "\n",
+ "parameters = [globals()[x] for x in parameter_names]\n",
+ "\n",
+ "constantsDark = {'Noise': 'BadPixelsDark',\n",
+ " 'Offset': 'BadPixelsDark'}\n",
+ "print('Bad pixels data: ', constantsDark)\n",
+ "\n",
+ "# Define parameters in order to perform loop over time stamps\n",
+ "start = datetime.datetime.now() if start_date.upper() == \"NOW\" else dateutil.parser.parse(\n",
+ " start_date)\n",
+ "end = datetime.datetime.now() if end_date.upper() == \"NOW\" else dateutil.parser.parse(\n",
+ " end_date)\n",
+ "\n",
+ "# Create output folder\n",
+ "os.makedirs(out_folder, exist_ok=True)\n",
+ "\n",
+ "# Get getector conditions\n",
+ "det = getattr(Detectors, db_module)\n",
+ "dconstants = getattr(Constants, dclass)(det.detector_type)\n",
+ "\n",
+ "print('CalDB Interface: {}'.format(cal_db_interface))\n",
+ "print('Start time at: ', start)\n",
+ "print('End time at: ', end)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "parameter_list = combine_lists(*parameters, names = parameter_names)\n",
+ "print(parameter_list)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "# Retrieve list of meta-data\n",
+ "constant_versions = []\n",
+ "constant_parameters = []\n",
+ "constantBP_versions = []\n",
+ "\n",
+ "# Loop over constants\n",
+ "for c, const in enumerate(constants):\n",
+ " \n",
+ " if use_existing != \"\":\n",
+ " break\n",
+ " \n",
+ " # Loop over parameters\n",
+ " for pars in parameter_list:\n",
+ " \n",
+ " if (const in [\"Offset\", \"Noise\", \"SlopesPC\"] or \"DARK\" in const.upper()):\n",
+ " dcond = Conditions.Dark\n",
+ " mcond = getattr(dcond, dclass)(**pars)\n",
+ " else:\n",
+ " dcond = Conditions.Illuminated\n",
+ " mcond = getattr(dcond, dclass)(**pars,\n",
+ " photon_energy=photon_energy)\n",
+ "\n",
+ " \n",
+ " \n",
+ " print('Request: ', const, 'with paramters:', pars)\n",
+ " # Request Constant versions for given parameters and module\n",
+ " data = get_from_db(det,\n",
+ " getattr(dconstants,\n",
+ " const)(),\n",
+ " copy.deepcopy(mcond), None,\n",
+ " cal_db_interface,\n",
+ " creation_time=start,\n",
+ " verbosity=0,\n",
+ " timeout=cal_db_timeout,\n",
+ " meta_only=True,\n",
+ " version_info=True)\n",
+ " \n",
+ " if not isinstance(data, list):\n",
+ " continue\n",
+ " \n",
+ " data = sorted(data, key=itemgetter('begin_at'))\n",
+ " print('Number of retrieved constants: {}'.format(len(data)) )\n",
+ " \n",
+ " if const in constantsDark:\n",
+ " # Request BP constant versions\n",
+ " dataBP = get_from_db(det,\n",
+ " getattr(dconstants, \n",
+ " constantsDark[const])(),\n",
+ " copy.deepcopy(mcond), None,\n",
+ " cal_db_interface,\n",
+ " creation_time=start,\n",
+ " verbosity=0,\n",
+ " timeout=cal_db_timeout,\n",
+ " meta_only=True,\n",
+ " version_info=True)\n",
+ " \n",
+ " if not isinstance(data, list) or not isinstance(dataBP, list):\n",
+ " continue\n",
+ " print('Number of retrieved darks: {}'.format(len(dataBP)) )\n",
+ " found_BPmatch = False\n",
+ " for d in data:\n",
+ " # Match proper BP constant version\n",
+ " # and get constant version within\n",
+ " # requested time range\n",
+ " if d is None:\n",
+ " print('Time or data is not found!')\n",
+ " continue\n",
+ "\n",
+ " dt = dateutil.parser.parse(d['begin_at'])\n",
+ "\n",
+ " if (dt.replace(tzinfo=None) > end or \n",
+ " (nconstants==0 and dt.replace(tzinfo=None) < start)):\n",
+ " continue\n",
+ " \n",
+ " if nconstants>0 and constant_parameters.count(pars)>nconstants-1:\n",
+ " break\n",
+ "\n",
+ " closest_BP = None\n",
+ " closest_BPtime = None\n",
+ "\n",
+ " for dBP in dataBP:\n",
+ " if dBP is None:\n",
+ " print(\"Bad pixels are not found!\")\n",
+ " continue\n",
+ "\n",
+ " dt = dateutil.parser.parse(d['begin_at'])\n",
+ " dBPt = dateutil.parser.parse(dBP['begin_at'])\n",
+ "\n",
+ " if dt == dBPt:\n",
+ " found_BPmatch = True\n",
+ " else:\n",
+ "\n",
+ " if np.abs(dBPt-dt).seconds < (max_time*60):\n",
+ " if closest_BP is None:\n",
+ " closest_BP = dBP\n",
+ " closest_BPtime = dBPt\n",
+ " else:\n",
+ " if np.abs(dBPt-dt) < np.abs(closest_BPtime-dt):\n",
+ " closest_BP = dBP\n",
+ " closest_BPtime = dBPt\n",
+ "\n",
+ " if dataBP.index(dBP) == len(dataBP)-1:\n",
+ " if closest_BP:\n",
+ " dBP = closest_BP\n",
+ " dBPt = closest_BPtime\n",
+ " found_BPmatch = True\n",
+ " else:\n",
+ " print('Bad pixels are not found!')\n",
+ "\n",
+ " if found_BPmatch:\n",
+ " print(\"Found constant {}: begin at {}\".format(const, dt))\n",
+ " print(\"Found bad pixels at {}\".format(dBPt))\n",
+ " constantBP_versions.append(dBP)\n",
+ " constant_versions.append(d)\n",
+ " constant_parameters.append(copy.deepcopy(pars))\n",
+ " found_BPmatch = False\n",
+ " break\n",
+ " else:\n",
+ " constant_versions += data\n",
+ " constant_parameters += [copy.deepcopy(pars)]*len(data)\n",
+ "\n",
+ "# Remove dublications\n",
+ "constant_versions_tmp = []\n",
+ "constant_parameters_tmp = []\n",
+ "for i, x in enumerate(constant_versions):\n",
+ " if x not in constant_versions_tmp:\n",
+ " constant_versions_tmp.append(x)\n",
+ " constant_parameters_tmp.append(constant_parameters[i])\n",
+ " \n",
+ "constant_versions=constant_versions_tmp\n",
+ "constant_parameters=constant_parameters_tmp\n",
+ " \n",
+ "print('Number of stored constant versions is {}'.format(len(constant_versions)))\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def get_rebined(a, rebin):\n",
+ " return a[:,:,0].reshape(\n",
+ " int(a.shape[0] / rebin[0]),\n",
+ " rebin[0],\n",
+ " int(a.shape[1] / rebin[1]),\n",
+ " rebin[1])\n",
+ " \n",
+ "def modify_const(const, data, isBP = False):\n",
+ " return data\n",
+ "\n",
+ "ret_constants = {}\n",
+ "constand_data = ConstantMetaData()\n",
+ "constant_BP = ConstantMetaData()\n",
+ "for i, constant_version in enumerate(constant_versions):\n",
+ "\n",
+ " const = constant_version['data_set_name'].split('/')[-2]\n",
+ " qm = db_module\n",
+ " \n",
+ " print(\"constant: {}, module {}\".format(const,qm))\n",
+ " \n",
+ " constand_data.retrieve_from_version_info(constant_version)\n",
+ " \n",
+ " # Convert parameters to dict\n",
+ " dpar = {p.name: p.value for p in constand_data.detector_condition.parameters}\n",
+ " \n",
+ " const = \"{}_{}_{}_{}\".format(const, \n",
+ " constant_parameters[i]['gain_setting'],\n",
+ " constant_parameters[i]['temperature'],\n",
+ " constant_parameters[i]['integration_time'])\n",
+ " \n",
+ " if not const in ret_constants:\n",
+ " ret_constants[const] = {}\n",
+ " if not qm in ret_constants[const]:\n",
+ " ret_constants[const][qm] = []\n",
+ " \n",
+ " cdata = constand_data.calibration_constant.data\n",
+ " ctime = constand_data.calibration_constant_version.begin_at\n",
+ " \n",
+ " cdata = modify_const(const, cdata)\n",
+ " \n",
+ " if len(constantBP_versions)>0:\n",
+ " constant_BP.retrieve_from_version_info(constantBP_versions[i])\n",
+ " cdataBP = constant_BP.calibration_constant.data\n",
+ " cdataBP = modify_const(const, cdataBP, True)\n",
+ " \n",
+ " if cdataBP.shape != cdata.shape:\n",
+ " print('Wrong bad pixel shape! {}, expected {}'.format(cdataBP.shape, cdata.shape))\n",
+ " continue\n",
+ " \n",
+ " # Apply bad pixel mask\n",
+ " cdataABP = np.copy(cdata)\n",
+ " cdataABP[cdataBP > 0] = np.nan\n",
+ " \n",
+ " # Create superpixels for constants with BP applied\n",
+ " cdataABP = get_rebined(cdataABP, spShape)\n",
+ " toStoreBP = np.nanmean(cdataABP, axis=(1, 3))\n",
+ " toStoreBPStd = np.nanstd(cdataABP, axis=(1, 3))\n",
+ "\n",
+ " # Prepare number of bad pixels per superpixels\n",
+ " cdataBP = get_rebined(cdataBP, spShape)\n",
+ " cdataNBP = np.nansum(cdataBP > 0, axis=(1, 3))\n",
+ " else:\n",
+ " toStoreBP = 0\n",
+ " toStoreBPStd = 0\n",
+ " cdataNBP = 0\n",
+ "\n",
+ " # Create superpixels for constants without BP applied\n",
+ " cdata = get_rebined(cdata, spShape)\n",
+ " toStoreStd = np.nanstd(cdata, axis=(1, 3))\n",
+ " toStore = np.nanmean(cdata, axis=(1, 3))\n",
+ " \n",
+ " # Convert parameters to dict\n",
+ " dpar = {p.name: p.value for p in constand_data.detector_condition.parameters}\n",
+ " \n",
+ " print(\"Store values in dict\", const, qm, ctime)\n",
+ " ret_constants[const][qm].append({'ctime': ctime,\n",
+ " 'nBP': cdataNBP,\n",
+ " 'dataBP': toStoreBP,\n",
+ " 'dataBPStd': toStoreBPStd,\n",
+ " 'data': toStore,\n",
+ " 'dataStd': toStoreStd,\n",
+ " 'mdata': dpar}) \n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "if use_existing == \"\":\n",
+ " print('Save data to /CalDBAna_{}_{}.h5'.format(dclass, db_module))\n",
+ " save_dict_to_hdf5(ret_constants,\n",
+ " '{}/CalDBAna_{}_{}.h5'.format(out_folder, dclass, db_module))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if use_existing == \"\":\n",
+ " fpath = '{}/CalDBAna_{}_*.h5'.format(out_folder, dclass)\n",
+ "else:\n",
+ " fpath = '{}/CalDBAna_{}_*.h5'.format(use_existing, dclass)\n",
+ "\n",
+ "print('Load data from {}'.format(fpath))\n",
+ "ret_constants = load_data_from_hdf5(fpath)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Parameters for plotting\n",
+ "\n",
+ "keys = {\n",
+ " 'Mean': ['data', '', 'Mean over pixels'],\n",
+ " 'std': ['dataStd', '', '$\\sigma$ over pixels'],\n",
+ " 'MeanBP': ['dataBP', 'Good pixels only', 'Mean over pixels'],\n",
+ " 'NBP': ['nBP', 'Fraction of BP', 'Fraction of BP'],\n",
+ " 'stdBP': ['dataBPStd', 'Good pixels only', '$\\sigma$ over pixels'],\n",
+ " 'stdASIC': ['', '', '$\\sigma$ over ASICs'],\n",
+ " 'stdCell': ['', '', '$\\sigma$ over Cells'],\n",
+ "}\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print('Plot calibration constants')\n",
+ "\n",
+ "# loop over constat type\n",
+ "for const, modules in ret_constants.items():\n",
+ "\n",
+ " const, gain, temp, int_time = const.split(\"_\")\n",
+ " print('Const: {}'.format(const))\n",
+ "\n",
+ " # loop over modules\n",
+ " mod_data = {}\n",
+ " mod_data['stdASIC'] = []\n",
+ " mod_data['stdCell'] = []\n",
+ " mod_names = []\n",
+ " mod_times = []\n",
+ "\n",
+ " # Loop over modules\n",
+ " for mod, data in modules.items():\n",
+ " print(mod)\n",
+ "\n",
+ " ctimes = np.array(data[\"ctime\"])\n",
+ " ctimes_ticks = [x.strftime('%y-%m-%d') for x in ctimes]\n",
+ "\n",
+ " if (\"mdata\" in data):\n",
+ " cmdata = np.array(data[\"mdata\"])\n",
+ " for i, tick in enumerate(ctimes_ticks):\n",
+ " ctimes_ticks[i] = ctimes_ticks[i] + \\\n",
+ " ', V={:1.0f}'.format(cmdata[i]['Sensor Temperature']) + \\\n",
+ " ', T={:1.0f}'.format(\n",
+ " cmdata[i]['Integration Time'])\n",
+ "\n",
+ " sort_ind = np.argsort(ctimes_ticks)\n",
+ " ctimes_ticks = list(np.array(ctimes_ticks)[sort_ind])\n",
+ "\n",
+ " # Create sorted by data dataset\n",
+ " rdata = {}\n",
+ " for key, item in keys.items():\n",
+ " if item[0] in data:\n",
+ " rdata[key] = np.array(data[item[0]])[sort_ind]\n",
+ "\n",
+ " nTimes = rdata['Mean'].shape[0]\n",
+ " nPixels = rdata['Mean'].shape[1] * rdata['Mean'].shape[2]\n",
+ " nBins = nPixels\n",
+ "\n",
+ " # Avoid to low values\n",
+ " if const in [\"Noise\", \"Offset\", \"Noise-e\"]:\n",
+ " rdata['Mean'][rdata['Mean'] < 0.1] = np.nan\n",
+ " if 'MeanBP' in rdata:\n",
+ " rdata['MeanBP'][rdata['MeanBP'] < 0.1] = np.nan\n",
+ " \n",
+ " if 'NBP' in rdata:\n",
+ " rdata[\"NBP\"] = rdata[\"NBP\"] / spShape[0] / spShape[1] * 100\n",
+ "\n",
+ " # Reshape: ASICs over cells for plotting\n",
+ " pdata = {}\n",
+ " for key in rdata:\n",
+ " if len(rdata[key].shape)<3:\n",
+ " continue\n",
+ " pdata[key] = rdata[key][:, :, :].reshape(nTimes, nBins).swapaxes(0, 1)\n",
+ "\n",
+ " # Summary over ASICs\n",
+ " adata = {}\n",
+ " for key in rdata:\n",
+ " if len(rdata[key].shape)<3:\n",
+ " continue\n",
+ " adata[key] = np.nanmean(rdata[key], axis=(1, 2))\n",
+ "\n",
+ " # Summary information over modules\n",
+ " for key in pdata:\n",
+ " if key not in mod_data:\n",
+ " mod_data[key] = []\n",
+ " mod_data[key].append(np.nanmean(pdata[key], axis=0))\n",
+ "\n",
+ " mod_data['stdASIC'].append(np.nanstd(rdata['Mean'], axis=(1, 2)))\n",
+ "\n",
+ " mod_names.append(mod)\n",
+ " mod_times.append(ctimes_ticks)\n",
+ "\n",
+ " # Plotting\n",
+ " for key in pdata:\n",
+ " if len(pdata[key].shape)<2:\n",
+ " continue\n",
+ "\n",
+ " if key == 'NBP':\n",
+ " unit = '[%]'\n",
+ " else:\n",
+ " unit = '[ADU]'\n",
+ " if const == 'Noise-e':\n",
+ " unit = '[$e^-$]'\n",
+ "\n",
+ " title = '{}, module {}, gain {} {}'.format(\n",
+ " const, mod, gain, keys[key][1])\n",
+ " cb_label = '{}, {} {}'.format(const, keys[key][2], unit)\n",
+ "\n",
+ " vmin,vmax = get_range(pdata[key][::-1].flatten(), plot_range)\n",
+ " hm_combine(pdata[key][::-1], htype=HMType.mro,\n",
+ " x_label='Creation Time', y_label='ASIC ID',\n",
+ " x_ticklabels=ctimes_ticks,\n",
+ " x_ticks=np.arange(len(ctimes_ticks))+0.3,\n",
+ " title=title, cb_label=cb_label,\n",
+ " vmin=vmin, vmax=vmax,\n",
+ " fname='{}/{}_{}_g{}_t{}_t{}_ASIC_{}.png'.format(\n",
+ " out_folder, const, mod.replace('_', ''), gain, temp, int_time, key),\n",
+ " pad=[0.125, 0.125, 0.12, 0.185])\n"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py
index 71da69695..fe9df1c8a 100644
--- a/xfel_calibrate/notebooks.py
+++ b/xfel_calibrate/notebooks.py
@@ -129,6 +129,12 @@ notebooks = {
"use function": "balance_sequences",
"cluster cores": 4},
},
+ "STATS_FROM_DB": {
+ "notebook": "notebooks/FastCCD/PlotFromCalDB_FastCCD_NBC.ipynb",
+ "concurrency": {"parameter": None,
+ "default concurrency": None,
+ "cluster cores": 1},
+ },
},
"JUNGFRAU": {
"DARK": {
--
GitLab