diff --git a/notebooks/Jungfrau/PlotFromCalDB_Jungfrau_NBC.ipynb b/notebooks/Jungfrau/PlotFromCalDB_Jungfrau_NBC.ipynb new file mode 100644 index 0000000000000000000000000000000000000000..530fb2dc566bc410ea24a87f5422fbaca297e289 --- /dev/null +++ b/notebooks/Jungfrau/PlotFromCalDB_Jungfrau_NBC.ipynb @@ -0,0 +1,577 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Statistical analysis of calibration factors#\n", + "\n", + "Author: Mikhail Karnevskiy, Steffen Hauf, Version 0.1\n", + "\n", + "Calibration constants for JungFrau detector from the data base with injection time between start_date and end_date are considered.\n", + "\n", + "To be visualized, calibration constants are averaged per group of pixels. Plots shows calibration constant over time for each constant.\n", + "\n", + "Values shown in plots are saved in h5 files." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "cluster_profile = \"noDB\" # The ipcluster profile to use\n", + "start_date = \"2019-06-30\" # date to start investigation interval from\n", + "end_date = \"2019-09-01\" # date to end investigation interval at, can be \"now\"\n", + "dclass=\"jungfrau\" # Detector class\n", + "db_modules = [\"Jungfrau_M125\", \"Jungfrau_M260\"] # detector entry in the DB to investigate\n", + "constants = [\"Noise\", \"Offset\"] # constants to plot\n", + "nconstants = 10 # Number of time stamps to plot. If not 0, overcome start_date.\n", + "bias_voltage = [90, 180]\n", + "memory_cells = [1]\n", + "pixels_x = [1024]\n", + "pixels_y = [512, 1024]\n", + "temperature = [291]\n", + "integration_time = [50, 250]\n", + "gain_setting = [0]\n", + "\n", + "parameter_names = ['bias_voltage', 'integration_time', 'pixels_x', 'pixels_y', 'gain_setting',\n", + " 'temperature', 'memory_cells'] # names of parameters\n", + "\n", + "max_time = 15\n", + "photon_energy = 9.2 # Photon energy of the beam\n", + "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_JF2/\" # output folder\n", + "use_existing = \"\" # If not empty, constants stored in given folder will be used\n", + "cal_db_interface = \"tcp://max-exfl016:8016\" # the database interface to use\n", + "cal_db_timeout = 180000 # timeout on caldb requests\",\n", + "range_offset = [1000., 2200] # plotting range for offset: high gain l, r, medium gain l, r \n", + "range_noise = [15, 20, 3, 7, 1, 6] # plotting range for noise: high gain l, r, medium gain l, r \n", + "plot_range = 3 # range for plotting in units of median absolute deviations\n", + "spShape = [256, 64] # Shape of superpixel" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "import copy\n", + "import datetime\n", + "import dateutil.parser\n", + "import numpy as np\n", + "import os\n", + "import sys\n", + "import warnings\n", + "warnings.filterwarnings('ignore')\n", + "\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "\n", + "from iCalibrationDB import Constants, Conditions, Detectors, ConstantMetaData\n", + "from cal_tools.tools import get_from_db, get_random_db_interface\n", + "from cal_tools.ana_tools import (save_dict_to_hdf5, load_data_from_hdf5, \n", + " HMType, hm_combine,\n", + " combine_lists, get_range)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Prepare variables\n", + "parameters = [globals()[x] for x in parameter_names]\n", + "\n", + "constantsDark = {'Noise': 'BadPixelsDark',\n", + " 'Offset': 'BadPixelsDark'}\n", + "print('Bad pixels data: ', constantsDark)\n", + "\n", + "# Define parameters in order to perform loop over time stamps\n", + "start = datetime.datetime.now() if start_date.upper() == \"NOW\" else dateutil.parser.parse(\n", + " start_date)\n", + "end = datetime.datetime.now() if end_date.upper() == \"NOW\" else dateutil.parser.parse(\n", + " end_date)\n", + "\n", + "# Create output folder\n", + "os.makedirs(out_folder, exist_ok=True)\n", + "\n", + "# Get getector conditions\n", + "dconstants = getattr(Constants, dclass)\n", + "\n", + "print('CalDB Interface: {}'.format(cal_db_interface))\n", + "print('Start time at: ', start)\n", + "print('End time at: ', end)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "parameter_list = combine_lists(*parameters, names = parameter_names)\n", + "print(parameter_list)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# Retrieve list of meta-data\n", + "constant_versions = []\n", + "constant_parameters = []\n", + "constantBP_versions = []\n", + "\n", + "# Loop over constants\n", + "for c, const in enumerate(constants):\n", + " \n", + " for db_module in db_modules:\n", + " det = getattr(Detectors, db_module)\n", + " \n", + " if use_existing != \"\":\n", + " break\n", + "\n", + " # Loop over parameters\n", + " for pars in parameter_list:\n", + "\n", + " if (const in [\"Offset\", \"Noise\", \"SlopesPC\"] or \"DARK\" in const.upper()):\n", + " dcond = Conditions.Dark\n", + " mcond = getattr(dcond, dclass)(**pars)\n", + " else:\n", + " dcond = Conditions.Illuminated\n", + " mcond = getattr(dcond, dclass)(**pars,\n", + " photon_energy=photon_energy)\n", + "\n", + "\n", + "\n", + " print('Request: ', const, 'with paramters:', pars)\n", + " # Request Constant versions for given parameters and module\n", + " data = get_from_db(det,\n", + " getattr(dconstants,\n", + " const)(),\n", + " copy.deepcopy(mcond), None,\n", + " cal_db_interface,\n", + " creation_time=start,\n", + " verbosity=0,\n", + " timeout=cal_db_timeout,\n", + " meta_only=True,\n", + " version_info=True)\n", + "\n", + " if not isinstance(data, list):\n", + " continue\n", + "\n", + " if const in constantsDark:\n", + " # Request BP constant versions\n", + " print('constantDark:', constantsDark[const], ) \n", + " dataBP = get_from_db(det,\n", + " getattr(dconstants, \n", + " constantsDark[const])(),\n", + " copy.deepcopy(mcond), None,\n", + " cal_db_interface,\n", + " creation_time=start,\n", + " verbosity=0,\n", + " timeout=cal_db_timeout,\n", + " meta_only=True,\n", + " version_info=True)\n", + "\n", + " if not isinstance(data, list) or not isinstance(dataBP, list):\n", + " continue\n", + "\n", + " found_BPmatch = False\n", + " for d in data:\n", + " # Match proper BP constant version\n", + " # and get constant version within\n", + " # requested time range\n", + " if d is None:\n", + " print('Time or data is not found!')\n", + " continue\n", + "\n", + " dt = dateutil.parser.parse(d['begin_at'])\n", + "\n", + " if dt.replace(tzinfo=None) > end or dt.replace(tzinfo=None) < start:\n", + " continue\n", + "\n", + " closest_BP = None\n", + " closest_BPtime = None\n", + "\n", + " for dBP in dataBP:\n", + " if dBP is None:\n", + " print(\"Bad pixels are not found!\")\n", + " continue\n", + "\n", + " dt = dateutil.parser.parse(d['begin_at'])\n", + " dBPt = dateutil.parser.parse(dBP['begin_at'])\n", + "\n", + " if dt == dBPt:\n", + " found_BPmatch = True\n", + " else:\n", + "\n", + " if np.abs(dBPt-dt).seconds < (max_time*60):\n", + " if closest_BP is None:\n", + " closest_BP = dBP\n", + " closest_BPtime = dBPt\n", + " else:\n", + " if np.abs(dBPt-dt) < np.abs(closest_BPtime-dt):\n", + " closest_BP = dBP\n", + " closest_BPtime = dBPt\n", + "\n", + " if dataBP.index(dBP) == len(dataBP)-1:\n", + " if closest_BP:\n", + " dBP = closest_BP\n", + " dBPt = closest_BPtime\n", + " found_BPmatch = True\n", + " else:\n", + " print('Bad pixels are not found!')\n", + "\n", + " if found_BPmatch:\n", + " print(\"Found constant {}: begin at {}\".format(const, dt))\n", + " print(\"Found bad pixels at {}\".format(dBPt))\n", + " constantBP_versions.append(dBP)\n", + " constant_versions.append(d)\n", + " constant_parameters.append(copy.deepcopy(pars))\n", + " found_BPmatch = False\n", + " break\n", + " else:\n", + " constant_versions += data\n", + " constant_parameters += [copy.deepcopy(pars)]*len(data)\n", + "\n", + "# Remove dublications\n", + "constant_versions_tmp = []\n", + "constant_parameters_tmp = []\n", + "for i, x in enumerate(constant_versions):\n", + " if x not in constant_versions_tmp:\n", + " constant_versions_tmp.append(x)\n", + " constant_parameters_tmp.append(constant_parameters[i])\n", + " \n", + "constant_versions=constant_versions_tmp\n", + "constant_parameters=constant_parameters_tmp\n", + "\n", + "print('Number of stored constant versions is {}'.format(len(constant_versions)))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def get_rebined(a, rebin):\n", + " return a.reshape(\n", + " int(a.shape[0] / rebin[0]),\n", + " rebin[0],\n", + " int(a.shape[1] / rebin[1]),\n", + " rebin[1],\n", + " a.shape[2],\n", + " a.shape[3])\n", + "\n", + "def modify_const(const, data, isBP = False):\n", + " return data\n", + "\n", + "ret_constants = {}\n", + "constant_data = ConstantMetaData()\n", + "constant_BP = ConstantMetaData()\n", + "\n", + "# sort over begin_at\n", + "idxs, _ = zip(*sorted(enumerate(constant_versions), \n", + " key=lambda x: x[1]['begin_at'], reverse=True))\n", + "\n", + "for i in idxs:\n", + " const = constant_versions[i]['data_set_name'].split('/')[-2]\n", + " qm = constant_versions[i]['physical_device']['name']\n", + " \n", + " if not const in ret_constants:\n", + " ret_constants[const] = {}\n", + " if not qm in ret_constants[const]:\n", + " ret_constants[const][qm] = []\n", + " \n", + " if nconstants>0 and len(ret_constants[const][qm])>=nconstants:\n", + " continue\n", + " \n", + " print(\"constant: {}, module {}\".format(const,qm))\n", + " constant_data.retrieve_from_version_info(constant_versions[i])\n", + " \n", + " cdata = constant_data.calibration_constant.data\n", + " ctime = constant_data.calibration_constant_version.begin_at\n", + " \n", + " cdata = modify_const(const, cdata)\n", + " \n", + " \n", + " if len(constantBP_versions)>0:\n", + " constant_BP.retrieve_from_version_info(constantBP_versions[i])\n", + " cdataBP = constant_BP.calibration_constant.data\n", + " cdataBP = modify_const(const, cdataBP, True)\n", + " \n", + " if cdataBP.shape != cdata.shape:\n", + " print('Wrong bad pixel shape! {}, expected {}'.format(cdataBP.shape, cdata.shape))\n", + " continue\n", + " \n", + " # Apply bad pixel mask\n", + " cdataABP = np.copy(cdata)\n", + " cdataABP[cdataBP > 0] = np.nan\n", + " \n", + " # Create superpixels for constants with BP applied\n", + " cdataABP = get_rebined(cdataABP, spShape)\n", + " toStoreBP = np.nanmean(cdataABP, axis=(1, 3))\n", + " toStoreBPStd = np.nanstd(cdataABP, axis=(1, 3))\n", + "\n", + " # Prepare number of bad pixels per superpixels\n", + " cdataBP = get_rebined(cdataBP, spShape)\n", + " cdataNBP = np.nansum(cdataBP > 0, axis=(1, 3))\n", + " else:\n", + " toStoreBP = 0\n", + " toStoreBPStd = 0\n", + " cdataNBP = 0\n", + "\n", + " # Create superpixels for constants without BP applied\n", + " cdata = get_rebined(cdata, spShape)\n", + " toStoreStd = np.nanstd(cdata, axis=(1, 3))\n", + " toStore = np.nanmean(cdata, axis=(1, 3))\n", + " \n", + " # Convert parameters to dict\n", + " dpar = {p.name: p.value for p in constant_data.detector_condition.parameters}\n", + " \n", + " print(\"Store values in dict\", const, qm, ctime)\n", + " ret_constants[const][qm].append({'ctime': ctime,\n", + " 'nBP': cdataNBP,\n", + " 'dataBP': toStoreBP,\n", + " 'dataBPStd': toStoreBPStd,\n", + " 'data': toStore,\n", + " 'dataStd': toStoreStd,\n", + " 'mdata': dpar}) \n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "if use_existing == \"\":\n", + " print('Save data to /CalDBAna_{}_{}.h5'.format(dclass, db_module))\n", + " save_dict_to_hdf5(ret_constants,\n", + " '{}/CalDBAna_{}_{}.h5'.format(out_folder, dclass, db_module))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if use_existing == \"\":\n", + " fpath = '{}/CalDBAna_{}_*.h5'.format(out_folder, dclass)\n", + "else:\n", + " fpath = '{}/CalDBAna_{}_*.h5'.format(use_existing, dclass)\n", + "\n", + "print('Load data from {}'.format(fpath))\n", + "ret_constants = load_data_from_hdf5(fpath)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Parameters for plotting\n", + "\n", + "# Define range for plotting\n", + "rangevals = {\n", + " \"OffsetEPix100\": [range_offset[0:2], range_offset[2:4]],\n", + " \"Noise10Hz\": [range_noise[0:2], range_noise[2:4], range_noise[4:6]],\n", + "}\n", + "\n", + "keys = {\n", + " 'Mean': ['data', '', 'Mean over pixels'],\n", + " 'std': ['dataStd', '', '$\\sigma$ over pixels'],\n", + " 'MeanBP': ['dataBP', 'Good pixels only', 'Mean over pixels'],\n", + " 'NBP': ['nBP', 'Fraction of BP', 'Number of BP'],\n", + " 'stdBP': ['dataBPStd', 'Good pixels only', '$\\sigma$ over pixels'],\n", + "}\n", + "\n", + "gain_name = ['High', 'Medium', 'Low']" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "print('Plot calibration constants')\n", + "\n", + "# loop over constat type\n", + "for const, modules in ret_constants.items():\n", + " \n", + " for gain in range(3):\n", + "\n", + " print('Const: {}'.format(const))\n", + "\n", + " # summary over modules\n", + " mod_data = {}\n", + " mod_names = []\n", + " mod_times = []\n", + " \n", + " # Loop over modules\n", + " for mod, data in modules.items():\n", + " print(mod)\n", + "\n", + " ctimes = np.array(data[\"ctime\"])\n", + " ctimes_ticks = [x.strftime('%y-%m-%d') for x in ctimes]\n", + "\n", + " if (\"mdata\" in data):\n", + " cmdata = np.array(data[\"mdata\"])\n", + " for i, tick in enumerate(ctimes_ticks):\n", + " ctimes_ticks[i] = ctimes_ticks[i] + \\\n", + " ', V={:1.0f}'.format(cmdata[i]['Sensor Temperature']) + \\\n", + " ', T={:1.0f}'.format(\n", + " cmdata[i]['Integration Time'])\n", + "\n", + " sort_ind = np.argsort(ctimes_ticks)\n", + " ctimes_ticks = list(np.array(ctimes_ticks)[sort_ind])\n", + "\n", + " # Create sorted by data dataset\n", + " rdata = {}\n", + " for key, item in keys.items():\n", + " if item[0] in data:\n", + " rdata[key] = np.array(data[item[0]])[sort_ind]\n", + "\n", + " nTimes = rdata['Mean'].shape[0]\n", + " nPixels = rdata['Mean'].shape[1] * rdata['Mean'].shape[2]\n", + " nBins = nPixels\n", + " \n", + " # Select gain\n", + " if const not in [\"Gain\", \"Noise-e\"]:\n", + " for key in rdata:\n", + " if len(rdata[key].shape)<5:\n", + " continue\n", + " rdata[key] = rdata[key][..., 0, gain]\n", + "\n", + " # Avoid to low values\n", + " if const in [\"Noise10Hz\", \"Offset10Hz\"]:\n", + " rdata['Mean'][rdata['Mean'] < 0.1] = np.nan\n", + " if 'MeanBP' in rdata:\n", + " rdata['MeanBP'][rdata['MeanBP'] < 0.1] = np.nan\n", + " if 'NBP' in rdata:\n", + " rdata['NBP'] = rdata['NBP'].astype(float)\n", + " rdata['NBP'][rdata['NBP'] == spShape[0]*spShape[1]] = np.nan\n", + "\n", + " # Reshape: ASICs over cells for plotting\n", + " pdata = {}\n", + " for key in rdata:\n", + " if len(rdata[key].shape)<3:\n", + " continue\n", + " pdata[key] = rdata[key].reshape(nTimes, nBins).swapaxes(0, 1)\n", + "\n", + " # Summary over ASICs\n", + " adata = {}\n", + " for key in rdata:\n", + " if len(rdata[key].shape)<3:\n", + " continue\n", + " adata[key] = np.nansum(rdata[key], axis=(1, 2))\n", + "\n", + " # Summary information over modules\n", + " for key in pdata:\n", + " if key not in mod_data:\n", + " mod_data[key] = []\n", + " if key == 'NBP':\n", + " mod_data[key].append(np.nansum(pdata[key], axis=0))\n", + " else:\n", + " mod_data[key].append(np.nanmean(pdata[key], axis=0))\n", + "\n", + " mod_names.append(mod)\n", + " mod_times.append(ctimes[sort_ind])\n", + " \n", + " # Plotting\n", + " for key in pdata:\n", + " \n", + " if len(pdata[key].shape)<2:\n", + " continue\n", + " \n", + " vmin,vmax = get_range(pdata[key][::-1].flatten(), plot_range)\n", + " #if const in rangevals and key in ['Mean', 'MeanBP']:\n", + " # vmin = rangevals[const][0][0]\n", + " # vmax = rangevals[const][0][1]\n", + "\n", + " if key == 'NBP':\n", + " unit = '[%]'\n", + " else:\n", + " unit = '[ADU]'\n", + " if const == 'Noise-e':\n", + " unit = '[$e^-$]'\n", + "\n", + " title = '{}, module {}, {}'.format(\n", + " const, mod, keys[key][1])\n", + " cb_label = '{}, {} {}'.format(const, keys[key][2], unit)\n", + "\n", + " hm_combine(pdata[key][::-1], htype=HMType.mro,\n", + " x_label='Creation Time', y_label='ASIC ID',\n", + " x_ticklabels=ctimes_ticks,\n", + " x_ticks=np.arange(len(ctimes_ticks))+0.3,\n", + " title=title, cb_label=cb_label,\n", + " vmin=vmin, vmax=vmax,\n", + " fname='{}/{}_{}_g{}_ASIC_{}.png'.format(\n", + " out_folder, const, mod.replace('_', ''), gain, key),\n", + " pad=[0.125, 0.125, 0.12, 0.185])\n", + "\n", + " \n", + " # Summary over modules\n", + " for key in mod_data:\n", + " \n", + " if key == 'NBP':\n", + " unit = ''\n", + " else:\n", + " unit = '[ADU]'\n", + "\n", + " title = '{}, All modules, {} gain, {}'.format(\n", + " const, gain_name[gain], keys[key][1])\n", + " \n", + " fig = plt.figure(figsize=(12,12) )\n", + " for i in range(len(mod_data[key])):\n", + " plt.scatter(mod_times[i], mod_data[key][i], label=mod_names[i])\n", + " plt.grid()\n", + " plt.xlabel('Creation Time')\n", + " plt.ylabel('{}, {} {}'.format(const, keys[key][2], unit)) \n", + " plt.legend(loc='best guess')\n", + " plt.title(title)\n", + " fig.savefig('{}/{}_{}_g{}_ASIC_{}.png'.format(\n", + " out_folder, const, 'All', gain, key))\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.7" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py index fe9df1c8a4354f8fa39da5d8d60385a94ec02c71..a004bdb0e2bc30d95687a032f0e8f6d1c9ca6c02 100644 --- a/xfel_calibrate/notebooks.py +++ b/xfel_calibrate/notebooks.py @@ -150,6 +150,13 @@ notebooks = { "use function": "balance_sequences", "cluster cores": 4}, }, + + "STATS_FROM_DB": { + "notebook": "notebooks/Jungfrau/PlotFromCalDB_Jungfrau_NBC.ipynb", + "concurrency": {"parameter": None, + "default concurrency": None, + "cluster cores": 1}, + }, }, "EPIX": { "DARK": {