From 4d5be7848b26d0f8b792ac41e35337b0a725ffa7 Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Mon, 21 Oct 2019 11:28:52 +0200
Subject: [PATCH] Add plotting for AGIPD and LPD for generic notebook
---
cal_tools/cal_tools/ana_tools.py | 10 +-
notebooks/generic/PlotFromCalDB_NBC.ipynb | 533 +++++++++++-----
.../generic/PlotFromCalDB_Summary_NBC.ipynb | 239 ++++----
reportservice/report_conf.yaml | 576 ++++++++++++++++--
xfel_calibrate/notebooks.py | 7 +
5 files changed, 1026 insertions(+), 339 deletions(-)
diff --git a/cal_tools/cal_tools/ana_tools.py b/cal_tools/cal_tools/ana_tools.py
index 0b36016b8..a01475006 100644
--- a/cal_tools/cal_tools/ana_tools.py
+++ b/cal_tools/cal_tools/ana_tools.py
@@ -359,16 +359,18 @@ class IMType(Enum):
ALL_BAD = -4e+4
NO_BPMAP = -5e+4
-def get_range(data, scale):
+
+def get_range(data, scale, threshold = -1000):
"""
Return a range calculated by median absolute deviations
:param data: numpy.array of data points
:param scale: range in units of median absolute deviations
- :return:
+ :param threshold: lower threshold for data to be considered
+ :return: Range [min, max] calculated by median absolute deviations
"""
- med = np.nanmedian(data)
- mad = np.nanmedian(np.abs(data.flatten() - med))
+ med = np.nanmedian(data[data>threshold])
+ mad = np.nanmedian(np.abs(data[data>threshold].flatten() - med))
return med - scale * mad, med + scale * mad
diff --git a/notebooks/generic/PlotFromCalDB_NBC.ipynb b/notebooks/generic/PlotFromCalDB_NBC.ipynb
index 88472fbbe..3e3ba7791 100644
--- a/notebooks/generic/PlotFromCalDB_NBC.ipynb
+++ b/notebooks/generic/PlotFromCalDB_NBC.ipynb
@@ -6,9 +6,9 @@
"source": [
"# Statistical analysis of calibration factors#\n",
"\n",
- "Author: Mikhail Karnevskiy, Steffen Hauf, Version 0.1\n",
+ "Author: Mikhail Karnevskiy, Version 0.2\n",
"\n",
- "Calibration constants for pnCCDdetector from the data base with injection time between start_date and end_date are considered.\n",
+ "Plot calibration constants retrieved from the cal. DB.\n",
"\n",
"To be visualized, calibration constants are averaged per group of pixels. Plots shows calibration constant over time for each constant.\n",
"\n",
@@ -24,32 +24,36 @@
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
"start_date = \"2019-06-30\" # date to start investigation interval from\n",
"end_date = \"NOW\" # date to end investigation interval at, can be \"now\"\n",
- "dclass=\"jungfrau\" # Detector class\n",
- "modules = [\"Jungfrau_M039\"] # detector entry in the DB to investigate\n",
- "constants = [\"Noise\", \"Offset\"] # constants to plot\n",
- "nconstants = 20 # Number of time stamps to plot. If not 0, overcome start_date.\n",
+ "dclass=\"LPD\" # Detector class\n",
+ "modules = [\"LPD1M1\"] # detector entry in the DB to investigate\n",
+ "submodules = [2] # module index of a modular detector (1 for Q1M1 of AGIPD), range allowed \n",
+ "constants = ['Noise'] # constants to plot\n",
+ "nconstants = 7 # Number of time stamps to plot. If not 0, overcome start_date.\n",
"max_time = 15 # max time margin in minutes to match bad pixels\n",
+ "nMemToShow = 32 # Number of memory cells to be shown in plots\n",
"\n",
"gain_setting = [0] # gain stages\n",
- "bias_voltage = [90, 180] # Bias voltage\n",
+ "bias_voltage = [250, 500] # Bias voltage\n",
"temperature = [291] # Operation temperature\n",
"integration_time = [250, 50] # Integration time\n",
"pixels_x=[1024] # number of pixels along X axis\n",
"pixels_y=[512, 1024] # number of pixels along Y axis\n",
"in_vacuum = [0] # 0 if detector is operated in room pressure\n",
- "memory_cells = [1] # number of memory cells\n",
- "parameter_names = ['bias_voltage', 'integration_time', 'temperature', \n",
- " 'gain_setting', 'memory_cells', 'pixels_x', 'pixels_y'] # names of parameters\n",
+ "memory_cells = [1, 512] # number of memory cells\n",
+ "acquisition_rate = [1.1] # aquisition rate\n",
+ "parameter_names = ['bias_voltage', 'memory_cells'] # names of parameters\n",
"\n",
- "separate_plot = ['integration_time'] # Plot on separate plots\n",
+ "separate_plot = ['gain_setting'] # Plot on separate plots\n",
"x_labels = ['Sensor Temperature', 'Integration Time'] # parameters to be shown on X axis: Acquisition rate, Memory cells, Sensor Temperature, Integration Time\n",
"photon_energy = 9.2 # Photon energy of the beam\n",
- "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_bla2/\" # output folder\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_bla3/\" # output folder\n",
"use_existing = \"\" # If not empty, constants stored in given folder will be used\n",
"cal_db_interface = \"tcp://max-exfl016:8016\" # the database interface to use\n",
"cal_db_timeout = 180000 # timeout on caldb requests\",\n",
"plot_range = 3 # range for plotting in units of median absolute deviations\n",
- "spShape = [256, 256] # Shape of superpixel"
+ "spShape = [64, 64] # Shape of superpixel\n",
+ "sp_name = 'ASIC IDs' # name of superpixel\n",
+ "gain_titles = ['High gain', 'Medium gain', 'Low gain'] # Title inset related to gain"
]
},
{
@@ -74,7 +78,7 @@
"from iCalibrationDB import Constants, Conditions, Detectors, ConstantMetaData\n",
"from cal_tools.tools import get_from_db, get_random_db_interface\n",
"from cal_tools.ana_tools import (save_dict_to_hdf5, load_data_from_hdf5, \n",
- " HMType, IMType, hm_combine,\n",
+ " HMType, IMType, hm_combine, combine_constants,\n",
" combine_lists, get_range)"
]
},
@@ -85,9 +89,28 @@
"outputs": [],
"source": [
"# Prepare variables\n",
+ "submodules = [\"Q{}M{}\".format(x // 4 + 1, x % 4 + 1) for x in submodules]\n",
+ "\n",
+ "# No submodules for small detectors\n",
+ "if dclass not in ['AGIPD', 'LPD']:\n",
+ " submodules = ['']\n",
+ "\n",
+ "# 0 is considered as None.\n",
+ "acquisition_rate = [x if x>0 else None for x in acquisition_rate]\n",
+ "\n",
+ "nMem = max(memory_cells) # Number of mem Cells to store\n",
+ "\n",
"parameters = [globals()[x] for x in parameter_names]\n",
"\n",
- "constantsDark = {'Noise': 'BadPixelsDark',\n",
+ "# Empty list from the command line may not work\n",
+ "if separate_plot == ['']:\n",
+ " separate_plot = []\n",
+ "\n",
+ "# Mapping between consatnts and their bad pixel maps \n",
+ "constantsDark = {\"SlopesFF\": 'BadPixelsFF',\n",
+ " 'SlopesPC': 'BadPixelsPC',\n",
+ " 'SlopesCI': 'BadPixelsCI',\n",
+ " 'Noise': 'BadPixelsDark',\n",
" 'Offset': 'BadPixelsDark'}\n",
"print('Bad pixels data: ', constantsDark)\n",
"\n",
@@ -134,6 +157,9 @@
" for db_module in modules:\n",
" det = getattr(Detectors, db_module)\n",
" \n",
+ " if dclass in ['AGIPD', 'LPD']:\n",
+ " det = getattr(det, submodules[0])\n",
+ " \n",
" # Get getector conditions\n",
" if dclass=='CCD':\n",
" dconstants = getattr(Constants, dclass)(det.detector_type)\n",
@@ -146,7 +172,7 @@
" # Loop over parameters\n",
" for pars in parameter_list:\n",
"\n",
- " if (const in [\"Offset\", \"Noise\", \"SlopesPC\"] or \"DARK\" in const.upper()):\n",
+ " if (const in [\"Offset\", \"Noise\", \"SlopesPC\", \"SlopesCI\"] or \"DARK\" in const.upper()):\n",
" dcond = Conditions.Dark\n",
" mcond = getattr(dcond, dclass)(**pars)\n",
" else:\n",
@@ -286,8 +312,24 @@
},
"outputs": [],
"source": [
+ "def prepare_to_store(a, nMem):\n",
+ " \"\"\"\n",
+ " Different constants for AGIPD and LPD may have different array shape.\n",
+ " This function unify array shape.\n",
+ " \"\"\"\n",
+ " if dclass in ['AGIPD', 'LPD']:\n",
+ " shape = list(a.shape[:2])+[nMem]\n",
+ " b = np.full(shape, np.nan)\n",
+ " b[:, :, :a.shape[2]] = a[:, :, :]\n",
+ " return b\n",
+ " else:\n",
+ " return a\n",
+ "\n",
"def get_rebined(a, rebin):\n",
- " if dclass==\"jungfrau\":\n",
+ " \"\"\"\n",
+ " Group of pixels are formed here for better visialization\n",
+ " \"\"\"\n",
+ " if dclass in ['AGIPD', 'LPD', 'jungfrau']:\n",
" return a.reshape(\n",
" int(a.shape[0] / rebin[0]),\n",
" rebin[0],\n",
@@ -305,8 +347,59 @@
" \n",
"\n",
"def modify_const(const, data, isBP = False):\n",
+ " \"\"\"\n",
+ " Shape of an array for some constants changes over time.\n",
+ " Modification is needed to unify shape of array and\n",
+ " make possible to show constants on the same plot.\n",
+ " \"\"\"\n",
" if dclass==\"jungfrau\" and data.shape[1] == 512:\n",
" data = data.swapaxes(0, 1)\n",
+ " return data\n",
+ " \n",
+ " if dclass==\"AGIPD\":\n",
+ " const = const.split('_')[0]\n",
+ " if const in ['SlopesFF']:\n",
+ " if (len(data.shape) == 4):\n",
+ " data = data[:, :, :, 0][..., None]\n",
+ " else:\n",
+ " data = data[..., None]\n",
+ "\n",
+ " if data.shape[2]<3:\n",
+ " data = data[:,:,0,None]\n",
+ "\n",
+ " if not isBP:\n",
+ " if data.shape[0] != 128:\n",
+ " data = data.swapaxes(0, 2).swapaxes(1, 3).swapaxes(2, 3)\n",
+ "\n",
+ " # Copy slope medium to be saved later\n",
+ " if const in ['SlopesPC']:\n",
+ " data[:, :, :, 1] = data[:, :, :, 3]\n",
+ " else:\n",
+ " if const in ['SlopesPC']:\n",
+ " if len(data.shape) == 3:\n",
+ " data = data[:, :, :, None].repeat(10, axis=3)\n",
+ "\n",
+ " if data.shape[0] != 128:\n",
+ " data = data.swapaxes(0, 1).swapaxes(1, 2)\n",
+ "\n",
+ " if len(data.shape) < 4:\n",
+ " print(data.shape, \"Unexpected shape!\")\n",
+ " return data\n",
+ " \n",
+ " if dclass==\"LPD\":\n",
+ " const = const.split('_')[0]\n",
+ " if const in ['SlopesFF']:\n",
+ " data = data[..., None, None]\n",
+ "\n",
+ " if(len(data.shape)==5):\n",
+ " data = data[:,:,:,:,0]\n",
+ "\n",
+ " if len(data.shape) < 4:\n",
+ " print(data.shape, \"Unexpected shape!\")\n",
+ "\n",
+ " if data.shape[0] != 256:\n",
+ " data = data.swapaxes(0, 2).swapaxes(1,3).swapaxes(2,3) \n",
+ " \n",
" return data\n",
"\n",
"ret_constants = {}\n",
@@ -323,12 +416,18 @@
" # fix naming for Jungfrau039\n",
" if qm == 'Jungfrau1':\n",
" qm = 'JungfrauM039'\n",
+ " # use submodule name for big detectors\n",
+ " if dclass in ['AGIPD', 'LPD']:\n",
+ " qm = submodules[0]\n",
" \n",
+ " # Add insets for parameters\n",
" for key in separate_plot:\n",
- " const = '{}_{}{}'.format(const, key[0], constant_parameters[i][key])\n",
- " # Constant for jungfrau already contains gain stages\n",
- " if dclass == \"jungfrau\":\n",
- " const += '_g0'\n",
+ " # Several constant already contains gain stages\n",
+ " if key == 'gain_setting' and dclass in ['AGIPD', 'LPD', 'jungfrau']:\n",
+ " val = 0\n",
+ " else:\n",
+ " val = constant_parameters[i][key]\n",
+ " const = '{}_{}{}'.format(const, key[0], val)\n",
" \n",
" if not const in ret_constants:\n",
" ret_constants[const] = {}\n",
@@ -351,7 +450,7 @@
" \n",
" if cdataBP.shape != cdata.shape:\n",
" print('Wrong bad pixel shape! {}, expected {}'.format(cdataBP.shape, cdata.shape))\n",
- " continue\n",
+ " cdataBP = np.full_like(cdata, IMType.NO_BPMAP.value)\n",
" \n",
" # Apply bad pixel mask\n",
" cdataABP = np.copy(cdata)\n",
@@ -379,10 +478,18 @@
" # Convert parameters to dict\n",
" dpar = {p.name: p.value for p in constant_data.detector_condition.parameters}\n",
" \n",
+ " # Several constants have dimensions running over gain.\n",
+ " # All gain stages are stored as separate arrays.\n",
" if len(toStore.shape)==4:\n",
" for i in range(3):\n",
- " if i>0:\n",
+ " if i>0 and 'gain_setting' in separate_plot:\n",
" const = const.replace('_g{}'.format(i-1), '_g{}'.format(i))\n",
+ " # FF has only high gain\n",
+ " if 'SlopesFF' in const and i>0:\n",
+ " continue\n",
+ " # PC only high and medium.\n",
+ " if 'SlopesPC' in const and i>1:\n",
+ " continue\n",
" \n",
" if not const in ret_constants:\n",
" ret_constants[const] = {}\n",
@@ -390,11 +497,11 @@
" ret_constants[const][qm] = []\n",
" print(\"Store values in dict\", const, qm, ctime)\n",
" ret_constants[const][qm].append({'ctime': ctime,\n",
- " 'nBP': cdataNBP[:,:,0,i],\n",
- " 'dataBP': toStoreBP[:,:,0,i],\n",
- " 'dataBPStd': toStoreBPStd[:,:,0,i],\n",
- " 'data': toStore[:,:,0,i],\n",
- " 'dataStd': toStoreStd[:,:,0,i],\n",
+ " 'nBP': prepare_to_store(cdataNBP[:,:,:,i], nMem),\n",
+ " 'dataBP': prepare_to_store(toStoreBP[:,:,:,i], nMem),\n",
+ " 'dataBPStd': prepare_to_store(toStoreBPStd[:,:,:,i], nMem),\n",
+ " 'data': prepare_to_store(toStore[:,:,:,i], nMem),\n",
+ " 'dataStd': prepare_to_store(toStoreStd[:,:,:,i], nMem),\n",
" 'mdata': dpar}) \n",
" \n",
" \n",
@@ -420,9 +527,9 @@
"outputs": [],
"source": [
"if use_existing == \"\":\n",
- " print('Save data to /CalDBAna_{}_{}.h5'.format(dclass, db_module))\n",
+ " print('Save data to {}/CalDBAna_{}_{}_{}.h5'.format(out_folder, dclass, db_module, submodules[0]))\n",
" save_dict_to_hdf5(ret_constants,\n",
- " '{}/CalDBAna_{}_{}.h5'.format(out_folder, dclass, db_module))"
+ " '{}/CalDBAna_{}_{}_{}.h5'.format(out_folder, dclass, db_module, submodules[0]))"
]
},
{
@@ -432,14 +539,109 @@
"outputs": [],
"source": [
"if use_existing == \"\":\n",
- " fpath = '{}/CalDBAna_{}_*.h5'.format(out_folder, dclass)\n",
+ " fpath = '{}/CalDBAna_{}_{}_{}.h5'.format(out_folder, dclass, db_module, submodules[0])\n",
"else:\n",
- " fpath = '{}/CalDBAna_{}_*.h5'.format(use_existing, dclass)\n",
+ " fpath = '{}/CalDBAna_{}_{}_{}.h5'.format(use_existing, dclass, db_module, submodules[0])\n",
"\n",
"print('Load data from {}'.format(fpath))\n",
"ret_constants = load_data_from_hdf5(fpath)"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# For AGIPD and LPD\n",
+ "# Combine FF and PC data to calculate Gain\n",
+ "# Estimate Noise in units of electrons\n",
+ "\n",
+ "ret_constants[\"Gain_g0\"] = {}\n",
+ "ret_constants[\"Noise-e_g0\"] = {}\n",
+ "\n",
+ "for mod in list(range(16)):\n",
+ " # The check is perform inside the for loop\n",
+ " # in order to use break\n",
+ " # This make code more readable\n",
+ " if (\"SlopesFF_g0\" not in ret_constants or\n",
+ " \"SlopesPC_g0\" not in ret_constants):\n",
+ " break\n",
+ "\n",
+ " qm = \"Q{}M{}\".format(mod // 4 + 1, mod % 4 + 1)\n",
+ " if (qm not in ret_constants[\"SlopesFF_g0\"] or\n",
+ " qm not in ret_constants[\"SlopesPC_g0\"]):\n",
+ " continue\n",
+ "\n",
+ " print(qm)\n",
+ " ret_constants[\"Gain_g0\"][qm] = {}\n",
+ "\n",
+ " dataFF = ret_constants[\"SlopesFF_g0\"][qm]\n",
+ " dataPC = ret_constants[\"SlopesPC_g0\"][qm]\n",
+ "\n",
+ " if (len(dataFF) == 0 or len(dataPC) == 0):\n",
+ " continue\n",
+ "\n",
+ " ctimesFF = np.array(dataFF[\"ctime\"])\n",
+ " ctimesPC = np.array(dataPC[\"ctime\"])\n",
+ "\n",
+ " ctime, icomb = combine_constants(ctimesFF, ctimesPC)\n",
+ "\n",
+ " cdataPC_vs_time = np.array(dataPC[\"data\"])[...]\n",
+ " cdataFF_vs_time = np.array(dataFF[\"data\"])[...]\n",
+ "\n",
+ " cdataFF_vs_time = np.nanmedian(cdataFF_vs_time, axis=3)[..., None]\n",
+ "\n",
+ " cdataFF_vs_time /= np.nanmedian(cdataFF_vs_time, axis=(1, 2, 3))[:, None,\n",
+ " None, None]\n",
+ " cdataPC_vs_time /= np.nanmedian(cdataPC_vs_time, axis=(1, 2, 3))[:, None,\n",
+ " None, None]\n",
+ "\n",
+ " gain_vs_time = []\n",
+ " for iFF, iPC in icomb:\n",
+ " gain_vs_time.append(cdataFF_vs_time[iFF] * cdataPC_vs_time[iPC])\n",
+ "\n",
+ " print('Shape of gain array: ', np.array(gain_vs_time).shape)\n",
+ " \n",
+ " ctime_ts = [t.timestamp() for t in ctime]\n",
+ " \n",
+ " ret_constants[\"Gain_g0\"][qm][\"ctime\"] = ctime\n",
+ " ret_constants[\"Gain_g0\"][qm][\"data\"] = np.array(gain_vs_time)\n",
+ "\n",
+ " if \"Noise_g0\" not in ret_constants:\n",
+ " continue\n",
+ "\n",
+ " if qm not in ret_constants[\"Noise_g0\"]:\n",
+ " continue\n",
+ "\n",
+ " dataN = ret_constants[\"Noise_g0\"][qm]\n",
+ " if len(dataN) == 0:\n",
+ " continue\n",
+ "\n",
+ " ret_constants[\"Noise-e_g0\"][qm] = {}\n",
+ " \n",
+ " ctimesG = np.array(ctime)\n",
+ " ctimesN = np.array(dataN[\"ctime\"])\n",
+ "\n",
+ " ctime, icomb = combine_constants(ctimesG, ctimesN)\n",
+ "\n",
+ " cdataG_vs_time = np.array(gain_vs_time)\n",
+ " cdataN_vs_time = np.array(dataN[\"data\"])[...]\n",
+ "\n",
+ " data_vs_time = []\n",
+ " for iG, iN in icomb:\n",
+ " data_vs_time.append(\n",
+ " cdataN_vs_time[iN] * adu_to_photon / cdataG_vs_time[iG])\n",
+ "\n",
+ " print('Shape of gain array: ',np.array(gain_vs_time).shape)\n",
+ " ctime_ts = [t.timestamp() for t in ctime]\n",
+ " ret_constants[\"Noise-e_g0\"][qm][\"ctime\"] = ctime\n",
+ " ret_constants[\"Noise-e_g0\"][qm][\"data\"] = np.array(data_vs_time)\n",
+ " \n",
+ "save_dict_to_hdf5({k:v for k,v in ret_constants.items() if k in ['Gain_g0', 'Noise-e_g0']},\n",
+ " '{}/CalDBAna_{}_Gain.h5'.format(out_folder, dclass))"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -452,7 +654,7 @@
" 'Mean': ['data', '', 'Mean over pixels'],\n",
" 'std': ['dataStd', '', '$\\sigma$ over pixels'],\n",
" 'MeanBP': ['dataBP', 'Good pixels only', 'Mean over pixels'],\n",
- " 'NBP': ['nBP', 'Fraction of BP', 'Number of BP'],\n",
+ " 'NBP': ['nBP', '', 'Fraction of BP'],\n",
" 'stdBP': ['dataBPStd', 'Good pixels only', '$\\sigma$ over pixels'],\n",
"}\n"
]
@@ -470,134 +672,171 @@
"# loop over constat type\n",
"for const, modules in ret_constants.items():\n",
" \n",
+ " # split key to constant name and list of insets\n",
" const = const.split(\"_\")\n",
- " if True:\n",
- "\n",
- " print('Const: {}'.format(const))\n",
+ " gain = [int(x[1]) for x in const if 'g' in x]\n",
+ " gain = gain[0] if len(gain)>0 else None\n",
+ " \n",
+ " print('Const: {}'.format(const))\n",
"\n",
- " # summary over modules\n",
- " mod_data = {}\n",
- " mod_names = []\n",
- " mod_times = []\n",
+ " # summary over modules\n",
+ " mod_data = {}\n",
+ " mod_names = []\n",
+ " mod_times = []\n",
" \n",
- " # Loop over modules\n",
- " for mod, data in modules.items():\n",
- " print('Module: {}'.format(mod))\n",
- "\n",
- " ctimes = np.array(data[\"ctime\"])\n",
- " ctimes_ticks = [x.strftime('%y-%m-%d') for x in ctimes]\n",
- "\n",
- " if (\"mdata\" in data):\n",
- " cmdata = np.array(data[\"mdata\"])\n",
- " for i, tick in enumerate(ctimes_ticks):\n",
- " for entr in x_labels:\n",
- " ctimes_ticks[i] += ', {}={}'.format(entr[0].upper(), \n",
- " cmdata[i].get(entr, None))\n",
- "\n",
- " sort_ind = np.argsort(ctimes_ticks)\n",
- " ctimes_ticks = list(np.array(ctimes_ticks)[sort_ind])\n",
- "\n",
- " # Create sorted by data dataset\n",
- " rdata = {}\n",
- " for key, item in keys.items():\n",
- " if item[0] in data:\n",
- " rdata[key] = np.array(data[item[0]])[sort_ind]\n",
- "\n",
- " nTimes = rdata['Mean'].shape[0]\n",
- " nPixels = rdata['Mean'].shape[1] * rdata['Mean'].shape[2]\n",
- " nBins = nPixels\n",
- " \n",
- " # Avoid to low values\n",
- " if const[0] in [\"Noise10Hz\", \"Offset10Hz\"]:\n",
- " rdata['Mean'][rdata['Mean'] < 0.1] = np.nan\n",
- " if 'MeanBP' in rdata:\n",
- " rdata['MeanBP'][rdata['MeanBP'] < 0.1] = np.nan\n",
- " if 'NBP' in rdata:\n",
- " rdata['NBP'] = rdata['NBP'].astype(float)\n",
- " rdata['NBP'][rdata['NBP'] == spShape[0]*spShape[1]] = np.nan\n",
- "\n",
- " # Reshape: ASICs over cells for plotting\n",
- " pdata = {}\n",
- " for key in rdata:\n",
- " if len(rdata[key].shape)<3:\n",
- " continue\n",
+ " # Loop over modules\n",
+ " for mod, data in modules.items():\n",
+ " print('Module: {}'.format(mod))\n",
+ "\n",
+ " ctimes = np.array(data[\"ctime\"])\n",
+ " ctimes_ticks = [x.strftime('%y-%m-%d') for x in ctimes]\n",
+ "\n",
+ " if (\"mdata\" in data):\n",
+ " cmdata = np.array(data[\"mdata\"])\n",
+ " for i, tick in enumerate(ctimes_ticks):\n",
+ " for entr in x_labels:\n",
+ " key = entr[0].upper()\n",
+ " val = cmdata[i].get(entr, None)\n",
+ " if val is not None:\n",
+ " ctimes_ticks[i] += ', {}={:.1f}'.format(key, val)\n",
+ " \n",
+ " sort_ind = np.argsort(ctimes_ticks)\n",
+ " ctimes_ticks = list(np.array(ctimes_ticks)[sort_ind])\n",
+ "\n",
+ " # Create sorted by data dataset\n",
+ " rdata = {}\n",
+ " for key, item in keys.items():\n",
+ " if item[0] in data:\n",
+ " rdata[key] = np.array(data[item[0]])[sort_ind]\n",
+ "\n",
+ " nTimes = rdata['Mean'].shape[0]\n",
+ " nPixels = rdata['Mean'].shape[1] * rdata['Mean'].shape[2]\n",
+ " nBins = nMemToShow * nPixels\n",
+ "\n",
+ " # Avoid too low values\n",
+ " if const[0] in [\"Noise10Hz\", \"Offset10Hz\"]:\n",
+ " rdata['Mean'][rdata['Mean'] < 0.1] = np.nan\n",
+ " if 'MeanBP' in rdata:\n",
+ " rdata['MeanBP'][rdata['MeanBP'] < 0.1] = np.nan\n",
+ " if 'NBP' in rdata:\n",
+ " rdata['NBP'] = rdata['NBP'].astype(float)\n",
+ " rdata['NBP'][rdata['NBP'] == spShape[0]*spShape[1]] = np.nan\n",
+ " rdata[\"NBP\"] = rdata[\"NBP\"] / (spShape[0] * spShape[1]) * 100\n",
+ "\n",
+ " # Reshape: ASICs over cells for plotting\n",
+ " pdata = {}\n",
+ " for key in rdata:\n",
+ " if len(rdata[key].shape)<3:\n",
+ " continue\n",
+ " if dclass in ['AGIPD', 'LPD']:\n",
+ " pdata[key] = rdata[key][:, :, :, :nMemToShow].reshape(\n",
+ " nTimes, nBins).swapaxes(0, 1)\n",
+ " else:\n",
" pdata[key] = rdata[key].reshape(nTimes, nBins).swapaxes(0, 1)\n",
"\n",
- " # Summary over ASICs\n",
- " adata = {}\n",
- " for key in rdata:\n",
- " if len(rdata[key].shape)<3:\n",
- " continue\n",
- " adata[key] = np.nansum(rdata[key], axis=(1, 2))\n",
- "\n",
- " # Summary information over modules\n",
- " for key in pdata:\n",
- " if key not in mod_data:\n",
- " mod_data[key] = []\n",
- " if key == 'NBP':\n",
- " mod_data[key].append(np.nansum(pdata[key], axis=0))\n",
- " else:\n",
- " mod_data[key].append(np.nanmean(pdata[key], axis=0))\n",
- "\n",
- " mod_names.append(mod)\n",
- " mod_times.append(ctimes[sort_ind])\n",
- " \n",
- " # Plotting\n",
- " for key in pdata:\n",
- " \n",
- " if len(pdata[key].shape)<2:\n",
- " continue\n",
- " \n",
- " vmin,vmax = get_range(pdata[key][::-1].flatten(), plot_range)\n",
- " if key == 'NBP':\n",
- " unit = '[%]'\n",
- " else:\n",
- " unit = '[ADU]'\n",
- "\n",
- " title = '{}, module {}, {}'.format(\n",
- " const[0], mod, keys[key][1])\n",
- " cb_label = '{}, {} {}'.format(const[0], keys[key][2], unit)\n",
- "\n",
- " fname = '{}/{}_{}'.format(out_folder, const[0], mod.replace('_', ''))\n",
- " for item in const[1:]:\n",
- " fname = '{}_{}'.format(fname, item)\n",
- " fname = '{}_ASIC_{}.png'.format(fname, key)\n",
- " \n",
- " hm_combine(pdata[key][::-1].astype(float), htype=HMType.mro,\n",
- " x_label='Creation Time', y_label='ASIC ID',\n",
- " x_ticklabels=ctimes_ticks,\n",
- " x_ticks=np.arange(len(ctimes_ticks))+0.3,\n",
- " title=title, cb_label=cb_label,\n",
- " vmin=vmin, vmax=vmax,\n",
- " fname=fname,\n",
- " pad=[0.125, 0.125, 0.12, 0.185])\n",
+ " # Summary over ASICs\n",
+ " adata = {}\n",
+ " for key in rdata:\n",
+ " if len(rdata[key].shape)<3 or nMemToShow==1:\n",
+ " continue\n",
+ " adata[key] = np.nanmean(rdata[key], axis=(1, 2)).swapaxes(0, 1)\n",
+ "\n",
+ " # Summary information over modules\n",
+ " for key in pdata:\n",
+ " if key not in mod_data:\n",
+ " mod_data[key] = []\n",
+ " if key == 'NBP':\n",
+ " mod_data[key].append(np.nansum(pdata[key], axis=0))\n",
+ " else:\n",
+ " mod_data[key].append(np.nanmean(pdata[key], axis=0))\n",
"\n",
- " \n",
- " # Summary over modules\n",
- " for key in mod_data:\n",
- " \n",
+ " mod_names.append(mod)\n",
+ " mod_times.append(ctimes[sort_ind])\n",
+ "\n",
+ " # Plotting\n",
+ " for key in pdata:\n",
+ "\n",
+ " if len(pdata[key].shape)<2:\n",
+ " continue\n",
+ "\n",
+ " vmin,vmax = get_range(pdata[key][::-1].flatten(), plot_range)\n",
" if key == 'NBP':\n",
- " unit = ''\n",
+ " unit = '[%]'\n",
+ " title = 'BadPixelsDark'\n",
" else:\n",
" unit = '[ADU]'\n",
+ " title = const[0]\n",
"\n",
- " title = '{}, All modules, {}'.format(\n",
- " const[0], keys[key][1])\n",
+ " title += ', module {}'.format(mod)\n",
+ " if keys[key][1] != '':\n",
+ " title += ', {}'.format(keys[key][1])\n",
+ " if gain is not None:\n",
+ " title += ', {}'.format(gain_titles[gain])\n",
" \n",
- " fname = '{}/{}_{}'.format(out_folder, const[0], 'all')\n",
+ " cb_label = '{}, {} {}'.format(const[0], keys[key][2], unit)\n",
+ "\n",
+ " fname = '{}/{}_{}'.format(out_folder, const[0], mod.replace('_', ''))\n",
" for item in const[1:]:\n",
" fname = '{}_{}'.format(fname, item)\n",
" fname = '{}_ASIC_{}.png'.format(fname, key)\n",
- " \n",
- " fig = plt.figure(figsize=(12,12) )\n",
- " for i in range(len(mod_data[key])):\n",
- " plt.scatter(mod_times[i], mod_data[key][i], label=mod_names[i])\n",
- " plt.grid()\n",
- " plt.xlabel('Creation Time')\n",
- " plt.ylabel('{}, {} {}'.format(const[0], keys[key][2], unit)) \n",
- " plt.legend(loc='best guess')\n",
- " plt.title(title)\n",
- " fig.savefig(fname)\n"
+ "\n",
+ " if nMemToShow>1:\n",
+ " htype=HMType.INSET_AXIS\n",
+ " else: \n",
+ " htype=HMType.mro\n",
+ "\n",
+ " hm_combine(pdata[key][::-1].astype(float), htype=htype,\n",
+ " x_label='Creation Time', y_label=sp_name,\n",
+ " x_ticklabels=ctimes_ticks,\n",
+ " x_ticks=np.arange(len(ctimes_ticks))+0.3,\n",
+ " title=title, cb_label=cb_label,\n",
+ " vmin=vmin, vmax=vmax,\n",
+ " fname=fname,\n",
+ " pad=[0.125, 0.125, 0.12, 0.185])\n",
+ "\n",
+ " if nMemToShow>1:\n",
+ " vmin,vmax = get_range(adata[key][::-1].flatten(), plot_range)\n",
+ " hm_combine(adata[key].astype(float), htype=HMType.mro,\n",
+ " x_label='Creation Time', y_label='Memory cell ID',\n",
+ " x_ticklabels=ctimes_ticks,\n",
+ " x_ticks=np.arange(len(ctimes_ticks))+0.3,\n",
+ " title=title, cb_label=cb_label,\n",
+ " fname=fname.replace('ASIC', 'MEM'),\n",
+ " vmin=vmin, vmax=vmax)\n",
+ "\n",
+ " plt.show()\n",
+ " # Summary over modules\n",
+ " for key in mod_data:\n",
+ " if dclass in ['AGIPD', 'LPD']:\n",
+ " continue\n",
+ "\n",
+ " if key == 'NBP':\n",
+ " unit = '[%]'\n",
+ " title = 'BadPixelsDark'\n",
+ " else:\n",
+ " unit = '[ADU]'\n",
+ " title = const[0]\n",
+ "\n",
+ " title += ', module {}'.format(mod)\n",
+ " if keys[key][1] != '':\n",
+ " title += ', {}'.format(keys[key][1])\n",
+ " if gain is not None:\n",
+ " title += ', {}'.format(gain_titles[gain])\n",
+ "\n",
+ " fname = '{}/{}_{}'.format(out_folder, const[0], 'all')\n",
+ " for item in const[1:]:\n",
+ " fname = '{}_{}'.format(fname, item)\n",
+ " fname = '{}_ASIC_{}.png'.format(fname, key)\n",
+ "\n",
+ " fig = plt.figure(figsize=(12,12) )\n",
+ " for i in range(len(mod_data[key])):\n",
+ " plt.scatter(mod_times[i], mod_data[key][i], label=mod_names[i])\n",
+ " plt.grid()\n",
+ " plt.xlabel('Creation Time')\n",
+ " plt.ylabel('{}, {} {}'.format(const[0], keys[key][2], unit)) \n",
+ " plt.legend(loc='best guess')\n",
+ " plt.title(title)\n",
+ " fig.savefig(fname)\n"
]
}
],
diff --git a/notebooks/generic/PlotFromCalDB_Summary_NBC.ipynb b/notebooks/generic/PlotFromCalDB_Summary_NBC.ipynb
index a071734e7..8d6d6e884 100644
--- a/notebooks/generic/PlotFromCalDB_Summary_NBC.ipynb
+++ b/notebooks/generic/PlotFromCalDB_Summary_NBC.ipynb
@@ -6,9 +6,13 @@
"source": [
"# Statistical analysis of calibration factors#\n",
"\n",
- "Author: Mikhail Karnevskiy, Steffen Hauf, Version 0.2\n",
+ "Author: Mikhail Karnevskiy, Version 0.2\n",
"\n",
- "Plot calibration constants for AGIPD1M1 detector aggregated within detector modules. Input information is taken from folder `use_existing`. Corresponding files are prepared by `PlotFromCalDB` notebook."
+ "Plot calibration constants retrieved from the cal. DB.\n",
+ "\n",
+ "To be visualized, calibration constants are averaged per group of pixels. Plots shows calibration constant over time for each constant.\n",
+ "\n",
+ "Values shown in plots are saved in h5 files."
]
},
{
@@ -18,8 +22,8 @@
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
- "out_folder = \"/gpfs/exfel/data/scratch/karnem/PlotCalDB/MID/AGIPD1M2/\" # Output folder, required\n",
- "use_existing = \"/gpfs/exfel/data/scratch/karnem/PlotCalDB/MID/AGIPD1M2/\" # Input folder\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/karnem/test_bla4/\" # Output folder, required\n",
+ "use_existing = \"/home/karnem/myscratch/PlotCalDB/SPB/AGIPD1M1/\" # Input folder\n",
"dclass = \"AGIPD\" # Detector class\n",
"nMemToShow = 32 # Number of memory cells to be shown in plots over ASICs\n",
"range_offset = [4000., 5500, 6500, 8500] # plotting range for offset: high gain l, r, medium gain l, r \n",
@@ -30,7 +34,9 @@
"range_slopesCI = [22.0, 27.0, -0.5, 1.5] # plotting range for slope CI: high gain l, r, medium gain l, r \n",
"range_slopesFF = [0.8, 1.2, 0.6, 1.2] # plotting range for slope FF: high gain l, r, medium gain l, r \n",
"plot_range = 3 # range for plotting in units of median absolute deviations\n",
- "x_labels = ['Sensor Bias Voltage', 'Memory cells'] # parameters to be shown on X axis"
+ "x_labels = ['Sensor Bias Voltage', 'Memory cells'] # parameters to be shown on X axis\n",
+ "spShape = [64, 64] # Shape of superpixel\n",
+ "gain_titles = ['High gain', 'Medium gain', 'Low gain'] # Title inset related to gain"
]
},
{
@@ -68,7 +74,7 @@
},
"outputs": [],
"source": [
- "print('Load data from {}/CalDBAna_{}_Q1M2.h5'.format(use_existing, dclass))\n",
+ "print('Load data from {}/CalDBAna_{}_*.h5'.format(use_existing, dclass))\n",
"ret_constants = load_data_from_hdf5(\n",
" '{}/CalDBAna_{}_*.h5'.format(use_existing, dclass))\n",
"\n",
@@ -100,20 +106,18 @@
" 'Mean': ['data', '', 'Mean over pixels'],\n",
" 'std': ['dataStd', '', '$\\sigma$ over pixels'],\n",
" 'MeanBP': ['dataBP', 'Good pixels only', 'Mean over pixels'],\n",
- " 'NBP': ['nBP', 'Fraction of BP', 'Fraction of BP'],\n",
+ " 'NBP': ['nBP', '', 'Fraction of BP'],\n",
" 'stdBP': ['dataBPStd', 'Good pixels only', '$\\sigma$ over pixels'],\n",
" 'stdASIC': ['', '', '$\\sigma$ over ASICs'],\n",
" 'stdCell': ['', '', '$\\sigma$ over Cells'],\n",
- "}\n",
- "\n",
- "gain_name = ['High', 'Medium', 'Low']"
+ "}\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
- "scrolled": false
+ "scrolled": true
},
"outputs": [],
"source": [
@@ -122,134 +126,127 @@
"# loop over constat type\n",
"for const, modules in ret_constants.items():\n",
"\n",
- " # Loop over gain\n",
- " for gain in range(2):\n",
- " print('Const: {}, gain : {}'.format(const, gain))\n",
- "\n",
- " if const in [\"Gain\", \"Noise-e\"] and gain == 1:\n",
- " continue\n",
- "\n",
- " # loop over modules\n",
- " mod_data = {}\n",
- " mod_data['stdASIC'] = []\n",
- " mod_data['stdCell'] = []\n",
- " mod_names = []\n",
- " mod_times = []\n",
+ " const = const.split(\"_\")\n",
+ " gain = [int(x[1]) for x in const if 'g' in x]\n",
+ " gain = gain[0] if len(gain)>0 else None\n",
+ " print('Const: {}, gain {}'.format(const, gain))\n",
"\n",
- " # Loop over modules\n",
- " for mod, data in modules.items():\n",
- " ctimes = np.array(data[\"ctime\"])\n",
- " ctimes_ticks = [x.strftime('%y-%m-%d') for x in ctimes]\n",
+ " # loop over modules\n",
+ " mod_data = {}\n",
+ " mod_data['stdASIC'] = []\n",
+ " mod_data['stdCell'] = []\n",
+ " mod_names = []\n",
+ " mod_times = []\n",
"\n",
- " if (\"mdata\" in data):\n",
- " cmdata = np.array(data[\"mdata\"])\n",
- " for i, tick in enumerate(ctimes_ticks):\n",
- " for entr in x_labels:\n",
- " ctimes_ticks[i] += ', {}={}'.format(entr[0].upper(), \n",
- " cmdata[i].get(entr, None))\n",
+ " # Loop over modules\n",
+ " for mod, data in modules.items():\n",
+ " ctimes = np.array(data[\"ctime\"])\n",
+ " ctimes_ticks = [x.strftime('%y-%m-%d') for x in ctimes]\n",
"\n",
- " sort_ind = np.argsort(ctimes_ticks)\n",
- " ctimes_ticks = list(np.array(ctimes_ticks)[sort_ind])\n",
+ " if (\"mdata\" in data):\n",
+ " cmdata = np.array(data[\"mdata\"])\n",
+ " for i, tick in enumerate(ctimes_ticks):\n",
+ " for entr in x_labels:\n",
+ " key = entr[0].upper()\n",
+ " val = cmdata[i].get(entr, None)\n",
+ " if val is not None:\n",
+ " ctimes_ticks[i] += ', {}={:.1f}'.format(key, val)\n",
"\n",
- " # Create sorted by data dataset\n",
- " rdata = {}\n",
- " for key, item in keys.items():\n",
- " if item[0] in data:\n",
- " rdata[key] = np.array(data[item[0]])[sort_ind]\n",
+ " sort_ind = np.argsort(ctimes_ticks)\n",
+ " ctimes_ticks = list(np.array(ctimes_ticks)[sort_ind])\n",
"\n",
- " nTimes = rdata['Mean'].shape[0]\n",
- " nPixels = rdata['Mean'].shape[1] * rdata['Mean'].shape[2]\n",
- " nBins = nMemToShow * nPixels\n",
+ " # Create sorted by data dataset\n",
+ " rdata = {}\n",
+ " for key, item in keys.items():\n",
+ " if item[0] in data:\n",
+ " rdata[key] = np.array(data[item[0]])[sort_ind]\n",
"\n",
- " # Select gain\n",
- " if const not in [\"Gain\", \"Noise-e\"]:\n",
- " for key in rdata:\n",
- " rdata[key] = rdata[key][..., gain]\n",
+ " nTimes = rdata['Mean'].shape[0]\n",
+ " nPixels = rdata['Mean'].shape[1] * rdata['Mean'].shape[2]\n",
+ " nBins = nMemToShow * nPixels\n",
"\n",
- " if 'NBP' in rdata:\n",
- " rdata[\"NBP\"] = rdata[\"NBP\"] / (64 * 64) * 100\n",
+ " if 'NBP' in rdata:\n",
+ " rdata[\"NBP\"] = rdata[\"NBP\"] / (spShape[0] * spShape[1]) * 100\n",
"\n",
- " # Reshape: ASICs over cells for plotting\n",
- " pdata = {}\n",
- " for key in rdata:\n",
- " pdata[key] = rdata[key][:, :, :, :nMemToShow].reshape(\n",
- " nTimes, nBins).swapaxes(0, 1)\n",
+ " # Reshape: ASICs over cells for plotting\n",
+ " pdata = {}\n",
+ " for key in rdata:\n",
+ " pdata[key] = rdata[key][:, :, :, :nMemToShow].reshape(\n",
+ " nTimes, nBins).swapaxes(0, 1)\n",
"\n",
- " # Summary information over modules\n",
- " for key in pdata:\n",
- " if key not in mod_data:\n",
- " mod_data[key] = []\n",
- " \n",
- " mod_data[key].append(np.nanmean(pdata[key], axis=0))\n",
- " # Avoid too low values\n",
- " if const in [\"Noise\", \"Offset\", \"Noise-e\"] and key in ['Mean', 'MeanBP']:\n",
- " mod_data[key][-1][mod_data[key][-1] == 0.0] = IMType.STRANGE_VAL.value\n",
- " if key=='NBP':\n",
- " if 'Mean' in mod_data:\n",
- " mod_data['Mean'][-1][mod_data[key][-1] == 100] = IMType.ALL_BAD.value\n",
- " if 'MeanBP' in mod_data:\n",
- " mod_data['MeanBP'][-1][mod_data[key][-1] == 100] = IMType.ALL_BAD.value\n",
- " mod_data[key][-1][mod_data[key][-1] == 100] = IMType.ALL_BAD.value\n",
- " \n",
+ " # Summary information over modules\n",
+ " for key in pdata:\n",
+ " if key not in mod_data:\n",
+ " mod_data[key] = []\n",
"\n",
- " mod_data['stdASIC'].append(np.nanstd(\n",
- " np.nanmean(rdata['Mean'][:, :, :, :nMemToShow], axis=(1, 2)), axis=1))\n",
- " mod_data['stdCell'].append(np.nanstd(\n",
- " np.nanmean(rdata['Mean'][:, :, :, :nMemToShow], axis=3), axis=(1, 2)))\n",
+ " mod_data[key].append(np.nanmean(pdata[key], axis=0))\n",
+ " # Avoid too low values\n",
+ " if const[0] in [\"Noise\", \"Offset\", \"Noise-e\"] and key in ['Mean', 'MeanBP']:\n",
+ " mod_data[key][-1][mod_data[key][-1] == 0.0] = IMType.STRANGE_VAL.value\n",
+ " if key=='NBP':\n",
+ " if 'Mean' in mod_data:\n",
+ " mod_data['Mean'][-1][mod_data[key][-1] == 100] = IMType.ALL_BAD.value\n",
+ " if 'MeanBP' in mod_data:\n",
+ " mod_data['MeanBP'][-1][mod_data[key][-1] == 100] = IMType.ALL_BAD.value\n",
+ " mod_data[key][-1][mod_data[key][-1] == 100] = IMType.ALL_BAD.value\n",
"\n",
- " mod_names.append(mod)\n",
- " mod_times.append(ctimes_ticks)\n",
"\n",
- " # Incert nans to get array-like list of data\n",
- " uTime = mod_times[0]\n",
- " for tlist in mod_times:\n",
- " uTime = sorted(multi_union(uTime, tlist))\n",
+ " mod_data['stdASIC'].append(np.nanstd(\n",
+ " np.nanmean(rdata['Mean'][:, :, :, :nMemToShow], axis=(1, 2)), axis=1))\n",
+ " mod_data['stdCell'].append(np.nanstd(\n",
+ " np.nanmean(rdata['Mean'][:, :, :, :nMemToShow], axis=3), axis=(1, 2)))\n",
"\n",
- " for i, tlist in enumerate(mod_times):\n",
- " for t, time in enumerate(uTime):\n",
- " if t == len(tlist) or time != tlist[t]:\n",
- " tlist.insert(t, time)\n",
- " for key in mod_data:\n",
- " mod_data[key][i] = np.insert(\n",
- " mod_data[key][i], t, IMType.NO_CONST.value )\n",
+ " mod_names.append(mod)\n",
+ " mod_times.append(ctimes_ticks)\n",
"\n",
- " # Plotting\n",
- " nModules = len(mod_names)\n",
- " mod_idx = np.argsort(mod_names)\n",
- " for key in mod_data:\n",
- " vmin,vmax = get_range(np.array(mod_data[key])[mod_idx][::-1].flatten(), plot_range)\n",
- " if const in rangevals and key in ['Mean', 'MeanBP']:\n",
- " vmin = rangevals[const][gain][0]\n",
- " vmax = rangevals[const][gain][1]\n",
+ " # Insert nans to get array-like list of data\n",
+ " uTime = mod_times[0]\n",
+ " for tlist in mod_times:\n",
+ " uTime = sorted(multi_union(uTime, tlist))\n",
"\n",
- " htype = None\n",
- " if const in ['SlopesFF', 'SlopesPC', 'SlopesCI']:\n",
- " htype = HMType.INSET_1D\n",
+ " for i, tlist in enumerate(mod_times):\n",
+ " for t, time in enumerate(uTime):\n",
+ " if t == len(tlist) or time != tlist[t]:\n",
+ " tlist.insert(t, time)\n",
+ " for key in mod_data:\n",
+ " mod_data[key][i] = np.insert(\n",
+ " mod_data[key][i], t, IMType.NO_CONST.value )\n",
"\n",
- " if key == 'NBP':\n",
- " unit = '[%]'\n",
- " else:\n",
- " unit = '[ADU]'\n",
- " if const == 'Noise-e':\n",
- " unit = '[$e^-$]'\n",
+ " # Plotting\n",
+ " nModules = len(mod_names)\n",
+ " mod_idx = np.argsort(mod_names)\n",
+ " for key in mod_data:\n",
+ " vmin,vmax = get_range(np.array(mod_data[key])[mod_idx][::-1].flatten(), plot_range)\n",
"\n",
- " title = '{}, All modules, {} gain, {}'.format(\n",
- " const, gain_name[gain], keys[key][1])\n",
- " cb_label = '{}, {} {}'.format(const, keys[key][2], unit)\n",
+ " htype = None\n",
+ " if const[0] in ['SlopesFF', 'SlopesPC', 'SlopesCI']:\n",
+ " htype = HMType.INSET_1D\n",
+ " \n",
+ " if key == 'NBP':\n",
+ " unit = '[%]'\n",
+ " title = 'BadPixelsDark'\n",
+ " else:\n",
+ " unit = '[ADU]'\n",
+ " title = const[0]\n",
"\n",
- " hm_combine(np.array(mod_data[key])[mod_idx][::-1],\n",
- " y_ticks=np.arange(nModules)[::-1]+0.8,\n",
- " y_ticklabels=np.array(mod_names)[mod_idx],\n",
- " x_label='Creation Time', y_label='Module ID',\n",
- " x_ticklabels=ctimes_ticks, x_ticks=np.arange(\n",
- " len(ctimes_ticks))+0.3,\n",
- " title=title, cb_label=cb_label,\n",
- " fname='{}/{}_all_g{}_{}.png'.format(\n",
- " out_folder, const, gain, key),\n",
- " vmin=vmin, vmax=vmax,\n",
- " pad=[0.125, 0.151, 0.12, 0.17], htype=htype)\n",
- " #break\n",
- " #break"
+ " title += ', All modules'\n",
+ " if keys[key][1] != '':\n",
+ " title += ', {}'.format(keys[key][1])\n",
+ " if gain is not None:\n",
+ " title += ', {}'.format(gain_titles[gain])\n",
+ " \n",
+ " cb_label = '{}, {} {}'.format(const[0], keys[key][2], unit)\n",
+ " hm_combine(np.array(mod_data[key])[mod_idx][::-1],\n",
+ " y_ticks=np.arange(nModules)[::-1]+0.8,\n",
+ " y_ticklabels=np.array(mod_names)[mod_idx],\n",
+ " x_label='Creation Time', y_label='Module ID',\n",
+ " x_ticklabels=ctimes_ticks, x_ticks=np.arange(\n",
+ " len(ctimes_ticks))+0.3,\n",
+ " title=title, cb_label=cb_label,\n",
+ " fname='{}/{}_all_g{}_{}.png'.format(\n",
+ " out_folder, const[0], gain, key),\n",
+ " vmin=vmin, vmax=vmax,\n",
+ " pad=[0.125, 0.151, 0.12, 0.17], htype=htype)\n"
]
}
],
diff --git a/reportservice/report_conf.yaml b/reportservice/report_conf.yaml
index 2fc336768..b8fbc0c3e 100644
--- a/reportservice/report_conf.yaml
+++ b/reportservice/report_conf.yaml
@@ -18,22 +18,110 @@ GLOBAL:
SPB:
AGIPD1M1:
det-type:
- - "AGIPD"
- - "STATS_FROM_DB"
+ - "GENERIC"
+ - "STATS_FROM_DB2"
+ modules:
+ - "AGIPD1M1"
start-date: "2019-01-01"
- end-date: "2019-12-12"
+ end-date: "NOW"
+ nconstants: 20
constants:
- "Noise"
- "SlopesFF"
- "SlopesPC"
- "Offset"
- modules: "0-16"
- bias-voltages:
+ dclass: "AGIPD"
+ submodules: "0-16"
+ bias-voltage:
- 300
- 500
- mem-cells:
+ memory-cells:
- 128
- 176
+ - 250
+ acquisition-rate:
+ - 1.1
+ - 2.2
+ - 4.5
+ photon-energy: 9.2
+ separate-plot:
+ - "gain_setting"
+ parameter-names:
+ - "bias_voltage"
+ - "acquisition_rate"
+ - "memory_cells"
+ spShape:
+ - 64
+ - 64
+ gain-titles:
+ - "High gain"
+ - "Medium gain"
+ - "Low gain"
+ x-labels:
+ - "Acquisition rate"
+ - "Memory cells"
+ sp-name: "ASICs id"
+ nMemToShow: 32
+ use-existing: "''"
+ out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+ cal-db-timeout: 180000
+ cal-db-interface: "tcp://max-exfl016:8015#8025"
+
+ JUNGFRAU:
+ det-type:
+ - "GENERIC"
+ - "STATS_FROM_DB"
+ start-date: "2019-01-01"
+ end-date: "NOW"
+ nconstants: 20
+ constants:
+ - "Noise"
+ - "Offset"
+ dclass: "jungfrau"
+ nMemToShow: 1
+ modules:
+ - "Jungfrau_M035"
+ - "Jungfrau_M203"
+ - "Jungfrau_M221"
+ bias-voltage:
+ - 90
+ - 180
+ memory-cells:
+ - 1
+ pixels-x:
+ - 1024
+ pixels-y:
+ - 512
+ - 1024
+ temperature:
+ - 291
+ integration-time:
+ - 50
+ - 250
+ gain-setting:
+ - 0
+ separate-plot:
+ - "integration_time"
+ - "gain_setting"
+ parameter-names:
+ - "bias_voltage"
+ - "integration_time"
+ - "pixels_x"
+ - "pixels_y"
+ - "gain_setting"
+ - "temperature"
+ - "memory_cells"
+ spShape:
+ - 256
+ - 64
+ gain-titles:
+ - "High gain"
+ - "Medium gain"
+ - "Low gain"
+ x-labels:
+ - "Sensor Temperature"
+ - "Integration Time"
+ sp-name: "Supercolumn 256*64"
photon-energy: 9.2
use-existing: "''"
out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
@@ -43,65 +131,67 @@ SPB:
MID:
AGIPD1M2:
det-type:
- - "AGIPD"
- - "STATS_FROM_DB"
- db-module: "AGIPD1M2"
+ - "GENERIC"
+ - "STATS_FROM_DB2"
+ modules:
+ - "AGIPD1M2"
start-date: "2019-01-01"
- end-date: "2019-12-12"
+ end-date: "NOW"
+ nconstants: 20
constants:
- "Noise"
- "SlopesFF"
- "SlopesPC"
- "Offset"
- modules: "0-16"
- bias-voltages:
+ dclass: "AGIPD"
+ submodules: "0-16"
+ bias-voltage:
- 300
- 500
- mem-cells:
+ memory-cells:
+ - 128
- 176
+ - 250
+ acquisition-rate:
+ - 1.1
+ - 2.2
+ - 4.5
photon-energy: 9.2
+ separate-plot:
+ - "gain_setting"
+ parameter-names:
+ - "bias_voltage"
+ - "acquisition_rate"
+ - "memory_cells"
+ spShape:
+ - 64
+ - 64
+ gain-titles:
+ - "High gain"
+ - "Medium gain"
+ - "Low gain"
+ x-labels:
+ - "Acquisition rate"
+ - "Memory cells"
+ sp-name: "ASICs id"
+ nMemToShow: 32
use-existing: "''"
out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
cal-db-timeout: 180000
cal-db-interface: "tcp://max-exfl016:8015#8025"
- range-offset:
- - 800.
- - 1500.
- - 600
- - 900
- range-noise:
- - 2.0
- - 16.
- - 1.0
- - 7.0
- range-gain:
- - 20
- - 30
- - 20
- - 30
- range-noise-e:
- - 100.
- - 600.
- - 100.
- - 600.
- range-slopesPC:
- - 0.95
- - 1.05
- - 0.0
- - 0.5
- range-slopesFF:
- - 0.8
- - 1.2
- - 0.8
- - 1.2
-
- EPIX01:
+
+ EPIX:
det-type:
- - "EPIX"
+ - "GENERIC"
- "STATS_FROM_DB"
- db-module: "ePix100_M15"
start-date: "2019-01-01"
- end-date: "2019-05-12"
+ end-date: "NOW"
+ nconstants: 20
+ dclass: "ePix100"
+ nMemToShow: 1
+ modules:
+ - "ePix100_M15"
+ - "ePix100_M18"
constants:
- "Noise"
- "Offset"
@@ -113,18 +203,322 @@ MID:
- 1
- 50
photon-energy: 9.2
+ separate-plot:
+ - "integration_time"
+ parameter-names:
+ - "bias_voltage"
+ - "integration_time"
+ - "temperature"
+ - "in_vacuum"
+ spShape:
+ - 354
+ - 96
+ x-labels:
+ - "Sensor Temperature"
+ - "Integration Time"
+ sp-name: "ASICs id"
+ use-existing: "''"
+ out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+ cal-db-timeout: 180000
+ cal-db-interface: "tcp://max-exfl016:8015#8025"
+
+
+FXE:
+ LPD1M1:
+ det-type:
+ - "GENERIC"
+ - "STATS_FROM_DB2"
+ modules:
+ - "LPD1M1"
+ start-date: "2019-01-01"
+ end-date: "NOW"
+ nconstants: 20
+ constants:
+ - "Noise"
+ - "SlopesFF"
+ - "SlopesCI"
+ - "Offset"
+ dclass: "LPD"
+ submodules: "0-16"
+ bias-voltage:
+ - 250
+ - 500
+ memory-cells:
+ - 1
+ - 128
+ - 512
+ photon-energy: 9.2
+ separate-plot:
+ - "gain_setting"
+ parameter-names:
+ - "bias_voltage"
+ - "memory_cells"
+ spShape:
+ - 64
+ - 64
+ gain-titles:
+ - "High gain"
+ - "Medium gain"
+ - "Low gain"
+ x-labels:
+ - "Memory cells"
+ sp-name: "ASICs id"
+ nMemToShow: 32
+ use-existing: "''"
+ out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+ cal-db-timeout: 180000
+ cal-db-interface: "tcp://max-exfl016:8015#8025"
+
+ JUNGFRAU:
+ det-type:
+ - "GENERIC"
+ - "STATS_FROM_DB"
+ start-date: "2019-01-01"
+ end-date: "NOW"
+ nconstants: 20
+ constants:
+ - "Noise"
+ - "Offset"
+ dclass: "jungfrau"
+ modules:
+ - "Jungfrau_M233"
+ - "Jungfrau_M125"
+ - "Jungfrau_M260"
+ bias-voltage:
+ - 90
+ - 180
+ memory-cells:
+ - 1
+ pixels-x:
+ - 1024
+ pixels-y:
+ - 512
+ - 1024
+ temperature:
+ - 291
+ integration-time:
+ - 50
+ - 250
+ gain-setting:
+ - 0
+ separate-plot:
+ - "integration_time"
+ - "gain_setting"
+ parameter-names:
+ - "bias_voltage"
+ - "integration_time"
+ - "pixels_x"
+ - "pixels_y"
+ - "gain_setting"
+ - "temperature"
+ - "memory_cells"
+ spShape:
+ - 256
+ - 64
+ gain-titles:
+ - "High gain"
+ - "Medium gain"
+ - "Low gain"
+ nMemToShow: 1
+ x-labels:
+ - "Sensor Temperature"
+ - "Integration Time"
+ sp-name: "Supercolumn 256*64"
+ photon-energy: 9.2
+ use-existing: "''"
+ out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+ cal-db-timeout: 180000
+ cal-db-interface: "tcp://max-exfl016:8015#8025"
+
+DETLAB:
+ FASTCCD:
+ det-type:
+ - "GENERIC"
+ - "STATS_FROM_DB"
+ start-date: "2019-01-01"
+ end-date: "NOW"
+ nconstants: 20
+ constants:
+ - "Noise"
+ - "Offset"
+ dclass: "CCD"
+ nMemToShow: 1
+ modules:
+ - "fastCCD1"
+ bias-voltage:
+ - 79
+ temperature:
+ - 235
+ - 216
+ - 245
+ integration-time:
+ - 1
+ - 50
+ gain-setting:
+ - 0
+ - 1
+ - 2
+ - 8
+ pixels-x:
+ - 1934
+ pixels-y:
+ - 960
+ separate-plot:
+ - "integration_time"
+ - "gain_setting"
+ - "temperature"
+ parameter-names:
+ - "bias_voltage"
+ - "integration_time"
+ - "pixels_x"
+ - "pixels_y"
+ - "gain_setting"
+ - "temperature"
+ spShape:
+ - 967
+ - 10
+ gain-titles:
+ - "gain 0x"
+ - "gain 1x"
+ - "gain 2x"
+ - "gain 8x"
+ x-labels:
+ - "Sensor Temperature"
+ - "Integration Time"
+ sp-name: "Supercolumn 967*10"
+ photon-energy: 9.2
+ use-existing: "''"
+ out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+ cal-db-timeout: 180000
+ cal-db-interface: "tcp://max-exfl016:8015#8025"
+
+
+SCS:
+ FASTCCD:
+ det-type:
+ - "GENERIC"
+ - "STATS_FROM_DB"
+ start-date: "2019-01-01"
+ end-date: "NOW"
+ nconstants: 20
+ constants:
+ - "Noise"
+ - "Offset"
+ dclass: "CCD"
+ nMemToShow: 1
+ modules:
+ - "fastCCD1"
+ bias-voltage:
+ - 79
+ temperature:
+ - 235
+ - 216
+ - 245
+ integration-time:
+ - 1
+ - 50
+ gain-setting:
+ - 0
+ - 1
+ - 2
+ - 8
+ pixels-x:
+ - 1934
+ pixels-y:
+ - 960
+ separate-plot:
+ - "integration_time"
+ - "gain_setting"
+ - "temperature"
+ parameter-names:
+ - "bias_voltage"
+ - "integration_time"
+ - "pixels_x"
+ - "pixels_y"
+ - "gain_setting"
+ - "temperature"
+ spShape:
+ - 967
+ - 10
+ gain-titles:
+ - "gain 0x"
+ - "gain 1x"
+ - "gain 2x"
+ - "gain 8x"
+ x-labels:
+ - "Sensor Temperature"
+ - "Integration Time"
+ sp-name: "Supercolumn 967*10"
+ photon-energy: 9.2
+ use-existing: "''"
+ out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+ cal-db-timeout: 180000
+ cal-db-interface: "tcp://max-exfl016:8015#8025"
+
+SQS:
+ PNCCD:
+ det-type:
+ - "GENERIC"
+ - "STATS_FROM_DB"
+ start-date: "2019-01-01"
+ end-date: "NOW"
+ nconstants: 20
+ constants:
+ - "Noise"
+ - "Offset"
+ dclass: "CCD"
+ nMemToShow: 1
+ modules:
+ - "PnCCD1"
+ bias-voltage:
+ - 300
+ temperature:
+ - 235
+ integration-time:
+ - 1
+ - 50
+ gain-setting:
+ - 0
+ pixels-x:
+ - 1024
+ pixels-y:
+ - 1024
+ separate-plot:
+ - "integration_time"
+ - "temperature"
+ parameter-names:
+ - "bias_voltage"
+ - "integration_time"
+ - "pixels_x"
+ - "pixels_y"
+ - "gain_setting"
+ - "temperature"
+ spShape:
+ - 256
+ - 256
+ x-labels:
+ - "Sensor Temperature"
+ - "Integration Time"
+ sp-name: "ASICs id"
+ photon-energy: 9.2
use-existing: "''"
out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
cal-db-timeout: 180000
cal-db-interface: "tcp://max-exfl016:8015#8025"
- EPIX02:
+HED:
+ EPIX:
det-type:
- - "EPIX"
+ - "GENERIC"
- "STATS_FROM_DB"
- db-module: "ePix100_M18"
start-date: "2019-01-01"
- end-date: "2019-05-12"
+ end-date: "NOW"
+ nconstants: 20
+ dclass: "ePix100"
+ nMemToShow: 1
+ modules:
+ - "ePix100_M16"
+ - "ePix100_M17"
constants:
- "Noise"
- "Offset"
@@ -136,35 +530,83 @@ MID:
- 1
- 50
photon-energy: 9.2
+ separate-plot:
+ - "integration_time"
+ parameter-names:
+ - "bias_voltage"
+ - "integration_time"
+ - "temperature"
+ - "in_vacuum"
+ spShape:
+ - 354
+ - 96
+ x-labels:
+ - "Sensor Temperature"
+ - "Integration Time"
+ sp-name: "superpixel id"
use-existing: "''"
out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
cal-db-timeout: 180000
cal-db-interface: "tcp://max-exfl016:8015#8025"
-FXE:
- LPD1M1:
+ JUNGFRAU:
det-type:
- - "LPD"
+ - "GENERIC"
- "STATS_FROM_DB"
- db-module: "LPD1M1"
- start-date: "2018-01-01"
- end-date: "2018-12-12"
+ start-date: "2019-01-01"
+ end-date: "NOW"
+ nconstants: 20
constants:
- "Noise"
- - "SlopesFF"
- - "SlopesCI"
- "Offset"
- modules: "0-16"
- bias-voltages:
- - 250
- - 500
- mem-cells:
+ dclass: "jungfrau"
+ nMemToShow: 1
+ modules:
+ - "Jungfrau_M039"
+ - "Jungfrau_M266"
+ bias-voltage:
+ - 90
+ - 180
+ memory-cells:
- 1
- - 128
- - 256
+ pixels-x:
+ - 1024
+ pixels-y:
- 512
+ - 1024
+ temperature:
+ - 291
+ integration-time:
+ - 50
+ - 250
+ gain-setting:
+ - 0
+ separate-plot:
+ - "integration_time"
+ - "gain_setting"
+ parameter-names:
+ - "bias_voltage"
+ - "integration_time"
+ - "pixels_x"
+ - "pixels_y"
+ - "gain_setting"
+ - "temperature"
+ - "memory_cells"
+ spShape:
+ - 256
+ - 64
+ gain-titles:
+ - "High gain"
+ - "Medium gain"
+ - "Low gain"
+ x-labels:
+ - "Sensor Temperature"
+ - "Integration Time"
+ sp-name: "Supercolumn 256*64"
photon-energy: 9.2
use-existing: "''"
out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
cal-db-timeout: 180000
cal-db-interface: "tcp://max-exfl016:8015#8025"
+
+
diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py
index 6baf79bd4..c5375bc0e 100644
--- a/xfel_calibrate/notebooks.py
+++ b/xfel_calibrate/notebooks.py
@@ -113,6 +113,13 @@ notebooks = {
"default concurrency": None,
"cluster cores": 1},
},
+ "STATS_FROM_DB2": {
+ "notebook": "notebooks/generic/PlotFromCalDB_NBC.ipynb",
+ "dep_notebooks": ["notebooks/generic/PlotFromCalDB_Summary_NBC.ipynb"],
+ "concurrency": {"parameter": "submodules",
+ "default concurrency": None,
+ "cluster cores": 1},
+ },
},
"TUTORIAL": {
"TEST": {
--
GitLab