From 89080e101bb5d2e27645e40f5888dd302ffed449 Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Wed, 10 Jul 2019 12:13:43 +0200
Subject: [PATCH] update plotting notebooks
---
cal_tools/cal_tools/ana_tools.py | 202 ++++--
notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb | 326 +++++----
.../PlotFromCalDB_Summary_AGIPD_NBC.ipynb | 286 ++++++++
notebooks/LPD/PlotFromCalDB.ipynb | 342 ---------
notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb | 652 ++++++++++++++++++
5 files changed, 1267 insertions(+), 541 deletions(-)
create mode 100644 notebooks/AGIPD/PlotFromCalDB_Summary_AGIPD_NBC.ipynb
delete mode 100644 notebooks/LPD/PlotFromCalDB.ipynb
create mode 100644 notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
diff --git a/cal_tools/cal_tools/ana_tools.py b/cal_tools/cal_tools/ana_tools.py
index 7d19a928e..689c94681 100644
--- a/cal_tools/cal_tools/ana_tools.py
+++ b/cal_tools/cal_tools/ana_tools.py
@@ -1,7 +1,6 @@
import h5py
import numpy as np
import matplotlib.pyplot as plt
-import seaborn
import datetime
import glob
@@ -43,21 +42,41 @@ def load_data_from_hdf5(filelist):
for mKey in f.get(cKey):
if mKey not in data[cKey]:
data[cKey][mKey] = {}
- data[cKey][mKey]["mdata"] = []
- # Loop over time stamps
+ # Loop over module data
for tKey in f.get(cKey + '/' + mKey):
- # Loop over metadata
- mdata = {}
- for mdKey in f.get(
- "/".join((cKey, mKey, tKey, 'mdata'))):
- mdata[mdKey] = f.get(
- "/".join(
- (cKey, mKey, tKey, 'mdata', mdKey))).value
- data[cKey][mKey]["mdata"].append(mdata)
+
+ if tKey == 'ctime':
+ if "ctime" not in data[cKey][mKey]:
+ data[cKey][mKey]["ctime"] = []
+ for timeKey in f.get("/".join((cKey, mKey, tKey))):
+ path = "/".join((cKey, mKey, tKey, timeKey))
+ value = f.get(path).value
+ value = datetime.datetime.fromtimestamp(value)
+ data[cKey][mKey]["ctime"].append(value)
+ continue
+
+ # Load ndarray if they are there
+ item = f.get("/".join((cKey, mKey, tKey)))
+ if isinstance(item, h5py._hl.dataset.Dataset):
+ val = item.value
+ data[cKey][mKey][tKey] = val
+ continue
# Loop over stored data
for dKay in f.get( "/".join((cKey, mKey, tKey))):
+
+ # Loop over metadata
if dKay == "mdata":
+ mdata = {}
+ for mdKey in f.get(
+ "/".join((cKey, mKey, tKey, 'mdata'))):
+ mdata[mdKey] = f.get(
+ "/".join(
+ (cKey, mKey, tKey, 'mdata',
+ mdKey))).value
+ if "mdata" not in data[cKey][mKey]:
+ data[cKey][mKey]["mdata"] = []
+ data[cKey][mKey]["mdata"].append(mdata)
continue
if dKay not in data[cKey][mKey]:
@@ -97,7 +116,13 @@ def recursively_save_dict_contents_to_group(h5file, path, dic):
elif isinstance(item, (list, tuple)):
for i, x in enumerate(item):
newpath = '/'.join((path, str(key), str(i)))
- recursively_save_dict_contents_to_group(h5file, newpath, x)
+ if isinstance(x, dict):
+ recursively_save_dict_contents_to_group(h5file, newpath, x)
+ elif isinstance(x, datetime.datetime):
+ h5file[newpath] = x.timestamp()
+ else:
+ raise ValueError('Cannot save a list of %s ' % type(item))
+
# Save strings, scalars or array of scalars
elif (np.isscalar(item) or
@@ -211,55 +236,6 @@ def combine_constants(ctimes_a, ctimes_b):
return ctime, icomb
-def plot_heatmap(data, x_title='Pixel X', y_title='Pixel Y', x_labels='',
- title='Average over pixels and cells', label='', fname='',
- vmax=None, vmin=None, ytickspos=None, yticks=None):
- """Plotting heatmap from 2D data
-
- Args:
- data: 2D dataset.
- x_title: Label for x axis
- y_title: Label for y axis
- title: Title on the top side of a plot
- x_labels: Ticks for X axis (list or array)
- yticks: Ticks for Y axis (list or array)
- ytickspos: Positions of ticks for Y axis (list or array)
- vmax: Maximum value of heatmap
- vmin: Minimum value of heatmap
- label: Label on the right side of a plot
- fname: Name of output file (including path)
- """
-
- fig = plt.figure(figsize=(12, 12))
- ax = fig.add_subplot(111)
-
- if vmin is None:
- vmin = np.nanmin(data[np.nonzero(data)])
-
- seaborn.heatmap(data, xticklabels=10, yticklabels=32, cmap='jet',
- vmin=vmin, vmax=vmax, cbar_kws={"label": label})
- ax.set_xticklabels(x_labels, minor=False, rotation=90)
- ax.figure.axes[-1].yaxis.label.set_size(20)
-
- plt.xticks(np.arange(data[0, :].size) + 0.3, )
-
- plt.xlabel(x_title, size=18)
- plt.ylabel(y_title, size=18)
- plt.title(title, size=18)
-
- if ytickspos is not None:
- plt.yticks(ytickspos, yticks)
-
- plt.subplots_adjust(wspace=0.2, hspace=0.2)
-
- if fname != '':
- fig.savefig(fname, bbox_inches='tight')
-
-
-# The following functions are used to combine two lists in one.
-# Elements present in both lists counted only once.
-# Output list is sorted.
-
def to_multiset(x):
"""
Convert list to multiset
@@ -324,3 +300,107 @@ def multi_intersect(a, b):
aa = to_multiset(a)
bb = to_multiset(b)
return from_multiset(aa & bb)
+
+
+def HMCombine(data, fname=None, type=1, **kwargs):
+ """
+ Plot heatmap for calibration report
+
+ :param data: data to plot
+ :param fname: file name of output file
+ :param type: type of figure
+ :param kwargs: other keywords supported by pyDetLib heatmapPlot
+ """
+
+ from XFELDetAna import xfelpyanatools as xana
+
+ fig = plt.figure(figsize=(10, 12))
+ ax = fig.add_subplot(111)
+
+ xana.heatmapPlot(data, add_panels=False, cmap='viridis',
+ cb_pad=0.6 if type == 1 else 0.1,
+ use_axis=ax,
+ **kwargs)
+ plt.setp(ax.yaxis.get_majorticklabels(), rotation=90)
+
+ pad = kwargs.get('pad', [0.125, 0.125, 0.1, 0.18])
+
+ pad_b = pad[0]
+ pad_l = pad[1]
+ pad_t = pad[2]
+ pad_r = pad[3]
+ mar = 0.003
+
+ vmin = kwargs.get('vmin', None)
+ vmax = kwargs.get('vmax', None)
+
+ h_frame = (1 - pad_b - pad_t)
+ w_frame = 1 - pad_l - pad_r
+
+ if type == 2:
+ ax.tick_params(axis='y', which='major', pad=50)
+ for y in range(data.shape[0]):
+
+ h_frame = (1 - pad_b - pad_t) / data.shape[0]
+ w_frame = 1 - pad_l - pad_r
+
+ mean = np.nanmean(data[y])
+ if vmin and vmax:
+ mean = np.nanmean(
+ data[y, np.where((data[y] > vmin) & (data[y] < vmax))])
+
+ ax = plt.axes(
+ [pad_l, pad_b + h_frame * y + mar, w_frame, h_frame - 2 * mar],
+ frame_on=False,
+ yticks=[mean],
+ yticklabels=['{:4.2f}'.format(mean)],
+ xticks=[],
+ xticklabels=[]
+ )
+ ax.tick_params(axis='y', which='major', pad=25)
+
+ if vmin and vmax:
+ ax.set_ylim([vmin, vmax])
+
+ d = [{'x': np.arange(data.shape[1]),
+ 'y': data[y],
+ 'drawstyle': 'steps-mid',
+ 'linestyle': 'dotted',
+ # 'marker': '^',
+ 'linewidth': 2,
+
+ 'color': 'white',
+ },
+ {'x': np.array([0, data.shape[1] - 1]),
+ 'y': np.array([mean, mean]),
+ 'drawstyle': 'steps-mid',
+ 'linewidth': 1.0,
+ 'linestyle': 'dotted',
+ 'color': 'white',
+ },
+ ]
+
+ y_lab = ''
+ if y == data.shape[0] / 2:
+ y_lab = 'Average over time'
+
+ _ = xana.simplePlot(d, # aspect=1.6,
+ x_label="",
+ y_label=y_lab,
+ use_axis=ax,
+ y_log=False)
+ elif type == 1:
+
+ ax2 = plt.axes([pad_l, pad_b, w_frame, h_frame / 16.], frame_on=False,
+ yticks=range(3), yticklabels=[32, 16, 0],
+ xticks=[], xticklabels=[]
+ )
+ ax2.yaxis.set_label_position("right")
+ ax2.set_ylabel('Memory cell ID', color='r')
+ ax2.tick_params('y', colors='r', right=True, left=False,
+ labelright=True, labelleft=False, )
+ else:
+ pass
+
+ if fname:
+ fig.savefig(fname, bbox_inches='tight')
diff --git a/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb b/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
index f87e744d5..ebd3e6629 100644
--- a/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
+++ b/notebooks/AGIPD/PlotFromCalDB_AGIPD_NBC.ipynb
@@ -33,10 +33,14 @@
"constants = [\"Noise\", \"SlopesFF\", \"SlopesPC\", \"Offset\"] # Constants to plot\n",
"modules = [0] # Modules, range allowed\n",
"bias_voltages = [300, 500] # Bias voltage\n",
- "max_cells = [32, 64, 74, 128, 176] # Number of used memory cells. Typically: 4,32,64,128,176.\n",
+ "mem_cells = [128, 176] # Number of used memory cells. Typically: 4,32,64,128,176.\n",
"photon_energy = 9.2 # Photon energy of the beam\n",
- "out_folder = \"/gpfs/exfel/data/scratch/karnem/testAGIPD16_9/\" # Output folder, required\n",
- "cal_db_timeout = 280000 # timeout on caldb requests\\n\","
+ "out_folder = \"/gpfs/exfel/data/scratch/karnem/testAGIPD16_14/\" # Output folder, required\n",
+ "use_existing = \"/gpfs/exfel/data/scratch/karnem/testAGIPD16_9/\" # Input folder\n",
+ "cal_db_timeout = 1200000 # timeout on caldb requests\",\n",
+ "detectorDB = \"AGIPD1M1\" # detector entry in the DB to investigate\n",
+ "dclass = \"AGIPD\" # Detector class\n",
+ "cal_db_interface = \"tcp://max-exfl016:8015#8025\" # the database interface to use"
]
},
{
@@ -52,7 +56,7 @@
"import numpy as np\n",
"import matplotlib\n",
"\n",
- "matplotlib.use(\"agg\")\n",
+ "##matplotlib.use(\"agg\")\n",
"import matplotlib.pyplot as plt\n",
"% matplotlib inline\n",
"\n",
@@ -78,17 +82,19 @@
"source": [
"# Prepare variables\n",
"interval = 1 # interval for evaluation in days\n",
- "detectorDB = \"AGIPD1M1\" # detector entry in the DB to investigate\n",
- "dclass = \"AGIPD\" # Detector class\n",
"\n",
"if modules[0] == -1:\n",
" modules = list(range(16))\n",
"\n",
+ "in_mod_names = []\n",
+ "for mod in modules:\n",
+ " in_mod_names.append(\"Q{}M{}\".format(mod // 4 + 1, mod % 4 + 1))\n",
+ "\n",
"constantsDark = {\"SlopesFF\": 'BadPixelsFF',\n",
" 'SlopesPC': 'BadPixelsPC',\n",
" 'Noise': 'BadPixelsDark',\n",
" 'Offset': 'BadPixelsDark'}\n",
- "print(constantsDark)\n",
+ "print('Bad pixels data: ', constantsDark)\n",
"\n",
"# Define parameters in order to perform loop over time stamps\n",
"start = datetime.datetime.now() if start_date.upper() == \"NOW\" else dateutil.parser.parse(\n",
@@ -105,8 +111,13 @@
"det = getattr(Detectors, detectorDB)\n",
"dconstants = getattr(Constants, dclass)\n",
"\n",
- "cal_db_interface = \"tcp://max-exfl016:80\" + str(np.random.randint(15, 25))\n",
- "print(cal_db_interface)\n"
+ "if \"#\" in cal_db_interface:\n",
+ " prot, serv, ran = cal_db_interface.split(\":\")\n",
+ " r1, r2 = ran.split(\"#\")\n",
+ " cal_db_interface = \":\".join(\n",
+ " [prot, serv, str(np.random.randint(int(r1), int(r2)))])\n",
+ "\n",
+ "print('CalDB Interface {}'.format(cal_db_interface))"
]
},
{
@@ -116,7 +127,18 @@
"collapsed": true
},
"outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
"source": [
+ "import copy\n",
+ "\n",
"def prepare_to_store(a, nMem):\n",
" shape = list(a.shape[:2])+[nMem,2]\n",
" b = np.full(shape, np.nan)\n",
@@ -125,9 +147,9 @@
"\n",
"def get_rebined(a, rebin):\n",
" return a.reshape(\n",
- " a.shape[0] / rebin[0],\n",
+ " int(a.shape[0] / rebin[0]),\n",
" rebin[0],\n",
- " a.shape[1] / rebin[1],\n",
+ " int(a.shape[1] / rebin[1]),\n",
" rebin[1],\n",
" a.shape[2],\n",
" a.shape[3])\n",
@@ -138,8 +160,11 @@
"\n",
"ret_constants = {}\n",
"\n",
+ "if use_existing != '':\n",
+ " mem_cells = []\n",
+ "\n",
"# Loop over max_memory cells\n",
- "for the_max_cells in max_cells:\n",
+ "for the_mem_cells in mem_cells:\n",
"\n",
" # Loop over bias voltages\n",
" for bias_voltage in bias_voltages:\n",
@@ -167,26 +192,27 @@
" \"SlopesPC\"] or \"DARK\" in const.upper()):\n",
" dcond = Conditions.Dark\n",
" mcond = getattr(dcond, dclass)(\n",
- " memory_cells=the_max_cells,\n",
+ " memory_cells=the_mem_cells,\n",
" bias_voltage=bias_voltage)\n",
" else:\n",
" dcond = Conditions.Illuminated\n",
" mcond = getattr(dcond, dclass)(\n",
- " memory_cells=the_max_cells,\n",
+ " memory_cells=the_mem_cells,\n",
" bias_voltage=bias_voltage,\n",
" photon_energy=photon_energy)\n",
"\n",
- " print('Request: ', const, qm, the_max_cells, bias_voltage,\n",
+ " print('Request: ', const, qm, the_mem_cells, bias_voltage,\n",
" creation_time)\n",
" cdata, ctime = get_from_db(getattr(det, qm),\n",
" getattr(dconstants, const)(),\n",
- " mcond,\n",
+ " copy.deepcopy(mcond),\n",
" None,\n",
" cal_db_interface,\n",
" creation_time=creation_time,\n",
" verbosity=0,\n",
- " timeout=cal_db_timeout)\n",
- "\n",
+ " timeout=cal_db_timeout,\n",
+ " meta_only=True )\n",
+ " \n",
" if ctime is None or cdata is None:\n",
" print('Time or Data is None')\n",
" break\n",
@@ -225,13 +251,14 @@
" cdataBP, ctimeBP = get_from_db(getattr(det, qm),\n",
" getattr(dconstants,\n",
" constantsDark[const])(),\n",
- " mcond,\n",
+ " copy.deepcopy(mcond),\n",
" None,\n",
" cal_db_interface,\n",
" creation_time=(\n",
" creation_time + step + step + step + step),\n",
" verbosity=0,\n",
- " timeout=cal_db_timeout)\n",
+ " timeout=cal_db_timeout,\n",
+ " meta_only=True)\n",
" ctimeBP = ctimeBP.calibration_constant_version.begin_at\n",
"\n",
" if cdataBP is None:\n",
@@ -314,8 +341,10 @@
},
"outputs": [],
"source": [
- "save_dict_to_hdf5(ret_constants,\n",
- " out_folder + '/CalDBAna_AGIPD_{}.h5'.format(modules[0]))\n"
+ "if use_existing == \"\":\n",
+ " print('Save data to /CalDBAna_{}_{}.h5'.format(dclass, modules[0]))\n",
+ " save_dict_to_hdf5(ret_constants,\n",
+ " '{}/CalDBAna_{}_{}.h5'.format(out_folder, dclass, modules[0]))"
]
},
{
@@ -327,7 +356,13 @@
},
"outputs": [],
"source": [
- "ret_constants = load_data_from_hdf5(out_folder + '/CalDBAna_AGIPD_*.h5')\n"
+ "if use_existing == \"\":\n",
+ " fpath = '{}/CalDBAna_{}_*.h5'.format(out_folder, dclass)\n",
+ "else:\n",
+ " fpath = '{}/CalDBAna_{}_*.h5'.format(use_existing, dclass)\n",
+ "\n",
+ "print('Load data from {}'.format(fpath))\n",
+ "ret_constants = load_data_from_hdf5(fpath)"
]
},
{
@@ -338,8 +373,9 @@
},
"outputs": [],
"source": [
- "# Combine Gain FF and PC\n",
+ "print ('Combine Gain FF and PC and Noise in [e-]')\n",
"ret_constants[\"Gain\"] = {}\n",
+ "ret_constants[\"Noise-e\"] = {}\n",
"\n",
"for module in list(range(16)):\n",
" if (\"SlopesFF\" not in ret_constants or\n",
@@ -381,40 +417,32 @@
" gain_vs_time.append(cdataFF_vs_time[iFF] * cdataPC_vs_time[iPC])\n",
"\n",
" print(np.array(gain_vs_time).shape)\n",
+ " \n",
+ " ctime_ts = [t.timestamp() for t in ctime]\n",
+ " \n",
" ret_constants[\"Gain\"][qm][\"ctime\"] = ctime\n",
- " ret_constants[\"Gain\"][qm][\"data\"] = gain_vs_time\n",
- " ret_constants[\"Gain\"][qm][\"dataBP\"] = gain_vs_time\n",
- " ret_constants[\"Gain\"][qm][\"nBP\"] = gain_vs_time\n",
- "\n",
- "# calculate Noise in terms of electrons\n",
- "ret_constants[\"Noise_e-\"] = {}\n",
- "for module in list(range(16)):\n",
- "\n",
- " if (\"Gain\" not in ret_constants or\n",
- " \"Noise\" not in ret_constants):\n",
- " break\n",
- "\n",
- " qm = \"Q{}M{}\".format(module // 4 + 1, module % 4 + 1)\n",
- " print(qm)\n",
+ " ret_constants[\"Gain\"][qm][\"data\"] = np.array(gain_vs_time)\n",
+ " ret_constants[\"Gain\"][qm][\"dataBP\"] = np.array(gain_vs_time)\n",
+ " ret_constants[\"Gain\"][qm][\"nBP\"] = np.array(gain_vs_time)\n",
"\n",
- " if (qm not in ret_constants[\"Gain\"] or\n",
- " qm not in ret_constants[\"Noise\"]):\n",
+ " if \"Noise\" not in ret_constants:\n",
" continue\n",
"\n",
- " ret_constants[\"Noise_e-\"][qm] = {}\n",
+ " if qm not in ret_constants[\"Noise\"]:\n",
+ " continue\n",
"\n",
- " dataG = ret_constants[\"Gain\"][qm]\n",
" dataN = ret_constants[\"Noise\"][qm]\n",
- "\n",
- " if (len(dataG) == 0 or len(dataN) == 0):\n",
+ " if len(dataN) == 0:\n",
" continue\n",
"\n",
- " ctimesG = np.array(dataG[\"ctime\"])\n",
+ " ret_constants[\"Noise-e\"][qm] = {}\n",
+ " \n",
+ " ctimesG = np.array(ctime)\n",
" ctimesN = np.array(dataN[\"ctime\"])\n",
"\n",
" ctime, icomb = combine_constants(ctimesG, ctimesN)\n",
"\n",
- " cdataG_vs_time = np.array(dataG[\"data\"])\n",
+ " cdataG_vs_time = np.array(gain_vs_time)\n",
" cdataN_vs_time = np.array(dataN[\"data\"])[..., 0]\n",
"\n",
" data_vs_time = []\n",
@@ -423,10 +451,49 @@
" cdataN_vs_time[iN] * (8000 / 3.6 / 67.) / cdataG_vs_time[iG])\n",
"\n",
" print(np.array(gain_vs_time).shape)\n",
- " ret_constants[\"Noise_e-\"][qm][\"ctime\"] = ctime\n",
- " ret_constants[\"Noise_e-\"][qm][\"data\"] = data_vs_time\n",
- " ret_constants[\"Noise_e-\"][qm][\"dataBP\"] = data_vs_time\n",
- " ret_constants[\"Noise_e-\"][qm][\"nBP\"] = data_vs_time\n"
+ " ctime_ts = [t.timestamp() for t in ctime]\n",
+ " ret_constants[\"Noise-e\"][qm][\"ctime\"] = ctime\n",
+ " ret_constants[\"Noise-e\"][qm][\"data\"] = np.array(data_vs_time)\n",
+ " ret_constants[\"Noise-e\"][qm][\"dataBP\"] = np.array(data_vs_time)\n",
+ " ret_constants[\"Noise-e\"][qm][\"nBP\"] = np.array(data_vs_time)\n",
+ " \n",
+ "save_dict_to_hdf5({k:v for k,v in ret_constants.items() if k in ['Gain', 'Noise-e']},\n",
+ " '{}/CalDBAna_{}_Gain.h5'.format(out_folder, dclass))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# Parameters for plotting\n",
+ "\n",
+ "# Define range for plotting\n",
+ "rangevals = {\n",
+ " \"Offset\": [[800., 1500], [600, 900]],\n",
+ " \"Noise\": [[2.0, 16], [1.0, 7.0]],\n",
+ " \"Gain\": [[20, 30], [20, 30]],\n",
+ " \"Noise-e\": [[100., 600.], [100., 600.]],\n",
+ " \"SlopesCI\": [[0.95, 1.05], [0.0, 0.5]],\n",
+ " \"SlopesFF\": [[0.8, 1.2], [0.8, 1.2]]\n",
+ "}\n",
+ "\n",
+ "nMemToShow = 32\n",
+ "\n",
+ "keys = {\n",
+ " 'Mean': ['data', '', 'Mean over pixels'],\n",
+ " 'std': ['dataStd', '', '$\\sigma$ over pixels'],\n",
+ " 'MeanBP': ['dataBP', 'Good pixels only', 'Mean over pixels'],\n",
+ " 'NBP': ['nBP', 'Fraction of BP', 'Fraction of BP'],\n",
+ " 'stdBP': ['dataBPStd', 'Good pixels only', '$\\sigma$ over pixels'],\n",
+ " 'stdASIC': ['', '', '$\\sigma$ over ASICs'],\n",
+ " 'stdCell': ['', '', '$\\sigma$ over Cells'],\n",
+ "}\n",
+ "\n",
+ "gain_name = ['High', 'Medium', 'Low']"
]
},
{
@@ -438,17 +505,19 @@
},
"outputs": [],
"source": [
- "# Plot calibration constants as heatmaps\n",
+ "print('Plot calibration constants')\n",
"\n",
"# loop over constat type\n",
"for const, modules in ret_constants.items():\n",
"\n",
" # Loop over gain\n",
" for gain in range(2):\n",
- " print(const, len(modules), gain)\n",
+ " print('Const: {}, gain {}'.format(const, gain))\n",
"\n",
- " if const in [\"Gain\", \"Noise_e-\"] and gain > 0:\n",
- " break\n",
+ " if const in [\"Gain\", \"Noise-e\"] and gain == 1:\n",
+ " continue\n",
+ " else:\n",
+ " pass\n",
"\n",
" # loop over modules\n",
" mod_data = {}\n",
@@ -457,21 +526,11 @@
" mod_names = []\n",
" mod_times = []\n",
"\n",
- " # Define range for plotting\n",
- " rangevals = {}\n",
- " rangevals[\"Offset\"] = [4000. + gain * 2500, 5500. + gain * 3000]\n",
- " rangevals[\"Noise\"] = [2.5 + gain * 5, 15. + gain * 2]\n",
- " rangevals[\"Gain\"] = [0.8, 1.2]\n",
- " rangevals[\"Noise_e-\"] = [85., 500.]\n",
- " rangevals[\"SlopesPC\"] = [20.0 + gain * (-19.4), 30. + gain * (-28.8)] \n",
- " rangevals[\"SlopesFF\"] = [0.8 + gain * (-0.2), 1.2]\n",
- " nMemToShow = 32\n",
- " \n",
- " keys = [['data', 'Mean'], ['dataStd', 'Std'], ['dataBP', 'Mean(BP)'],\n",
- " ['nBP', 'NBP'], ['dataBPStd', 'Std(BP)']]\n",
- " \n",
" # Loop over modules\n",
" for mod, data in modules.items():\n",
+ " if mod not in in_mod_names:\n",
+ " continue\n",
+ "\n",
" print(mod)\n",
"\n",
" ctimes = np.array(data[\"ctime\"])\n",
@@ -481,54 +540,56 @@
" cmdata = np.array(data[\"mdata\"])\n",
" for i, tick in enumerate(ctimes_ticks):\n",
" ctimes_ticks[i] = ctimes_ticks[i] + \\\n",
- " ', V={:1.0f}'.format( cmdata[i]['Sensor Bias Voltage']) + \\\n",
- " ', M={:1.0f}'.format(cmdata[i]['Memory cells'])\n",
+ " ', V={:1.0f}'.format(cmdata[i]['Sensor Bias Voltage']) + \\\n",
+ " ', M={:1.0f}'.format(\n",
+ " cmdata[i]['Memory cells'])\n",
"\n",
" sort_ind = np.argsort(ctimes_ticks)\n",
" ctimes_ticks = list(np.array(ctimes_ticks)[sort_ind])\n",
- " \n",
+ "\n",
+ " # Create sorted by data dataset\n",
" rdata = {}\n",
- " for key in keys:\n",
- " if key[0] in data:\n",
- " rdata[key[1]] = np.array(data[key[0]])[sort_ind]\n",
- " \n",
- " print(rdata['Mean'].shape)\n",
+ " for key, item in keys.items():\n",
+ " if item[0] in data:\n",
+ " rdata[key] = np.array(data[item[0]])[sort_ind]\n",
+ "\n",
+ " # print(rdata['Mean'].shape)\n",
" nTimes = rdata['Mean'].shape[0]\n",
" nPixels = rdata['Mean'].shape[1] * rdata['Mean'].shape[2]\n",
" nBins = nMemToShow * nPixels\n",
"\n",
" # Select gain\n",
- " if const not in [\"Gain\", \"Noise_e-\"]:\n",
+ " if const not in [\"Gain\", \"Noise-e\"]:\n",
" for key in rdata:\n",
" rdata[key] = rdata[key][..., gain]\n",
"\n",
" # Avoid to low values\n",
- " if const in [\"Noise\", \"Offset\", \"Noise_e-\"]:\n",
+ " if const in [\"Noise\", \"Offset\", \"Noise-e\"]:\n",
" rdata['Mean'][rdata['Mean'] < 0.1] = np.nan\n",
- " if 'Mean(BP)' in rdata:\n",
- " rdata['Mean(BP)'][rdata['Mean(BP)'] < 0.1] = np.nan\n",
- " \n",
+ " if 'MeanBP' in rdata:\n",
+ " rdata['MeanBP'][rdata['MeanBP'] < 0.1] = np.nan\n",
+ "\n",
" if 'NBP' in rdata:\n",
" rdata[\"NBP\"][rdata[\"NBP\"] == 4096] = np.nan\n",
" rdata[\"NBP\"] = rdata[\"NBP\"] / (64 * 64) * 100\n",
- " \n",
- " # ASICs over cells.\n",
+ "\n",
+ " # Reshape: ASICs over cells for plotting\n",
" pdata = {}\n",
" for key in rdata:\n",
- " pdata[key] = rdata[key][:, :, :, :nMemToShow].reshape(nTimes,nBins).swapaxes(0, 1)\n",
+ " pdata[key] = rdata[key][:, :, :, :nMemToShow].reshape(\n",
+ " nTimes, nBins).swapaxes(0, 1)\n",
"\n",
" # Summary over ASICs\n",
" adata = {}\n",
" for key in rdata:\n",
" adata[key] = np.nanmean(rdata[key], axis=(1, 2)).swapaxes(0, 1)\n",
- " \n",
- " \n",
- " # Summary information over modules \n",
+ "\n",
+ " # Summary information over modules\n",
" for key in pdata:\n",
" if key not in mod_data:\n",
" mod_data[key] = []\n",
" mod_data[key].append(np.nanmean(pdata[key], axis=0))\n",
- " \n",
+ "\n",
" mod_data['stdASIC'].append(np.nanstd(\n",
" np.nanmean(rdata['Mean'][:, :, :, :nMemToShow], axis=(1, 2)), axis=1))\n",
" mod_data['stdCell'].append(np.nanstd(\n",
@@ -541,56 +602,45 @@
" for key in pdata:\n",
" vmin = None\n",
" vmax = None\n",
- " if const in rangevals and key in ['Mean', 'Mean(BP)']:\n",
- " vmin = rangevals[const][0]\n",
- " vmax = rangevals[const][1]\n",
- " plot_heatmap(pdata[key],\n",
- " x_title='Creation Time', y_title='ASIC devided to Momery cells',\n",
- " x_labels=ctimes_ticks, title='{} for {}, module {}, gain {}'.format(key, const, mod, gain),\n",
- " label='Mean for {}'.format(const), \n",
- " fname='{}/{}_ASIC_cell_{}_{}_g{}.pdf'.format(out_folder, const, key, mod, gain),\n",
- " vmin=vmin, vmax=vmax,\n",
- " ytickspos=np.arange(nBins, step=nMemToShow), yticks=np.arange(nPixels)[::-1]+1)\n",
- " \n",
- "\n",
- " plot_heatmap(adata[key],\n",
- " x_title='Creation Time', y_title='Momery cells',\n",
- " x_labels=ctimes_ticks, title='{} for {}, module {}, gain {}'.format(key, const, mod, gain),\n",
- " label='Mean for {}'.format(const), \n",
- " fname='{}/{}_cell_{}_{}_g{}.pdf'.format(out_folder, const, key, mod, gain),\n",
- " vmin=vmin, vmax=vmax,\n",
- " )\n",
- "\n",
- "\n",
- " # Incert nans to get array-like list of data\n",
- " uTime = mod_times[0]\n",
- " for tlist in mod_times:\n",
- " uTime = sorted(multi_union(uTime, tlist))\n",
- "\n",
- " for i, tlist in enumerate(mod_times):\n",
- " for t, time in enumerate(uTime):\n",
- " if t == len(tlist) or time != tlist[t]:\n",
- " tlist.insert(t, time)\n",
- " for key in mod_data:\n",
- " mod_data[key][i] = np.insert(mod_data[key][i], t, np.nan)\n",
- "\n",
- " # Plotting\n",
- " nModules = len(mod_names)\n",
- " for key in mod_data:\n",
- " vmin = None\n",
- " vmax = None\n",
- " if const in rangevals and key in ['Mean', 'Mean(BP)']:\n",
- " vmin = rangevals[const][0]\n",
- " vmax = rangevals[const][1]\n",
- " plot_heatmap(np.array(mod_data[key]),\n",
- " x_title='Creation Time', y_title='Modules',\n",
- " x_labels=ctimes_ticks,\n",
- " title='{} for {}, all modules, gain {}'.format(key, const, gain),\n",
- " label='{} for {}'.format(key, const),\n",
- " fname='{}/{}_all_{}_g{}.pdf'.format(out_folder, const, key, gain),\n",
- " vmin=vmin, vmax=vmax,\n",
- " ytickspos=np.arange(nModules)[::-1]+0.8, yticks=mod_names)\n",
- " \n"
+ " if const in rangevals and key in ['Mean', 'MeanBP']:\n",
+ " vmin = rangevals[const][gain][0]\n",
+ " vmax = rangevals[const][gain][1]\n",
+ "\n",
+ " if key == 'NBP':\n",
+ " unit = '[%]'\n",
+ " else:\n",
+ " unit = '[ADU]'\n",
+ " if const == 'Noise-e':\n",
+ " unit = '[$e^-$]'\n",
+ "\n",
+ " title = '{}, module {}, {} gain, {}'.format(\n",
+ " const, mod, gain_name[gain], keys[key][1])\n",
+ " cb_label = '{}, {} {}'.format(const, keys[key][2], unit)\n",
+ "\n",
+ " HMCombine(pdata[key][::-1],\n",
+ " x_label='Creation Time', y_label='ASIC ID',\n",
+ " x_ticklabels=ctimes_ticks,\n",
+ " x_ticks=np.arange(len(ctimes_ticks))+0.3,\n",
+ " title=title, cb_label=cb_label,\n",
+ " vmin=vmin, vmax=vmax,\n",
+ " fname='{}/{}_{}_g{}_ASIC_{}.png'.format(\n",
+ " out_folder, const, mod, gain, key),\n",
+ " y_ticks=np.arange(nBins, step=nMemToShow)+16,\n",
+ " y_ticklabels=np.arange(nPixels)[::-1]+1,\n",
+ " pad=[0.125, 0.125, 0.12, 0.185])\n",
+ "\n",
+ " HMCombine(adata[key], type=0,\n",
+ " x_label='Creation Time', y_label='Memory cell ID',\n",
+ " x_ticklabels=ctimes_ticks,\n",
+ " x_ticks=np.arange(len(ctimes_ticks))+0.3,\n",
+ " title=title, cb_label=cb_label,\n",
+ " fname='{}/{}_{}_g{}_MEM_{}.png'.format(\n",
+ " out_folder, const, mod, gain, key),\n",
+ " vmin=vmin, vmax=vmax)\n",
+ "\n",
+ " # break\n",
+ " #break\n",
+ " #break"
]
}
],
@@ -610,7 +660,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.6"
+ "version": "3.4.3"
}
},
"nbformat": 4,
diff --git a/notebooks/AGIPD/PlotFromCalDB_Summary_AGIPD_NBC.ipynb b/notebooks/AGIPD/PlotFromCalDB_Summary_AGIPD_NBC.ipynb
new file mode 100644
index 000000000..016a8ef0d
--- /dev/null
+++ b/notebooks/AGIPD/PlotFromCalDB_Summary_AGIPD_NBC.ipynb
@@ -0,0 +1,286 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Statistical analysis of calibration factors#\n",
+ "\n",
+ "Author: Mikhail Karnevskiy, Steffen Hauf, Version 0.2\n",
+ "\n",
+ "Plot calibration constants for AGIPD1M1 detector aggregated within detector modules. Input information is taken from folder `use_existing`. Corresponding files are prepared by `PlotFromCalDB` notebook."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "cluster_profile = \"noDB\" # The ipcluster profile to use\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/karnem/testAGIPD16_14/\" # Output folder, required\n",
+ "use_existing = \"/gpfs/exfel/data/scratch/karnem/testAGIPD16_21/\" # Input folder\n",
+ "dclass = \"AGIPD\" # Detector class"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "from cal_tools.ana_tools import *\n",
+ "import numpy as np\n",
+ "import matplotlib\n",
+ "matplotlib.use(\"agg\")\n",
+ "% matplotlib inline\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "import warnings\n",
+ "warnings.filterwarnings('ignore')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "if use_existing == '':\n",
+ " use_existing = out_folder"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "print('Load data from {}/CalDBAna_{}_*.h5'.format(use_existing, dclass))\n",
+ "ret_constants = load_data_from_hdf5(\n",
+ " '{}/CalDBAna_{}_*.h5'.format(use_existing, dclass))\n",
+ "\n",
+ "print('Evaluated data from {}/CalDBAna_{}_Gain*.h5'.format(out_folder, dclass))\n",
+ "ret_constants.update(load_data_from_hdf5(\n",
+ " '{}/CalDBAna_{}_Gain*.h5'.format(out_folder, dclass)))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# Parameters for plotting\n",
+ "\n",
+ "# Define range for plotting\n",
+ "rangevals = {\n",
+ " \"Offset\": [[4000., 5500], [6500, 8500]],\n",
+ " \"Noise\": [[2.5, 15], [7.5, 17.0]],\n",
+ " \"Gain\": [[0.8, 1.2], [0.8, 1.2]],\n",
+ " \"Noise-e\": [[85., 500.], [85., 500.]],\n",
+ " \"SlopesCI\": [[0.95, 1.05], [0.0, 0.5]],\n",
+ " \"SlopesPC\": [[22.0, 27.0], [-0.5, 1.5]],\n",
+ " \"SlopesFF\": [[0.8, 1.2], [0.6, 1.2]]\n",
+ "}\n",
+ "\n",
+ "nMemToShow = 32\n",
+ "\n",
+ "keys = {\n",
+ " 'Mean': ['data', '', 'Mean over pixels'],\n",
+ " 'std': ['dataStd', '', '$\\sigma$ over pixels'],\n",
+ " 'MeanBP': ['dataBP', 'Good pixels only', 'Mean over pixels'],\n",
+ " 'NBP': ['nBP', 'Fraction of BP', 'Fraction of BP'],\n",
+ " 'stdBP': ['dataBPStd', 'Good pixels only', '$\\sigma$ over pixels'],\n",
+ " 'stdASIC': ['', '', '$\\sigma$ over ASICs'],\n",
+ " 'stdCell': ['', '', '$\\sigma$ over Cells'],\n",
+ "}\n",
+ "\n",
+ "gain_name = ['High', 'Medium', 'Low']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "print('Plot calibration constants')\n",
+ "\n",
+ "# loop over constat type\n",
+ "for const, modules in ret_constants.items():\n",
+ "\n",
+ " # Loop over gain\n",
+ " for gain in range(2):\n",
+ " print('Const: {}, gain : {}'.format(const, gain))\n",
+ "\n",
+ " if const in [\"Gain\", \"Noise-e\"] and gain == 1:\n",
+ " continue\n",
+ "\n",
+ " # loop over modules\n",
+ " mod_data = {}\n",
+ " mod_data['stdASIC'] = []\n",
+ " mod_data['stdCell'] = []\n",
+ " mod_names = []\n",
+ " mod_times = []\n",
+ "\n",
+ " # Loop over modules\n",
+ " for mod, data in modules.items():\n",
+ " ctimes = np.array(data[\"ctime\"])\n",
+ " ctimes_ticks = [x.strftime('%y-%m-%d') for x in ctimes]\n",
+ "\n",
+ " if (\"mdata\" in data):\n",
+ " cmdata = np.array(data[\"mdata\"])\n",
+ " for i, tick in enumerate(ctimes_ticks):\n",
+ " ctimes_ticks[i] = ctimes_ticks[i] + \\\n",
+ " ', V={:1.0f}'.format(cmdata[i]['Sensor Bias Voltage']) + \\\n",
+ " ', M={:1.0f}'.format(\n",
+ " cmdata[i]['Memory cells'])\n",
+ "\n",
+ " sort_ind = np.argsort(ctimes_ticks)\n",
+ " ctimes_ticks = list(np.array(ctimes_ticks)[sort_ind])\n",
+ "\n",
+ " # Create sorted by data dataset\n",
+ " rdata = {}\n",
+ " for key, item in keys.items():\n",
+ " if item[0] in data:\n",
+ " rdata[key] = np.array(data[item[0]])[sort_ind]\n",
+ "\n",
+ " nTimes = rdata['Mean'].shape[0]\n",
+ " nPixels = rdata['Mean'].shape[1] * rdata['Mean'].shape[2]\n",
+ " nBins = nMemToShow * nPixels\n",
+ "\n",
+ " # Select gain\n",
+ " if const not in [\"Gain\", \"Noise-e\"]:\n",
+ " for key in rdata:\n",
+ " rdata[key] = rdata[key][..., gain]\n",
+ "\n",
+ " # Avoid to low values\n",
+ " if const in [\"Noise\", \"Offset\", \"Noise-e\"]:\n",
+ " rdata['Mean'][rdata['Mean'] < 0.1] = np.nan\n",
+ " if 'MeanBP' in rdata:\n",
+ " rdata['MeanBP'][rdata['MeanBP'] < 0.1] = np.nan\n",
+ "\n",
+ " if 'NBP' in rdata:\n",
+ " rdata[\"NBP\"][rdata[\"NBP\"] == 4096] = np.nan\n",
+ " rdata[\"NBP\"] = rdata[\"NBP\"] / (64 * 64) * 100\n",
+ "\n",
+ " # Reshape: ASICs over cells for plotting\n",
+ " pdata = {}\n",
+ " for key in rdata:\n",
+ " pdata[key] = rdata[key][:, :, :, :nMemToShow].reshape(\n",
+ " nTimes, nBins).swapaxes(0, 1)\n",
+ "\n",
+ " # Summary information over modules\n",
+ " for key in pdata:\n",
+ " if key not in mod_data:\n",
+ " mod_data[key] = []\n",
+ " mod_data[key].append(np.nanmean(pdata[key], axis=0))\n",
+ "\n",
+ " mod_data['stdASIC'].append(np.nanstd(\n",
+ " np.nanmean(rdata['Mean'][:, :, :, :nMemToShow], axis=(1, 2)), axis=1))\n",
+ " mod_data['stdCell'].append(np.nanstd(\n",
+ " np.nanmean(rdata['Mean'][:, :, :, :nMemToShow], axis=3), axis=(1, 2)))\n",
+ "\n",
+ " mod_names.append(mod)\n",
+ " mod_times.append(ctimes_ticks)\n",
+ "\n",
+ " # Incert nans to get array-like list of data\n",
+ " uTime = mod_times[0]\n",
+ " for tlist in mod_times:\n",
+ " uTime = sorted(multi_union(uTime, tlist))\n",
+ "\n",
+ " for i, tlist in enumerate(mod_times):\n",
+ " for t, time in enumerate(uTime):\n",
+ " if t == len(tlist) or time != tlist[t]:\n",
+ " tlist.insert(t, time)\n",
+ " for key in mod_data:\n",
+ " mod_data[key][i] = np.insert(\n",
+ " mod_data[key][i], t, np.nan)\n",
+ "\n",
+ " # Plotting\n",
+ " nModules = len(mod_names)\n",
+ " mod_idx = np.argsort(mod_names)\n",
+ " for key in mod_data:\n",
+ " vmin = None\n",
+ " vmax = None\n",
+ " if const in rangevals and key in ['Mean', 'MeanBP']:\n",
+ " vmin = rangevals[const][gain][0]\n",
+ " vmax = rangevals[const][gain][1]\n",
+ " else:\n",
+ " vmin = np.nanmin(np.array(mod_data[key]))\n",
+ " vmax = np.nanmean(\n",
+ " np.array(mod_data[key])) + 2*np.nanstd(np.array(mod_data[key]))\n",
+ "\n",
+ " if const in ['SlopesFF', 'SlopesPC', 'SlopesCI']:\n",
+ " htype = 2\n",
+ " else:\n",
+ " htype = 0\n",
+ "\n",
+ " if key == 'NBP':\n",
+ " unit = '[%]'\n",
+ " else:\n",
+ " unit = '[ADU]'\n",
+ " if const == 'Noise-e':\n",
+ " unit = '[$e^-$]'\n",
+ "\n",
+ " title = '{}, All modules, {} gain, {}'.format(\n",
+ " const, gain_name[gain], keys[key][1])\n",
+ " cb_label = '{}, {} {}'.format(const, keys[key][2], unit)\n",
+ "\n",
+ " HMCombine(np.array(mod_data[key])[mod_idx][::-1],\n",
+ " y_ticks=np.arange(nModules)[::-1]+0.8,\n",
+ " y_ticklabels=np.array(mod_names)[mod_idx],\n",
+ " x_label='Creation Time', y_label='Module ID',\n",
+ " x_ticklabels=ctimes_ticks, x_ticks=np.arange(\n",
+ " len(ctimes_ticks))+0.3,\n",
+ " title=title, cb_label=cb_label,\n",
+ " fname='{}/{}_all_g{}_{}.png'.format(\n",
+ " out_folder, const, gain, key),\n",
+ " vmin=vmin, vmax=vmax,\n",
+ " pad=[0.125, 0.151, 0.12, 0.17], type=htype)\n",
+ "\n",
+ " #plt.show()\n",
+ " #break\n",
+ " #break\n",
+ " #break"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.4.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/notebooks/LPD/PlotFromCalDB.ipynb b/notebooks/LPD/PlotFromCalDB.ipynb
deleted file mode 100644
index fab2014d5..000000000
--- a/notebooks/LPD/PlotFromCalDB.ipynb
+++ /dev/null
@@ -1,342 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Statistical analysis of calibration factors#\n",
- "\n",
- "Author: Mikhail Karnevskiy, Steffen Hauf, Version 0.1\n",
- "\n",
- "A description of the notebook."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": [
- "cluster_profile = \"noDB\" # The ipcluster profile to use\n",
- "start_date = \"2018-07-01\" # date to start investigation interval from\n",
- "end_date = \"now\" # date to end investigation interval at, can be \"now\"\n",
- "interval = 3 # interval for evaluation in days\n",
- "detectorDB = \"LPD1M1\" # detector entry in the DB to investigate\n",
- "constants = [\"Offset\", \"Noise\"] # constants to plot\n",
- "cal_db_interface = \"tcp://max-exfl016:8015\" # the database interface to use\n",
- "bias_voltage = 500\n",
- "max_cells = 128\n",
- "modules = [3]\n",
- "out_folder = \"/gpfs/exfel/data/scratch/karnem/test/\""
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false,
- "scrolled": true
- },
- "outputs": [],
- "source": [
- "import sys\n",
- "from datetime import datetime\n",
- "import warnings\n",
- "warnings.filterwarnings('ignore')\n",
- "\n",
- "#print (sys.path)\n",
- "\n",
- "import os\n",
- "import h5py\n",
- "import numpy as np\n",
- "import matplotlib\n",
- "matplotlib.use(\"agg\")\n",
- "import matplotlib.pyplot as plt\n",
- "%matplotlib inline\n",
- "\n",
- "\n",
- "\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
- "from cal_tools.tools import gain_map_files, parse_runs, run_prop_seq_from_path, get_notebook_name, get_dir_creation_date, get_constant_from_db\n",
- "from cal_tools.influx import InfluxLogger\n",
- "from cal_tools.enums import BadPixels\n",
- "from cal_tools.plotting import show_overview, plot_badpix_3d, create_constant_overview\n",
- "\n",
- "detectorDB = detectorDB.upper()\n",
- "dclass = \"AGIPD\" if \"AGIPD\" in detectorDB else \"LPD\"\n",
- "\n",
- "import datetime\n",
- "import dateutil.parser\n",
- "\n",
- "\n",
- "\n",
- "if modules[0] == -1:\n",
- " modules = list(range(16))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "import seaborn"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": [
- "all_ready_printed = {}\n",
- "def get_constant_from_db(device, constant, condition, empty_constant,\n",
- " cal_db_interface, creation_time = None, print_once=True):\n",
- " from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
- " \n",
- " if device:\n",
- " metadata = ConstantMetaData() \n",
- " metadata.calibration_constant = constant\n",
- " metadata.detector_condition = condition\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
- " start=creation_time)\n",
- "\n",
- " try:\n",
- " metadata.retrieve(cal_db_interface, when=creation_time.isoformat())\n",
- " if constant.name not in all_ready_printed or not print_once:\n",
- " all_ready_printed[constant.name] = True\n",
- " print(\"{} was injected on: {}\".format(constant.name, metadata.calibration_constant_version.begin_at)) \n",
- " return constant.data\n",
- " except Exception as e:\n",
- " return empty_constant\n",
- " else:\n",
- " return empty_constant"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false,
- "scrolled": true
- },
- "outputs": [],
- "source": [
- "dt = dateutil.parser.parse(start_date)\n",
- "end = datetime.datetime.now() if end_date.upper() == \"NOW\" else dateutil.parser.parse(end_date)\n",
- "step = datetime.timedelta(days=interval)\n",
- "\n",
- "det = getattr(Detectors, detectorDB)\n",
- "dconstants = getattr(Constants, dclass)\n",
- "ret_constants = {}\n",
- "while dt < end:\n",
- " creation_time = dt\n",
- " print(\"Retreiving data from: {}\".format(creation_time.isoformat()))\n",
- " for const in constants:\n",
- " if not const in ret_constants:\n",
- " ret_constants[const] = {}\n",
- " dcond = Conditions.Dark if (const in [\"Offset\", \"Noise\"] or \"DARK\" in const.upper()) else Conditions.Illuminated\n",
- " for i in modules:\n",
- " qm = \"Q{}M{}\".format(i//4+1, i%4+1)\n",
- " cdata = get_constant_from_db(getattr(det, qm),\n",
- " getattr(dconstants, const)(),\n",
- " getattr(dcond, dclass)(memory_cells=max_cells, bias_voltage=bias_voltage),\n",
- " None,\n",
- " cal_db_interface,\n",
- " creation_time=creation_time)\n",
- "\n",
- " print(\"Found constant {}: {}\".format(const, cdata is not None))\n",
- "\n",
- " if not qm in ret_constants[const]:\n",
- " ret_constants[const][qm] = []\n",
- " \n",
- " if cdata is not None:\n",
- " carr = np.zeros((5, max_cells, 3))\n",
- " carr_glob = np.zeros((5, 3))\n",
- " carr_px = np.zeros((cdata.shape[0], max_cells, 3, 2))\n",
- " for g in range(3):\n",
- " td = np.nanmean(cdata[...,g], axis=(0,1))\n",
- " carr[0,:td.shape[0],g] = td\n",
- "\n",
- " td = np.nanmedian(cdata[...,g], axis=(0,1))\n",
- " carr[1,:td.shape[0],g] = td\n",
- "\n",
- " td = np.nanmin(cdata[...,g], axis=(0,1))\n",
- " carr[2,:td.shape[0],g] = td\n",
- "\n",
- " td = np.nanmax(cdata[...,g], axis=(0,1))\n",
- " carr[3,:td.shape[0],g] = td\n",
- " \n",
- " td = np.nanstd(cdata[...,g], axis=(0,1))\n",
- " carr[4,:td.shape[0],g] = td\n",
- "\n",
- " td = np.nanmean(cdata[...,g])\n",
- " carr_glob[0, g] = td\n",
- "\n",
- " td = np.nanmedian(cdata[...,g])\n",
- " carr_glob[1, g] = td\n",
- "\n",
- " td = np.nanmin(cdata[...,g])\n",
- " carr_glob[2, g] = td\n",
- "\n",
- " td = np.nanmax(cdata[...,g])\n",
- " carr_glob[3, g] = td\n",
- " \n",
- " td = np.nanstd(cdata[...,g])\n",
- " carr_glob[3, g] = td\n",
- " \n",
- " carr_px[...,g, 0] = np.nanmedian(cdata[...,g], axis=0)\n",
- " carr_px[...,g, 1] = np.nanmedian(cdata[...,g], axis=1)\n",
- "\n",
- " ret_constants[const][qm].append((creation_time,(carr, carr_glob, carr_px)))\n",
- " \n",
- " dt += step"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false,
- "scrolled": false
- },
- "outputs": [],
- "source": [
- "import matplotlib.ticker as ticker\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "types = [\"mean\", \"median\", \"min\", \"max\", \"std\"]\n",
- "colors = [\"red\", \"green\", \"orange\", \"blue\"]\n",
- "skip = [False, False, False, False, False]\n",
- "# loop over constat type\n",
- "for const, modules in ret_constants.items():\n",
- " fig = plt.figure(figsize=(15,7))\n",
- " tt = 0\n",
- " print (const)\n",
- "\n",
- " # loop over type of stat parameter\n",
- " for typ in range(len(types)):\n",
- " if skip[typ]:\n",
- " continue\n",
- " ax = plt.subplot2grid((np.count_nonzero(~np.array(skip)), 1), (tt, 0))\n",
- " \n",
- " # loop over modules\n",
- " for mod, data in modules.items():\n",
- " ctimes, cd = list(zip(*data))\n",
- " pmm, glob, _ = list(zip(*cd))\n",
- " pma = np.array(pmm)\n",
- " ga = np.array(glob)\n",
- " d = pma[:,typ,:,:]\n",
- "\n",
- " if np.allclose(d, 0):\n",
- " continue\n",
- " dd = pma[:,typ,:,:]#-pma[0,typ,:,:])/pma[0,typ,:,:]\n",
- " y = dd.flatten()\n",
- " x = np.repeat(np.array(ctimes)[:,None],\n",
- " dd[0,:,:].size, axis=1).flatten()\n",
- " hue = np.repeat(np.array(['gain 0', 'gain 1', 'gain 2'])[:,None],\n",
- " dd[:,:,0].size, axis=1).swapaxes(0,1).flatten()\n",
- " seaborn.violinplot(x, y, hue, scale=\"width\", dodge=False, saturation=0.7)\n",
- "\n",
- " #ax.set_ylim(-0.25, .25)\n",
- " \n",
- " if typ != len(types)-1:\n",
- " ax.axes.get_xaxis().set_visible(False)\n",
- " else:\n",
- " def format_date(x, pos=None):\n",
- " return ctimes[x].strftime('%d-%m')\n",
- " ax.xaxis.set_major_formatter(ticker.FuncFormatter(format_date))\n",
- " ax.set_xlabel(\"Date\")\n",
- " ax.set_ylabel(\"{}\".format(types[typ]))\n",
- " \n",
- " tt += 1\n",
- " plt.subplots_adjust(wspace=0.2, hspace=0.2)\n",
- " \n",
- " if out_folder != \"\":\n",
- " fig.savefig(\"{}/{}_time_development.pdf\".format(out_folder, const), \n",
- " bbox_inches='tight')\n",
- " \n",
- "\n",
- " fig = plt.figure(figsize=(15,7))\n",
- " ax = plt.subplot2grid((1, 1), (0, 0))\n",
- " \n",
- " # loop over modules\n",
- " for mod, data in modules.items():\n",
- " ctimes, cd = list(zip(*data))\n",
- " _, _, px = list(zip(*cd))\n",
- " px = np.array(px)\n",
- " print (px.shape)\n",
- " y = px[:,:,5,0,:].flatten()\n",
- " x = np.repeat(np.array(ctimes)[:,None],\n",
- " px[0,:,5,0,:].size, axis=1).flatten()\n",
- " hue = np.repeat(np.array(['px','py'])[:,None], px[:,:,5,0,0].size, \n",
- " axis=1).swapaxes(0,1).flatten()\n",
- "\n",
- " seaborn.violinplot(x, y, hue, palette=\"muted\", split=True)\n",
- "\n",
- " def format_date(x, pos=None):\n",
- " return ctimes[x].strftime('%d-%m')\n",
- " ax.xaxis.set_major_formatter(ticker.FuncFormatter(format_date))\n",
- " ax.set_xlabel(\"Date\")\n",
- " ax.set_ylabel(\"Median over pixels\")\n",
- " \n",
- " plt.subplots_adjust(wspace=0.2, hspace=0.2)\n",
- " \n",
- " if out_folder != \"\":\n",
- " fig.savefig(\"{}/{}_pxtime_development.pdf\".format(out_folder, const),\n",
- " bbox_inches='tight')\n",
- " "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": []
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.6.6"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb b/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
new file mode 100644
index 000000000..abdbaf26b
--- /dev/null
+++ b/notebooks/LPD/PlotFromCalDB_LPD_NBC.ipynb
@@ -0,0 +1,652 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Statistical analysis of calibration factors#\n",
+ "\n",
+ "Author: Mikhail Karnevskiy, Steffen Hauf, Version 0.2\n",
+ "\n",
+ "Calibration constants for LPD1M1 detector from the data base with injection time between start_date and end_date are considered.\n",
+ "\n",
+ "To be visualized, calibration constants are averaged per ASICs. Plots shows calibration constant over time for each constant and for each module. Summary plots overall modules are created.\n",
+ "\n",
+ "In additional gain-slopes flat-field and pulse-capacitor are combined to relative-gain constant and presented as well. Noise in electron units is derived using gain factors and presented.\n",
+ "\n",
+ "Values shown in plots are saved in h5 files.\n",
+ "\n",
+ "All presented values corresponds to high and medium gain stages."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "cluster_profile = \"noDB\" # The ipcluster profile to use\n",
+ "start_date = \"2018-01-30\" # Date to start investigation interval from\n",
+ "end_date = \"2018-12-12\" # Date to end investigation interval at, can be \"now\"\n",
+ "constants = [\"Offset\", \"Noise\", \"SlopesFF\", \"SlopesCI\"] # constants to plot\n",
+ "modules = [1] # Modules, range allowed\n",
+ "bias_voltages = [250, 500] # Bias voltage\n",
+ "mem_cells = [1, 128, 256, 512] # Number of used memory cells. Typically: 4,32,64,128,176.\n",
+ "photon_energy = 9.2 # Photon energy of the beam\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/karnem/testLPD_16/\" # Output folder, required\n",
+ "use_existing = \"\" # Input folder\n",
+ "cal_db_timeout = 180000 # timeout on caldb requests\",\n",
+ "detectorDB = \"LPD1M1\" # detector entry in the DB to investigate\n",
+ "dclass = \"LPD\" # Detector class\n",
+ "cal_db_interface = \"tcp://max-exfl016:8015#8025\" # the database interface to use"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import numpy as np\n",
+ "import matplotlib\n",
+ "\n",
+ "##matplotlib.use(\"agg\")\n",
+ "import matplotlib.pyplot as plt\n",
+ "% matplotlib inline\n",
+ "\n",
+ "import sys\n",
+ "import datetime\n",
+ "import dateutil.parser\n",
+ "import warnings\n",
+ "\n",
+ "warnings.filterwarnings('ignore')\n",
+ "\n",
+ "from iCalibrationDB import Constants, Conditions, Detectors\n",
+ "from cal_tools.tools import get_from_db\n",
+ "from cal_tools.ana_tools import *\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "# Prepare variables\n",
+ "interval = 1 # interval for evaluation in days\n",
+ "\n",
+ "if modules[0] == -1:\n",
+ " modules = list(range(16))\n",
+ "\n",
+ "in_mod_names = []\n",
+ "for mod in modules:\n",
+ " in_mod_names.append(\"Q{}M{}\".format(mod // 4 + 1, mod % 4 + 1))\n",
+ "\n",
+ "constantsDark = {\"SlopesFF\": 'BadPixelsFF',\n",
+ " 'SlopesCI': 'BadPixelsCI',\n",
+ " 'Noise': 'BadPixelsDark',\n",
+ " 'Offset': 'BadPixelsDark'}\n",
+ "print('Bad pixels data: ', constantsDark)\n",
+ "\n",
+ "# Define parameters in order to perform loop over time stamps\n",
+ "start = datetime.datetime.now() if start_date.upper() == \"NOW\" else dateutil.parser.parse(\n",
+ " start_date)\n",
+ "end = datetime.datetime.now() if end_date.upper() == \"NOW\" else dateutil.parser.parse(\n",
+ " end_date)\n",
+ "step = datetime.timedelta(hours=interval)\n",
+ "dt = end\n",
+ "\n",
+ "# Create output folder\n",
+ "os.makedirs(out_folder, exist_ok=True)\n",
+ "\n",
+ "# Get getector conditions\n",
+ "det = getattr(Detectors, detectorDB)\n",
+ "dconstants = getattr(Constants, dclass)\n",
+ "\n",
+ "if \"#\" in cal_db_interface:\n",
+ " prot, serv, ran = cal_db_interface.split(\":\")\n",
+ " r1, r2 = ran.split(\"#\")\n",
+ " cal_db_interface = \":\".join(\n",
+ " [prot, serv, str(np.random.randint(int(r1), int(r2)))])\n",
+ "\n",
+ "print('CalDB Interface {}'.format(cal_db_interface))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "import copy\n",
+ "\n",
+ "def prepare_to_store(a, nMem):\n",
+ " shape = list(a.shape[:2])+[nMem,2]\n",
+ " b = np.full(shape, np.nan)\n",
+ " b[:, :, :a.shape[2]] = a[:, :, :, :2]\n",
+ " return b\n",
+ "\n",
+ "def get_rebined(a, rebin):\n",
+ " return a.reshape(\n",
+ " int(a.shape[0] / rebin[0]),\n",
+ " rebin[0],\n",
+ " int(a.shape[1] / rebin[1]),\n",
+ " rebin[1],\n",
+ " a.shape[2],\n",
+ " a.shape[3])\n",
+ " \n",
+ "\n",
+ "nMem = 512 # Number of mem Cells to store\n",
+ "spShape = (64,64) # Shape of superpixel\n",
+ "\n",
+ "ret_constants = {}\n",
+ "\n",
+ "if use_existing != '':\n",
+ " mem_cells = []\n",
+ "\n",
+ "# Loop over max_memory cells\n",
+ "for the_mem_cells in mem_cells:\n",
+ "\n",
+ " # Loop over bias voltages\n",
+ " for bias_voltage in bias_voltages:\n",
+ "\n",
+ " # Loop over constants\n",
+ " for c, const in enumerate(constants):\n",
+ "\n",
+ " if not const in ret_constants:\n",
+ " ret_constants[const] = {}\n",
+ "\n",
+ " # Loop over modules\n",
+ " for module in modules:\n",
+ "\n",
+ " dt = end\n",
+ "\n",
+ " qm = \"Q{}M{}\".format(module // 4 + 1, module % 4 + 1)\n",
+ " if not qm in ret_constants[const]:\n",
+ " ret_constants[const][qm] = []\n",
+ "\n",
+ " # Loop over time stamps\n",
+ " while dt > start:\n",
+ " creation_time = dt\n",
+ "\n",
+ " if (const in [\"Offset\", \"Noise\",\n",
+ " \"SlopesCI\"] or \"DARK\" in const.upper()):\n",
+ " dcond = Conditions.Dark\n",
+ " mcond = getattr(dcond, dclass)(\n",
+ " memory_cells=the_mem_cells,\n",
+ " bias_voltage=bias_voltage)\n",
+ " else:\n",
+ " dcond = Conditions.Illuminated\n",
+ " mcond = getattr(dcond, dclass)(\n",
+ " memory_cells=the_mem_cells,\n",
+ " bias_voltage=bias_voltage,\n",
+ " photon_energy=photon_energy)\n",
+ "\n",
+ " print('Request: ', const, qm, the_mem_cells, bias_voltage,\n",
+ " creation_time)\n",
+ " cdata, ctime = get_from_db(getattr(det, qm),\n",
+ " getattr(dconstants, const)(),\n",
+ " copy.deepcopy(mcond),\n",
+ " None,\n",
+ " cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " verbosity=0,\n",
+ " timeout=cal_db_timeout,\n",
+ " meta_only=True )\n",
+ " \n",
+ " if ctime is None or cdata is None:\n",
+ " print('Time or Data is None')\n",
+ " break\n",
+ "\n",
+ " ctime = ctime.calibration_constant_version.begin_at\n",
+ "\n",
+ " print(\"Found constant {}: begin at {}\".format(const,\n",
+ " cdata is not None),\n",
+ " ctime)\n",
+ " print(cdata.shape)\n",
+ "\n",
+ " if const in ['SlopesFF']:\n",
+ " cdata = cdata[...,None,None]\n",
+ " \n",
+ " if(len(cdata.shape)==5):\n",
+ " cdata = cdata[:,:,:,:,0]\n",
+ "\n",
+ " if len(cdata.shape) < 4:\n",
+ " print(cdata.shape, \"Unexpected shape !!!!!!!!!\")\n",
+ "\n",
+ " if cdata.shape[0] != 256:\n",
+ " cdata = cdata.swapaxes(0, 2).swapaxes(1,3).swapaxes(2,3) \n",
+ "\n",
+ " print(cdata.shape)\n",
+ "\n",
+ " # Request bad pixel mask\n",
+ " if const in constantsDark:\n",
+ " #for par in mcond.parameters:\n",
+ " # print (par.name, par.value)\n",
+ " print('RequestBP ',\n",
+ " (creation_time + step + step + step + step))\n",
+ " cdataBP, ctimeBP = get_from_db(getattr(det, qm),\n",
+ " getattr(dconstants,\n",
+ " constantsDark[const])(),\n",
+ " copy.deepcopy(mcond),\n",
+ " None,\n",
+ " cal_db_interface,\n",
+ " creation_time=(\n",
+ " creation_time + step + step + step + step),\n",
+ " verbosity=0,\n",
+ " timeout=cal_db_timeout,\n",
+ " meta_only=True)\n",
+ " ctimeBP = ctimeBP.calibration_constant_version.begin_at\n",
+ "\n",
+ " if cdataBP is None:\n",
+ " print(\"Bad pixels are not found !!!!!!!!!\")\n",
+ " ctime = ctime.replace(tzinfo=None)\n",
+ " dt = ctime - step\n",
+ " continue\n",
+ "\n",
+ " print(\"Found BP {}\".format(ctimeBP))\n",
+ " print(cdataBP.shape)\n",
+ "\n",
+ " if const in ['SlopesFF']:\n",
+ " cdataBP = cdataBP[...,None,None]\n",
+ " \n",
+ " if(len(cdataBP.shape)==5):\n",
+ " cdataBP = cdataBP[:,:,:,:,0]\n",
+ "\n",
+ " if cdataBP.shape[0] != 256:\n",
+ " cdataBP = cdataBP.swapaxes(0, 2).swapaxes(1,3).swapaxes(2,3) \n",
+ "\n",
+ " if (len(cdataBP.shape) < 4):\n",
+ " print(cdataBP.shape, \"Unexpected shape !!!!!!!!!\")\n",
+ "\n",
+ " if cdataBP.shape != cdata.shape:\n",
+ " print(cdataBP.shape, 'Wrong BP shape!!!')\n",
+ " ctime = ctime.replace(tzinfo=None)\n",
+ " dt = ctime - step\n",
+ " continue\n",
+ "\n",
+ " # Apply bad pixel mask\n",
+ " cdataABP = np.copy(cdata)\n",
+ " cdataABP[cdataBP > 0] = np.nan\n",
+ "\n",
+ " # Create superpixels for constants with BP applied\n",
+ " cdataABP = get_rebined(cdataABP, spShape)\n",
+ " toStoreBP = prepare_to_store(np.nanmean(cdataABP, axis=(1, 3)), nMem)\n",
+ " toStoreBPStd = prepare_to_store(np.nanstd(cdataABP, axis=(1, 3)), nMem)\n",
+ " \n",
+ " # Prepare number of bad pixels per superpixels\n",
+ " cdataBP = get_rebined(cdataBP, spShape)\n",
+ " cdataNBP = prepare_to_store(np.nansum(cdataBP > 0, axis=(1, 3)), nMem)\n",
+ " else:\n",
+ " toStoreExtBP = 0\n",
+ " cdataBPExt = 0\n",
+ "\n",
+ " # Create superpixels for constants without BP applied\n",
+ " cdata = get_rebined(cdata, spShape)\n",
+ " toStoreStd = prepare_to_store(np.nanstd(cdata, axis=(1, 3)), nMem)\n",
+ " toStore = prepare_to_store(np.nanmean(cdata, axis=(1, 3)), nMem)\n",
+ "\n",
+ " # Convert parameters to dict\n",
+ " dpar = {}\n",
+ " for par in mcond.parameters:\n",
+ " dpar[par.name] = par.value\n",
+ "\n",
+ " print(\"Store values in dict\", const, qm, ctime)\n",
+ " ret_constants[const][qm].append({'ctime': ctime,\n",
+ " 'nBP': cdataNBP,\n",
+ " 'dataBP': toStoreBP,\n",
+ " 'dataBPStd': toStoreBPStd,\n",
+ " 'data': toStore,\n",
+ " 'dataStd': toStoreStd,\n",
+ " 'mdata': dpar})\n",
+ " \n",
+ " ctime = ctime.replace(tzinfo=None)\n",
+ " dt = ctime - step\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "if use_existing == \"\":\n",
+ " print('Save data to /CalDBAna_{}_{}.h5'.format(dclass, modules[0]))\n",
+ " save_dict_to_hdf5(ret_constants,\n",
+ " '{}/CalDBAna_{}_{}.h5'.format(out_folder, dclass, modules[0]))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "if use_existing == \"\":\n",
+ " fpath = '{}/CalDBAna_{}_*.h5'.format(out_folder, dclass)\n",
+ "else:\n",
+ " fpath = '{}/CalDBAna_{}_*.h5'.format(use_existing, dclass)\n",
+ "\n",
+ "print('Load data from {}'.format(fpath))\n",
+ "ret_constants = load_data_from_hdf5(fpath)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "print ('Combine Gain FF and CI and Noise in [e-]')\n",
+ "ret_constants[\"Gain\"] = {}\n",
+ "ret_constants[\"Noise-e\"] = {}\n",
+ "\n",
+ "for module in list(range(16)):\n",
+ " if (\"SlopesFF\" not in ret_constants or\n",
+ " \"SlopesPC\" not in ret_constants):\n",
+ " break\n",
+ "\n",
+ " qm = \"Q{}M{}\".format(module // 4 + 1, module % 4 + 1)\n",
+ " print(qm)\n",
+ "\n",
+ " if (qm not in ret_constants[\"SlopesFF\"] or\n",
+ " qm not in ret_constants[\"SlopesCI\"]):\n",
+ " continue\n",
+ "\n",
+ " ret_constants[\"Gain\"][qm] = {}\n",
+ "\n",
+ " dataFF = ret_constants[\"SlopesFF\"][qm]\n",
+ " dataPC = ret_constants[\"SlopesCI\"][qm]\n",
+ "\n",
+ " if (len(dataFF) == 0 or len(dataPC) == 0):\n",
+ " continue\n",
+ "\n",
+ " ctimesFF = np.array(dataFF[\"ctime\"])\n",
+ " ctimesPC = np.array(dataPC[\"ctime\"])\n",
+ "\n",
+ " ctime, icomb = combine_constants(ctimesFF, ctimesPC)\n",
+ "\n",
+ " cdataPC_vs_time = np.array(dataPC[\"data\"])[..., 0]\n",
+ " cdataFF_vs_time = np.array(dataFF[\"data\"])[..., 0]\n",
+ "\n",
+ " cdataFF_vs_time = np.nanmedian(cdataFF_vs_time, axis=3)[..., None]\n",
+ "\n",
+ " cdataFF_vs_time /= np.nanmedian(cdataFF_vs_time, axis=(1, 2, 3))[:, None,\n",
+ " None, None]\n",
+ " cdataPC_vs_time /= np.nanmedian(cdataPC_vs_time, axis=(1, 2, 3))[:, None,\n",
+ " None, None]\n",
+ "\n",
+ " gain_vs_time = []\n",
+ " for iFF, iPC in icomb:\n",
+ " gain_vs_time.append(cdataFF_vs_time[iFF] * cdataPC_vs_time[iPC])\n",
+ "\n",
+ " print(np.array(gain_vs_time).shape)\n",
+ " \n",
+ " ctime_ts = [t.timestamp() for t in ctime]\n",
+ " \n",
+ " ret_constants[\"Gain\"][qm][\"ctime\"] = ctime\n",
+ " ret_constants[\"Gain\"][qm][\"data\"] = np.array(gain_vs_time)\n",
+ " ret_constants[\"Gain\"][qm][\"dataBP\"] = np.array(gain_vs_time)\n",
+ " ret_constants[\"Gain\"][qm][\"nBP\"] = np.array(gain_vs_time)\n",
+ "\n",
+ " if \"Noise\" not in ret_constants:\n",
+ " continue\n",
+ "\n",
+ " if qm not in ret_constants[\"Noise\"]:\n",
+ " continue\n",
+ "\n",
+ " dataN = ret_constants[\"Noise\"][qm]\n",
+ " if len(dataN) == 0:\n",
+ " continue\n",
+ "\n",
+ " ret_constants[\"Noise-e\"][qm] = {}\n",
+ " \n",
+ " ctimesG = np.array(ctime)\n",
+ " ctimesN = np.array(dataN[\"ctime\"])\n",
+ "\n",
+ " ctime, icomb = combine_constants(ctimesG, ctimesN)\n",
+ "\n",
+ " cdataG_vs_time = np.array(gain_vs_time)\n",
+ " cdataN_vs_time = np.array(dataN[\"data\"])[..., 0]\n",
+ "\n",
+ " data_vs_time = []\n",
+ " for iG, iN in icomb:\n",
+ " data_vs_time.append(\n",
+ " cdataN_vs_time[iN] * (8000 / 3.6 / 67.) / cdataG_vs_time[iG])\n",
+ "\n",
+ " print(np.array(gain_vs_time).shape)\n",
+ " ctime_ts = [t.timestamp() for t in ctime]\n",
+ " ret_constants[\"Noise-e\"][qm][\"ctime\"] = ctime\n",
+ " ret_constants[\"Noise-e\"][qm][\"data\"] = np.array(data_vs_time)\n",
+ " ret_constants[\"Noise-e\"][qm][\"dataBP\"] = np.array(data_vs_time)\n",
+ " ret_constants[\"Noise-e\"][qm][\"nBP\"] = np.array(data_vs_time)\n",
+ " \n",
+ "save_dict_to_hdf5({k:v for k,v in ret_constants.items() if k in ['Gain', 'Noise-e']},\n",
+ " '{}/CalDBAna_{}_Gain.h5'.format(out_folder, dclass))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# Parameters for plotting\n",
+ "\n",
+ "# Define range for plotting\n",
+ "rangevals = {\n",
+ " \"Offset\": [[800., 1500], [600, 900]],\n",
+ " \"Noise\": [[2.0, 16], [1.0, 7.0]],\n",
+ " \"Gain\": [[20, 30], [20, 30]],\n",
+ " \"Noise-e\": [[100., 600.], [100., 600.]],\n",
+ " \"SlopesCI\": [[0.95, 1.05], [0.0, 0.5]],\n",
+ " \"SlopesFF\": [[0.8, 1.2], [0.8, 1.2]]\n",
+ "}\n",
+ "\n",
+ "nMemToShow = 32\n",
+ "\n",
+ "keys = {\n",
+ " 'Mean': ['data', '', 'Mean over pixels'],\n",
+ " 'std': ['dataStd', '', '$\\sigma$ over pixels'],\n",
+ " 'MeanBP': ['dataBP', 'Good pixels only', 'Mean over pixels'],\n",
+ " 'NBP': ['nBP', 'Fraction of BP', 'Fraction of BP'],\n",
+ " 'stdBP': ['dataBPStd', 'Good pixels only', '$\\sigma$ over pixels'],\n",
+ " 'stdASIC': ['', '', '$\\sigma$ over ASICs'],\n",
+ " 'stdCell': ['', '', '$\\sigma$ over Cells'],\n",
+ "}\n",
+ "\n",
+ "gain_name = ['High', 'Medium', 'Low']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "print('Plot calibration constants')\n",
+ "\n",
+ "# loop over constat type\n",
+ "for const, modules in ret_constants.items():\n",
+ "\n",
+ " # Loop over gain\n",
+ " for gain in range(2):\n",
+ " print('Const: {}, gain {}'.format(const, gain))\n",
+ "\n",
+ " if const in [\"Gain\", \"Noise-e\"] and gain == 1:\n",
+ " continue\n",
+ " else:\n",
+ " pass\n",
+ "\n",
+ " # loop over modules\n",
+ " mod_data = {}\n",
+ " mod_data['stdASIC'] = []\n",
+ " mod_data['stdCell'] = []\n",
+ " mod_names = []\n",
+ " mod_times = []\n",
+ "\n",
+ " # Loop over modules\n",
+ " for mod, data in modules.items():\n",
+ " \n",
+ " if mod not in in_mod_names:\n",
+ " continue\n",
+ "\n",
+ " print(mod)\n",
+ "\n",
+ " ctimes = np.array(data[\"ctime\"])\n",
+ " ctimes_ticks = [x.strftime('%m-%d') for x in ctimes]\n",
+ "\n",
+ " if (\"mdata\" in data):\n",
+ " cmdata = np.array(data[\"mdata\"])\n",
+ " for i, tick in enumerate(ctimes_ticks):\n",
+ " ctimes_ticks[i] = ctimes_ticks[i] + \\\n",
+ " ', V={:1.0f}'.format(cmdata[i]['Sensor Bias Voltage']) + \\\n",
+ " ', M={:1.0f}'.format(\n",
+ " cmdata[i]['Memory cells'])\n",
+ "\n",
+ " sort_ind = np.argsort(ctimes_ticks)\n",
+ " ctimes_ticks = list(np.array(ctimes_ticks)[sort_ind])\n",
+ "\n",
+ " # Create sorted by data dataset\n",
+ " rdata = {}\n",
+ " for key, item in keys.items():\n",
+ " if item[0] in data:\n",
+ " rdata[key] = np.array(data[item[0]])[sort_ind]\n",
+ "\n",
+ " # print(rdata['Mean'].shape)\n",
+ " nTimes = rdata['Mean'].shape[0]\n",
+ " nPixels = rdata['Mean'].shape[1] * rdata['Mean'].shape[2]\n",
+ " nBins = nMemToShow * nPixels\n",
+ "\n",
+ " # Select gain\n",
+ " if const not in [\"Gain\", \"Noise-e\"]:\n",
+ " for key in rdata:\n",
+ " rdata[key] = rdata[key][..., gain]\n",
+ "\n",
+ " # Avoid to low values\n",
+ " if const in [\"Noise\", \"Offset\", \"Noise-e\"]:\n",
+ " rdata['Mean'][rdata['Mean'] < 0.1] = np.nan\n",
+ " if 'MeanBP' in rdata:\n",
+ " rdata['MeanBP'][rdata['MeanBP'] < 0.1] = np.nan\n",
+ "\n",
+ " if 'NBP' in rdata:\n",
+ " rdata[\"NBP\"][rdata[\"NBP\"] == 4096] = np.nan\n",
+ " rdata[\"NBP\"] = rdata[\"NBP\"] / (64 * 64) * 100\n",
+ "\n",
+ " # Reshape: ASICs over cells for plotting\n",
+ " pdata = {}\n",
+ " for key in rdata:\n",
+ " pdata[key] = rdata[key][:, :, :, :nMemToShow].reshape(\n",
+ " nTimes, nBins).swapaxes(0, 1)\n",
+ "\n",
+ " # Summary over ASICs\n",
+ " adata = {}\n",
+ " for key in rdata:\n",
+ " adata[key] = np.nanmean(rdata[key], axis=(1, 2)).swapaxes(0, 1)\n",
+ "\n",
+ " # Summary information over modules\n",
+ " for key in pdata:\n",
+ " if key not in mod_data:\n",
+ " mod_data[key] = []\n",
+ " mod_data[key].append(np.nanmean(pdata[key], axis=0))\n",
+ "\n",
+ " mod_data['stdASIC'].append(np.nanstd(\n",
+ " np.nanmean(rdata['Mean'][:, :, :, :nMemToShow], axis=(1, 2)), axis=1))\n",
+ " mod_data['stdCell'].append(np.nanstd(\n",
+ " np.nanmean(rdata['Mean'][:, :, :, :nMemToShow], axis=3), axis=(1, 2)))\n",
+ "\n",
+ " mod_names.append(mod)\n",
+ " mod_times.append(ctimes_ticks)\n",
+ "\n",
+ " # Plotting\n",
+ " for key in pdata:\n",
+ " vmin = None\n",
+ " vmax = None\n",
+ " if const in rangevals and key in ['Mean', 'MeanBP']:\n",
+ " vmin = rangevals[const][gain][0]\n",
+ " vmax = rangevals[const][gain][1]\n",
+ "\n",
+ " if key == 'NBP':\n",
+ " unit = '[%]'\n",
+ " else:\n",
+ " unit = '[ADU]'\n",
+ " if const == 'Noise-e':\n",
+ " unit = '[$e^-$]'\n",
+ "\n",
+ " title = '{}, module {}, {} gain, {}'.format(\n",
+ " const, mod, gain_name[gain], keys[key][1])\n",
+ " cb_label = '{}, {} {}'.format(const, keys[key][2], unit)\n",
+ "\n",
+ " HMCombine(pdata[key][::-1],\n",
+ " x_label='Creation Time', y_label='ASIC ID',\n",
+ " x_ticklabels=ctimes_ticks,\n",
+ " x_ticks=np.arange(len(ctimes_ticks))+0.3,\n",
+ " title=title, cb_label=cb_label,\n",
+ " vmin=vmin, vmax=vmax,\n",
+ " fname='{}/{}_{}_g{}_ASIC_{}.png'.format(\n",
+ " out_folder, const, mod, gain, key),\n",
+ " y_ticks=np.arange(nBins, step=nMemToShow)+16,\n",
+ " y_ticklabels=np.arange(nPixels)[::-1]+1,\n",
+ " pad=[0.125, 0.125, 0.12, 0.185])\n",
+ "\n",
+ " HMCombine(adata[key], type=0,\n",
+ " x_label='Creation Time', y_label='Memory cell ID',\n",
+ " x_ticklabels=ctimes_ticks,\n",
+ " x_ticks=np.arange(len(ctimes_ticks))+0.3,\n",
+ " title=title, cb_label=cb_label,\n",
+ " fname='{}/{}_{}_g{}_MEM_{}.png'.format(\n",
+ " out_folder, const, mod, gain, key),\n",
+ " vmin=vmin, vmax=vmax)\n",
+ "\n",
+ " #break\n",
+ " #break\n",
+ " #break\n",
+ " #break"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.4.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
--
GitLab