From 656be091ef0e7136ded0be1c7df530a5ff4bdcd8 Mon Sep 17 00:00:00 2001
From: Mikhail Karnevskiy <karnem@max-exfl001.desy.de>
Date: Mon, 6 Aug 2018 09:54:11 +0200
Subject: [PATCH] Add python script to visualise statistics from calibration DB
---
notebooks/LPD/PlotFromCalDB.py | 215 +++++++++++++++++++++++++++++++++
1 file changed, 215 insertions(+)
create mode 100644 notebooks/LPD/PlotFromCalDB.py
diff --git a/notebooks/LPD/PlotFromCalDB.py b/notebooks/LPD/PlotFromCalDB.py
new file mode 100644
index 000000000..9845a140a
--- /dev/null
+++ b/notebooks/LPD/PlotFromCalDB.py
@@ -0,0 +1,215 @@
+import sys
+import warnings
+import os
+import h5py
+import numpy as np
+import matplotlib
+import matplotlib.pyplot as plt
+import matplotlib.ticker as ticker
+import datetime
+import dateutil.parser
+
+import seaborn
+from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions
+from cal_tools.cal_tools import gain_map_files, parse_runs, run_prop_seq_from_path, get_notebook_name, get_dir_creation_date, get_constant_from_db
+from cal_tools.influx import InfluxLogger
+from cal_tools.enums import BadPixels
+from cal_tools.plotting import show_overview, plot_badpix_3d, create_constant_overview
+
+warnings.filterwarnings('ignore')
+matplotlib.use("agg")
+
+
+start_date = "2018-07-25" # date to start investigation interval from
+end_date = "now" # date to end investigation interval at, can be "now"
+interval = 3 # interval for evaluation in days
+detector = "LPD1M1" # detector to investigate
+constants = ["Offset", "Noise"] # constants to plot
+cal_db_interface = "tcp://max-exfl016:8015" # the database interface to use
+bias_voltage = 500
+max_cells = 128
+modules = [3]
+out_folder = "/gpfs/exfel/data/scratch/karnem/test/"
+all_ready_printed = {}
+
+
+detector = detector.upper()
+dclass = "AGIPD" if "AGIPD" in detector else "LPD"
+
+
+if modules[0] == -1:
+ modules = list(range(16))
+
+
+def get_constant_from_db(device, constant, condition, empty_constant,
+ cal_db_interface, creation_time = None,
+ print_once=True):
+
+ if device:
+ metadata = ConstantMetaData()
+ metadata.calibration_constant = constant
+ metadata.detector_condition = condition
+ if creation_time is None:
+ metadata.calibration_constant_version = Versions.Now(device=device)
+ else:
+ metadata.calibration_constant_version = \
+ Versions.Timespan(device=device, start=creation_time)
+
+ try:
+ print ("Send requiests")
+ metadata.retrieve(cal_db_interface, when=creation_time.isoformat())
+ if constant.name not in all_ready_printed or not print_once:
+ all_ready_printed[constant.name] = True
+ print("{} was injected on: {}".format(constant.name,
+ metadata.calibration_constant_version.begin_at))
+ return constant.data
+ except Exception as e:
+ return empty_constant
+ else:
+ return empty_constant
+
+
+dt = dateutil.parser.parse(start_date)
+end = datetime.datetime.now() if end_date.upper() == "NOW" else dateutil.parser.parse(end_date)
+step = datetime.timedelta(days=interval)
+
+det = getattr(Detectors, detector)
+dconstants = getattr(Constants, dclass)
+ret_constants = {}
+while dt < end:
+ creation_time = dt
+ print("Retreiving data from: {}".format(creation_time.isoformat()))
+ for const in constants:
+ if not const in ret_constants:
+ ret_constants[const] = {}
+ if (const in ["Offset", "Noise"] or "DARK" in const.upper()):
+ dcond = Conditions.Dark
+ else:
+ Conditions.Illuminated
+
+ for i in modules:
+ qm = "Q{}M{}".format(i//4+1, i%4+1)
+ cdata = get_constant_from_db(getattr(det, qm),
+ getattr(dconstants, const)(),
+ getattr(dcond, dclass)(memory_cells=max_cells,
+ bias_voltage=bias_voltage),
+ None,
+ cal_db_interface,
+ creation_time=creation_time)
+
+ print("Found constant {}: {}".format(const, cdata is not None))
+
+ if not qm in ret_constants[const]:
+ ret_constants[const][qm] = []
+
+ if cdata is not None:
+ carr = np.zeros((5, max_cells, 3))
+ carr_glob = np.zeros((5, 3))
+ carr_rad = np.zeros((cdata.shape[0], max_cells, 3))
+ for g in range(3):
+ td = np.nanmean(cdata[...,g], axis=(0,1))
+ print (td.shape)
+ carr[0,:td.shape[0],g] = td
+
+ td = np.nanmedian(cdata[...,g], axis=(0,1))
+ carr[1,:td.shape[0],g] = td
+
+ td = np.nanmin(cdata[...,g], axis=(0,1))
+ carr[2,:td.shape[0],g] = td
+
+ td = np.nanmax(cdata[...,g], axis=(0,1))
+ carr[3,:td.shape[0],g] = td
+
+ td = np.nanstd(cdata[...,g], axis=(0,1))
+ carr[4,:td.shape[0],g] = td
+
+ td = np.nanmean(cdata[...,g])
+ carr_glob[0, g] = td
+
+ td = np.nanmedian(cdata[...,g])
+ carr_glob[1, g] = td
+
+ td = np.nanmin(cdata[...,g])
+ carr_glob[2, g] = td
+
+ td = np.nanmax(cdata[...,g])
+ carr_glob[3, g] = td
+
+ td = np.nanstd(cdata[...,g])
+ carr_glob[3, g] = td
+
+ carr_rad[...,g] = np.nanmedian(cdata[...,g], axis=0)
+
+ ret_constants[const][qm].append((creation_time,(carr, carr_glob, carr_rad)))
+
+ dt += step
+
+
+types = ["mean", "median", "min", "max", "std"]
+colors = ["red", "green", "orange", "blue"]
+skip = [False, False, False, False, False]
+
+# loop over constat type
+for const, modules in ret_constants.items():
+ fig = plt.figure(figsize=(15,7))
+ tt = 0
+ print (const)
+
+ # loop over type of stat parameter
+ for typ in range(len(types)):
+ if skip[typ]:
+ continue
+ ax = plt.subplot2grid((np.count_nonzero(~np.array(skip)), 1), (tt, 0))
+
+ # loop over modules
+ for mod, data in modules.items():
+ ctimes, cd = list(zip(*data))
+ pmm, glob, _ = list(zip(*cd))
+ pma = np.array(pmm)
+ ga = np.array(glob)
+
+ d = pma[:,typ,:,:]
+ print (pma.shape, ga.shape, d.shape)
+
+ if np.allclose(d, 0):
+ continue
+ x = []
+ y = []
+ hue = []
+
+ # loop over gain
+ for g in range(3):
+ dd = (pma[:,typ,:,g]-pma[0,typ,:,g])/pma[0,typ,:,g]
+ print (pma[0,typ,:,g])
+ dd[~np.isfinite(dd)] = 0
+ x.append(np.repeat(np.array(ctimes)[:,None], dd.shape[1], axis=1).flatten())
+ print (x)
+ y.append(dd.flatten())
+ hue.append(np.ones(dd.shape).flatten()*g)
+ x = np.concatenate(x)
+ y = np.concatenate(y)
+ hue = np.concatenate(hue)
+ print (x.shape)
+ seaborn.violinplot(x, y, hue, scale="width", dodge=False, saturation=0.7)
+ #for i in range(max_cells):
+ #ax.scatter(ctimes, (pma[:,typ,i,g]-pma[0,typ,i,g])/pma[0,typ,i,g], marker='.', color=colors[g], alpha=0.5)
+ #ax.plot(ctimes, (ga[:,typ,g]-ga[0,typ,g])/ga[0,typ,g], color=colors[g], alpha=0.5)
+
+ ax.set_ylim(-0.25, .25)
+
+ if typ != len(types)-1:
+ ax.axes.get_xaxis().set_visible(False)
+ else:
+ def format_date(x, pos=None):
+
+ return ctimes[x].strftime('%d-%m')
+ ax.xaxis.set_major_formatter(ticker.FuncFormatter(format_date))
+ ax.set_xlabel("Date")
+ ax.set_ylabel("{}".format(types[typ]))
+
+ tt += 1
+ plt.subplots_adjust(wspace=0.2, hspace=0.2)
+
+ if out_folder != "":
+ fig.savefig("{}/{}_time_development.pdf".format(out_folder, const), bbox_inches='tight')
+
--
GitLab