diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
index 672e8e9bc089a6abf787658d52d2e70e3624c006..4185ab714d4a91cf10990ac4be728f1ab1d0e5a7 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
@@ -36,7 +36,6 @@
"out_folder = \"/gpfs/exfel/data/scratch/jsztuk/test/pc\" # path to output to, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"runs = [92, 93, 94, 95, 96, 97, 98, 99] # runs to use, required, range allowed\n",
- "n_sequences = 5 # number of sequence files, starting for 0 to evaluate\n",
"\n",
"modules = [-1] # modules to work on, required, range allowed\n",
"karabo_da = [\"all\"]\n",
@@ -47,9 +46,11 @@
"instrument_source_template = '{}/DET/{}:xtdf' # path in the HDF5 file to images\n",
"receiver_template = \"{}CH0\" # inset for receiver devices\n",
"\n",
- "use_dir_creation_date = True\n",
- "delta_time = 0 # offset to the creation time (e.g. useful in case we want to force the system to use diff. dark constants)\n",
+ "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
+ "creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
+ "cal_db_timeout = 3000000 # timeout on caldb requests\"\n",
"cal_db_interface = \"tcp://max-exfl-cal001:8019\" # the database interface to use\n",
+ "\n",
"local_output = True # output constants locally\n",
"db_output = False # output constants to database\n",
"\n",
@@ -62,6 +63,7 @@
"\n",
"fit_hook = False # fit a hook function to medium gain slope --> run without hook\n",
"high_res_badpix_3d = False # set this to True if you need high-resolution 3d bad pixel plots. Runtime: ~ 1h\n",
+ "save_plots = False # set to True if you desire saving plots to output folder\n",
"\n",
"hg_range = [30,210]#[0,-150] # range for linear fit. If upper edge is negative use clustering result (bound_hg - 20)\n",
"mg_range = [-277,600]#[-350,600] # range for linear fit. If lower edge is negative use clustering result (bound_mg + 20)\n",
@@ -80,10 +82,10 @@
"import warnings\n",
"from datetime import datetime, timedelta\n",
"from functools import partial\n",
+ "from dateutil import parser\n",
"\n",
"warnings.filterwarnings('ignore')\n",
"\n",
- "import dateutil.parser\n",
"import h5py\n",
"import matplotlib\n",
"import numpy as np\n",
@@ -91,7 +93,7 @@
"\n",
"import matplotlib.pyplot as plt\n",
"from matplotlib import gridspec\n",
- "from matplotlib.colors import LogNorm, PowerNorm\n",
+ "from matplotlib.colors import LogNorm\n",
"import matplotlib.patches as patches\n",
"from mpl_toolkits.axes_grid1 import AxesGrid\n",
"\n",
@@ -102,19 +104,14 @@
"\n",
"from cal_tools.agipdlib import AgipdCtrl\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.plotting import plot_badpix_3d, show_overview\n",
+ "from cal_tools.plotting import plot_badpix_3d\n",
"from cal_tools.tools import (\n",
- " gain_map_files,\n",
+ " calcat_creation_time,\n",
" get_constant_from_db_and_time,\n",
" get_dir_creation_date,\n",
- " get_notebook_name,\n",
- " get_pdu_from_db,\n",
- " get_report,\n",
" module_index_to_qm,\n",
- " parse_runs,\n",
- " send_to_db,\n",
")\n",
- "from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions\n",
+ "from iCalibrationDB import Conditions, Constants\n",
"\n",
"# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
"view = Client(profile=cluster_profile)[:]\n",
@@ -182,11 +179,12 @@
"metadata": {},
"outputs": [],
"source": [
- "# Define creation time\n",
- "creation_time = None\n",
- "if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, first_run)\n",
- " creation_time = creation_time + timedelta(hours=delta_time)\n",
+ "# Evaluate creation time\n",
+ "creation_time = calcat_creation_time(in_folder, runs[0], creation_time)\n",
+ "offset = parser.parse(creation_date_offset)\n",
+ "delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
+ "creation_time += delta\n",
+ "print(f\"Creation time: {creation_time}\\n\")\n",
"\n",
"# Read AGIPD parameter conditions.\n",
"if acq_rate < 0.:\n",
@@ -886,10 +884,10 @@
" fig1.show()\n",
" fig2.show()\n",
" fig3.show()\n",
- "\n",
- " fig1.savefig(\"{}/module_{}_{}_pixel_plot.png\".format(out_folder, mod, roi))\n",
- " fig2.savefig(\"{}/module_{}_{}_pixel_plot_gain.png\".format(out_folder, mod, roi))\n",
- " fig3.savefig(\"{}/module_{}_{}_pixel_plot_fits.png\".format(out_folder, mod, roi))"
+ " if save_plots:\n",
+ " fig1.savefig(\"{}/module_{}_{}_pixel_plot.png\".format(out_folder, mod, roi))\n",
+ " fig2.savefig(\"{}/module_{}_{}_pixel_plot_gain.png\".format(out_folder, mod, roi))\n",
+ " fig3.savefig(\"{}/module_{}_{}_pixel_plot_fits.png\".format(out_folder, mod, roi))"
]
},
{
@@ -1415,35 +1413,6 @@
" mask[bad_fits_med] |= BadPixels.CI_EVAL_ERROR.value\n"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "def slope_dict_to_arr(d):\n",
- " key_to_index = {\n",
- " \"ml\": 0,\n",
- " \"bl\": 1,\n",
- " \"devl\": 2,\n",
- " \"mh\": 3,\n",
- " \"bh\": 4,\n",
- " \"oh\": 5,\n",
- " \"ch\": 6,\n",
- " \"ah\": 7,\n",
- " \"devh\": 8,\n",
- " \"tresh\": 9,\n",
- " }\n",
- "\n",
- " arr = np.zeros([11]+list(d[\"ml\"].shape), np.float32)\n",
- " for key, item in d.items():\n",
- " if key not in key_to_index:\n",
- " continue\n",
- " arr[key_to_index[key],...] = item\n",
- " \n",
- " return arr"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -1461,65 +1430,6 @@
" store_file.close()"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Read report path and create file location tuple to add with the injection\n",
- "proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
- "file_loc = proposal + ' ' + ' '.join(list(map(str,runs)))\n",
- "\n",
- "report = get_report(metadata_folder)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "md = None\n",
- "\n",
- "# set the operating condition\n",
- "condition = Conditions.Dark.AGIPD(memory_cells=mem_cells, \n",
- " bias_voltage=bias_voltage,\n",
- " acquisition_rate=acq_rate, \n",
- " gain_setting=gain_setting,\n",
- " integration_time=integration_time)\n",
- "\n",
- "db_modules = get_pdu_from_db(karabo_id, karabo_da, Constants.AGIPD.SlopesPC(),\n",
- " condition, cal_db_interface,\n",
- " snapshot_at=creation_time)\n",
- "\n",
- "for pdu, (qm, r) in zip(db_modules, fres.items()):\n",
- " for const in [\"SlopesPC\", \"BadPixelsPC\"]:\n",
- "\n",
- " dbconst = getattr(Constants.AGIPD, const)()\n",
- "\n",
- " if const == \"SlopesPC\":\n",
- " dbconst.data = slope_dict_to_arr(r)\n",
- " else:\n",
- " dbconst.data = bad_pixels[qm]\n",
- "\n",
- " if db_output:\n",
- " md = send_to_db(pdu, karabo_id, dbconst, condition,\n",
- " file_loc, report, cal_db_interface,\n",
- " creation_time=creation_time,\n",
- " variant=1)\n",
- " # TODO: check if this can replace other written function of this notebook.\n",
- " #if local_output:\n",
- " # md = save_const_to_h5(pdu, karabo_id, dconst, condition, dconst.data, \n",
- " # file_loc, report, creation_time, out_folder)\n",
- "\n",
- "print(\"Constants parameter conditions are:\\n\")\n",
- "print(f\"• memory_cells: {mem_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
- " f\"• acquisition_rate: {acq_rate}\\n• gain_setting: {gain_setting}\\n\"\n",
- " f\"• integration_time: {integration_time}\\n\"\n",
- " f\"• creation_time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
- ]
- },
{
"cell_type": "markdown",
"metadata": {},
@@ -1944,13 +1854,6 @@
" ax.set_xlabel(\"PC scan point (#)\")\n",
" ax.grid(lw=1.5)"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..a690dc0c5662465ea645e4ba44859736cb6ba42a
--- /dev/null
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb
@@ -0,0 +1,740 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Pulse Capacitor Characterisation Summary\n",
+ "\n",
+ "This notebook is used as a dependency notebook for a pulse capacitor characterisation to provide summary for all modules of the AGIPD."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = \"/gpfs/exfel/d/raw/SPB/202331/p900376/\" # path to input data, required\n",
+ "out_folder = \"/gpfs/exfel/exp/SPB/202331/p900376/usr/PC/agipd12/202cells0.5MHz_gs0_20clk/\" # path to output to, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
+ "runs = [] # runs to use, required, range allowed\n",
+ "\n",
+ "karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for the control device e.g. \"MID_EXP_AGIPD1M1\", or \"SPB_IRU_AGIPD1M1\"\n",
+ "karabo_id = \"SPB_DET_AGIPD1M-1\"\n",
+ "ctrl_source_template = '{}/MDL/FPGA_COMP' # path to control information\n",
+ "\n",
+ "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
+ "creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
+ "cal_db_timeout = 3000000 # timeout on caldb requests\"\n",
+ "cal_db_interface = \"tcp://max-exfl-cal001:8015#8045\"\n",
+ "db_output = False\n",
+ "\n",
+ "bias_voltage = -1 # detector bias voltage, negative values for auto-detection.\n",
+ "mem_cells = -1 # number of memory cells used, negative values for auto-detection.\n",
+ "acq_rate = -1. # the detector acquisition rate, negative values for auto-detection.\n",
+ "gain_setting = -1 # gain setting can have value 0 or 1, negative values for auto-detection.\n",
+ "integration_time = -1 # integration time, negative values for auto-detection."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import warnings\n",
+ "warnings.filterwarnings('ignore')\n",
+ "\n",
+ "import os\n",
+ "from dateutil import parser\n",
+ "from datetime import timedelta\n",
+ "\n",
+ "import h5py\n",
+ "import matplotlib.pyplot as plt\n",
+ "import matplotlib.gridspec as gridspec\n",
+ "import matplotlib.cm as cm\n",
+ "import numpy as np\n",
+ "import tabulate\n",
+ "import multiprocessing\n",
+ "\n",
+ "from cal_tools.agipdlib import AgipdCtrl\n",
+ "from cal_tools.ana_tools import get_range\n",
+ "from cal_tools.tools import (\n",
+ " calcat_creation_time,\n",
+ " module_index_to_qm,\n",
+ " get_from_db,\n",
+ " get_pdu_from_db,\n",
+ " get_report,\n",
+ " send_to_db\n",
+ ")\n",
+ "\n",
+ "from extra_data import RunDirectory\n",
+ "from extra_geom import AGIPD_1MGeometry, AGIPD_500K2GGeometry\n",
+ "from IPython.display import Latex, display\n",
+ "from XFELDetAna.plotting.simpleplot import simplePlot\n",
+ "\n",
+ "from iCalibrationDB import Conditions, Constants\n",
+ "\n",
+ "%matplotlib inline"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Evaluate creation time\n",
+ "creation_time = calcat_creation_time(in_folder, runs[0], creation_time)\n",
+ "offset = parser.parse(creation_date_offset)\n",
+ "delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
+ "creation_time += delta\n",
+ "print(f\"Creation time: {creation_time}\\n\")\n",
+ "\n",
+ "# Get operation conditions\n",
+ "ctrl_source = ctrl_source_template.format(karabo_id_control)\n",
+ "run_folder = f'{in_folder}/r{runs[0]:04d}/'\n",
+ "raw_dc = RunDirectory(run_folder)\n",
+ "\n",
+ "# Read operating conditions from AGIPD00 files\n",
+ "instrument_src_mod = [\n",
+ " s for s in list(raw_dc.all_sources) if \"0CH\" in s][0]\n",
+ "ctrl_src = [\n",
+ " s for s in list(raw_dc.all_sources) if ctrl_source in s][0]\n",
+ "\n",
+ "agipd_cond = AgipdCtrl(\n",
+ " run_dc=raw_dc,\n",
+ " image_src=instrument_src_mod,\n",
+ " ctrl_src=ctrl_src,\n",
+ " raise_error=False, # to be able to process very old data without mosetting value\n",
+ ")\n",
+ "if mem_cells == -1:\n",
+ " mem_cells = agipd_cond.get_num_cells()\n",
+ "if mem_cells is None:\n",
+ " raise ValueError(f\"No raw images found in {run_folder}\")\n",
+ "if acq_rate == -1.:\n",
+ " acq_rate = agipd_cond.get_acq_rate()\n",
+ "if gain_setting == -1:\n",
+ " gain_setting = agipd_cond.get_gain_setting(creation_time)\n",
+ "if bias_voltage == -1:\n",
+ " bias_voltage = agipd_cond.get_bias_voltage(karabo_id_control)\n",
+ "if integration_time == -1:\n",
+ " integration_time = agipd_cond.get_integration_time()\n",
+ "\n",
+ "# Evaluate detector instance for mapping\n",
+ "instrument = karabo_id.split(\"_\")[0]\n",
+ "if instrument == \"HED\":\n",
+ " nmods = 8\n",
+ "else:\n",
+ " nmods = 16\n",
+ "\n",
+ "print(f\"Using {creation_time} as creation time\\n\")\n",
+ "print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Memory cells: {mem_cells}\\n\"\n",
+ " f\"• Acquisition rate: {acq_rate}\\n• Gain setting: {gain_setting}\\n• Integration time: {integration_time}\\n\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "# ml - high gain slope\n",
+ "# bl - high gain intercept\n",
+ "# devl - absolute relative deviation from linearity for high gain\n",
+ "# mh - medium gain slope\n",
+ "# bh - medium gain intercept\n",
+ "# oh, ch, ah - parameters of hook function fit to medium gain (only if requested)\n",
+ "# devh - absolute relative deviation from linearity for linear part of medium gain\n",
+ "\n",
+ "keys = ['ml', 'bl', 'mh', 'bh', 'BadPixelsPC']\n",
+ "keys_file = [\"ml\", \"bl\", \"devl\", \"mh\", \"bh\", \"oh\", \"ch\", \"ah\", \"devh\"]\n",
+ " \n",
+ "fit_data = {}\n",
+ "bad_pixels = {}\n",
+ "modules = []\n",
+ "karabo_da = []\n",
+ "\n",
+ "for mod in range(nmods):\n",
+ " qm = module_index_to_qm(mod)\n",
+ " fit_data[mod] = {}\n",
+ " constants_file = f'{out_folder}/agipd_pc_store_{\"_\".join([str(run) for run in runs])}_{mod}_{mod}.h5'\n",
+ " \n",
+ " if os.path.exists(constants_file):\n",
+ " print(f'Trying to find: {constants_file}')\n",
+ " print(f'Data available for module {qm}\\n')\n",
+ " with h5py.File(constants_file, 'r') as hf:\n",
+ " bad_pixels[mod] = hf[f'/{qm}/BadPixelsPC/0/data'][()]\n",
+ " for key in keys_file:\n",
+ " fit_data[mod][key]= hf[f'/{qm}/{key}/0/data'][()]\n",
+ " \n",
+ " modules.append(mod)\n",
+ " karabo_da.append(f\"AGIPD{mod:02d}\")\n",
+ " else:\n",
+ " print(f\"No fit data available for module {qm}\\n\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def slope_dict_to_arr(d):\n",
+ " key_to_index = {\n",
+ " \"ml\": 0,\n",
+ " \"bl\": 1,\n",
+ " \"devl\": 2,\n",
+ " \"mh\": 3,\n",
+ " \"bh\": 4,\n",
+ " \"oh\": 5,\n",
+ " \"ch\": 6,\n",
+ " \"ah\": 7,\n",
+ " \"devh\": 8,\n",
+ " \"tresh\": 9\n",
+ " }\n",
+ "\n",
+ " arr = np.zeros([11]+list(d[\"ml\"].shape), np.float32)\n",
+ " for key, item in d.items():\n",
+ " if key not in key_to_index:\n",
+ " continue\n",
+ " arr[key_to_index[key],...] = item\n",
+ " \n",
+ " return arr"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# set the operating condition\n",
+ "condition = Conditions.Dark.AGIPD(memory_cells=mem_cells, \n",
+ " bias_voltage=bias_voltage,\n",
+ " acquisition_rate=acq_rate, \n",
+ " gain_setting=gain_setting,\n",
+ " integration_time=integration_time)\n",
+ "\n",
+ "db_modules = get_pdu_from_db(karabo_id, karabo_da, Constants.AGIPD.SlopesPC(),\n",
+ " condition, cal_db_interface,\n",
+ " snapshot_at=creation_time)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
+ "file_loc = proposal + ' ' + ' '.join(list(map(str,runs)))\n",
+ "\n",
+ "report = get_report(metadata_folder)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "md = None\n",
+ "\n",
+ "if db_output:\n",
+ " for mod, pdu in zip(modules, db_modules):\n",
+ " for const in [\"SlopesPC\", \"BadPixelsPC\"]:\n",
+ "\n",
+ " dbconst = getattr(Constants.AGIPD, const)()\n",
+ "\n",
+ " if const == \"SlopesPC\":\n",
+ " dbconst.data = slope_dict_to_arr(fit_data[mod])\n",
+ " else:\n",
+ " dbconst.data = bad_pixels[mod]\n",
+ "\n",
+ "\n",
+ " md = send_to_db(pdu, karabo_id, dbconst, condition,\n",
+ " file_loc, report, cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " variant=1)\n",
+ "\n",
+ " print(\"Constants injected with the following conditions:\\n\")\n",
+ " print(f\"• memory_cells: {mem_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
+ " f\"• acquisition_rate: {acq_rate}\\n• gain_setting: {gain_setting}\\n\"\n",
+ " f\"• integration_time: {integration_time}\\n\"\n",
+ " f\"• creation_time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")\n",
+ "else:\n",
+ " print('Injection to DB not requested.')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Remove keys which won't be used for comparison plots and add BP to the rest of data \n",
+ "for mod in modules:\n",
+ " fit_data[mod]['BadPixelsPC'] = bad_pixels[mod]\n",
+ " \n",
+ " for key in keys_file:\n",
+ " if key not in keys:\n",
+ " del fit_data[mod][key]\n",
+ " \n",
+ " for key in keys:\n",
+ " fit_data[mod][key] = fit_data[mod][key].swapaxes(1,2) "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def retrieve_old_PC(mod):\n",
+ " dconst = getattr(Constants.AGIPD, 'SlopesPC')()\n",
+ " old_PC = get_from_db(karabo_id=karabo_id,\n",
+ " karabo_da=karabo_da[mod],\n",
+ " constant=dconst,\n",
+ " condition=condition,\n",
+ " empty_constant=None,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time-timedelta(seconds=1) if creation_time else None,\n",
+ " strategy=\"pdu_prior_in_time\",\n",
+ " verbosity=1,\n",
+ " timeout=cal_db_timeout)\n",
+ " return old_PC\n",
+ "\n",
+ "with multiprocessing.Pool(processes=len(modules)) as pool:\n",
+ " old_PC_consts = pool.map(retrieve_old_PC, range(len(modules)))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Create the arrays that will be used for figures.\n",
+ "# Each array correponds to the data for all processed modules.\n",
+ "\n",
+ "pixel_range = [0,0,512,128]\n",
+ "const_data = {}\n",
+ "old_const = {}\n",
+ "const_order = [0, 1, 3, 4]\n",
+ "\n",
+ "for (key, c) in zip(keys, const_order):\n",
+ " const_data[key] = np.full((nmods, mem_cells, 512, 128), np.nan)\n",
+ " old_const[key] = np.full((nmods, mem_cells, 512, 128), np.nan)\n",
+ " for cnt, i in enumerate(modules):\n",
+ " if key in fit_data[i]:\n",
+ " const_data[key][i,:,pixel_range[0]:pixel_range[2],\n",
+ " pixel_range[1]:pixel_range[3]] = fit_data[i][key]\n",
+ " if old_PC_consts[0][0]:\n",
+ " old_const[key][i,:,pixel_range[0]:pixel_range[2],\n",
+ " pixel_range[1]:pixel_range[3]] = old_PC_consts[cnt][0][c].swapaxes(1,2)\n",
+ "\n",
+ "const_data['BadPixelsPC'] = np.full((nmods, mem_cells, 512, 128), np.nan)\n",
+ "for i in modules:\n",
+ " const_data['BadPixelsPC'][i,:,pixel_range[0]:pixel_range[2],\n",
+ " pixel_range[1]:pixel_range[3]] = fit_data[i]['BadPixelsPC']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#Define AGIPD geometry\n",
+ "if instrument == \"HED\":\n",
+ " geom = AGIPD_500K2GGeometry.from_origin()\n",
+ "else:\n",
+ " geom = AGIPD_1MGeometry.from_quad_positions(quad_pos=[\n",
+ " (-525, 625),\n",
+ " (-550, -10),\n",
+ " (520, -160),\n",
+ " (542.5, 475),\n",
+ " ])"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Summary across pixels ##\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "gain_data = {'HG': {},\n",
+ " 'MG': {},\n",
+ " 'BP': {}\n",
+ " }\n",
+ "\n",
+ "old_gain_data = {'HG': {},\n",
+ " 'MG': {}\n",
+ " }\n",
+ " \n",
+ "for key in ['ml', 'bl']:\n",
+ " gain_data['HG'][key] = const_data[key]\n",
+ " old_gain_data['HG'][key] = old_const[key]\n",
+ "for key in ['mh', 'bh']:\n",
+ " gain_data['MG'][key] = const_data[key]\n",
+ " old_gain_data['MG'][key] = old_const[key]\n",
+ "gain_data['BP']['BadPixelsPC'] = const_data['BadPixelsPC']"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def plot_definition(data, g, key):\n",
+ " titel = ['Difference to previous', 'Percentage difference']\n",
+ " gs = gridspec.GridSpec(1, 2)\n",
+ " fig = plt.figure(figsize=(15, 7))\n",
+ " plt.suptitle(f'{g}', fontsize=16)\n",
+ " \n",
+ " for pos in range(0,2):\n",
+ " vmin, vmax = get_range(data[pos], 2)\n",
+ " vmax = max(vmax, abs(vmin))\n",
+ " ax = fig.add_subplot(gs[0, pos])\n",
+ " ticks = [-1, 0, 1] if np.isnan(vmin) else [vmin, (vmin+vmax)/2, vmax]\n",
+ " geom.plot_data_fast(data[pos],\n",
+ " vmin=vmin, vmax=vmax, ax=ax, cmap=\"RdBu\", figsize=(13,7),\n",
+ " colorbar={'shrink': 1,\n",
+ " 'pad': 0.04,\n",
+ " 'fraction': 0.1,\n",
+ " \n",
+ " })\n",
+ " colorbar = ax.images[0].colorbar\n",
+ " colorbar.ax.set_yticklabels([\"{:.1f}\".format(tk) for tk in colorbar.get_ticks()])\n",
+ " if pos == 1:\n",
+ " colorbar.set_label('%')\n",
+ " ax.set_title(f\"{titel[pos]}: {key}\", fontsize=14)\n",
+ " ax.set_xlabel(\"Columns\", fontsize=13)\n",
+ " ax.set_ylabel(\"Rows\", fontsize=13)\n",
+ "\n",
+ "def plot_diff_consts(old_const, new_const, g, ratio=False):\n",
+ " if ratio:\n",
+ " old_data = old_const['HG']['ml'] / old_const['MG']['mh']\n",
+ " new_data = new_const['HG']['ml'] / new_const['MG']['mh']\n",
+ " data1 = np.nanmean((new_data - old_data), axis=1)\n",
+ " data2 = np.nanmean((new_data - old_data)/old_data*100, axis=1)\n",
+ " data = [data1, data2]\n",
+ " key ='Slopes ratio HG/MG'\n",
+ " plot_definition(data, g, key)\n",
+ " else:\n",
+ " for i, key in enumerate(old_const[g].keys()):\n",
+ " data1 = np.nanmean((new_const[g][key] - old_const[g][key]), axis=1)\n",
+ " data2 = np.nanmean((new_const[g][key] - old_const[g][key])/old_const[g][key]*100, axis=1)\n",
+ " data = [data1, data2]\n",
+ " plot_definition(data, g, key)\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "for gain in old_gain_data.keys():\n",
+ " plot_diff_consts(old_gain_data, gain_data, gain)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "plot_diff_consts(old_gain_data, gain_data, 'Ratio HG/MG', ratio=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "g_label = ['High gain', 'Medium gain', 'Bad pixels PC']\n",
+ "for idx, g in enumerate(gain_data.keys()):\n",
+ " gs = gridspec.GridSpec(1, 2)\n",
+ " fig = plt.figure(figsize=(15, 7))\n",
+ " plt.suptitle(f'{g_label[idx]}', fontsize=16)\n",
+ " \n",
+ " for i, key in enumerate(gain_data[g].keys()):\n",
+ " if key is 'BadPixelsPC':\n",
+ " data = np.nanmean(gain_data[g][key]>0, axis=1)\n",
+ " vmin, vmax = (0,1)\n",
+ " ax = fig.add_subplot(gs[0, :])\n",
+ " ticks = [0, 0.5, 1]\n",
+ " \n",
+ " else:\n",
+ " data = np.nanmean(gain_data[g][key], axis=1)\n",
+ " vmin, vmax = get_range(data, 5)\n",
+ " ax = fig.add_subplot(gs[0, i])\n",
+ " geom.plot_data_fast(data,\n",
+ " vmin=vmin, vmax=vmax, ax=ax, cmap=\"jet\", figsize=(13,7),\n",
+ " colorbar={'shrink': 1,\n",
+ " 'pad': 0.04,\n",
+ " 'fraction': 0.1,\n",
+ " \n",
+ " })\n",
+ " colorbar = ax.images[0].colorbar\n",
+ " ax.set_title(key, fontsize=14)\n",
+ " ax.set_xlabel('Columns', fontsize=13)\n",
+ " ax.set_ylabel('Rows', fontsize=13)\n",
+ " \n",
+ " plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Summary across cells ##\n",
+ "\n",
+ "Good pixels only."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "ratio = gain_data['HG']['ml'] / gain_data['MG']['mh']\n",
+ "\n",
+ "fig = plt.figure(figsize=(7, 7))\n",
+ "ax = fig.add_subplot(111)\n",
+ "data = np.nanmean(ratio, axis=1)\n",
+ "vmin, vmax = get_range(data, 5)\n",
+ "ax = geom.plot_data_fast(data,\n",
+ " vmin=vmin, vmax=vmax, ax=ax, cmap=\"jet\", figsize=(6,7),\n",
+ " colorbar={'shrink': 1,\n",
+ " 'pad': 0.04,\n",
+ " 'fraction': 0.1\n",
+ " })\n",
+ "colorbar = ax.images[0].colorbar\n",
+ "colorbar.set_label('HG slope / MG slope', fontsize=13)\n",
+ "ax.set_title('High/Medium Gain Slope Ratio', fontsize=14)\n",
+ "ax.set_xlabel('Columns', fontsize=13)\n",
+ "ax.set_ylabel('Rows', fontsize=13)\n",
+ "\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "for idx, g in enumerate(gain_data.keys()):\n",
+ " \n",
+ " for key in gain_data[g].keys():\n",
+ " data = np.copy(gain_data[g][key])\n",
+ " if key=='BadPixelsPC':\n",
+ " data = data>0\n",
+ " else:\n",
+ " data[gain_data['BP']['BadPixelsPC']>0] = np.nan\n",
+ "\n",
+ " d = []\n",
+ " for i in range(nmods):\n",
+ " d.append({'x': np.arange(data[i].shape[0]),\n",
+ " 'y': np.nanmean(data[i], axis=(1,2)),\n",
+ " 'drawstyle': 'steps-pre',\n",
+ " 'label': f'{i}',\n",
+ " 'linewidth': 2,\n",
+ " 'linestyle': '--' if i>7 else '-'\n",
+ " })\n",
+ "\n",
+ " fig = plt.figure(figsize=(12, 6))\n",
+ " plt.suptitle(f'{g_label[idx]} - {key}', fontsize=16)\n",
+ " ax = fig.add_subplot(111)\n",
+ "\n",
+ " _ = simplePlot(d, xrange=(-12, 510),\n",
+ " x_label='Memory Cell ID',\n",
+ " y_label=key,\n",
+ " use_axis=ax,\n",
+ " legend='top-left-frame-ncol8',)\n",
+ " ylim = ax.get_ylim()\n",
+ " ax.set_ylim(ylim[0], ylim[1] + np.abs(ylim[1]-ylim[0])*0.2)\n",
+ " ax.grid()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "d = []\n",
+ "for i in range(nmods):\n",
+ " d.append({'x': np.arange(ratio[i].shape[0]),\n",
+ " 'y': np.nanmean(ratio[i], axis=(1,2)),\n",
+ " 'drawstyle': 'steps-pre',\n",
+ " 'label': f'{i}',\n",
+ " 'linewidth': 2,\n",
+ " 'linestyle': '--' if i>7 else '-'\n",
+ " })\n",
+ "\n",
+ "fig = plt.figure(figsize=(12, 6))\n",
+ "plt.suptitle('High/Medium Gain Slope Ratio', fontsize=16)\n",
+ "ax = fig.add_subplot(111)\n",
+ "\n",
+ "_ = simplePlot(d, xrange=(-12, 510),\n",
+ " x_label='Memory Cell ID',\n",
+ " y_label='Gain ratio ml/mh',\n",
+ " use_axis=ax,\n",
+ " legend='top-left-frame-ncol8',)\n",
+ "ylim = ax.get_ylim()\n",
+ "ax.set_ylim(ylim[0], ylim[1] + np.abs(ylim[1]-ylim[0])*0.2)\n",
+ "ax.grid()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "table = []\n",
+ "ratio_old = old_gain_data['HG']['ml'] / old_gain_data['MG']['mh']\n",
+ "for mod in modules:\n",
+ " \n",
+ " table.append((mod,\n",
+ " f\"{np.nanmean(ratio[mod]):0.1f} +- {np.nanstd(ratio[mod]):0.2f}\",\n",
+ " f\"{np.nanmean(ratio_old[mod]):0.1f} +- {np.nanstd(ratio_old[mod]):0.2f}\",\n",
+ " f\"{np.nanmean(gain_data['BP']['BadPixelsPC'][mod]>0)*100:0.1f} ({np.nansum(gain_data['BP']['BadPixelsPC'][mod]>0)})\"\n",
+ " ))\n",
+ "\n",
+ "all_HM = []\n",
+ "all_HM_old = []\n",
+ "for mod in modules:\n",
+ " all_HM.extend(ratio[mod])\n",
+ " all_HM_old.extend(ratio_old[mod])\n",
+ "all_HM = np.array(all_HM)\n",
+ "all_HM_old = np.array(all_HM_old)\n",
+ "\n",
+ "all_MSK = np.array([list(msk) for msk in gain_data['BP']['BadPixelsPC']])\n",
+ "\n",
+ "table.append(('overall',\n",
+ " f\"{np.nanmean(all_HM):0.1f} +- {np.nanstd(all_HM):0.2f}\",\n",
+ " f\"{np.nanmean(all_HM_old):0.1f} +- {np.nanstd(all_HM_old):0.2f}\",\n",
+ " f\"{np.nanmean(all_MSK>0)*100:0.1f} ({np.nansum(all_MSK>0)})\"\n",
+ " ))\n",
+ "\n",
+ "md = display(Latex(tabulate.tabulate(table, tablefmt='latex',\n",
+ " headers=[\"Module\", \n",
+ " \"HG/MG Ratio\",\n",
+ " \"Previous HG/MG Ratio\",\n",
+ " \"Bad pixels [%(Count)]\"])))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Summary high-medium gain ratio (good pixels only) + histograms"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "colors = cm.rainbow(np.linspace(0, 1, nmods))\n",
+ "\n",
+ "gs = gridspec.GridSpec(1, 2)\n",
+ "fig = plt.figure(figsize=(17, 8))\n",
+ "\n",
+ "\n",
+ "ratio[gain_data['BP']['BadPixelsPC'] > 0] = np.nan\n",
+ "data = np.nanmean(ratio, axis=1)\n",
+ "vmin, vmax = get_range(data, 5)\n",
+ "ax = fig.add_subplot(gs[0, 0])\n",
+ "geom.plot_data_fast(data,\n",
+ " vmin=vmin, vmax=vmax, ax=ax, cmap=\"jet\", figsize=(12.5,7),\n",
+ " colorbar={'shrink': 1,\n",
+ " 'pad': 0.04,\n",
+ " 'fraction': 0.1\n",
+ " })\n",
+ "colorbar = ax.images[0].colorbar\n",
+ "colorbar.set_label('HG/MG', fontsize=12)\n",
+ "ax.set_xlabel('Columns', fontsize=12)\n",
+ "ax.set_ylabel('Rows', fontsize=12)\n",
+ "\n",
+ "ax = fig.add_subplot(gs[0,1])\n",
+ "for mod in modules:\n",
+ " h, e = np.histogram(ratio[mod].flatten(), bins=100, range=(vmin, vmax))\n",
+ " ax.plot(e[:-1], h, color=colors[mod],linewidth=2, label=f'{mod}', alpha=0.8)\n",
+ " ax.set_xlabel('High/Medium Gain Ratio', fontsize=13)\n",
+ " ax.set_ylabel('Counts', fontsize=13)\n",
+ " plt.ticklabel_format(axis='y', style='sci', scilimits=(0,0))\n",
+ "ax.grid()\n",
+ "ax.legend()\n",
+ "plt.show()"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "pycalibration",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.11"
+ },
+ "latex_envs": {
+ "LaTeX_envs_menu_present": true,
+ "autocomplete": true,
+ "bibliofile": "biblio.bib",
+ "cite_by": "apalike",
+ "current_citInitial": 1,
+ "eqLabelWithNumbers": true,
+ "eqNumInitial": 1,
+ "hotkeys": {
+ "equation": "Ctrl-E",
+ "itemize": "Ctrl-I"
+ },
+ "labels_anchors": false,
+ "latex_user_defs": false,
+ "report_style_numbering": false,
+ "user_envs_cfg": false
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index 63231b4dff9c60cf7a62a808ef06b270da51e7a3..e1f308faab5228ef145cfa19b57f891d43630e4d 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -14,6 +14,8 @@ notebooks = {
},
"PC": {
"notebook": "notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb",
+ "dep_notebooks": [
+ "notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb"],
"concurrency": {"parameter": "modules",
"default concurrency": 16,
"cluster cores": 32},