From 874f16a3cc5645ec02059c61c6a4923dae3b7d25 Mon Sep 17 00:00:00 2001
From: Vratko Rovensky <vratko.rovensky@xfel.eu>
Date: Thu, 19 Sep 2024 15:41:01 +0200
Subject: [PATCH] [AGIPD] [PC] various fixes in PC notebooks
---
.../AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb | 39 +++++++++++------
.../Chracterize_AGIPD_Gain_PC_Summary.ipynb | 43 +++++++++++++------
src/cal_tools/plotting.py | 2 +-
src/xfel_calibrate/notebooks.py | 2 +-
4 files changed, 60 insertions(+), 26 deletions(-)
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
index b24865a33..d8bd254d8 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
@@ -32,16 +32,16 @@
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
- "in_folder = '/gpfs/exfel/exp/SPB/202331/p900376/raw' # path to input data, required\n",
- "out_folder = \"/gpfs/exfel/exp/HED/202430/p900438/scratch/rovensky/pc\" # path to output to, required\n",
+ "in_folder = '/gpfs/exfel/exp/HED/202430/p900438/raw/' # path to input data, required\n",
+ "out_folder = \"/gpfs/exfel/exp/MID/202431/p900465/scratch/rovensky/\" # path to output to, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
- "runs = [61, 62, 63, 64, 65, 66, 67, 68] # runs to use, required, range allowed\n",
+ "runs = [72, 73, 74, 75, 76, 77, 78, 79] # runs to use, required, range allowed\n",
"\n",
"modules = [-1] # modules to work on, required, range allowed\n",
"karabo_da = [\"all\"]\n",
- "karabo_da_control = \"AGIPD1MCTRL00\" # karabo DA for control infromation\n",
- "karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for the control device e.g. \"MID_EXP_AGIPD1M1\", or \"SPB_IRU_AGIPD1M1\"\n",
- "karabo_id = \"SPB_DET_AGIPD1M-1\"\n",
+ "karabo_da_control = \"AGIPD65KCTRL00\" # karabo DA for control infromation\n",
+ "karabo_id_control = \"HED_DET_AGIPD65K1\" # karabo-id for the control device e.g. \"MID_EXP_AGIPD1M1\", or \"SPB_IRU_AGIPD1M1\"\n",
+ "karabo_id = \"HED_DET_AGIPD65K1\"\n",
"ctrl_source_template = '{}/MDL/FPGA_COMP' # path to control information\n",
"instrument_source_template = '{}/DET/{}:xtdf' # path in the HDF5 file to images\n",
"receiver_template = \"{}CH0\" # inset for receiver devices\n",
@@ -65,9 +65,22 @@
"high_res_badpix_3d = False # set this to True if you need high-resolution 3d bad pixel plots. Runtime: ~ 1h\n",
"save_plots = False # set to True if you desire saving plots to output folder\n",
"\n",
- "hg_range = [30,170]#[0,-150] # range for linear fit. If upper edge is negative use clustering result (bound_hg - 20)\n",
- "mg_range = [240,590]#[-350,600] # range for linear fit. If lower edge is negative use clustering result (bound_mg + 20)\n",
- "sigma_dev_cut = 5.0 # parameters outside the range median +- sigma_dev_cut*MAD are replaced with the median"
+ "hg_range = [30,180]#[0,-150] # range for linear fit. If upper edge is negative use clustering result (bound_hg - 20)\n",
+ "mg_range = [340,580]#[-350,600] # range for linear fit. If lower edge is negative use clustering result (bound_mg + 20)\n",
+ "sigma_dev_cut = 5.0 # parameters outside the range median +- sigma_dev_cut*MAD are replaced with the median\n",
+ "\n",
+ "# This is used if modules is not specified:\n",
+ "def find_modules(in_folder, runs, modules):\n",
+ " if (modules is not None) and modules != [-1]:\n",
+ " return modules\n",
+ " from pathlib import Path\n",
+ " import re\n",
+ " modules = set()\n",
+ " for file in Path(in_folder, f'r{runs[0]:04d}').iterdir():\n",
+ " m = re.search(r'-AGIPD(\\d{2})-', file.name)\n",
+ " if m:\n",
+ " modules.add(int(m.group(1)))\n",
+ " return sorted(modules)"
]
},
{
@@ -656,6 +669,7 @@
" bound_m = ms[1]\n",
" threshold = (np.mean(yd[labels[0]])+np.mean(yd[labels[2]])) / 2\n",
" \n",
+ " \n",
" #### Transition region gain is set based on dark thresholds. ####\n",
" msk = yd[labels[1]] < thresh_trans\n",
" label_filter = np.where(labels[1])[0]\n",
@@ -668,7 +682,8 @@
" print('Could not determine maximum value.')\n",
" pass\n",
" ############################################################### \n",
- " \n",
+ " else:\n",
+ " threshold = 0\n",
" if bound is None or bound < 20: #and False:\n",
" ya = dig[:,cell, pix[0], pix[1]][vidx]\n",
" msa, labels, centers = calc_m_cluster2(x, ya, 25, -10, 25)\n",
@@ -1843,7 +1858,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -1857,7 +1872,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.11"
+ "version": "3.11.9"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb
index 8c99eb084..96104d375 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb
@@ -15,13 +15,13 @@
"metadata": {},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/SPB/202431/p900461/raw/\" # path to input data, required\n",
- "out_folder = \"/gpfs/exfel/exp/SPB/202431/p900461/usr/PC/352cells1.1MHz_gs0_20clk/\" # path to output to, required\n",
+ "in_folder = \"/gpfs/exfel/exp/MID/202431/p900465/raw/\" # path to input data, required\n",
+ "out_folder = \"/gpfs/exfel/exp/MID/202431/p900465/scratch/rovensky/PC_test/202cells0.5MHz_gs0_20clk/M0/\" # path to output to, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"runs = [] # runs to use, required, range allowed\n",
"\n",
- "karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for the control device e.g. \"MID_EXP_AGIPD1M1\", or \"SPB_IRU_AGIPD1M1\"\n",
- "karabo_id = \"SPB_DET_AGIPD1M-1\"\n",
+ "karabo_id_control = \"MID_EXP_AGIPD1M1\" # karabo-id for the control device e.g. \"MID_EXP_AGIPD1M1\", or \"SPB_IRU_AGIPD1M1\"\n",
+ "karabo_id = \"MID_DET_AGIPD1M-1\"\n",
"ctrl_source_template = '{}/MDL/FPGA_COMP' # path to control information\n",
"\n",
"creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
@@ -267,7 +267,7 @@
" md = send_to_db(pdu, karabo_id, dbconst, condition,\n",
" file_loc, report, cal_db_interface,\n",
" creation_time=creation_time,\n",
- " variant=1)\n",
+ " variant=1, doraise=True)\n",
"\n",
" print(\"Constants injected with the following conditions:\\n\")\n",
" print(f\"• memory_cells: {mem_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
@@ -305,7 +305,7 @@
"def retrieve_old_PC(mod):\n",
" dconst = getattr(Constants.AGIPD, 'SlopesPC')()\n",
" old_PC, _ = get_from_db(karabo_id=karabo_id,\n",
- " karabo_da=karabo_da[mod],\n",
+ " karabo_da=f\"AGIPD{mod:02d}\",\n",
" constant=dconst,\n",
" condition=condition,\n",
" empty_constant=None,\n",
@@ -316,8 +316,26 @@
" timeout=cal_db_timeout)\n",
" return old_PC\n",
"\n",
+ "# List of all existing modules of a detector instance to retrieve old constants for all modules\n",
+ "strings = [s.split(\"/\")[-1] for s in list(raw_dc.detector_sources)]\n",
+ "all_modules = [int(s[:s.index(\"C\")]) for s in strings]\n",
+ "all_modules.sort()\n",
+ "\n",
"with multiprocessing.Pool(processes=len(modules)) as pool:\n",
- " old_PC_consts = pool.map(retrieve_old_PC, range(len(modules)))"
+ " old_PC_consts = pool.map(retrieve_old_PC, all_modules)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# This is needed in case of single modules where module number is different from 0\n",
+ "if nmods == 1:\n",
+ " module_positions = [0]\n",
+ "else:\n",
+ " module_positions = modules"
]
},
{
@@ -337,16 +355,17 @@
"for (key, c) in zip(keys, const_order):\n",
" const_data[key] = np.full((nmods, mem_cells, 512, 128), np.nan)\n",
" old_const[key] = np.full((nmods, mem_cells, 512, 128), np.nan)\n",
- " for cnt, mn in enumerate(modules):\n",
+ " \n",
+ " for cnt, mn in zip(module_positions, modules):\n",
" if key in fit_data[mn]:\n",
" const_data[key][cnt,:,pixel_range[0]:pixel_range[2],\n",
" pixel_range[1]:pixel_range[3]] = fit_data[mn][key]\n",
" if np.any(old_PC_consts[cnt]):\n",
" old_const[key][cnt,:,pixel_range[0]:pixel_range[2],\n",
- " pixel_range[1]:pixel_range[3]] = old_PC_consts[cnt][c].swapaxes(1,2)\n",
+ " pixel_range[1]:pixel_range[3]] = old_PC_consts[mn][c].swapaxes(1,2)\n",
"\n",
"const_data['BadPixelsPC'] = np.full((nmods, mem_cells, 512, 128), np.nan)\n",
- "for cnt, mn in enumerate(modules):\n",
+ "for cnt, mn in zip(module_positions, modules):\n",
" const_data['BadPixelsPC'][cnt,:,pixel_range[0]:pixel_range[2],\n",
" pixel_range[1]:pixel_range[3]] = fit_data[mn]['BadPixelsPC']"
]
@@ -687,7 +706,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -701,7 +720,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.18"
+ "version": "3.11.9"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
diff --git a/src/cal_tools/plotting.py b/src/cal_tools/plotting.py
index 838e1fc17..1ef0d3898 100644
--- a/src/cal_tools/plotting.py
+++ b/src/cal_tools/plotting.py
@@ -146,7 +146,7 @@ def plot_badpix_3d(data, definitions, title=None, rebin_fac=2, azim=22.5):
colors[d == k] = c[1]
fig = plt.figure(figsize=(15, 10))
- ax = fig.gca(projection="3d")
+ ax = fig.add_subplot(projection="3d")
ax.voxels(xx*rebin_fac, yy*rebin_fac, zz, voxels, facecolors=colors)
ax.view_init(elev=25., azim=azim)
ax.set_xlabel("pixels")
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index c08f4b7ac..a2aa8f02e 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -17,7 +17,7 @@ notebooks = {
"dep_notebooks": [
"notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb"],
"concurrency": {"parameter": "modules",
- "default concurrency": 16,
+ "use function": "find_modules",
"cluster cores": 32},
},
"CS_MERGE": {
--
GitLab