From ef15c49e1671ab113dc60d4ed1c4a6f0f853b063 Mon Sep 17 00:00:00 2001
From: Vratko Rovensky <vratko.rovensky@xfel.eu>
Date: Wed, 20 Nov 2024 15:07:03 +0100
Subject: [PATCH 01/10] Fixes and improvments for HZ FF processing
---
notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb | 36 +++++-----
...aracterize_AGIPD_Gain_FlatFields_NBC.ipynb | 36 +++++-----
...terize_AGIPD_Gain_FlatFields_Summary.ipynb | 70 ++++++++++++-------
3 files changed, 85 insertions(+), 57 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb b/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb
index caca0264e..91d470d01 100644
--- a/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb
+++ b/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb
@@ -17,14 +17,15 @@
"metadata": {},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/MID/202030/p900137/proc/\" # the folder to read data from, required\n",
- "out_folder = \"/gpfs/exfel/exp/MID/202030/p900137/scratch/karnem/r0319_0322_0342_v50\" # the folder to output to, required\n",
+ "in_folder = \"/gpfs/exfel/exp/HED/202431/p900478/proc/\" # the folder to read data from, required\n",
+ "out_folder = \"/gpfs/exfel/exp/HED/202431/p900478/scratch/\" # the folder to output to, required\n",
"sequences = [-1] # module to consider, set to -1 for all, range allowed\n",
- "modules = [5] # module to consider, range allowed\n",
- "runs = [319] # list of run numbers, required\n",
- "cells_list = ['range(0,15)'] # list of lists or any python expressions, which can be converted to a list. E.g. 'range(0,15,5)' 'list(range(0,5))+list(range(50,60))'\n",
+ "modules = [81] # module to consider, range allowed\n",
+ "runs = [44] # list of run numbers, required\n",
+ "cells_list = ['range(0,3)'] # list of lists or any python expressions, which can be converted to a list. E.g. 'range(0,15,5)' 'list(range(0,5))+list(range(50,60))'\n",
+ "roi = [0, 512, 0, 128] # ROI considered for the analysis of data quality\n",
"\n",
- "karabo_id = \"MID_DET_AGIPD1M-1\" # karabo karabo_id\n",
+ "karabo_id = \"HED_TST_AGIPDHZ1\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
"receiver_id = \"{}CH0\" # inset for receiver devices\n",
"path_template = 'CORR-R{:04d}-AGIPD{:02d}-S{}.h5' # the template to use to access data\n",
@@ -32,7 +33,7 @@
"\n",
"n_bins_adu = 1 # number of bins per ADU\n",
"h_range = [-50, 450] # range of the histogram in ADU\n",
- "n_cells = 202 # total number of memory cells (used to create summary file)\n",
+ "n_cells = 352 # total number of memory cells (used to create summary file)\n",
"\n",
"hist_std_tresh = 10 # Threshold for histogram standard deviation\n",
"hist_mean_tresh = 15 # Threshold for histogram mean\n",
@@ -451,14 +452,14 @@
"metadata": {},
"outputs": [],
"source": [
- "# Perfomr analysis of the data quality\n",
+ "# Perform analysis of the data quality\n",
"rshist = np.reshape(total_hist, (n_cells, n_bins, 512, 128))\n",
"\n",
"x = np.linspace(h_range[0], h_range[1], n_bins)\n",
"mids = 0.5*(x[1:] + x[:-1])\n",
- "\n",
- "h_sums = np.sum(rshist, axis=1)\n",
- "hist_norm = rshist / h_sums[:, None, :, :]\n",
+ "roi_rshist = rshist[..., roi[0]:roi[1], roi[2]:roi[3]]\n",
+ "h_sums = np.sum(roi_rshist, axis=1)\n",
+ "hist_norm = roi_rshist / h_sums[:, None, :, :]\n",
"hist_mean = np.sum(hist_norm[:, :n_bins-1, ...] *\n",
" mids[None, :, None, None], axis=1)\n",
"\n",
@@ -491,7 +492,9 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "scrolled": false
+ },
"outputs": [],
"source": [
"fig = plt.figure(figsize=(10, 5))\n",
@@ -613,7 +616,7 @@
"outputs": [],
"source": [
"# Plot for single pixel and all memory cells.\n",
- "xpix= 23\n",
+ "xpix= 470\n",
"ypix= 44\n",
"\n",
"x = np.arange(h_range[0],h_range[1] , 1)\n",
@@ -634,14 +637,15 @@
"ax.set_xlim(-50,300)\n",
"for color, y in zip(colors, rshist[:,:,xpix,ypix]):\n",
" ax.plot(x, y, color=color,linewidth=0.2)\n",
- "plt.grid() \n",
+ "plt.grid()\n",
+ "plt.yscale('log')\n",
"plt.show()"
]
}
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -655,7 +659,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.11"
+ "version": "3.11.9"
}
},
"nbformat": 4,
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
index fa1d218a0..26b0bc44e 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
@@ -14,20 +14,20 @@
"metadata": {},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/SPB/202030/p900138/scratch/karnem/r0203_r0204_v01/\" # the folder to read histograms from, required\n",
+ "in_folder = \"/gpfs/exfel/exp/HED/202431/p900478/scratch/HISTS/air/352cells2.2MHz_gs0_20clk_fixedHG/HED_TST_AGIPDHZ1/\" # the folder to read histograms from, required\n",
"out_folder = \"\" # the folder to output to, required\n",
"hist_file_template = \"hists_m{:02d}_sum.h5\" # the template to use to access histograms\n",
- "modules = [10] # modules to correct, set to -1 for all, range allowed\n",
+ "modules = [81] # modules to correct, set to -1 for all, range allowed\n",
"\n",
- "raw_folder = \"/gpfs/exfel/exp/MID/202030/p900137/raw\" # Path to raw image data used to create histograms\n",
- "proc_folder = \"\" # Path to corrected image data used to create histograms\n",
+ "raw_folder = \"/gpfs/exfel/exp/HED/202431/p900478/raw\" # Path to raw image data used to create histograms\n",
+ "proc_folder = \"/gpfs/exfel/exp/HED/202431/p900478/proc\" # Path to corrected image data used to create histograms\n",
"\n",
- "run = 449 # of the run of image data used to create histograms\n",
+ "run = 280 # of the run of image data used to create histograms\n",
"\n",
- "karabo_id = \"MID_DET_AGIPD1M-1\" # karabo karabo_id\n",
+ "karabo_id = \"HED_DET_AGIPD65K1\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
"ctrl_source_template = '{}/MDL/FPGA_COMP' # path to control information\n",
- "karabo_id_control = \"MID_IRU_AGIPD1M1\" # karabo-id for control device\n",
+ "karabo_id_control = \"HED_TST_AGIPDHZ\" # karabo-id for control device\n",
"karabo_da_control = 'AGIPD1MCTRL00' # karabo DA for control infromation\n",
"\n",
"use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
@@ -37,21 +37,21 @@
"db_output = False # output constants to database\n",
"\n",
"# Fit parameters\n",
- "peak_range = [-30, 30, 35, 70, 95, 135, 145, 220] # where to look for the peaks, [a0, b0, a1, b1, ...] exactly 8 elements\n",
- "peak_width_range = [0, 30, 0, 35, 0, 40, 0, 45] # fit limits on the peak widths, [a0, b0, a1, b1, ...] exactly 8 elements\n",
+ "peak_range = [-30, 20, 90, 130, 150, 180, 180, 250] # where to look for the peaks, [a0, b0, a1, b1, ...] exactly 8 elements\n",
+ "peak_width_range = [0, 20, 0, 30, 0, 30, 0, 45] # fit limits on the peak widths, [a0, b0, a1, b1, ...] exactly 8 elements\n",
"peak_norm_range = [0.0, -1, 0, -1, 0, -1, 0, -1] # \n",
"\n",
"# Bad-pixel thresholds (gain evaluation error). Contribute to BadPixel bit \"Gain_Evaluation_Error\"\n",
"peak_lim = [-30, 30] # Limit of position of noise peak\n",
- "d0_lim = [10, 80] # hard limits for distance between noise and first peak\n",
+ "d0_lim = [70, 130] # hard limits for distance between noise and first peak\n",
"peak_width_lim = [0.9, 1.55, 0.95, 1.65] # hard limits on the peak widths for first and second peak, in units of the noise peak. 4 parameters.\n",
- "chi2_lim = [0, 3.0] # Hard limit on chi2/nDOF value\n",
+ "chi2_lim = [0, 4.0] # Hard limit on chi2/nDOF value\n",
"\n",
"intensity_lim = 15 # Threshold on standard deviation of a histogram in ADU. Contribute to BadPixel bit \"No_Entry\"\n",
- "gain_lim = [0.8, 1.2] # Threshold on gain in relative number. Contribute to BadPixel bit \"Gain_deviation\"\n",
+ "gain_lim = [0.6, 2.0] # Threshold on gain in relative number. Contribute to BadPixel bit \"Gain_deviation\"\n",
"\n",
"cell_range = [1, 3] # range of cell to be considered, [0,0] for all\n",
- "pixel_range = [0, 0, 32, 32] # range of pixels x1,y1,x2,y2 to consider [0,0,512,128] for all\n",
+ "pixel_range = [0, 0, 512, 128] # range of pixels x1,y1,x2,y2 to consider [0,0,512,128] for all\n",
"max_bins = 0 # Maximum number of bins to consider, 0 for all bins\n",
"batch_size = [1, 8, 8] # batch size: [cell,x,y]\n",
"fit_range = [0, 0] # range of a histogram considered for fitting in ADU. Dynamically evaluated in case [0,0]\n",
@@ -59,6 +59,7 @@
"fix_peaks = False # Fix distance between photon peaks\n",
"do_minos = False # This is additional feature of minuit to evaluate errors. \n",
"sigma_limit = 0. # If >0, repeat fit keeping only bins within mu +- sigma_limit*sigma\n",
+ "log_plot = False\n",
"\n",
"# Detector conditions\n",
"# NOTE: The below parameters are needed for the summary notebook when running through xfel-calibrate.\n",
@@ -257,7 +258,7 @@
"metadata": {},
"outputs": [],
"source": [
- "hist = hist_data['hist'][1,:,1, 1]\n",
+ "hist = hist_data['hist'][1,:,490, 44]\n",
"prev, shapes = get_starting_parameters(x, hist, peak_range, n_peaks=n_peaks_fit)\n",
"\n",
"if fit_range == [0, 0]:\n",
@@ -372,6 +373,9 @@
"ax1.set_xlabel(\"Signal [ADU]\")\n",
"ax1.set_ylabel(\"Counts\")\n",
"ax1.legend(ncol=2)\n",
+ "if log_plot:\n",
+ " ax1.set_yscale('log')\n",
+ " ax2.set_yscale('log')\n",
"\n",
"_ = xana.simplePlot(\n",
" d,\n",
@@ -750,7 +754,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -764,7 +768,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.11"
+ "version": "3.11.9"
}
},
"nbformat": 4,
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
index a09618fed..67a2b7036 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
@@ -15,16 +15,16 @@
"outputs": [],
"source": [
"in_folder = \"\" # in this notebook, in_folder is not used as the data source is in the destination folder\n",
- "out_folder = \"\" # the folder to output to, required\n",
+ "out_folder = \"/gpfs/exfel/exp/HED/202431/p900478/scratch/rovensky/temp/HZ2/\" # the folder to output to, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"hist_file_template = \"hists_m{:02d}_sum.h5\"\n",
- "proc_folder = \"\" # Path to corrected image data used to create histograms and validation plots\n",
- "raw_folder = \"\" # folder of raw data. This is used to save information of source data of generated constants, required\n",
- "run = 53 # runs of image data used to create histograms\n",
+ "proc_folder = \"/gpfs/exfel/exp/HED/202431/p900478/proc\" # Path to corrected image data used to create histograms and validation plots\n",
+ "raw_folder = \"/gpfs/exfel/exp/HED/202431/p900478/raw\" # folder of raw data. This is used to save information of source data of generated constants, required\n",
+ "run = 44 # runs of image data used to create histograms\n",
"\n",
- "karabo_id = \"SPB_DET_AGIPD1M-1\" # karabo karabo_id\n",
+ "karabo_id = \"HED_TST_AGIPDHZ2\" # karabo karabo_id\n",
"ctrl_source_template = '{}/MDL/FPGA_COMP' # path to control information\n",
- "karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for control device\n",
+ "karabo_id_control = \"HED_TST_AGIPDHZ\" # karabo-id for control device\n",
"\n",
"use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
"cal_db_interface = \"tcp://max-exfl-cal001:8015#8045\" # the database interface to use\n",
@@ -33,15 +33,17 @@
"db_output = False # output constants to database\n",
"\n",
"# Fit parameters\n",
- "peak_range = [-30,30,35,65,80,130,145,200] # where to look for the peaks, [a0, b0, a1, b1, ...] exactly 8 elements\n",
+ "peak_range = [-20, 20, 30, 90, 110, 160, 160, 240] # where to look for the peaks, [a0, b0, a1, b1, ...] exactly 8 elements\n",
"\n",
"# Bad-pixel thresholds\n",
- "d0_lim = [10, 70] # hard limits for d0 value (distance between noise and first peak)\n",
- "gain_lim = [0.80, 1.2] # Threshold on gain in relative number. Contribute to BadPixel bit \"Gain_deviation\"\n",
+ "d0_lim = [30, 90] # hard limits for d0 value (distance between noise and first peak)\n",
+ "gain_lim = [0.6, 2.0] # Threshold on gain in relative number. Contribute to BadPixel bit \"Gain_deviation\"\n",
+ "sigma_limit = 1.\n",
"\n",
- "cell_range = [0,2] # range of cell to be considered, [0,0] for all\n",
+ "cell_range = [1,3] # range of cell to be considered, [0,0] for all\n",
"pixel_range = [0,0,512,128] # range of pixels x1,y1,x2,y2 to consider [0,0,512,128] for all\n",
"n_peaks_fit = 4 # Number of gaussian peaks to fit including noise peak\n",
+ "log_plot = False\n",
"\n",
"# Detector conditions\n",
"mem_cells = -1 # number of memory cells used, negative values for auto-detection.\n",
@@ -84,7 +86,7 @@
" get_dir_creation_date,\n",
" get_pdu_from_db,\n",
" get_report,\n",
- " module_index_to_qm,\n",
+ "# module_index_to_qm,\n",
" send_to_db\n",
")\n",
"from cal_tools.plotting import agipd_single_module_geometry\n",
@@ -192,7 +194,7 @@
" \n",
"for f in constants_files:\n",
" mod = int(f.stem.split(\"m\")[-1])\n",
- " qm = module_index_to_qm(mod)\n",
+ "# qm = module_index_to_qm(mod)\n",
" fit_data[mod] = {}\n",
" with h5py.File(f, 'r') as hf:\n",
" shape = hf['data/g0mean'].shape\n",
@@ -334,8 +336,8 @@
"# each constant type has its corresponding condition and data.\n",
"if local_output:\n",
" for mod, pdu in zip(modules, db_modules):\n",
- " qm = module_index_to_qm(mod)\n",
- " file = f\"{out_folder}/slopesff_bpmask_module_{qm}.h5\"\n",
+ "# qm = module_index_to_qm(mod)\n",
+ " file = f\"{out_folder}/slopesff_bpmask_module_{mod}.h5\"\n",
" dic = {\n",
" pdu:{\n",
" 'SlopesFF': {\n",
@@ -574,10 +576,10 @@
" \"\"\"\n",
" starts_at = modules[0] if nmods == 1 else 0\n",
" if tid:\n",
- " tid, data = run_data.select('*/DET/*', source).train_from_id(tid)\n",
+ " tid, data = run_data.select(f'{karabo_id}/DET/*', source).train_from_id(tid)\n",
" return tid, stack_detector_data(data, source, modules=nmods, starts_at=starts_at)\n",
" else:\n",
- " for tid, data in run_data.select('*/DET/*', source).trains(require_all=True):\n",
+ " for tid, data in run_data.select(f'{karabo_id}/DET/*', source).trains(require_all=True):\n",
" return tid, stack_detector_data(data, source, modules=nmods, starts_at=starts_at)\n",
" return None, None\n",
" \n",
@@ -602,9 +604,10 @@
"# this is needed if LitFrame is enabled in DAQ to avoid shape mismatch \n",
"# and correction of the right cells\n",
"if np.diff(cell_range)[0] == mem_cells:\n",
+ " starts_at = modules[0] if nmods == 1 else 0\n",
" sel = run_data.select(f'{karabo_id}/DET/{modules[0]}CH0:xtdf', 'image.cellId')\n",
" _, cell = sel.train_from_index(0)\n",
- " stacked_cells = stack_detector_data(cell, 'image.cellId')[:, 0]\n",
+ " stacked_cells = stack_detector_data(cell, 'image.cellId', starts_at=starts_at)[:, 0]\n",
" \n",
" rel_corr = rel_corr[stacked_cells[0]:stacked_cells[-1]+1]\n",
"\n",
@@ -629,7 +632,7 @@
"fig = plt.figure(figsize=(20,20))\n",
"ax = fig.add_subplot(111)\n",
"odata = np.nanmean(orig, axis=0)\n",
- "vmin, vmax = get_range(odata, 5)\n",
+ "vmin, vmax = get_range(odata[np.max(const_data['mask'], axis=1)==0.], 5)\n",
"ax = geom.plot_data_fast(odata, ax=ax, cmap=\"jet\", vmin=vmin, vmax=vmax, figsize=(20,20))\n",
"_ = ax.set_title(\"Original data, mean across one train\")"
]
@@ -654,6 +657,21 @@
"_ = ax.set_title(\"Corrected data, mean across one train\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "corrected[const_data['mask'].swapaxes(0,1)>0] = np.nan\n",
+ "fig = plt.figure(figsize=(20,20))\n",
+ "ax = fig.add_subplot(111)\n",
+ "cdata = np.nanmean(corrected, axis=0)\n",
+ "vmin, vmax = get_range(cdata, 5)\n",
+ "ax = geom.plot_data_fast(cdata, ax=ax, cmap=\"jet\", vmin=vmin, vmax=vmax, figsize=(20,20))\n",
+ "_ = ax.set_title(\"Corrected data (BP excluded), mean across one train\")\n"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -679,7 +697,7 @@
" olap[i] = np.abs(laplace(odata[i].astype(float)/omax))\n",
"\n",
"fig = plt.figure(figsize=(20,10))\n",
- "vmin, vmax = get_range(olap, 2)\n",
+ "vmin, vmax = get_range(clap, 2)\n",
"\n",
"ax = fig.add_subplot(121)\n",
"ax = geom.plot_data_fast(olap, ax=ax, cmap=\"jet\", vmin=vmin, vmax=vmax, )\n",
@@ -719,27 +737,29 @@
"\n",
"# get the bin centers from the bin edges \n",
"xc=xe[:-1]+(xe[1]-xe[0])/2\n",
- "pars, _ = get_starting_parameters(xc, y, peak_range,4)\n",
- "minuit = fit_n_peaks(xc, y, pars, x_range,fix_d01=False,sigma_limit=1)\n",
+ "pars, _ = get_starting_parameters(xc, y, peak_range,n_peaks_fit)\n",
+ "minuit = fit_n_peaks(xc, y, pars, x_range,n_peaks=n_peaks_fit,fix_d01=False,sigma_limit=1)\n",
"\n",
"pc = minuit.args\n",
"resc=minuit.fitarg\n",
"\n",
- "yfc = gaussian_sum(xc,4, *pc)\n",
+ "yfc = gaussian_sum(xc,n_peaks_fit, *pc)\n",
"plt.plot(xc, yfc, label='corrected fit')\n",
"\n",
"y,_, _ = ax.hist(orig[sel].flatten(), bins=nb, range=x_range, label='original',alpha=0.5)\n",
- "pars, _ = get_starting_parameters(xc, y, peak_range,4)\n",
- "minuit = fit_n_peaks(xc, y, pars, x_range,fix_d01=False,sigma_limit=1)\n",
+ "pars, _ = get_starting_parameters(xc, y, peak_range,n_peaks_fit)\n",
+ "minuit = fit_n_peaks(xc, y, pars, x_range,n_peaks=n_peaks_fit,fix_d01=False,sigma_limit=1)\n",
"\n",
"po = minuit.args\n",
"reso=minuit.fitarg\n",
"\n",
- "yfo = gaussian_sum(xc,4, *po)\n",
+ "yfo = gaussian_sum(xc,n_peaks_fit, *po)\n",
"plt.plot(xc, yfo, label='original fit')\n",
"\n",
"plt.title(f\"Signal spectrum, first train\")\n",
"plt.xlabel('[ADU]')\n",
+ "if log_plot:\n",
+ " plt.yscale('log')\n",
"plt.legend()\n",
"plt.show()"
]
--
GitLab
From 53a139542309607200c0c8bfe0b7940a46e77698 Mon Sep 17 00:00:00 2001
From: Vratko Rovensky <vratko.rovensky@xfel.eu>
Date: Thu, 21 Nov 2024 16:59:44 +0100
Subject: [PATCH 02/10] Improving plotting
---
...aracterize_AGIPD_Gain_FlatFields_NBC.ipynb | 28 ++++----
...terize_AGIPD_Gain_FlatFields_Summary.ipynb | 70 +++++++++++--------
2 files changed, 53 insertions(+), 45 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
index 26b0bc44e..c205bac25 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
@@ -50,25 +50,25 @@
"intensity_lim = 15 # Threshold on standard deviation of a histogram in ADU. Contribute to BadPixel bit \"No_Entry\"\n",
"gain_lim = [0.6, 2.0] # Threshold on gain in relative number. Contribute to BadPixel bit \"Gain_deviation\"\n",
"\n",
- "cell_range = [1, 3] # range of cell to be considered, [0,0] for all\n",
+ "cell_range = [261, 263] # range of cell to be considered, [0,0] for all\n",
"pixel_range = [0, 0, 512, 128] # range of pixels x1,y1,x2,y2 to consider [0,0,512,128] for all\n",
"max_bins = 0 # Maximum number of bins to consider, 0 for all bins\n",
"batch_size = [1, 8, 8] # batch size: [cell,x,y]\n",
- "fit_range = [0, 0] # range of a histogram considered for fitting in ADU. Dynamically evaluated in case [0,0]\n",
- "n_peaks_fit = 4 # Number of gaussian peaks to fit including noise peak\n",
+ "fit_range = [-30, 150] # range of a histogram considered for fitting in ADU. Dynamically evaluated in case [0,0]\n",
+ "n_peaks_fit = 2 # Number of gaussian peaks to fit including noise peak\n",
"fix_peaks = False # Fix distance between photon peaks\n",
"do_minos = False # This is additional feature of minuit to evaluate errors. \n",
- "sigma_limit = 0. # If >0, repeat fit keeping only bins within mu +- sigma_limit*sigma\n",
- "log_plot = False\n",
+ "sigma_limit = 1.5 # If >0, repeat fit keeping only bins within mu +- sigma_limit*sigma\n",
+ "log_plot = True\n",
"\n",
"# Detector conditions\n",
"# NOTE: The below parameters are needed for the summary notebook when running through xfel-calibrate.\n",
- "mem_cells = -1 # number of memory cells used, negative values for auto-detection. \n",
- "bias_voltage = 300 # Bias voltage. \n",
- "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine.\n",
- "gain_setting = -1 # the gain setting, negative values for auto-detection.\n",
- "photon_energy = 8.05 # photon energy in keV.\n",
- "integration_time = -1 # integration time, negative values for auto-detection."
+ "mem_cells = 2 # number of memory cells used, negative values for auto-detection. \n",
+ "bias_voltage = 200 # Bias voltage. \n",
+ "acq_rate = 2.2 # the detector acquisition rate, use 0 to try to auto-determine.\n",
+ "gain_setting = 0 # the gain setting, negative values for auto-detection.\n",
+ "photon_energy = 24 # photon energy in keV.\n",
+ "integration_time = 20 # integration time, negative values for auto-detection."
]
},
{
@@ -258,7 +258,7 @@
"metadata": {},
"outputs": [],
"source": [
- "hist = hist_data['hist'][1,:,490, 44]\n",
+ "hist = hist_data['hist'][1,:,470, 44]\n",
"prev, shapes = get_starting_parameters(x, hist, peak_range, n_peaks=n_peaks_fit)\n",
"\n",
"if fit_range == [0, 0]:\n",
@@ -374,8 +374,8 @@
"ax1.set_ylabel(\"Counts\")\n",
"ax1.legend(ncol=2)\n",
"if log_plot:\n",
- " ax1.set_yscale('log')\n",
- " ax2.set_yscale('log')\n",
+ " ax1.set_yscale('symlog')\n",
+ " ax2.set_yscale('symlog')\n",
"\n",
"_ = xana.simplePlot(\n",
" d,\n",
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
index 67a2b7036..fe278a7d4 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
@@ -15,14 +15,14 @@
"outputs": [],
"source": [
"in_folder = \"\" # in this notebook, in_folder is not used as the data source is in the destination folder\n",
- "out_folder = \"/gpfs/exfel/exp/HED/202431/p900478/scratch/rovensky/temp/HZ2/\" # the folder to output to, required\n",
+ "out_folder = \"/gpfs/exfel/exp/HED/202431/p900478/scratch/rovensky/temp/HZ1/\" # the folder to output to, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"hist_file_template = \"hists_m{:02d}_sum.h5\"\n",
"proc_folder = \"/gpfs/exfel/exp/HED/202431/p900478/proc\" # Path to corrected image data used to create histograms and validation plots\n",
"raw_folder = \"/gpfs/exfel/exp/HED/202431/p900478/raw\" # folder of raw data. This is used to save information of source data of generated constants, required\n",
- "run = 44 # runs of image data used to create histograms\n",
+ "run = 280 # runs of image data used to create histograms\n",
"\n",
- "karabo_id = \"HED_TST_AGIPDHZ2\" # karabo karabo_id\n",
+ "karabo_id = \"HED_TST_AGIPDHZ1\" # karabo karabo_id\n",
"ctrl_source_template = '{}/MDL/FPGA_COMP' # path to control information\n",
"karabo_id_control = \"HED_TST_AGIPDHZ\" # karabo-id for control device\n",
"\n",
@@ -33,25 +33,25 @@
"db_output = False # output constants to database\n",
"\n",
"# Fit parameters\n",
- "peak_range = [-20, 20, 30, 90, 110, 160, 160, 240] # where to look for the peaks, [a0, b0, a1, b1, ...] exactly 8 elements\n",
+ "peak_range = [-30, 20, 90, 130, 150, 180, 180, 250] # where to look for the peaks, [a0, b0, a1, b1, ...] exactly 8 elements\n",
"\n",
"# Bad-pixel thresholds\n",
- "d0_lim = [30, 90] # hard limits for d0 value (distance between noise and first peak)\n",
- "gain_lim = [0.6, 2.0] # Threshold on gain in relative number. Contribute to BadPixel bit \"Gain_deviation\"\n",
- "sigma_limit = 1.\n",
+ "d0_lim = [70, 150] # hard limits for d0 value (distance between noise and first peak)\n",
+ "gain_lim = [0.6, 1.5] # Threshold on gain in relative number. Contribute to BadPixel bit \"Gain_deviation\"\n",
+ "sigma_limit = 1.5\n",
"\n",
- "cell_range = [1,3] # range of cell to be considered, [0,0] for all\n",
+ "cell_range = [71,73] # range of cell to be considered, [0,0] for all\n",
"pixel_range = [0,0,512,128] # range of pixels x1,y1,x2,y2 to consider [0,0,512,128] for all\n",
- "n_peaks_fit = 4 # Number of gaussian peaks to fit including noise peak\n",
- "log_plot = False\n",
+ "n_peaks_fit = 2 # Number of gaussian peaks to fit including noise peak\n",
+ "log_plot = True\n",
"\n",
"# Detector conditions\n",
- "mem_cells = -1 # number of memory cells used, negative values for auto-detection.\n",
- "bias_voltage = 0. # Bias voltage\n",
- "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
- "gain_setting = -1 # the gain setting, negative values for auto-detection.\n",
- "photon_energy = 8.05 # photon energy in keV\n",
- "integration_time = -1 # integration time, negative values for auto-detection."
+ "mem_cells = 2 # number of memory cells used, negative values for auto-detection.\n",
+ "bias_voltage = 200 # Bias voltage\n",
+ "acq_rate = 2.2 # the detector acquisition rate, use 0 to try to auto-determine\n",
+ "gain_setting = 0 # the gain setting, negative values for auto-detection.\n",
+ "photon_energy = 24 # photon energy in keV\n",
+ "integration_time = 20 # integration time, negative values for auto-detection."
]
},
{
@@ -414,7 +414,7 @@
"outputs": [],
"source": [
"for key in const_data.keys():\n",
- " fig = plt.figure(figsize=(20,20))\n",
+ " fig = plt.figure(figsize=(15,7))\n",
" ax = fig.add_subplot(111)\n",
" if key=='mask':\n",
" data = np.nanmean(const_data[key]>0, axis=1)\n",
@@ -422,7 +422,7 @@
" else:\n",
" data = np.nanmean(const_data[key], axis=1)\n",
" vmin, vmax = get_range(data, 5)\n",
- " ax = geom.plot_data_fast(data, ax=ax, cmap=\"jet\", vmin=vmin, vmax=vmax, figsize=(20,20))\n",
+ " ax = geom.plot_data_fast(data, ax=ax, cmap=\"jet\", vmin=vmin, vmax=vmax, figsize=(13,7))\n",
" _ = ax.set_title(labels[key])"
]
},
@@ -449,7 +449,7 @@
" 'gain [ADU/keV]':const_data[f\"gain\"],\n",
" 'gain relative to module mean':const_data[f\"gain\"]/module_mean,\n",
" }\n",
- "fig = plt.figure(figsize=(16,5))\n",
+ "fig = plt.figure(figsize=(12,4))\n",
"for i, (par, data) in enumerate(dsets.items()):\n",
" ax = fig.add_subplot(1, 3, i+1)\n",
" plt_range= np.nanmin(data), np.nanmax(data)\n",
@@ -486,7 +486,9 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "scrolled": false
+ },
"outputs": [],
"source": [
"for key in const_data.keys():\n",
@@ -506,7 +508,7 @@
" 'linestyle': '--' if i>7 else '-'\n",
" })\n",
"\n",
- " fig = plt.figure(figsize=(15, 6))\n",
+ " fig = plt.figure(figsize=(12, 6))\n",
" ax = fig.add_subplot(111)\n",
"\n",
" _ = simplePlot(d, xrange=(-12, 510),\n",
@@ -629,11 +631,11 @@
},
"outputs": [],
"source": [
- "fig = plt.figure(figsize=(20,20))\n",
+ "fig = plt.figure(figsize=(15,7))\n",
"ax = fig.add_subplot(111)\n",
"odata = np.nanmean(orig, axis=0)\n",
"vmin, vmax = get_range(odata[np.max(const_data['mask'], axis=1)==0.], 5)\n",
- "ax = geom.plot_data_fast(odata, ax=ax, cmap=\"jet\", vmin=vmin, vmax=vmax, figsize=(20,20))\n",
+ "ax = geom.plot_data_fast(odata, ax=ax, cmap=\"jet\", vmin=vmin, vmax=vmax, figsize=(13,7))\n",
"_ = ax.set_title(\"Original data, mean across one train\")"
]
},
@@ -650,10 +652,10 @@
"metadata": {},
"outputs": [],
"source": [
- "fig = plt.figure(figsize=(20,20))\n",
+ "fig = plt.figure(figsize=(15,7))\n",
"ax = fig.add_subplot(111)\n",
"cdata = np.nanmean(corrected, axis=0)\n",
- "ax = geom.plot_data_fast(cdata, ax=ax, cmap=\"jet\", vmin=vmin, vmax=vmax, figsize=(20,20))\n",
+ "ax = geom.plot_data_fast(cdata, ax=ax, cmap=\"jet\", vmin=vmin, vmax=vmax, figsize=(13,7))\n",
"_ = ax.set_title(\"Corrected data, mean across one train\")"
]
},
@@ -664,11 +666,11 @@
"outputs": [],
"source": [
"corrected[const_data['mask'].swapaxes(0,1)>0] = np.nan\n",
- "fig = plt.figure(figsize=(20,20))\n",
+ "fig = plt.figure(figsize=(15,7))\n",
"ax = fig.add_subplot(111)\n",
"cdata = np.nanmean(corrected, axis=0)\n",
"vmin, vmax = get_range(cdata, 5)\n",
- "ax = geom.plot_data_fast(cdata, ax=ax, cmap=\"jet\", vmin=vmin, vmax=vmax, figsize=(20,20))\n",
+ "ax = geom.plot_data_fast(cdata, ax=ax, cmap=\"jet\", vmin=vmin, vmax=vmax, figsize=(13,7))\n",
"_ = ax.set_title(\"Corrected data (BP excluded), mean across one train\")\n"
]
},
@@ -696,15 +698,21 @@
" clap[i] = np.abs(laplace(cdata[i].astype(float)/cmax))\n",
" olap[i] = np.abs(laplace(odata[i].astype(float)/omax))\n",
"\n",
- "fig = plt.figure(figsize=(20,10))\n",
+ "fig = plt.figure(figsize=(15,10))\n",
+ "ax_cbar = fig.add_axes([0.15, 0.08, 0.7, 0.04])\n",
+ "\n",
"vmin, vmax = get_range(clap, 2)\n",
"\n",
"ax = fig.add_subplot(121)\n",
- "ax = geom.plot_data_fast(olap, ax=ax, cmap=\"jet\", vmin=vmin, vmax=vmax, )\n",
+ "ax = geom.plot_data_fast(olap, ax=ax, cmap=\"jet\", vmin=vmin, vmax=vmax, \n",
+ " colorbar={'cax': ax_cbar,\n",
+ " 'shrink': 1,\n",
+ " 'pad': 0.1,\n",
+ " 'orientation': 'horizontal'})\n",
"_ = ax.set_title(\"Laplace (original data)\")\n",
"\n",
"ax = fig.add_subplot(122)\n",
- "ax = geom.plot_data_fast(clap, ax=ax, cmap=\"jet\", vmin=vmin, vmax=vmax, )\n",
+ "ax = geom.plot_data_fast(clap, ax=ax, cmap=\"jet\", vmin=vmin, vmax=vmax, colorbar=False)\n",
"_ = ax.set_title(\"Laplace (gain corrected data)\")"
]
},
@@ -759,7 +767,7 @@
"plt.title(f\"Signal spectrum, first train\")\n",
"plt.xlabel('[ADU]')\n",
"if log_plot:\n",
- " plt.yscale('log')\n",
+ " plt.yscale('symlog')\n",
"plt.legend()\n",
"plt.show()"
]
--
GitLab
From eb56bf6f62c35550a13b48833dcbec24945691a3 Mon Sep 17 00:00:00 2001
From: Vratko Rovensky <vratko.rovensky@xfel.eu>
Date: Wed, 4 Dec 2024 09:36:39 +0100
Subject: [PATCH 03/10] PC adaptations for HZ
---
.../AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb | 36 ++++++++++---------
1 file changed, 19 insertions(+), 17 deletions(-)
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
index d8bd254d8..a5fc853ba 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
@@ -551,7 +551,7 @@
"outputs": [],
"source": [
"test_pixels = []\n",
- "tpix_range1 = [(0,16), (0,64)]\n",
+ "tpix_range1 = [(450,466), (0,64)]\n",
"for i in range(*tpix_range1[0]):\n",
" for j in range(*tpix_range1[1]):\n",
" test_pixels.append((j,i))\n",
@@ -600,10 +600,11 @@
" ymin = y.min()\n",
" \n",
" #### Transition region gain is set based on dark thresholds. ####\n",
- " msk = yd[labels[1]] < thresh_trans\n",
- " label_filter = np.where(labels[1])[0]\n",
- " labels[1][label_filter[msk]] = False\n",
- " labels[0][label_filter[msk]] = True\n",
+ " if 'HZ' not in karabo_id: # gain discrimination for HZ modules is not reliable\n",
+ " msk = yd[labels[1]] < thresh_trans\n",
+ " label_filter = np.where(labels[1])[0]\n",
+ " labels[1][label_filter[msk]] = False\n",
+ " labels[0][label_filter[msk]] = True\n",
" ################################################################\n",
" \n",
" for i, lbl in enumerate(labels):\n",
@@ -671,16 +672,17 @@
" \n",
" \n",
" #### Transition region gain is set based on dark thresholds. ####\n",
- " msk = yd[labels[1]] < thresh_trans\n",
- " label_filter = np.where(labels[1])[0]\n",
- " labels[1][label_filter[msk]] = False\n",
- " labels[0][label_filter[msk]] = True\n",
- " try:\n",
- " if np.max(x[labels[0]] > 220): # step value above which dark threholds are not considered \n",
- " labels[0][label_filter[msk]] = False\n",
- " except ValueError:\n",
- " print('Could not determine maximum value.')\n",
- " pass\n",
+ " if 'HZ' not in karabo_id: # gain discrimination for HZ modules is not reliable\n",
+ " msk = yd[labels[1]] < thresh_trans\n",
+ " label_filter = np.where(labels[1])[0]\n",
+ " labels[1][label_filter[msk]] = False\n",
+ " labels[0][label_filter[msk]] = True\n",
+ " try:\n",
+ " if np.max(x[labels[0]] > 220): # step value above which dark threholds are not considered \n",
+ " labels[0][label_filter[msk]] = False\n",
+ " except ValueError:\n",
+ " print('Could not determine maximum value.')\n",
+ " pass\n",
" ############################################################### \n",
" else:\n",
" threshold = 0\n",
@@ -903,7 +905,7 @@
"metadata": {},
"outputs": [],
"source": [
- "tpix_range1 = [(250,254), (60,63)]\n",
+ "tpix_range1 = [(450,454), (60,63)]\n",
"test_cells = [4, 38]\n",
"roi = 'ROI1'\n",
"evaluate_fitting_ROI(modules, res, tpix_range1, test_cells, roi)"
@@ -922,7 +924,7 @@
"metadata": {},
"outputs": [],
"source": [
- "tpix_range2 = [(96,128), (32,64)]\n",
+ "tpix_range2 = [(480,500), (32,64)]\n",
"roi2 = 'ROI2'\n",
"evaluate_fitting_ROI(modules, res, tpix_range2, test_cells, roi2)"
]
--
GitLab
From ce63dd48a2cd2d5fbc4ecbe011d9a7d6ea0aa9f8 Mon Sep 17 00:00:00 2001
From: Vratko Rovensky <vratko.rovensky@xfel.eu>
Date: Thu, 9 Jan 2025 16:58:05 +0100
Subject: [PATCH 04/10] Cleaning up
---
notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb | 14 ++---
...aracterize_AGIPD_Gain_FlatFields_NBC.ipynb | 50 ++++++++---------
...terize_AGIPD_Gain_FlatFields_Summary.ipynb | 56 +++++++++++--------
.../AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb | 51 ++++++++++++++---
.../Chracterize_AGIPD_Gain_PC_Summary.ipynb | 9 +--
5 files changed, 113 insertions(+), 67 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb b/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb
index 91d470d01..05b4971ac 100644
--- a/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb
+++ b/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb
@@ -17,15 +17,15 @@
"metadata": {},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/HED/202431/p900478/proc/\" # the folder to read data from, required\n",
- "out_folder = \"/gpfs/exfel/exp/HED/202431/p900478/scratch/\" # the folder to output to, required\n",
+ "in_folder = \"/gpfs/exfel/exp/MID/202030/p900137/proc/\" # the folder to read data from, required\n",
+ "out_folder = \"/gpfs/exfel/exp/MID/202030/p900137/scratch/karnem/r0319_0322_0342_v50\" # the folder to output to, required\n",
"sequences = [-1] # module to consider, set to -1 for all, range allowed\n",
- "modules = [81] # module to consider, range allowed\n",
- "runs = [44] # list of run numbers, required\n",
- "cells_list = ['range(0,3)'] # list of lists or any python expressions, which can be converted to a list. E.g. 'range(0,15,5)' 'list(range(0,5))+list(range(50,60))'\n",
+ "modules = [5] # module to consider, range allowed\n",
+ "runs = [319] # list of run numbers, required\n",
+ "cells_list = ['range(0,15)'] # list of lists or any python expressions, which can be converted to a list. E.g. 'range(0,15,5)' 'list(range(0,5))+list(range(50,60))'\n",
"roi = [0, 512, 0, 128] # ROI considered for the analysis of data quality\n",
"\n",
- "karabo_id = \"HED_TST_AGIPDHZ1\" # karabo karabo_id\n",
+ "karabo_id = \"MID_DET_AGIPD1M-1\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
"receiver_id = \"{}CH0\" # inset for receiver devices\n",
"path_template = 'CORR-R{:04d}-AGIPD{:02d}-S{}.h5' # the template to use to access data\n",
@@ -33,7 +33,7 @@
"\n",
"n_bins_adu = 1 # number of bins per ADU\n",
"h_range = [-50, 450] # range of the histogram in ADU\n",
- "n_cells = 352 # total number of memory cells (used to create summary file)\n",
+ "n_cells = 202 # total number of memory cells (used to create summary file)\n",
"\n",
"hist_std_tresh = 10 # Threshold for histogram standard deviation\n",
"hist_mean_tresh = 15 # Threshold for histogram mean\n",
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
index c205bac25..8c343d87d 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
@@ -14,20 +14,20 @@
"metadata": {},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/HED/202431/p900478/scratch/HISTS/air/352cells2.2MHz_gs0_20clk_fixedHG/HED_TST_AGIPDHZ1/\" # the folder to read histograms from, required\n",
+ "in_folder = \"/gpfs/exfel/exp/SPB/202030/p900138/scratch/karnem/r0203_r0204_v01/\" # the folder to read histograms from, required\n",
"out_folder = \"\" # the folder to output to, required\n",
"hist_file_template = \"hists_m{:02d}_sum.h5\" # the template to use to access histograms\n",
- "modules = [81] # modules to correct, set to -1 for all, range allowed\n",
+ "modules = [10] # modules to correct, set to -1 for all, range allowed\n",
"\n",
- "raw_folder = \"/gpfs/exfel/exp/HED/202431/p900478/raw\" # Path to raw image data used to create histograms\n",
- "proc_folder = \"/gpfs/exfel/exp/HED/202431/p900478/proc\" # Path to corrected image data used to create histograms\n",
+ "raw_folder = \"/gpfs/exfel/exp/MID/202030/p900137/raw\" # Path to raw image data used to create histograms\n",
+ "proc_folder = \"\" # Path to corrected image data used to create histograms\n",
"\n",
- "run = 280 # of the run of image data used to create histograms\n",
+ "run = 449 # of the run of image data used to create histograms\n",
"\n",
- "karabo_id = \"HED_DET_AGIPD65K1\" # karabo karabo_id\n",
+ "karabo_id = \"MID_DET_AGIPD1M-1\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
"ctrl_source_template = '{}/MDL/FPGA_COMP' # path to control information\n",
- "karabo_id_control = \"HED_TST_AGIPDHZ\" # karabo-id for control device\n",
+ "karabo_id_control = \"MID_IRU_AGIPD1M1\" # karabo-id for control device\n",
"karabo_da_control = 'AGIPD1MCTRL00' # karabo DA for control infromation\n",
"\n",
"use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
@@ -37,38 +37,38 @@
"db_output = False # output constants to database\n",
"\n",
"# Fit parameters\n",
- "peak_range = [-30, 20, 90, 130, 150, 180, 180, 250] # where to look for the peaks, [a0, b0, a1, b1, ...] exactly 8 elements\n",
- "peak_width_range = [0, 20, 0, 30, 0, 30, 0, 45] # fit limits on the peak widths, [a0, b0, a1, b1, ...] exactly 8 elements\n",
+ "peak_range = [-30, 30, 35, 70, 95, 135, 145, 220] # where to look for the peaks, [a0, b0, a1, b1, ...] exactly 8 elements\n",
+ "peak_width_range = [0, 30, 0, 35, 0, 40, 0, 45] # fit limits on the peak widths, [a0, b0, a1, b1, ...] exactly 8 elements\n",
"peak_norm_range = [0.0, -1, 0, -1, 0, -1, 0, -1] # \n",
"\n",
"# Bad-pixel thresholds (gain evaluation error). Contribute to BadPixel bit \"Gain_Evaluation_Error\"\n",
"peak_lim = [-30, 30] # Limit of position of noise peak\n",
- "d0_lim = [70, 130] # hard limits for distance between noise and first peak\n",
+ "d0_lim = [10, 80] # hard limits for distance between noise and first peak\n",
"peak_width_lim = [0.9, 1.55, 0.95, 1.65] # hard limits on the peak widths for first and second peak, in units of the noise peak. 4 parameters.\n",
- "chi2_lim = [0, 4.0] # Hard limit on chi2/nDOF value\n",
+ "chi2_lim = [0, 3.0] # Hard limit on chi2/nDOF value\n",
"\n",
"intensity_lim = 15 # Threshold on standard deviation of a histogram in ADU. Contribute to BadPixel bit \"No_Entry\"\n",
- "gain_lim = [0.6, 2.0] # Threshold on gain in relative number. Contribute to BadPixel bit \"Gain_deviation\"\n",
+ "gain_lim = [0.8, 1.2] # Threshold on gain in relative number. Contribute to BadPixel bit \"Gain_deviation\"\n",
"\n",
- "cell_range = [261, 263] # range of cell to be considered, [0,0] for all\n",
- "pixel_range = [0, 0, 512, 128] # range of pixels x1,y1,x2,y2 to consider [0,0,512,128] for all\n",
+ "cell_range = [1, 3] # range of cell to be considered, [0,0] for all\n",
+ "pixel_range = [0, 0, 32, 32] # range of pixels x1,y1,x2,y2 to consider [0,0,512,128] for all\n",
"max_bins = 0 # Maximum number of bins to consider, 0 for all bins\n",
"batch_size = [1, 8, 8] # batch size: [cell,x,y]\n",
- "fit_range = [-30, 150] # range of a histogram considered for fitting in ADU. Dynamically evaluated in case [0,0]\n",
- "n_peaks_fit = 2 # Number of gaussian peaks to fit including noise peak\n",
+ "fit_range = [0, 0] # range of a histogram considered for fitting in ADU. Dynamically evaluated in case [0,0]\n",
+ "n_peaks_fit = 4 # Number of gaussian peaks to fit including noise peak\n",
"fix_peaks = False # Fix distance between photon peaks\n",
"do_minos = False # This is additional feature of minuit to evaluate errors. \n",
- "sigma_limit = 1.5 # If >0, repeat fit keeping only bins within mu +- sigma_limit*sigma\n",
- "log_plot = True\n",
+ "sigma_limit = 0. # If >0, repeat fit keeping only bins within mu +- sigma_limit*sigma\n",
+ "log_plot = False\n",
"\n",
"# Detector conditions\n",
"# NOTE: The below parameters are needed for the summary notebook when running through xfel-calibrate.\n",
- "mem_cells = 2 # number of memory cells used, negative values for auto-detection. \n",
- "bias_voltage = 200 # Bias voltage. \n",
- "acq_rate = 2.2 # the detector acquisition rate, use 0 to try to auto-determine.\n",
- "gain_setting = 0 # the gain setting, negative values for auto-detection.\n",
- "photon_energy = 24 # photon energy in keV.\n",
- "integration_time = 20 # integration time, negative values for auto-detection."
+ "mem_cells = -1 # number of memory cells used, negative values for auto-detection. \n",
+ "bias_voltage = 300 # Bias voltage. \n",
+ "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine.\n",
+ "gain_setting = -1 # the gain setting, negative values for auto-detection.\n",
+ "photon_energy = 8.05 # photon energy in keV.\n",
+ "integration_time = -1 # integration time, negative values for auto-detection."
]
},
{
@@ -258,7 +258,7 @@
"metadata": {},
"outputs": [],
"source": [
- "hist = hist_data['hist'][1,:,470, 44]\n",
+ "hist = hist_data['hist'][1,:, 470, 44]\n",
"prev, shapes = get_starting_parameters(x, hist, peak_range, n_peaks=n_peaks_fit)\n",
"\n",
"if fit_range == [0, 0]:\n",
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
index fe278a7d4..26bdb158a 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
@@ -15,16 +15,16 @@
"outputs": [],
"source": [
"in_folder = \"\" # in this notebook, in_folder is not used as the data source is in the destination folder\n",
- "out_folder = \"/gpfs/exfel/exp/HED/202431/p900478/scratch/rovensky/temp/HZ1/\" # the folder to output to, required\n",
+ "out_folder = \"\" # the folder to output to, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"hist_file_template = \"hists_m{:02d}_sum.h5\"\n",
- "proc_folder = \"/gpfs/exfel/exp/HED/202431/p900478/proc\" # Path to corrected image data used to create histograms and validation plots\n",
- "raw_folder = \"/gpfs/exfel/exp/HED/202431/p900478/raw\" # folder of raw data. This is used to save information of source data of generated constants, required\n",
- "run = 280 # runs of image data used to create histograms\n",
+ "proc_folder = \"\" # Path to corrected image data used to create histograms and validation plots\n",
+ "raw_folder = \"\" # folder of raw data. This is used to save information of source data of generated constants, required\n",
+ "run = 53 # runs of image data used to create histograms\n",
"\n",
- "karabo_id = \"HED_TST_AGIPDHZ1\" # karabo karabo_id\n",
+ "karabo_id = \"SPB_DET_AGIPD1M-1\" # karabo karabo_id\n",
"ctrl_source_template = '{}/MDL/FPGA_COMP' # path to control information\n",
- "karabo_id_control = \"HED_TST_AGIPDHZ\" # karabo-id for control device\n",
+ "karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for control device\n",
"\n",
"use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
"cal_db_interface = \"tcp://max-exfl-cal001:8015#8045\" # the database interface to use\n",
@@ -33,25 +33,27 @@
"db_output = False # output constants to database\n",
"\n",
"# Fit parameters\n",
- "peak_range = [-30, 20, 90, 130, 150, 180, 180, 250] # where to look for the peaks, [a0, b0, a1, b1, ...] exactly 8 elements\n",
+ "peak_range = [-30, 30, 35, 65, 80, 130, 145, 200] # where to look for the peaks, [a0, b0, a1, b1, ...] exactly 8 elements\n",
+ "peak_width_range = [0, 30, 0, 35, 0, 40, 0, 45] # fit limits on the peak widths, [a0, b0, a1, b1, ...] exactly 8 elements\n",
+ "peak_norm_range = [0.0, -1, 0, -1, 0, -1, 0, -1] # \n",
"\n",
"# Bad-pixel thresholds\n",
- "d0_lim = [70, 150] # hard limits for d0 value (distance between noise and first peak)\n",
- "gain_lim = [0.6, 1.5] # Threshold on gain in relative number. Contribute to BadPixel bit \"Gain_deviation\"\n",
+ "d0_lim = [10, 70] # hard limits for d0 value (distance between noise and first peak)\n",
+ "gain_lim = [0.8, 1.2] # Threshold on gain in relative number. Contribute to BadPixel bit \"Gain_deviation\"\n",
"sigma_limit = 1.5\n",
"\n",
- "cell_range = [71,73] # range of cell to be considered, [0,0] for all\n",
+ "cell_range = [0,2] # range of cell to be considered, [0,0] for all\n",
"pixel_range = [0,0,512,128] # range of pixels x1,y1,x2,y2 to consider [0,0,512,128] for all\n",
- "n_peaks_fit = 2 # Number of gaussian peaks to fit including noise peak\n",
- "log_plot = True\n",
+ "n_peaks_fit = 4 # Number of gaussian peaks to fit including noise peak\n",
+ "log_plot = False\n",
"\n",
"# Detector conditions\n",
- "mem_cells = 2 # number of memory cells used, negative values for auto-detection.\n",
- "bias_voltage = 200 # Bias voltage\n",
- "acq_rate = 2.2 # the detector acquisition rate, use 0 to try to auto-determine\n",
- "gain_setting = 0 # the gain setting, negative values for auto-detection.\n",
- "photon_energy = 24 # photon energy in keV\n",
- "integration_time = 20 # integration time, negative values for auto-detection."
+ "mem_cells = -1 # number of memory cells used, negative values for auto-detection.\n",
+ "bias_voltage = 0. # Bias voltage\n",
+ "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
+ "gain_setting = -1 # the gain setting, negative values for auto-detection.\n",
+ "photon_energy = 8.05 # photon energy in keV\n",
+ "integration_time = -1 # integration time, negative values for auto-detection."
]
},
{
@@ -79,6 +81,7 @@
" fit_n_peaks,\n",
" gaussian_sum,\n",
" get_starting_parameters,\n",
+ " set_par_limits\n",
")\n",
"from cal_tools.ana_tools import get_range, save_dict_to_hdf5\n",
"from cal_tools.enums import BadPixels\n",
@@ -107,7 +110,10 @@
"metadata": {},
"outputs": [],
"source": [
- "peak_range = np.reshape(peak_range,(4,2))"
+ "peak_range = np.reshape(peak_range,(4,2))\n",
+ "peak_width_range = np.reshape(peak_width_range,(4,2))\n",
+ "peak_norm_range = [None if x == -1 else x for x in peak_norm_range]\n",
+ "peak_norm_range = np.reshape(peak_norm_range,(4,2))"
]
},
{
@@ -649,7 +655,9 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "scrolled": true
+ },
"outputs": [],
"source": [
"fig = plt.figure(figsize=(15,7))\n",
@@ -746,7 +754,9 @@
"# get the bin centers from the bin edges \n",
"xc=xe[:-1]+(xe[1]-xe[0])/2\n",
"pars, _ = get_starting_parameters(xc, y, peak_range,n_peaks_fit)\n",
- "minuit = fit_n_peaks(xc, y, pars, x_range,n_peaks=n_peaks_fit,fix_d01=False,sigma_limit=1)\n",
+ "set_par_limits(pars, peak_range, peak_norm_range,\n",
+ " peak_width_range, n_peaks=n_peaks_fit)\n",
+ "minuit = fit_n_peaks(xc, y, pars, x_range,n_peaks=n_peaks_fit,fix_d01=False,sigma_limit=sigma_limit)\n",
"\n",
"pc = minuit.args\n",
"resc=minuit.fitarg\n",
@@ -756,7 +766,9 @@
"\n",
"y,_, _ = ax.hist(orig[sel].flatten(), bins=nb, range=x_range, label='original',alpha=0.5)\n",
"pars, _ = get_starting_parameters(xc, y, peak_range,n_peaks_fit)\n",
- "minuit = fit_n_peaks(xc, y, pars, x_range,n_peaks=n_peaks_fit,fix_d01=False,sigma_limit=1)\n",
+ "set_par_limits(pars, peak_range, peak_norm_range,\n",
+ " peak_width_range, n_peaks=n_peaks_fit)\n",
+ "minuit = fit_n_peaks(xc, y, pars, x_range,n_peaks=n_peaks_fit,fix_d01=False,sigma_limit=sigma_limit)\n",
"\n",
"po = minuit.args\n",
"reso=minuit.fitarg\n",
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
index a5fc853ba..e0e0226a4 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
@@ -33,7 +33,7 @@
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
"in_folder = '/gpfs/exfel/exp/HED/202430/p900438/raw/' # path to input data, required\n",
- "out_folder = \"/gpfs/exfel/exp/MID/202431/p900465/scratch/rovensky/\" # path to output to, required\n",
+ "out_folder = \"/gpfs/exfel/exp/HED/202430/p900438/scratch/rovensky/pc/\" # path to output to, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"runs = [72, 73, 74, 75, 76, 77, 78, 79] # runs to use, required, range allowed\n",
"\n",
@@ -124,6 +124,7 @@
" get_dir_creation_date,\n",
" module_index_to_qm,\n",
")\n",
+ "\n",
"from iCalibrationDB import Conditions, Constants\n",
"\n",
"# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
@@ -196,6 +197,13 @@
" bias_voltage = agipd_cond.get_bias_voltage(karabo_id_control)\n",
"if integration_time < 0:\n",
" integration_time = agipd_cond.get_integration_time()\n",
+ " \n",
+ "if \"AGIPD1M\" in karabo_id:\n",
+ " nmods = 16\n",
+ "elif \"AGIPD500K\" in karabo_id:\n",
+ " nmods = 8\n",
+ "elif \"AGIPD\" in karabo_id: # single module detector\n",
+ " nmods = 1\n",
"\n",
"print(f\"Acquisition rate: {acq_rate} MHz\")\n",
"print(f\"Memory cells: {mem_cells}\")\n",
@@ -358,7 +366,7 @@
"metadata": {},
"outputs": [],
"source": [
- "burst_of_interest = [100, 200, 550]\n",
+ "burst_of_interest = [100, 200, -1]\n",
"cell_of_interest = 4\n",
"for module in modules:\n",
" fig = PC_image_plotting(res, burst_of_interest, cell_of_interest)"
@@ -504,6 +512,7 @@
"offsets = {}\n",
"noises = {}\n",
"thresholds = {}\n",
+ "BPdark = {}\n",
"for k_da, mod in zip(karabo_da, modules):\n",
"\n",
" offset, when = get_constant_from_db_and_time(karabo_id, k_da,\n",
@@ -541,7 +550,19 @@
" integration_time=integration_time),\n",
" np.zeros((128, 512, mem_cells, 3)),\n",
" cal_db_interface, creation_time=creation_time)\n",
- " thresholds[mod] = np.array(threshold.data)"
+ " thresholds[mod] = np.array(threshold.data)\n",
+ " \n",
+ " BP, when = get_constant_from_db_and_time(karabo_id, k_da,\n",
+ " Constants.AGIPD.BadPixelsDark(),\n",
+ " Conditions.Dark.AGIPD(\n",
+ " memory_cells=mem_cells,\n",
+ " bias_voltage=bias_voltage,\n",
+ " acquisition_rate=acq_rate,\n",
+ " gain_setting=gain_setting,\n",
+ " integration_time=integration_time),\n",
+ " np.zeros((128, 512, mem_cells, 3)),\n",
+ " cal_db_interface, creation_time=creation_time)\n",
+ " BPdark[mod] = np.array(BP.data)"
]
},
{
@@ -1196,7 +1217,7 @@
"for i, r in zip(modules, res):\n",
" offset = offsets[i]\n",
" noise = noises[i]\n",
- " qm = module_index_to_qm(i)\n",
+ " qm = i if nmods < 8 else module_index_to_qm(i) \n",
" dig, ana, cellId = r\n",
" thresh_trans = thresholds[i][..., 0]\n",
" \n",
@@ -1285,6 +1306,22 @@
"8: hook function - deviation"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Here we mask BP from darks to avoid biasing median calculation for PC BP\n",
+ "for mod in modules:\n",
+ " BPd = np.moveaxis(BPdark[mod], 2, 0)\n",
+ " for i, key in enumerate(['ml', 'bl', 'mh', 'bh']): \n",
+ " if i < 2:\n",
+ " fres[mod][key][BPd[..., 0] > 0] = np.nan\n",
+ " else:\n",
+ " fres[mod][key][BPd[..., 1] > 0] = np.nan"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -1685,9 +1722,9 @@
"metadata": {},
"outputs": [],
"source": [
- "one_photon = 73 # ADU (10 keV photon)\n",
+ "one_photon = 10*FF_gain # ADU (10 keV photon)\n",
"test_pixels = []\n",
- "tpix_range1 = [(0,8), (0,8)]\n",
+ "tpix_range1 = [(450,458), (30,38)]\n",
"for i in range(*tpix_range1[0]):\n",
" for j in range(*tpix_range1[1]):\n",
" test_pixels.append((j,i))\n",
@@ -1708,7 +1745,7 @@
" H = [0, 0, 0, 0]\n",
" H2 = [0, 0, 0, 0]\n",
" Ha = [0, 0, 0, 0]\n",
- " qm = module_index_to_qm(mod)\n",
+ " qm = mod if nmods < 8 else module_index_to_qm(mod)\n",
" cdata = fres[qm]\n",
" ex, ey, ea = None, None, None\n",
" medml = np.nanmean(cdata['ml'])\n",
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb
index 153938056..2932f3d16 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb
@@ -164,7 +164,7 @@
" \n",
"for f in constants_files:\n",
" mod = int(f.stem.split(\"_\")[-1])\n",
- " qm = module_index_to_qm(mod)\n",
+ " qm = mod if nmods < 8 else module_index_to_qm(mod)\n",
" fit_data[mod] = {}\n",
" with h5py.File(f, 'r') as hf:\n",
" bad_pixels[mod] = hf[f'/{qm}/BadPixelsPC/0/data'][()]\n",
@@ -324,11 +324,8 @@
"metadata": {},
"outputs": [],
"source": [
- "# This is needed in case of single modules where module number is different from 0\n",
- "if nmods == 1:\n",
- " module_positions = [0]\n",
- "else:\n",
- " module_positions = modules"
+ "# This is needed for plotting in case of single modules where module number is different from 0\n",
+ "module_positions = [0] if nmods == 1 else modules"
]
},
{
--
GitLab
From 697173822ce8c45e0102bb2c69c328ab99e8531e Mon Sep 17 00:00:00 2001
From: Vratko Rovensky <vratko.rovensky@xfel.eu>
Date: Fri, 10 Jan 2025 15:26:34 +0100
Subject: [PATCH 05/10] Update indexing bug
---
notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
index e0e0226a4..970e2f705 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
@@ -1313,13 +1313,13 @@
"outputs": [],
"source": [
"# Here we mask BP from darks to avoid biasing median calculation for PC BP\n",
- "for mod in modules:\n",
+ "for mod, qm in zip(modules, fres.keys()):\n",
" BPd = np.moveaxis(BPdark[mod], 2, 0)\n",
" for i, key in enumerate(['ml', 'bl', 'mh', 'bh']): \n",
" if i < 2:\n",
- " fres[mod][key][BPd[..., 0] > 0] = np.nan\n",
+ " fres[qm][key][BPd[..., 0] > 0] = np.nan\n",
" else:\n",
- " fres[mod][key][BPd[..., 1] > 0] = np.nan"
+ " fres[qm][key][BPd[..., 1] > 0] = np.nan"
]
},
{
--
GitLab
From 512a5f3abf81225bf7a551174d5d6be33980630c Mon Sep 17 00:00:00 2001
From: Vratko Rovensky <vratko.rovensky@xfel.eu>
Date: Fri, 10 Jan 2025 19:07:08 +0100
Subject: [PATCH 06/10] udate ROI for check plot
---
notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
index 970e2f705..17623e47f 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
@@ -1724,7 +1724,7 @@
"source": [
"one_photon = 10*FF_gain # ADU (10 keV photon)\n",
"test_pixels = []\n",
- "tpix_range1 = [(450,458), (30,38)]\n",
+ "tpix_range1 = [(455,458), (30,38)]\n",
"for i in range(*tpix_range1[0]):\n",
" for j in range(*tpix_range1[1]):\n",
" test_pixels.append((j,i))\n",
--
GitLab
From 1217e0b136061ed64034e8b9338e80ac8324b733 Mon Sep 17 00:00:00 2001
From: Vratko Rovensky <vratko.rovensky@xfel.eu>
Date: Mon, 13 Jan 2025 14:29:39 +0100
Subject: [PATCH 07/10] Fix file naming to be compatible with previous
versions.
---
...terize_AGIPD_Gain_FlatFields_Summary.ipynb | 26 ++++++-------------
1 file changed, 8 insertions(+), 18 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
index 26bdb158a..46dbbf5e4 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
@@ -89,7 +89,7 @@
" get_dir_creation_date,\n",
" get_pdu_from_db,\n",
" get_report,\n",
- "# module_index_to_qm,\n",
+ " module_index_to_qm,\n",
" send_to_db\n",
")\n",
"from cal_tools.plotting import agipd_single_module_geometry\n",
@@ -200,7 +200,6 @@
" \n",
"for f in constants_files:\n",
" mod = int(f.stem.split(\"m\")[-1])\n",
- "# qm = module_index_to_qm(mod)\n",
" fit_data[mod] = {}\n",
" with h5py.File(f, 'r') as hf:\n",
" shape = hf['data/g0mean'].shape\n",
@@ -342,8 +341,8 @@
"# each constant type has its corresponding condition and data.\n",
"if local_output:\n",
" for mod, pdu in zip(modules, db_modules):\n",
- "# qm = module_index_to_qm(mod)\n",
- " file = f\"{out_folder}/slopesff_bpmask_module_{mod}.h5\"\n",
+ " qm = mod if nmods < 8 else module_index_to_qm(mod)\n",
+ " file = f\"{out_folder}/slopesff_bpmask_module_{qm}.h5\"\n",
" dic = {\n",
" pdu:{\n",
" 'SlopesFF': {\n",
@@ -368,10 +367,7 @@
"outputs": [],
"source": [
"# This is needed in case of single modules where module number is different from 0\n",
- "if nmods == 1:\n",
- " module_positions = [0]\n",
- "else:\n",
- " module_positions = modules"
+ "module_positions = [0] if nmods == 1 else modules"
]
},
{
@@ -414,9 +410,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"for key in const_data.keys():\n",
@@ -492,9 +486,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"for key in const_data.keys():\n",
@@ -632,9 +624,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"fig = plt.figure(figsize=(15,7))\n",
@@ -889,5 +879,5 @@
}
},
"nbformat": 4,
- "nbformat_minor": 1
+ "nbformat_minor": 4
}
--
GitLab
From a2821f52e29ee11d9e59df9662df103f5f2f143d Mon Sep 17 00:00:00 2001
From: Vratko Rovensky <vratko.rovensky@xfel.eu>
Date: Sun, 2 Feb 2025 14:31:04 +0100
Subject: [PATCH 08/10] fix modules in ff summary.
---
.../AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
index 46dbbf5e4..b127241da 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
@@ -607,7 +607,7 @@
" starts_at = modules[0] if nmods == 1 else 0\n",
" sel = run_data.select(f'{karabo_id}/DET/{modules[0]}CH0:xtdf', 'image.cellId')\n",
" _, cell = sel.train_from_index(0)\n",
- " stacked_cells = stack_detector_data(cell, 'image.cellId', starts_at=starts_at)[:, 0]\n",
+ " stacked_cells = stack_detector_data(cell, 'image.cellId', starts_at=starts_at)[:, modules[0]]\n",
" \n",
" rel_corr = rel_corr[stacked_cells[0]:stacked_cells[-1]+1]\n",
"\n",
@@ -858,7 +858,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.11.9"
+ "version": "3.9.16"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
--
GitLab
From b3b45e60124ad08b042c3415ba89af59cb4c119e Mon Sep 17 00:00:00 2001
From: Vratko Rovensky <vratko.rovensky@xfel.eu>
Date: Fri, 7 Feb 2025 09:40:42 +0100
Subject: [PATCH 09/10] FF Summarry: point to index in corr. plot
---
.../AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
index b127241da..8ac85becd 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
@@ -607,7 +607,7 @@
" starts_at = modules[0] if nmods == 1 else 0\n",
" sel = run_data.select(f'{karabo_id}/DET/{modules[0]}CH0:xtdf', 'image.cellId')\n",
" _, cell = sel.train_from_index(0)\n",
- " stacked_cells = stack_detector_data(cell, 'image.cellId', starts_at=starts_at)[:, modules[0]]\n",
+ " stacked_cells = stack_detector_data(cell, 'image.cellId', starts_at=starts_at)[:, module_positions[0]]\n",
" \n",
" rel_corr = rel_corr[stacked_cells[0]:stacked_cells[-1]+1]\n",
"\n",
@@ -858,7 +858,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.9.16"
+ "version": "3.11.9"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
--
GitLab
From 78f5795e43e17fc5e8c3a4bad0b4032a10680dc7 Mon Sep 17 00:00:00 2001
From: Vratko Rovensky <vratko.rovensky@xfel.eu>
Date: Mon, 24 Feb 2025 18:07:08 +0100
Subject: [PATCH 10/10] Update condition for QM notation.
---
notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb | 1 -
.../AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb | 2 +-
notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb | 5 ++---
notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb | 2 +-
4 files changed, 4 insertions(+), 6 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb b/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb
index 05b4971ac..4114ef362 100644
--- a/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb
+++ b/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb
@@ -638,7 +638,6 @@
"for color, y in zip(colors, rshist[:,:,xpix,ypix]):\n",
" ax.plot(x, y, color=color,linewidth=0.2)\n",
"plt.grid()\n",
- "plt.yscale('log')\n",
"plt.show()"
]
}
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
index 6b24be2fa..92f27d999 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_Summary.ipynb
@@ -345,7 +345,7 @@
"# each constant type has its corresponding condition and data.\n",
"if local_output:\n",
" for mod, pdu in zip(modules, db_modules):\n",
- " qm = mod if nmods < 8 else module_index_to_qm(mod)\n",
+ " qm = mod if nmods == 1 else module_index_to_qm(mod)\n",
" file = f\"{out_folder}/slopesff_bpmask_module_{qm}.h5\"\n",
" dic = {\n",
" pdu:{\n",
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
index 17623e47f..a3706a07f 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
@@ -124,7 +124,6 @@
" get_dir_creation_date,\n",
" module_index_to_qm,\n",
")\n",
- "\n",
"from iCalibrationDB import Conditions, Constants\n",
"\n",
"# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
@@ -1217,7 +1216,7 @@
"for i, r in zip(modules, res):\n",
" offset = offsets[i]\n",
" noise = noises[i]\n",
- " qm = i if nmods < 8 else module_index_to_qm(i) \n",
+ " qm = i if nmods == 1 else module_index_to_qm(i) \n",
" dig, ana, cellId = r\n",
" thresh_trans = thresholds[i][..., 0]\n",
" \n",
@@ -1745,7 +1744,7 @@
" H = [0, 0, 0, 0]\n",
" H2 = [0, 0, 0, 0]\n",
" Ha = [0, 0, 0, 0]\n",
- " qm = mod if nmods < 8 else module_index_to_qm(mod)\n",
+ " qm = mod if nmods == 1 else module_index_to_qm(mod)\n",
" cdata = fres[qm]\n",
" ex, ey, ea = None, None, None\n",
" medml = np.nanmean(cdata['ml'])\n",
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb
index 2932f3d16..89f0e7a6a 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb
@@ -164,7 +164,7 @@
" \n",
"for f in constants_files:\n",
" mod = int(f.stem.split(\"_\")[-1])\n",
- " qm = mod if nmods < 8 else module_index_to_qm(mod)\n",
+ " qm = mod if nmods == 1 else module_index_to_qm(mod)\n",
" fit_data[mod] = {}\n",
" with h5py.File(f, 'r') as hf:\n",
" bad_pixels[mod] = hf[f'/{qm}/BadPixelsPC/0/data'][()]\n",
--
GitLab