diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
index 4185ab714d4a91cf10990ac4be728f1ab1d0e5a7..b24865a33407ec186192ff08b4cbea1a7879e232 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
@@ -32,10 +32,10 @@
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
- "in_folder = '/gpfs/exfel/exp/SPB/202130/p900188/raw/' # path to input data, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/jsztuk/test/pc\" # path to output to, required\n",
+ "in_folder = '/gpfs/exfel/exp/SPB/202331/p900376/raw' # path to input data, required\n",
+ "out_folder = \"/gpfs/exfel/exp/HED/202430/p900438/scratch/rovensky/pc\" # path to output to, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
- "runs = [92, 93, 94, 95, 96, 97, 98, 99] # runs to use, required, range allowed\n",
+ "runs = [61, 62, 63, 64, 65, 66, 67, 68] # runs to use, required, range allowed\n",
"\n",
"modules = [-1] # modules to work on, required, range allowed\n",
"karabo_da = [\"all\"]\n",
@@ -65,8 +65,8 @@
"high_res_badpix_3d = False # set this to True if you need high-resolution 3d bad pixel plots. Runtime: ~ 1h\n",
"save_plots = False # set to True if you desire saving plots to output folder\n",
"\n",
- "hg_range = [30,210]#[0,-150] # range for linear fit. If upper edge is negative use clustering result (bound_hg - 20)\n",
- "mg_range = [-277,600]#[-350,600] # range for linear fit. If lower edge is negative use clustering result (bound_mg + 20)\n",
+ "hg_range = [30,170]#[0,-150] # range for linear fit. If upper edge is negative use clustering result (bound_hg - 20)\n",
+ "mg_range = [240,590]#[-350,600] # range for linear fit. If lower edge is negative use clustering result (bound_mg + 20)\n",
"sigma_dev_cut = 5.0 # parameters outside the range median +- sigma_dev_cut*MAD are replaced with the median"
]
},
@@ -124,34 +124,18 @@
"metadata": {},
"outputs": [],
"source": [
- "cells = mem_cells\n",
- "path_temp = in_folder+\"/r{:04d}/\"\n",
- "image_name_temp = 'RAW-R{:04d}-AGIPD{:02d}-S{:05d}.h5'\n",
- "print(\"Parameters are:\")\n",
- "if mem_cells < 0:\n",
- " print(\"Memory cells: auto-detection on\")\n",
- "else:\n",
- " print(\"Memory cells set by user: {}\".format(mem_cells))\n",
- "print(\"Runs: {}\".format(runs))\n",
- "print(\"Modules: {}\".format(modules))\n",
+ "# Get operation conditions\n",
+ "ctrl_source = ctrl_source_template.format(karabo_id_control)\n",
+ "run_folder = f'{in_folder}/r{runs[0]:04d}/'\n",
+ "raw_dc = RunDirectory(run_folder)\n",
+ "instrument_src_mod = list(raw_dc.detector_sources)[0]\n",
"\n",
- "instrument = karabo_id.split(\"_\")[0]\n",
- "\n",
- "\n",
- "if instrument == \"HED\":\n",
- " nmods = 8\n",
- "else:\n",
- " nmods = 16\n",
- "\n",
- "print(f\"Detector in use is {karabo_id}\")\n",
- "\n",
- "\n",
- "if karabo_da == [\"all\"]:\n",
- " if modules[0] == -1:\n",
- " modules = list(range(nmods))\n",
- " karabo_da = [\"AGIPD{:02d}\".format(i) for i in modules]\n",
- "else:\n",
- " modules = [int(x[-2:]) for x in karabo_da]"
+ "agipd_cond = AgipdCtrl(\n",
+ " run_dc=RunDirectory(f'{in_folder}/r{runs[0]:04d}'),\n",
+ " image_src=instrument_src_mod,\n",
+ " ctrl_src=ctrl_source,\n",
+ " raise_error=False, # to be able to process very old data without gain_setting value\n",
+ ")"
]
},
{
@@ -160,17 +144,18 @@
"metadata": {},
"outputs": [],
"source": [
- "first_run = runs[0]\n",
- "channel = 0\n",
- "ctrl_src = ctrl_source_template.format(karabo_id_control)\n",
- "instrument_src = instrument_source_template.format(karabo_id, receiver_template).format(channel)\n",
+ "if karabo_da == [\"all\"]:\n",
+ " if modules[0] == -1:\n",
+ " strings = [s.split(\"/\")[-1] for s in list(raw_dc.detector_sources)]\n",
+ " modules = [int(s[:s.index(\"C\")]) for s in strings]\n",
+ " modules.sort()\n",
+ " karabo_da = [\"AGIPD{:02d}\".format(i) for i in modules]\n",
+ "else:\n",
+ " modules = [int(x[-2:]) for x in karabo_da]\n",
"\n",
- "agipd_cond = AgipdCtrl(\n",
- " run_dc=RunDirectory(f'{in_folder}/r{first_run:04d}'),\n",
- " image_src=instrument_src,\n",
- " ctrl_src=ctrl_src,\n",
- " raise_error=False, # to be able to process very old data without gain_setting value\n",
- ")"
+ "print(\"Runs: {}\".format(runs))\n",
+ "print(\"Modules: {}\".format(modules))\n",
+ "print(f\"Detector in use is {karabo_id}\")"
]
},
{
@@ -1222,7 +1207,7 @@
" for j in range(dig.shape[2]):\n",
" inp.append((dig[:,cell,j,:], ana[:,cell,j,:], offset[j,:,cell,:], noise[j,:,cell,:], thresh_trans[j,:,cell]))\n",
"\n",
- " p = partial(calibrate_single_row, cells, fit_hook, [hg_range, mg_range])\n",
+ " p = partial(calibrate_single_row, mem_cells, fit_hook, [hg_range, mg_range])\n",
" frs = view.map_sync(p, inp)\n",
"\n",
" for j, fr in enumerate(frs):\n",
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb
index cbf7052e18d63088ead8d2c62e874bc9d9f8f70e..0511729a0c449fbf52a42368e25e7e14ebae96a4 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_Summary.ipynb
@@ -15,13 +15,13 @@
"metadata": {},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/d/raw/SPB/202331/p900376/\" # path to input data, required\n",
- "out_folder = \"/gpfs/exfel/exp/SPB/202331/p900376/usr/PC/temp_test/12on/\" # path to output to, required\n",
+ "in_folder = \"/gpfs/exfel/exp/HED/202430/p900438/raw/\" # path to input data, required\n",
+ "out_folder = \"/gpfs/exfel/exp/HED/202430/p900438/scratch/rovensky/pc/\" # path to output to, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"runs = [] # runs to use, required, range allowed\n",
"\n",
- "karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for the control device e.g. \"MID_EXP_AGIPD1M1\", or \"SPB_IRU_AGIPD1M1\"\n",
- "karabo_id = \"SPB_DET_AGIPD1M-1\"\n",
+ "karabo_id_control = \"HED_DET_AGIPD65K1\" # karabo-id for the control device e.g. \"MID_EXP_AGIPD1M1\", or \"SPB_IRU_AGIPD1M1\"\n",
+ "karabo_id = \"HED_DET_AGIPD65K1\"\n",
"ctrl_source_template = '{}/MDL/FPGA_COMP' # path to control information\n",
"\n",
"creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
@@ -46,7 +46,7 @@
"import warnings\n",
"warnings.filterwarnings('ignore')\n",
"\n",
- "import os\n",
+ "from pathlib import Path\n",
"from dateutil import parser\n",
"from datetime import timedelta\n",
"\n",
@@ -68,6 +68,7 @@
" get_report,\n",
" send_to_db\n",
")\n",
+ "from cal_tools.plotting import agipd_single_module_geometry\n",
"\n",
"from extra_data import RunDirectory\n",
"from extra_geom import AGIPD_1MGeometry, AGIPD_500K2GGeometry\n",
@@ -96,17 +97,12 @@
"ctrl_source = ctrl_source_template.format(karabo_id_control)\n",
"run_folder = f'{in_folder}/r{runs[0]:04d}/'\n",
"raw_dc = RunDirectory(run_folder)\n",
- "\n",
- "# Read operating conditions from AGIPD00 files\n",
- "instrument_src_mod = [\n",
- " s for s in list(raw_dc.all_sources) if \"0CH\" in s][0]\n",
- "ctrl_src = [\n",
- " s for s in list(raw_dc.all_sources) if ctrl_source in s][0]\n",
+ "instrument_src_mod = list(raw_dc.detector_sources)[0]\n",
"\n",
"agipd_cond = AgipdCtrl(\n",
" run_dc=raw_dc,\n",
" image_src=instrument_src_mod,\n",
- " ctrl_src=ctrl_src,\n",
+ " ctrl_src=ctrl_source,\n",
" raise_error=False, # to be able to process very old data without mosetting value\n",
")\n",
"if mem_cells == -1:\n",
@@ -123,11 +119,22 @@
" integration_time = agipd_cond.get_integration_time()\n",
"\n",
"# Evaluate detector instance for mapping\n",
- "instrument = karabo_id.split(\"_\")[0]\n",
- "if instrument == \"HED\":\n",
- " nmods = 8\n",
- "else:\n",
- " nmods = 16\n",
+ "instance = karabo_id_control.split(\"_\")[-1]\n",
+ "agipd_instances = {'AGIPD65K1': [1, agipd_single_module_geometry()],\n",
+ " 'AGIPD500K2G': [8, AGIPD_500K2GGeometry.from_origin()], \n",
+ " 'AGIPD1M1': [16, AGIPD_1MGeometry.from_quad_positions(quad_pos=[(-525, 625),\n",
+ " (-550, -10),\n",
+ " (520, -160),\n",
+ " (542.5, 475),\n",
+ " ])]\n",
+ " }\n",
+ "\n",
+ "try:\n",
+ " nmods = agipd_instances[instance][0]\n",
+ " geom = agipd_instances[instance][1]\n",
+ "except KeyError as ke:\n",
+ " print('The provided AGIPD instance is not recognised. Available instances are: \\\n",
+ " AGIPD65K1,\\nAGIPD500K2G,\\nAGIPD1M1')\n",
"\n",
"print(f\"Using {creation_time} as creation time\\n\")\n",
"print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Memory cells: {mem_cells}\\n\"\n",
@@ -138,9 +145,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"# ml - high gain slope\n",
@@ -158,24 +163,24 @@
"bad_pixels = {}\n",
"modules = []\n",
"karabo_da = []\n",
+ "constants_files = []\n",
"\n",
- "for mod in range(nmods):\n",
+ "out_folder = Path(out_folder)\n",
+ "constants_files = sorted(out_folder.glob('*.h5'), \n",
+ " key=lambda f: (len(f.stem), f.stem)) \n",
+ " \n",
+ "for f in constants_files:\n",
+ " mod = int(f.stem.split(\"_\")[-1])\n",
" qm = module_index_to_qm(mod)\n",
" fit_data[mod] = {}\n",
- " constants_file = f'{out_folder}/agipd_pc_store_{\"_\".join([str(run) for run in runs])}_{mod}_{mod}.h5'\n",
- " \n",
- " if os.path.exists(constants_file):\n",
- " print(f'Trying to find: {constants_file}')\n",
- " print(f'Data available for module {qm}\\n')\n",
- " with h5py.File(constants_file, 'r') as hf:\n",
- " bad_pixels[mod] = hf[f'/{qm}/BadPixelsPC/0/data'][()]\n",
- " for key in keys_file:\n",
- " fit_data[mod][key]= hf[f'/{qm}/{key}/0/data'][()]\n",
+ " with h5py.File(f, 'r') as hf:\n",
+ " bad_pixels[mod] = hf[f'/{qm}/BadPixelsPC/0/data'][()]\n",
+ " for key in keys_file:\n",
+ " fit_data[mod][key]= hf[f'/{qm}/{key}/0/data'][()]\n",
" \n",
- " modules.append(mod)\n",
- " karabo_da.append(f\"AGIPD{mod:02d}\")\n",
- " else:\n",
- " print(f\"No fit data available for module {qm}\\n\")"
+ " modules.append(mod)\n",
+ " karabo_da.append(f\"AGIPD{mod:02d}\")\n",
+ " print(f'Data available for AGIPD{mod:02d}')"
]
},
{
@@ -332,36 +337,18 @@
"for (key, c) in zip(keys, const_order):\n",
" const_data[key] = np.full((nmods, mem_cells, 512, 128), np.nan)\n",
" old_const[key] = np.full((nmods, mem_cells, 512, 128), np.nan)\n",
- " for cnt, i in enumerate(modules):\n",
- " if key in fit_data[i]:\n",
- " const_data[key][i,:,pixel_range[0]:pixel_range[2],\n",
- " pixel_range[1]:pixel_range[3]] = fit_data[i][key]\n",
+ " for cnt, mn in enumerate(modules):\n",
+ " if key in fit_data[mn]:\n",
+ " const_data[key][cnt,:,pixel_range[0]:pixel_range[2],\n",
+ " pixel_range[1]:pixel_range[3]] = fit_data[mn][key]\n",
" if np.any(old_PC_consts[0][0]):\n",
- " old_const[key][i,:,pixel_range[0]:pixel_range[2],\n",
+ " old_const[key][cnt,:,pixel_range[0]:pixel_range[2],\n",
" pixel_range[1]:pixel_range[3]] = old_PC_consts[cnt][0][c].swapaxes(1,2)\n",
"\n",
"const_data['BadPixelsPC'] = np.full((nmods, mem_cells, 512, 128), np.nan)\n",
- "for i in modules:\n",
- " const_data['BadPixelsPC'][i,:,pixel_range[0]:pixel_range[2],\n",
- " pixel_range[1]:pixel_range[3]] = fit_data[i]['BadPixelsPC']"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "#Define AGIPD geometry\n",
- "if instrument == \"HED\":\n",
- " geom = AGIPD_500K2GGeometry.from_origin()\n",
- "else:\n",
- " geom = AGIPD_1MGeometry.from_quad_positions(quad_pos=[\n",
- " (-525, 625),\n",
- " (-550, -10),\n",
- " (520, -160),\n",
- " (542.5, 475),\n",
- " ])"
+ "for cnt, mn in enumerate(modules):\n",
+ " const_data['BadPixelsPC'][cnt,:,pixel_range[0]:pixel_range[2],\n",
+ " pixel_range[1]:pixel_range[3]] = fit_data[mn]['BadPixelsPC']"
]
},
{
@@ -417,7 +404,6 @@
" colorbar={'shrink': 1,\n",
" 'pad': 0.04,\n",
" 'fraction': 0.1,\n",
- " \n",
" })\n",
" colorbar = ax.images[0].colorbar\n",
" colorbar.ax.set_yticklabels([\"{:.1f}\".format(tk) for tk in colorbar.get_ticks()])\n",
@@ -546,7 +532,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "scrolled": true
+ "scrolled": false
},
"outputs": [],
"source": [
@@ -564,7 +550,7 @@
" d.append({'x': np.arange(data[i].shape[0]),\n",
" 'y': np.nanmean(data[i], axis=(1,2)),\n",
" 'drawstyle': 'steps-pre',\n",
- " 'label': f'{i}',\n",
+ " 'label': f'{modules[i]}',\n",
" 'linewidth': 2,\n",
" 'linestyle': '--' if i>7 else '-'\n",
" })\n",
@@ -592,9 +578,9 @@
"d = []\n",
"for i in range(nmods):\n",
" d.append({'x': np.arange(ratio[i].shape[0]),\n",
- " 'y': np.nanmean(ratio[i], axis=(1,2)),\n",
+ " 'y': np.nanmedian(ratio[i], axis=(1,2)),\n",
" 'drawstyle': 'steps-pre',\n",
- " 'label': f'{i}',\n",
+ " 'label': f'{modules[i]}',\n",
" 'linewidth': 2,\n",
" 'linestyle': '--' if i>7 else '-'\n",
" })\n",
@@ -621,17 +607,17 @@
"source": [
"table = []\n",
"ratio_old = old_gain_data['HG']['ml'] / old_gain_data['MG']['mh']\n",
- "for mod in modules:\n",
+ "for cnt, mod in enumerate(modules):\n",
" \n",
" table.append((mod,\n",
- " f\"{np.nanmean(ratio[mod]):0.1f} +- {np.nanstd(ratio[mod]):0.2f}\",\n",
- " f\"{np.nanmean(ratio_old[mod]):0.1f} +- {np.nanstd(ratio_old[mod]):0.2f}\",\n",
- " f\"{np.nanmean(gain_data['BP']['BadPixelsPC'][mod]>0)*100:0.1f} ({np.nansum(gain_data['BP']['BadPixelsPC'][mod]>0)})\"\n",
+ " f\"{np.nanmedian(ratio[cnt]):0.1f} +- {np.nanstd(ratio[cnt]):0.2f}\",\n",
+ " f\"{np.nanmedian(ratio_old[cnt]):0.1f} +- {np.nanstd(ratio_old[cnt]):0.2f}\",\n",
+ " f\"{np.nanmedian(gain_data['BP']['BadPixelsPC'][cnt]>0)*100:0.1f} ({np.nansum(gain_data['BP']['BadPixelsPC'][cnt]>0)})\"\n",
" ))\n",
"\n",
"all_HM = []\n",
"all_HM_old = []\n",
- "for mod in modules:\n",
+ "for mod in range(len(modules)):\n",
" all_HM.extend(ratio[mod])\n",
" all_HM_old.extend(ratio_old[mod])\n",
"all_HM = np.array(all_HM)\n",
@@ -687,9 +673,9 @@
"ax.set_ylabel('Rows', fontsize=12)\n",
"\n",
"ax = fig.add_subplot(gs[0,1])\n",
- "for mod in modules:\n",
- " h, e = np.histogram(ratio[mod].flatten(), bins=100, range=(vmin, vmax))\n",
- " ax.plot(e[:-1], h, color=colors[mod],linewidth=2, label=f'{mod}', alpha=0.8)\n",
+ "for cnt, mod in enumerate(modules):\n",
+ " h, e = np.histogram(ratio[cnt].flatten(), bins=100, range=(vmin, vmax))\n",
+ " ax.plot(e[:-1], h, color=colors[cnt],linewidth=2, label=f'{mod}', alpha=0.8)\n",
" ax.set_xlabel('High/Medium Gain Ratio', fontsize=13)\n",
" ax.set_ylabel('Counts', fontsize=13)\n",
" plt.ticklabel_format(axis='y', style='sci', scilimits=(0,0))\n",