From 41585b8b1d1f74e7531c4c5db43fda0780d07817 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Mon, 7 Feb 2022 17:11:47 +0100
Subject: [PATCH] JF/update_correction_plots
---
...Jungfrau_Gain_Correct_and_Verify_NBC.ipynb | 306 ++++++++++++------
1 file changed, 209 insertions(+), 97 deletions(-)
diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
index 9bd9950e4..9054b0a49 100644
--- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
@@ -39,7 +39,6 @@
"# Parameters affecting corrected data.\n",
"overwrite = True # set to True if existing data should be overwritten\n",
"relative_gain = True # do relative gain correction\n",
- "plt_images = 100 # Number of images to plot after applying selected corrections.\n",
"limit_images = 0 # ONLY FOR TESTING. process only first N images, Use 0 to process all.\n",
"cell_id_preview = 15 # cell Id used for preview in single-shot plots\n",
"\n",
@@ -82,6 +81,9 @@
"import tabulate\n",
"from IPython.display import Latex, Markdown, display\n",
"from extra_data import H5File, RunDirectory\n",
+ "from extra_data import RunDirectory, stack_detector_data, by_id\n",
+ "from extra_data import components\n",
+ "from extra_geom import JUNGFRAUGeometry\n",
"from matplotlib.colors import LogNorm\n",
"\n",
"from cal_tools import h5_copy_except\n",
@@ -194,6 +196,7 @@
" print(f\"Run is in burst mode.\\n\"\n",
" f\"Storage cell start: {sc_start:02d}\")\n",
"except KeyError as e:\n",
+ " sc_start = None\n",
" print(\"WARNING: KeyError while reading number of memory cells.\")\n",
" if mem_cells == 0:\n",
" memory_cells = 1\n",
@@ -216,6 +219,25 @@
"print(f\"Number of memory cells are {memory_cells}\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Validate selected cell_id_preview\n",
+ "cell_idx_preview = cell_id_preview\n",
+ "if memory_cells == 1:\n",
+ " cell_idx_preview = 0\n",
+ " if cell_id_preview != sc_start:\n",
+ " print(f\"WARNING: The selected cell_id_preview {cell_id_preview} is not available, \"\n",
+ " f\"switching to cell `{sc_start}` to preview.\")\n",
+ " cell_id_preview = sc_start\n",
+ "else:\n",
+ " if cell_id_preview > 15:\n",
+ " raise ValueError(\"Selected cell_id_preview is greater than available 16 memory cells.\")"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -237,6 +259,15 @@
" gain_mode=gain_mode,\n",
")\n",
"\n",
+ "\n",
+ "db_modules = get_pdu_from_db(\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=karabo_da,\n",
+ " constant=Constants.jungfrau.Offset(),\n",
+ " condition=condition,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " snapshot_at=creation_time)\n",
+ "\n",
"def get_constants_for_module(karabo_da: str):\n",
" \"\"\" Get calibration constants for given module of Jungfrau\n",
"\n",
@@ -456,11 +487,6 @@
"metadata": {},
"outputs": [],
"source": [
- "fim_data = {}\n",
- "gim_data = {}\n",
- "rim_data = {}\n",
- "msk_data = {}\n",
- "\n",
"# Loop over modules\n",
"for local_karabo_da, mapped_files_module in mapped_files.items():\n",
" instrument_src_kda = instrument_src.format(int(local_karabo_da[-2:]))\n",
@@ -497,13 +523,10 @@
" print(f\"\\t- WARNING: {sequence_file.name} has {n_trains - dshape[0]} \"\n",
" \"trains with empty data.\")\n",
"\n",
- " # Just in case if n_imgs is less than the chosen plt_images.\n",
- " plt_images = min(plt_images, n_imgs)\n",
- "\n",
" # load constants from the constants dictionary.\n",
" offset_map, mask, gain_map = constants[local_karabo_da]\n",
"\n",
- " # Allocate shared arrays.\n",
+ " # Allocate shared arrays for corrected data.\n",
" data_corr = context.alloc(shape=dshape, dtype=np.float32)\n",
" mask_corr = context.alloc(shape=dshape, dtype=np.uint32)\n",
"\n",
@@ -557,16 +580,7 @@
" )\n",
" save_reduced_rois(ofile, data_corr, mask_corr, local_karabo_da)\n",
"\n",
- " step_timer.done_step(f'Saving data time.')\n",
- "\n",
- " # Prepare plotting arrays\n",
- " step_timer.start()\n",
- "\n",
- " fim_data[local_karabo_da] = data_corr[:plt_images, cell_idx_preview, ...].copy()\n",
- " msk_data[local_karabo_da] = mask_corr[:plt_images, cell_idx_preview, ...].copy()\n",
- " gim_data[local_karabo_da] = gain[:plt_images, cell_idx_preview, ...].copy()\n",
- "\n",
- " step_timer.done_step(f'Preparing plotting data of {plt_images} images.')"
+ " step_timer.done_step(f'Saving data time.')"
]
},
{
@@ -598,6 +612,49 @@
" sys.exit(0)"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Positions are given in pixels\n",
+ "mod_width = (256 * 4) + (2 * 3) # inc. 2px gaps between tiles\n",
+ "mod_height = (256 * 2) + 2\n",
+ "if karabo_id == \"SPB_IRDA_JF4M\":\n",
+ " # The first 4 modules are rotated 180 degrees relative to the others.\n",
+ " # We pass the bottom, beam-right corner of the module regardless of its\n",
+ " # orientation, requiring a subtraction from the symmetric positions we'd\n",
+ " # otherwise calculate.\n",
+ " x_start, y_start = 1125, 1078\n",
+ " module_pos = [\n",
+ " (x_start - mod_width, y_start - mod_height - (i * (mod_height + 33)))\n",
+ " for i in range(4)\n",
+ " ] + [\n",
+ " (-x_start, -y_start + (i * (mod_height + 33))) for i in range(4)\n",
+ " ]\n",
+ " orientations = [(-1, -1) for _ in range(4)] + [(1, 1) for _ in range(4)]\n",
+ "elif karabo_id == \"FXE_XAD_JF1M\":\n",
+ " module_pos = ((-mod_width//2, 33),(-mod_width//2, -mod_height -33))\n",
+ " orientations = [(-1,-1), (1,1)]\n",
+ "else:\n",
+ " module_pos = ((-mod_width//2,-mod_height//2),)\n",
+ " orientations = None\n",
+ "\n",
+ "geom = JUNGFRAUGeometry.from_module_positions(module_pos, orientations=orientations)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "jf_inp = components.JUNGFRAU(RunDirectory(f'{in_folder}/r{run:04d}/', '*S00000*'), detector_name=karabo_id)\n",
+ "raw = jf_inp.get_array(\"data.adc\")[:, :, cell_idx_preview, ...].values\n",
+ "gain = jf_inp.get_array(\"data.gain\")[:, :, cell_idx_preview, ...].values"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -630,28 +687,16 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "db_modules = get_pdu_from_db(\n",
- " karabo_id=karabo_id,\n",
- " karabo_da=karabo_da,\n",
- " constant=Constants.jungfrau.Offset(),\n",
- " condition=condition,\n",
- " cal_db_interface=cal_db_interface,\n",
- " snapshot_at=creation_time)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
"outputs": [],
"source": [
"for pdu, mod in zip(db_modules, karabo_da):\n",
+ " modno = jf_inp.source_to_modno[instrument_source_template.format(karabo_id, mod)] - 1\n",
" h, ex, ey = np.histogram2d(\n",
- " rim_data[mod].flatten(),\n",
- " gim_data[mod].flatten(),\n",
+ " raw[modno].flatten(),\n",
+ " gain[modno].flatten(),\n",
" bins=[100, 4],\n",
" range=[[0, 10000], [0, 4]],\n",
" )\n",
@@ -677,18 +722,29 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " fig = plt.figure(figsize=(20, 10))\n",
- " ax = fig.add_subplot(111)\n",
- "\n",
- " im = ax.imshow(\n",
- " np.mean(rim_data[mod],axis=0),\n",
- " vmin=min(0.75*np.median(rim_data[mod][rim_data[mod] > 0]), 2000),\n",
- " vmax=max(1.5*np.median(rim_data[mod][rim_data[mod] > 0]), 16000),\n",
- " cmap=\"jet\")\n",
- "\n",
- " ax.set_title(f'Module {mod} ({pdu})')\n",
- " cb = fig.colorbar(im, ax=ax)"
+ "jf_out = components.JUNGFRAU(RunDirectory(out_folder, f'*{run}*S00000*'), detector_name=karabo_id)\n",
+ "corrected = jf_out.get_array(\"data.adc\")[:, :, cell_idx_preview, ...].values\n",
+ "mask = jf_out.get_array(\"data.mask\")[:, :, cell_idx_preview, ...].values"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fig = plt.figure(figsize=(18, 20))\n",
+ "ax = fig.add_subplot(111)\n",
+ "raw_mean = np.mean(raw, axis=1)\n",
+ "geom.plot_data_fast(\n",
+ " raw_mean,\n",
+ " ax=ax,\n",
+ " vmin=min(0.75*np.median(raw_mean[raw_mean > 0]), 2000),\n",
+ " vmax=max(1.5*np.median(raw_mean[raw_mean > 0]), 16000),\n",
+ " cmap=\"jet\",\n",
+ " colorbar={'shrink': 0.5, 'pad': 0.02}\n",
+ ")\n",
+ "plt.show()"
]
},
{
@@ -706,25 +762,25 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " fig = plt.figure(figsize=(20, 10))\n",
- " ax = fig.add_subplot(111)\n",
- "\n",
- " im = ax.imshow(\n",
- " np.mean(fim_data[mod],axis=0),\n",
- " vmin=min(0.75*np.median(fim_data[mod][fim_data[mod] > 0]), -0.5),\n",
- " vmax=max(2.*np.median(fim_data[mod][fim_data[mod] > 0]), 100),\n",
- " cmap=\"jet\")\n",
- "\n",
- " ax.set_title(f'Module {mod} ({pdu})', size=18)\n",
- " cb = fig.colorbar(im, ax=ax)"
+ "fig = plt.figure(figsize=(20, 25))\n",
+ "ax = fig.add_subplot(111)\n",
+ "corrected_mean = np.mean(corrected, axis=1)\n",
+ "geom.plot_data_fast(\n",
+ " corrected_mean,\n",
+ " ax=ax,\n",
+ " vmin= min(0.75*np.median(corrected_mean[corrected_mean > 0]), -0.5),\n",
+ " vmax = max(2.*np.median(corrected_mean[corrected_mean > 0]), 100),\n",
+ " cmap=\"jet\",\n",
+ " colorbar={'shrink': 0.5, 'pad': 0.02}\n",
+ ")\n",
+ "plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "### Single Train Preview ###\n",
+ "### Single Train RAW Preview ###\n",
"\n",
"A single image from the first train"
]
@@ -735,18 +791,47 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " fig = plt.figure(figsize=(20, 10))\n",
- " ax = fig.add_subplot(111)\n",
- "\n",
- " im = ax.imshow(\n",
- " fim_data[mod][0, ...],\n",
- " vmin=min(0.75*np.median(fim_data[mod][0, ...]), -0.5),\n",
- " vmax=max(2.*np.median(fim_data[mod][0, ...]), 100),\n",
- " cmap=\"jet\")\n",
+ "fig = plt.figure(figsize=(18, 20))\n",
+ "ax = fig.add_subplot(111)\n",
+ "raw_first_train = raw[:, 0, ...]\n",
+ "geom.plot_data_fast(\n",
+ " raw_first_train,\n",
+ " ax=ax,\n",
+ " vmin=min(0.75*np.median(raw_first_train[raw_first_train > 0]), 2000),\n",
+ " vmax=max(1.5*np.median(raw_first_train[raw_first_train > 0]), 16000),\n",
+ " cmap=\"jet\",\n",
+ " colorbar={'shrink': 0.5, 'pad': 0.02}\n",
+ ")\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Single Train CORRECTED Preview ###\n",
"\n",
- " ax.set_title(f'Module {mod} ({pdu})', size=18)\n",
- " cb = fig.colorbar(im, ax=ax)"
+ "A single image from the first train"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fig = plt.figure(figsize=(20, 25))\n",
+ "ax = fig.add_subplot(111)\n",
+ "corrected_first_train = corrected[:, 0, ...]\n",
+ "geom.plot_data_fast(\n",
+ " corrected_first_train,\n",
+ " ax=ax,\n",
+ " vmin= min(0.75*np.median(corrected_first_train[corrected_first_train > 0]), -0.5),\n",
+ " vmax = max(2.*np.median(corrected_first_train[corrected_first_train > 0]), 100),\n",
+ " cmap=\"jet\",\n",
+ " colorbar={'shrink': 0.5, 'pad': 0.02}\n",
+ ")\n",
+ "plt.show()"
]
},
{
@@ -759,30 +844,34 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "tags": []
+ },
"outputs": [],
"source": [
"for pdu, mod in zip(db_modules, karabo_da):\n",
+ " modno = jf_out.source_to_modno[instrument_source_template.format(karabo_id, mod)] - 1\n",
" fig = plt.figure(figsize=(20,10))\n",
" ax = fig.add_subplot(211)\n",
+ " corrected_flatten = corrected[modno].flatten()\n",
" h = ax.hist(\n",
- " fim_data[mod].flatten(),\n",
+ " corrected_flatten,\n",
" bins=1000,\n",
" range=(-100, 1000),\n",
" log=True,\n",
" )\n",
- " l = ax.set_xlabel(\"Signal (keV)\")\n",
+ " l = ax.set_xlabel(\"Signal (keV) range[-100, 1000]\")\n",
" l = ax.set_ylabel(\"Counts\")\n",
" _ = ax.set_title(f'Module {mod} ({pdu})')\n",
"\n",
" ax = fig.add_subplot(212)\n",
" h = ax.hist(\n",
- " fim_data[mod].flatten(),\n",
+ " corrected_flatten,\n",
" bins=1000,\n",
" range=(-1000, 10000),\n",
" log=True,\n",
" )\n",
- " l = ax.set_xlabel(\"Signal (keV)\")\n",
+ " l = ax.set_xlabel(\"Signal (keV) range[-1000, 10000]\")\n",
" l = ax.set_ylabel(\"Counts\")\n",
" _ = ax.set_title(f'Module {mod} ({pdu})')"
]
@@ -791,6 +880,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
+ "\n",
"### Maximum GAIN Preview ###\n",
"\n",
"The per pixel maximum of the first train of the GAIN data"
@@ -802,14 +892,16 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " fig = plt.figure(figsize=(20, 10))\n",
- " ax = fig.add_subplot(111)\n",
- " im = ax.imshow(\n",
- " np.max(gim_data[mod], axis=0),\n",
- " vmin=0, vmax=3, cmap=\"jet\")\n",
- " ax.set_title(f'Module {mod} ({pdu})', size=18)\n",
- " cb = fig.colorbar(im, ax=ax)"
+ "fig = plt.figure(figsize=(18, 20))\n",
+ "ax = fig.add_subplot(111)\n",
+ "gain_max = np.max(gain, axis=1)\n",
+ "geom.plot_data_fast(\n",
+ " gain_max,\n",
+ " ax=ax,\n",
+ " vmin=0, vmax=min(np.max(gain_max), 5), cmap=\"jet\",\n",
+ " colorbar={'shrink': 0.5, 'pad': 0.02}\n",
+ ")\n",
+ "plt.show()"
]
},
{
@@ -839,9 +931,26 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "### Single Image Bad Pixels ###\n",
+ "### Single Image Bad Pixels ###"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(Markdown(f\"#### Bad pixel image for the first train and cell `{sc_start}`:\\n\"))\n",
"\n",
- "A single image bad pixel map for the first image of the first train"
+ "fig = plt.figure(figsize=(18, 20))\n",
+ "ax = fig.add_subplot(111)\n",
+ "geom.plot_data_fast(\n",
+ " np.log2(mask[:, 0, ...]),\n",
+ " ax=ax,\n",
+ " vmin=0, vmax=32, cmap=\"jet\",\n",
+ " colorbar={'shrink': 0.5, 'pad': 0.02},\n",
+ ")\n",
+ "plt.show()"
]
},
{
@@ -850,14 +959,17 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " fig = plt.figure(figsize=(20, 10))\n",
- " ax = fig.add_subplot(111)\n",
- " im = ax.imshow(\n",
- " np.log2(msk_data[mod][0,...]),\n",
- " vmin=0, vmax=32, cmap=\"jet\")\n",
- " ax.set_title(f'Module {mod} ({pdu})', size=18)\n",
- " cb = fig.colorbar(im, ax=ax)"
+ "display(Markdown(f\"#### Bad pixel precentage for cell `{sc_start}`:\\n\"))\n",
+ "\n",
+ "fig = plt.figure(figsize=(18, 20))\n",
+ "ax = fig.add_subplot(111)\n",
+ "geom.plot_data_fast(\n",
+ " np.mean(mask>0, axis=1),\n",
+ " ax=ax,\n",
+ " vmin=0, vmax=1, cmap=\"jet\",\n",
+ " colorbar={'shrink': 0.5, 'pad': 0.02},\n",
+ ")\n",
+ "plt.show()"
]
}
],
--
GitLab