diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
index 9bd9950e4c85fafcbd770e76dbb55927218230fa..8f91e9263fbb1efd967861462b289bddefc3314d 100644
--- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
@@ -19,7 +19,7 @@
"source": [
"in_folder = \"/gpfs/exfel/exp/SPB/202130/p900204/raw\" # the folder to read data from, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove\" # the folder to output to, required\n",
- "run = 112 # run to process, required\n",
+ "run = 91 # run to process, required\n",
"sequences = [-1] # sequences to correct, set to [-1] for all, range allowed\n",
"sequences_per_node = 1 # number of sequence files per cluster node if run as slurm job, set to 0 to not run SLURM parallel\n",
"\n",
@@ -39,9 +39,7 @@
"# Parameters affecting corrected data.\n",
"overwrite = True # set to True if existing data should be overwritten\n",
"relative_gain = True # do relative gain correction\n",
- "plt_images = 100 # Number of images to plot after applying selected corrections.\n",
"limit_images = 0 # ONLY FOR TESTING. process only first N images, Use 0 to process all.\n",
- "cell_id_preview = 15 # cell Id used for preview in single-shot plots\n",
"\n",
"# Parameters for retrieving calibration constants\n",
"manual_slow_data = False # if true, use manually entered bias_voltage and integration_time values\n",
@@ -53,6 +51,8 @@
"\n",
"# Parameters for plotting\n",
"skip_plots = False # exit after writing corrected files\n",
+ "plot_images = -1 # Number of images to plot for RAW and CORRECTED plots.\n",
+ "cell_id_preview = 15 # cell Id used for preview in single-shot plots\n",
"\n",
"# Parameters for ROI selection and reduction\n",
"roi_definitions = [-1] # List with groups of 6 values defining ROIs, e.g. [3, 120, 180, 200, 550, -2] for module 3 (JNGFR03), slice 120:180, 200:550, average along axis -2 (slow scan, or -1 for fast scan)\n",
@@ -81,7 +81,8 @@
"import pasha as psh\n",
"import tabulate\n",
"from IPython.display import Latex, Markdown, display\n",
- "from extra_data import H5File, RunDirectory\n",
+ "from extra_data import H5File, RunDirectory, by_id, components\n",
+ "from extra_geom import JUNGFRAUGeometry\n",
"from matplotlib.colors import LogNorm\n",
"\n",
"from cal_tools import h5_copy_except\n",
@@ -456,11 +457,6 @@
"metadata": {},
"outputs": [],
"source": [
- "fim_data = {}\n",
- "gim_data = {}\n",
- "rim_data = {}\n",
- "msk_data = {}\n",
- "\n",
"# Loop over modules\n",
"for local_karabo_da, mapped_files_module in mapped_files.items():\n",
" instrument_src_kda = instrument_src.format(int(local_karabo_da[-2:]))\n",
@@ -497,13 +493,10 @@
" print(f\"\\t- WARNING: {sequence_file.name} has {n_trains - dshape[0]} \"\n",
" \"trains with empty data.\")\n",
"\n",
- " # Just in case if n_imgs is less than the chosen plt_images.\n",
- " plt_images = min(plt_images, n_imgs)\n",
- "\n",
" # load constants from the constants dictionary.\n",
" offset_map, mask, gain_map = constants[local_karabo_da]\n",
"\n",
- " # Allocate shared arrays.\n",
+ " # Allocate shared arrays for corrected data.\n",
" data_corr = context.alloc(shape=dshape, dtype=np.float32)\n",
" mask_corr = context.alloc(shape=dshape, dtype=np.uint32)\n",
"\n",
@@ -527,8 +520,6 @@
" else:\n",
" cell_idx_preview = 0\n",
"\n",
- " rim_data[local_karabo_da] = data[:plt_images, cell_idx_preview, ...].copy()\n",
- "\n",
" context.map(correct_train, data)\n",
" step_timer.done_step(f'Correction time.')\n",
"\n",
@@ -557,16 +548,7 @@
" )\n",
" save_reduced_rois(ofile, data_corr, mask_corr, local_karabo_da)\n",
"\n",
- " step_timer.done_step(f'Saving data time.')\n",
- "\n",
- " # Prepare plotting arrays\n",
- " step_timer.start()\n",
- "\n",
- " fim_data[local_karabo_da] = data_corr[:plt_images, cell_idx_preview, ...].copy()\n",
- " msk_data[local_karabo_da] = mask_corr[:plt_images, cell_idx_preview, ...].copy()\n",
- " gim_data[local_karabo_da] = gain[:plt_images, cell_idx_preview, ...].copy()\n",
- "\n",
- " step_timer.done_step(f'Preparing plotting data of {plt_images} images.')"
+ " step_timer.done_step(f'Saving data time.')"
]
},
{
@@ -604,27 +586,84 @@
"metadata": {},
"outputs": [],
"source": [
- "def do_2d_plot(data, edges, y_axis, x_axis, title):\n",
- " fig = plt.figure(figsize=(10, 10))\n",
- " ax = fig.add_subplot(111)\n",
- " extent = [\n",
- " np.min(edges[1]),\n",
- " np.max(edges[1]),\n",
- " np.min(edges[0]),\n",
- " np.max(edges[0]),\n",
+ "# Positions are given in pixels\n",
+ "mod_width = (256 * 4) + (2 * 3) # inc. 2px gaps between tiles\n",
+ "mod_height = (256 * 2) + 2\n",
+ "if karabo_id == \"SPB_IRDA_JF4M\":\n",
+ " # The first 4 modules are rotated 180 degrees relative to the others.\n",
+ " # We pass the bottom, beam-right corner of the module regardless of its\n",
+ " # orientation, requiring a subtraction from the symmetric positions we'd\n",
+ " # otherwise calculate.\n",
+ " x_start, y_start = 1125, 1078\n",
+ " module_pos = [\n",
+ " (x_start - mod_width, y_start - mod_height - (i * (mod_height + 33)))\n",
+ " for i in range(4)\n",
+ " ] + [\n",
+ " (-x_start, -y_start + (i * (mod_height + 33))) for i in range(4)\n",
" ]\n",
+ " orientations = [(-1, -1) for _ in range(4)] + [(1, 1) for _ in range(4)]\n",
+ "elif karabo_id == \"FXE_XAD_JF1M\":\n",
+ " module_pos = ((-mod_width//2, 33),(-mod_width//2, -mod_height -33))\n",
+ " orientations = [(-1,-1), (1,1)]\n",
+ "else:\n",
+ " module_pos = ((-mod_width//2,-mod_height//2),)\n",
+ " orientations = None\n",
"\n",
- " im = ax.imshow(\n",
- " data[::-1, :],\n",
- " extent=extent,\n",
- " aspect=\"auto\",\n",
- " norm=LogNorm(vmin=1, vmax=np.max(data))\n",
+ "geom = JUNGFRAUGeometry.from_module_positions(module_pos, orientations=orientations, asic_gap=0)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "first_seq = 0 if sequences == [-1] else sequences[0]\n",
+ "\n",
+ "with RunDirectory(out_folder, f\"*{run}*S{first_seq:05d}*\") as corr_dc:\n",
+ " # Reading CORR data for plotting.\n",
+ " jf_corr = components.JUNGFRAU(\n",
+ " corr_dc,\n",
+ " detector_name=karabo_id,\n",
+ " ).select_trains(np.s_[:plot_images])\n",
+ " tid, jf_corr_data = next(iter(jf_corr.trains(require_all=True)))\n",
+ "\n",
+ "# Shape = [modules, trains, cells, x, y]\n",
+ "corrected = jf_corr.get_array(\"data.adc\")[:, :, cell_idx_preview, ...].values\n",
+ "corrected_train = jf_corr_data[\"data.adc\"][\n",
+ " :, cell_idx_preview, ...\n",
+ "].values # loose the train axis.\n",
+ "\n",
+ "mask = jf_corr.get_array(\"data.mask\")[:, :, cell_idx_preview, ...].values\n",
+ "mask_train = jf_corr_data[\"data.mask\"][:, cell_idx_preview, ...].values\n",
+ "\n",
+ "with RunDirectory(f\"{in_folder}/r{run:04d}/\", f\"*S{first_seq:05d}*\") as raw_dc:\n",
+ "\n",
+ " # Reading RAW data for plotting.\n",
+ " jf_raw = components.JUNGFRAU(raw_dc, detector_name=karabo_id).select_trains(\n",
+ " np.s_[:plot_images]\n",
" )\n",
- " ax.set_xlabel(x_axis)\n",
- " ax.set_ylabel(y_axis)\n",
- " ax.set_title(title)\n",
- " cb = fig.colorbar(im)\n",
- " cb.set_label(\"Counts\")"
+ "\n",
+ "raw = jf_raw.get_array(\"data.adc\")[:, :, cell_idx_preview, ...].values\n",
+ "raw_train = (\n",
+ " jf_raw.select_trains(by_id[[tid]])\n",
+ " .get_array(\"data.adc\")[:, 0, cell_idx_preview, ...]\n",
+ " .values\n",
+ ")\n",
+ "\n",
+ "gain = jf_raw.get_array(\"data.gain\")[:, :, cell_idx_preview, ...].values\n",
+ "gain_train_cells = (\n",
+ " jf_raw.select_trains(by_id[[tid]]).get_array(\"data.gain\")[:, :, :, ...].values\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Single Train Preview ###\n",
+ "\n",
+ "A single image from the first train"
]
},
{
@@ -639,7 +678,17 @@
" constant=Constants.jungfrau.Offset(),\n",
" condition=condition,\n",
" cal_db_interface=cal_db_interface,\n",
- " snapshot_at=creation_time)"
+ " snapshot_at=creation_time,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Mean RAW Preview ###\n",
+ "\n",
+ "The per pixel mean of the sequence file of RAW data"
]
},
{
@@ -648,27 +697,27 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " h, ex, ey = np.histogram2d(\n",
- " rim_data[mod].flatten(),\n",
- " gim_data[mod].flatten(),\n",
- " bins=[100, 4],\n",
- " range=[[0, 10000], [0, 4]],\n",
- " )\n",
- " do_2d_plot(\n",
- " h, (ex, ey),\n",
- " \"Signal (ADU)\",\n",
- " \"Gain Bit Value (high gain=0[00], medium gain=1[01], low gain=3[11])\",\n",
- " f\"Module {mod} ({pdu})\")"
+ "fig, ax = plt.subplots(figsize=(18, 10))\n",
+ "raw_mean = np.mean(raw, axis=1)\n",
+ "geom.plot_data_fast(\n",
+ " raw_mean,\n",
+ " ax=ax,\n",
+ " vmin=min(0.75*np.median(raw_mean[raw_mean > 0]), 2000),\n",
+ " vmax=max(1.5*np.median(raw_mean[raw_mean > 0]), 16000),\n",
+ " cmap=\"jet\",\n",
+ " colorbar={'shrink': 1, 'pad': 0.01},\n",
+ ")\n",
+ "ax.set_title(f'{karabo_id} - Mean RAW', size=18)\n",
+ "plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "### Mean RAW Preview ###\n",
+ "### Mean CORRECTED Preview ###\n",
"\n",
- "The per pixel mean of the sequence file of RAW data"
+ "The per pixel mean of the sequence file of CORR data"
]
},
{
@@ -677,27 +726,41 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " fig = plt.figure(figsize=(20, 10))\n",
- " ax = fig.add_subplot(111)\n",
- "\n",
- " im = ax.imshow(\n",
- " np.mean(rim_data[mod],axis=0),\n",
- " vmin=min(0.75*np.median(rim_data[mod][rim_data[mod] > 0]), 2000),\n",
- " vmax=max(1.5*np.median(rim_data[mod][rim_data[mod] > 0]), 16000),\n",
- " cmap=\"jet\")\n",
+ "fig, ax = plt.subplots(figsize=(18, 10))\n",
+ "corrected_mean = np.mean(corrected, axis=1)\n",
+ "geom.plot_data_fast(\n",
+ " corrected_mean,\n",
+ " ax=ax,\n",
+ " vmin= min(0.75*np.median(corrected_mean[corrected_mean > 0]), -0.5),\n",
+ " vmax = max(2.*np.median(corrected_mean[corrected_mean > 0]), 100),\n",
+ " cmap=\"jet\",\n",
+ " colorbar={'shrink': 1, 'pad': 0.01},\n",
+ ")\n",
+ "ax.set_title(f'{karabo_id} - Mean CORRECTED', size=18)\n",
"\n",
- " ax.set_title(f'Module {mod} ({pdu})')\n",
- " cb = fig.colorbar(im, ax=ax)"
+ "plt.show()"
]
},
{
- "cell_type": "markdown",
+ "cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
- "### Mean CORRECTED Preview ###\n",
+ "display(Markdown((f\"#### A single image from train {tid}\")))\n",
"\n",
- "The per pixel mean of the sequence file of CORR data"
+ "fig, ax = plt.subplots(figsize=(18, 10))\n",
+ "geom.plot_data_fast(\n",
+ " corrected_train,\n",
+ " ax=ax,\n",
+ " vmin=min(0.75 * np.median(corrected_train[corrected_train > 0]), -0.5),\n",
+ " vmax=max(2.0 * np.median(corrected_train[corrected_train > 0]), 100),\n",
+ " cmap=\"jet\",\n",
+ " colorbar={\"shrink\": 1, \"pad\": 0.01},\n",
+ ")\n",
+ "ax.set_title(f\"{karabo_id} - CORRECTED train: {tid}\", size=18)\n",
+ "\n",
+ "plt.show()"
]
},
{
@@ -706,27 +769,34 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " fig = plt.figure(figsize=(20, 10))\n",
+ "def do_2d_plot(data, edges, y_axis, x_axis, title):\n",
+ " fig = plt.figure(figsize=(10, 10))\n",
" ax = fig.add_subplot(111)\n",
+ " extent = [\n",
+ " np.min(edges[1]),\n",
+ " np.max(edges[1]),\n",
+ " np.min(edges[0]),\n",
+ " np.max(edges[0]),\n",
+ " ]\n",
"\n",
" im = ax.imshow(\n",
- " np.mean(fim_data[mod],axis=0),\n",
- " vmin=min(0.75*np.median(fim_data[mod][fim_data[mod] > 0]), -0.5),\n",
- " vmax=max(2.*np.median(fim_data[mod][fim_data[mod] > 0]), 100),\n",
- " cmap=\"jet\")\n",
- "\n",
- " ax.set_title(f'Module {mod} ({pdu})', size=18)\n",
- " cb = fig.colorbar(im, ax=ax)"
+ " data[::-1, :],\n",
+ " extent=extent,\n",
+ " aspect=\"auto\",\n",
+ " norm=LogNorm(vmin=1, vmax=np.max(data))\n",
+ " )\n",
+ " ax.set_xlabel(x_axis)\n",
+ " ax.set_ylabel(y_axis)\n",
+ " ax.set_title(title)\n",
+ " cb = fig.colorbar(im)\n",
+ " cb.set_label(\"Counts\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "### Single Train Preview ###\n",
- "\n",
- "A single image from the first train"
+ "### Gain Bit Value"
]
},
{
@@ -735,18 +805,20 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " fig = plt.figure(figsize=(20, 10))\n",
- " ax = fig.add_subplot(111)\n",
- "\n",
- " im = ax.imshow(\n",
- " fim_data[mod][0, ...],\n",
- " vmin=min(0.75*np.median(fim_data[mod][0, ...]), -0.5),\n",
- " vmax=max(2.*np.median(fim_data[mod][0, ...]), 100),\n",
- " cmap=\"jet\")\n",
- "\n",
- " ax.set_title(f'Module {mod} ({pdu})', size=18)\n",
- " cb = fig.colorbar(im, ax=ax)"
+ "for i, (pdu, mod) in enumerate(zip(db_modules, karabo_da)):\n",
+ " h, ex, ey = np.histogram2d(\n",
+ " raw[i].flatten(),\n",
+ " gain[i].flatten(),\n",
+ " bins=[100, 4],\n",
+ " range=[[0, 10000], [0, 4]],\n",
+ " )\n",
+ " do_2d_plot(\n",
+ " h,\n",
+ " (ex, ey),\n",
+ " \"Signal (ADU)\",\n",
+ " \"Gain Bit Value (high gain=0[00], medium gain=1[01], low gain=3[11])\",\n",
+ " f\"Module {mod} ({pdu})\",\n",
+ " )"
]
},
{
@@ -762,29 +834,19 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " fig = plt.figure(figsize=(20,10))\n",
- " ax = fig.add_subplot(211)\n",
- " h = ax.hist(\n",
- " fim_data[mod].flatten(),\n",
- " bins=1000,\n",
- " range=(-100, 1000),\n",
- " log=True,\n",
- " )\n",
- " l = ax.set_xlabel(\"Signal (keV)\")\n",
- " l = ax.set_ylabel(\"Counts\")\n",
- " _ = ax.set_title(f'Module {mod} ({pdu})')\n",
- "\n",
- " ax = fig.add_subplot(212)\n",
- " h = ax.hist(\n",
- " fim_data[mod].flatten(),\n",
- " bins=1000,\n",
- " range=(-1000, 10000),\n",
- " log=True,\n",
- " )\n",
- " l = ax.set_xlabel(\"Signal (keV)\")\n",
- " l = ax.set_ylabel(\"Counts\")\n",
- " _ = ax.set_title(f'Module {mod} ({pdu})')"
+ "for i, (pdu, mod) in enumerate(zip(db_modules, karabo_da)): \n",
+ " fig, axs = plt.subplots(nrows=2, ncols=1, figsize=(18, 10))\n",
+ " corrected_flatten = corrected[i].flatten()\n",
+ " for ax, hist_range in zip(axs, [(-100, 1000), (-1000, 10000)]):\n",
+ " h = ax.hist(\n",
+ " corrected_flatten,\n",
+ " bins=1000,\n",
+ " range=hist_range,\n",
+ " log=True,\n",
+ " )\n",
+ " l = ax.set_xlabel(\"Signal (keV)\")\n",
+ " l = ax.set_ylabel(\"Counts\")\n",
+ " _ = ax.set_title(f'Module {mod} ({pdu})')"
]
},
{
@@ -802,14 +864,17 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " fig = plt.figure(figsize=(20, 10))\n",
- " ax = fig.add_subplot(111)\n",
- " im = ax.imshow(\n",
- " np.max(gim_data[mod], axis=0),\n",
- " vmin=0, vmax=3, cmap=\"jet\")\n",
- " ax.set_title(f'Module {mod} ({pdu})', size=18)\n",
- " cb = fig.colorbar(im, ax=ax)"
+ "display(Markdown((f\"#### The per pixel maximum of train {tid} of the GAIN data\")))\n",
+ "\n",
+ "fig, ax = plt.subplots(figsize=(18, 10))\n",
+ "gain_max = np.max(gain_train_cells, axis=(1, 2))\n",
+ "geom.plot_data_fast(\n",
+ " gain_max,\n",
+ " ax=ax,\n",
+ " cmap=\"jet\",\n",
+ " colorbar={'shrink': 1, 'pad': 0.01},\n",
+ ")\n",
+ "plt.show()"
]
},
{
@@ -850,14 +915,16 @@
"metadata": {},
"outputs": [],
"source": [
- "for pdu, mod in zip(db_modules, karabo_da):\n",
- " fig = plt.figure(figsize=(20, 10))\n",
- " ax = fig.add_subplot(111)\n",
- " im = ax.imshow(\n",
- " np.log2(msk_data[mod][0,...]),\n",
- " vmin=0, vmax=32, cmap=\"jet\")\n",
- " ax.set_title(f'Module {mod} ({pdu})', size=18)\n",
- " cb = fig.colorbar(im, ax=ax)"
+ "display(Markdown(f\"#### Bad pixels image for train {tid}\"))\n",
+ "\n",
+ "fig, ax = plt.subplots(figsize=(18, 10))\n",
+ "geom.plot_data_fast(\n",
+ " np.log2(mask_train),\n",
+ " ax=ax,\n",
+ " vmin=0, vmax=32, cmap=\"jet\",\n",
+ " colorbar={'shrink': 1, 'pad': 0.01},\n",
+ ")\n",
+ "plt.show()"
]
}
],