From 21994635bfdfccdc6f360ccb76c417d4874fb362 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Wed, 11 Sep 2024 14:36:28 +0200
Subject: [PATCH] Update plots and add new masked_corrected plot - Also remove
the constants_file part as it is outdated and was needed for new detector
integration only
---
.../Gotthard2/Correction_Gotthard2_NBC.ipynb | 213 +++++++++---------
1 file changed, 108 insertions(+), 105 deletions(-)
diff --git a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
index b2276642a..f1961bf82 100644
--- a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
+++ b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
@@ -73,7 +73,6 @@
"creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
"\n",
"# Parameters affecting corrected data.\n",
- "constants_file = \"\" # Use constants in given constant file path. /gpfs/exfel/data/scratch/ahmedk/dont_remove/gotthard2/constants/calibration_constants_GH2.h5\n",
"offset_correction = True # apply offset correction. This can be disabled to only apply LUT or apply LUT and gain correction for non-linear differential results.\n",
"gain_correction = True # apply gain correction.\n",
"chunks_data = 1 # HDF chunk size for pixel data in number of frames.\n",
@@ -288,40 +287,30 @@
"# Used for old FXE (p003225) runs before adding Gotthard2 to CALCAT\n",
"const_data = dict()\n",
"\n",
- "if constants_file:\n",
- " for mod in karabo_da:\n",
- " const_data[mod] = dict()\n",
- " # load constants temporarily using defined local paths.\n",
- " with h5py.File(constants_file, \"r\") as cfile:\n",
- " const_data[mod][\"LUTGotthard2\"] = cfile[\"LUT\"][()]\n",
- " const_data[mod][\"OffsetGotthard2\"] = cfile[\"offset_map\"][()].astype(np.float32)\n",
- " const_data[mod][\"RelativeGainGotthard2\"] = cfile[\"gain_map\"][()].astype(np.float32)\n",
- " const_data[mod][\"Mask\"] = cfile[\"bpix_ff\"][()].astype(np.uint32)\n",
- "else:\n",
- " constant_names = [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"]\n",
- " if gain_correction:\n",
- " constant_names += [\"RelativeGainGotthard2\", \"BadPixelsFFGotthard2\"]\n",
+ "constant_names = [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"]\n",
+ "if gain_correction:\n",
+ " constant_names += [\"RelativeGainGotthard2\", \"BadPixelsFFGotthard2\"]\n",
"\n",
- " g2_metadata = g2_cal.metadata(calibrations=constant_names)\n",
- " # Display retrieved calibration constants timestamps\n",
- " g2_cal.display_markdown_retrieved_constants(metadata=g2_metadata)\n",
+ "g2_metadata = g2_cal.metadata(calibrations=constant_names)\n",
+ "# Display retrieved calibration constants timestamps\n",
+ "g2_cal.display_markdown_retrieved_constants(metadata=g2_metadata)\n",
"\n",
- " # Validate the constants availability and raise/warn correspondingly.\n",
- " for mod, calibrations in g2_metadata.items():\n",
+ "# Validate the constants availability and raise/warn correspondingly.\n",
+ "for mod, calibrations in g2_metadata.items():\n",
"\n",
- " dark_constants = {\"LUTGotthard2\"}\n",
- " if offset_correction:\n",
- " dark_constants |= {\"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"}\n",
+ " dark_constants = {\"LUTGotthard2\"}\n",
+ " if offset_correction:\n",
+ " dark_constants |= {\"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"}\n",
"\n",
- " missing_dark_constants = dark_constants - set(calibrations)\n",
- " if missing_dark_constants:\n",
- " karabo_da.remove(mod)\n",
- " warning(f\"Dark constants {missing_dark_constants} are not available to correct {mod}.\") # noqa\n",
+ " missing_dark_constants = dark_constants - set(calibrations)\n",
+ " if missing_dark_constants:\n",
+ " karabo_da.remove(mod)\n",
+ " warning(f\"Dark constants {missing_dark_constants} are not available to correct {mod}.\") # noqa\n",
"\n",
- " missing_gain_constants = {\n",
- " \"BadPixelsFFGotthard2\", \"RelativeGainGotthard2\"} - set(calibrations)\n",
- " if gain_correction and missing_gain_constants:\n",
- " warning(f\"Gain constants {missing_gain_constants} are not retrieved for mod {mod}.\")\n",
+ " missing_gain_constants = {\n",
+ " \"BadPixelsFFGotthard2\", \"RelativeGainGotthard2\"} - set(calibrations)\n",
+ " if gain_correction and missing_gain_constants:\n",
+ " warning(f\"Gain constants {missing_gain_constants} are not retrieved for mod {mod}.\")\n",
"\n",
"if not karabo_da:\n",
" raise ValueError(\"Dark constants are not available for all modules.\")"
@@ -344,26 +333,25 @@
"const_data = g2_cal.ndarray_map(metadata=g2_metadata)\n",
"\n",
"# Prepare constant arrays.\n",
- "if not constants_file:\n",
- " for mod in karabo_da:\n",
- " # Create the mask array.\n",
- " bpix = const_data[mod].get(\"BadPixelsDarkGotthard2\")\n",
- " if bpix is None:\n",
- " bpix = np.zeros((1280, 2, 3), dtype=np.uint32)\n",
- " if const_data[mod].get(\"BadPixelsFFGotthard2\") is not None:\n",
- " bpix |= const_data[mod][\"BadPixelsFFGotthard2\"]\n",
- " const_data[mod][\"Mask\"] = bpix\n",
- "\n",
- " # Prepare empty arrays for missing constants.\n",
- " if const_data[mod].get(\"OffsetGotthard2\") is None:\n",
- " const_data[mod][\"OffsetGotthard2\"] = np.zeros(\n",
- " (1280, 2, 3), dtype=np.float32)\n",
- "\n",
- " if const_data[mod].get(\"RelativeGainGotthard2\") is None:\n",
- " const_data[mod][\"RelativeGainGotthard2\"] = np.ones(\n",
- " (1280, 2, 3), dtype=np.float32)\n",
- " const_data[mod][\"RelativeGainGotthard2\"] = const_data[mod][\"RelativeGainGotthard2\"].astype( # noqa\n",
- " np.float32, copy=False) # Old gain constants are not float32."
+ "for mod in karabo_da:\n",
+ " # Create the mask array.\n",
+ " bpix = const_data[mod].get(\"BadPixelsDarkGotthard2\")\n",
+ " if bpix is None:\n",
+ " bpix = np.zeros((1280, 2, 3), dtype=np.uint32)\n",
+ " if const_data[mod].get(\"BadPixelsFFGotthard2\") is not None:\n",
+ " bpix |= const_data[mod][\"BadPixelsFFGotthard2\"]\n",
+ " const_data[mod][\"Mask\"] = bpix\n",
+ "\n",
+ " # Prepare empty arrays for missing constants.\n",
+ " if const_data[mod].get(\"OffsetGotthard2\") is None:\n",
+ " const_data[mod][\"OffsetGotthard2\"] = np.zeros(\n",
+ " (1280, 2, 3), dtype=np.float32)\n",
+ "\n",
+ " if const_data[mod].get(\"RelativeGainGotthard2\") is None:\n",
+ " const_data[mod][\"RelativeGainGotthard2\"] = np.ones(\n",
+ " (1280, 2, 3), dtype=np.float32)\n",
+ " const_data[mod][\"RelativeGainGotthard2\"] = const_data[mod][\"RelativeGainGotthard2\"].astype( # noqa\n",
+ " np.float32, copy=False) # Old gain constants are not float32."
]
},
{
@@ -572,6 +560,7 @@
" tid, mod_dcs[corr_data_source][\"train_corr_data\"] = next(\n",
" out_dc[corr_data_source, \"data.adc\"].trains()\n",
" )\n",
+ " mask = out_dc[corr_data_source, \"data.mask\"].train_from_id(tid)[1]\n",
"\n",
"if gh2_detector == \"25um\":\n",
" mod_dcs[corr_data_source][\"train_raw_data\"] = np.zeros((data_corr.shape[1], 1280 * 2), dtype=np.float32)\n",
@@ -597,39 +586,47 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "1b379438-eb1d-42b2-ac83-eb8cf88c46db",
+ "id": "bcb2c472",
"metadata": {},
"outputs": [],
"source": [
+ "def plot_mean_data(data, title_format, y_label, fig_size=(10, 6)):\n",
+ " _, ax = plt.subplots(figsize=fig_size)\n",
+ " mean_data = np.mean(data, axis=0)\n",
+ " \n",
+ " ax.plot(mean_data)\n",
+ " ax.set_title(title_format, fontsize=12)\n",
+ " ax.set_xlabel(\"Strip #\", size=12)\n",
+ " ax.set_ylabel(y_label, size=12)\n",
+ " plt.xticks(fontsize=10)\n",
+ " plt.yticks(fontsize=10)\n",
+ " plt.show()\n",
+ "\n",
+ "# Display markdown headers\n",
"display(Markdown(\"### Mean RAW and CORRECTED across pulses for one train:\"))\n",
"display(Markdown(f\"Train: {tid}\"))\n",
"\n",
+ "# Set title format based on gh2_detector\n",
"if gh2_detector == \"50um\":\n",
- " title = f\"{{}} data for {karabo_da} ({db_modules})\"\n",
+ " title_format = f\"{{}} data for {karabo_da} ({db_modules})\"\n",
"else:\n",
- " title = f\"Interleaved {{}} data for {karabo_da} ({db_modules})\"\n",
+ " title_format = f\"Interleaved {{}} data for {karabo_da} ({db_modules})\"\n",
"\n",
"step_timer.start()\n",
"\n",
- "fig, ax = plt.subplots(figsize=(15, 15))\n",
+ "# Plot RAW data\n",
"raw_data = mod_dcs[corr_data_source][\"train_raw_data\"]\n",
- "im = ax.plot(np.mean(raw_data, axis=0))\n",
- "ax.set_title(title.format(\"RAW\"), fontsize=20)\n",
- "ax.set_xlabel(\"Strip #\", size=20)\n",
- "ax.set_ylabel(\"12-bit ADC output\", size=20)\n",
- "plt.xticks(fontsize=20)\n",
- "plt.yticks(fontsize=20)\n",
- "pass\n",
- "\n",
- "fig, ax = plt.subplots(figsize=(15, 15))\n",
+ "plot_mean_data(raw_data, title_format.format(\"RAW\"), \"12-bit ADC output\")\n",
+ "\n",
+ "# Plot CORRECTED data\n",
"corr_data = mod_dcs[corr_data_source][\"train_corr_data\"]\n",
- "im = ax.plot(np.mean(corr_data, axis=0))\n",
- "ax.set_title(title.format(\"CORRECTED\"), fontsize=20)\n",
- "ax.set_xlabel(\"Strip #\", size=20)\n",
- "ax.set_ylabel(\"10-bit KeV. output\", size=20)\n",
- "plt.xticks(fontsize=20)\n",
- "plt.yticks(fontsize=20)\n",
- "pass\n",
+ "plot_mean_data(corr_data, title_format.format(\"CORRECTED\"), \"10-bit KeV. output\")\n",
+ "\n",
+ "# Plot CORRECTED MASKED data\n",
+ "corr_data_masked = corr_data.copy()\n",
+ "corr_data_masked[mask!=0] = np.nan\n",
+ "plot_mean_data(corr_data_masked, title_format.format(\"CORRECTED MASKED\"), \"10-bit KeV. output\")\n",
+ "\n",
"step_timer.done_step(\"Plotting mean data\")"
]
},
@@ -641,23 +638,27 @@
"outputs": [],
"source": [
"display(Markdown(f\"### RAW and CORRECTED strips across pulses for train {tid}\"))\n",
- "\n",
"step_timer.start()\n",
"for plt_data, dname in zip(\n",
- " [\"train_raw_data\", \"train_corr_data\"], [\"RAW\", \"CORRECTED\"]\n",
+ " [raw_data, corr_data, corr_data_masked], [\"RAW\", \"CORRECTED\", \"CORRECTED MASKED\"]\n",
"):\n",
- " fig, ax = plt.subplots(figsize=(15, 15))\n",
- " plt.rcParams.update({\"font.size\": 20})\n",
- "\n",
+ " fig, ax = plt.subplots(figsize=(10, 6))\n",
+ " plt.rcParams.update({\"font.size\": 12})\n",
+ " vmin, vmax = np.nanpercentile(plt_data, [1, 99])\n",
+ " cb_aspect = max(2, (vmax-vmin) / 25)\n",
" heatmapPlot(\n",
- " mod_dcs[corr_data_source][plt_data],\n",
+ " plt_data,\n",
" y_label=\"Pulses\",\n",
" x_label=\"Strips\",\n",
- " title=title.format(dname),\n",
+ " title=title_format.format(dname),\n",
" use_axis=ax,\n",
- " cb_pad=0.8,\n",
+ " cb_aspect=cb_aspect, # Adjusted for vertical colorbar\n",
+ " # cb_size=0.1,\n",
+ " vmin=vmin, vmax=vmax,\n",
+ " cmap='viridis',\n",
" )\n",
" pass\n",
+ "\n",
"step_timer.done_step(\"Plotting RAW and CORRECTED data for one train\")"
]
},
@@ -695,39 +696,41 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "e5f0d4d8-e32c-4f2c-8469-4ebbfd3f644c",
+ "id": "e2ecafe1",
"metadata": {},
"outputs": [],
"source": [
+ "def plot_pulses(data, odd_pulse, even_pulse, title, y_label, fig_size=(10, 6)):\n",
+ " fig, ax = plt.subplots(figsize=fig_size)\n",
+ " ax.plot(data[odd_pulse], label=f\"Odd Pulse {odd_pulse}\")\n",
+ " if even_pulse:\n",
+ " ax.plot(data[even_pulse], label=f\"Even Pulse {even_pulse}\")\n",
+ " ax.set_title(title, fontsize=12)\n",
+ " ax.set_xlabel(\"Strip #\", size=12)\n",
+ " ax.set_ylabel(y_label, size=12)\n",
+ " plt.xticks(fontsize=12)\n",
+ " plt.yticks(fontsize=12)\n",
+ " ax.legend()\n",
+ " plt.show()\n",
+ "\n",
+ "# Display markdown headers\n",
"display(Markdown(\"### RAW and CORRECTED even/odd pulses for one train:\"))\n",
"display(Markdown(f\"Train: {tid}\"))\n",
- "fig, ax = plt.subplots(figsize=(15, 15))\n",
- "raw_data = mod_dcs[corr_data_source][\"train_raw_data\"]\n",
- "corr_data = mod_dcs[corr_data_source][\"train_corr_data\"]\n",
"\n",
- "ax.plot(raw_data[odd_pulse], label=f\"Odd Pulse {odd_pulse}\")\n",
- "if even_pulse:\n",
- " ax.plot(raw_data[even_pulse], label=f\"Even Pulse {even_pulse}\")\n",
- "\n",
- "ax.set_title(title.format(\"RAW\"), fontsize=20)\n",
- "ax.set_xlabel(\"Strip #\", size=20)\n",
- "ax.set_ylabel(\"12-bit ADC RAW\", size=20)\n",
- "plt.xticks(fontsize=20)\n",
- "plt.yticks(fontsize=20)\n",
- "ax.legend()\n",
- "pass\n",
- "\n",
- "fig, ax = plt.subplots(figsize=(15, 15))\n",
- "ax.plot(corr_data[odd_pulse], label=f\"Odd Pulse {odd_pulse}\")\n",
- "if even_pulse:\n",
- " ax.plot(corr_data[even_pulse], label=f\"Even Pulse {even_pulse}\")\n",
- "ax.set_title(title.format(\"CORRECTED\"), fontsize=20)\n",
- "ax.set_xlabel(\"Strip #\", size=20)\n",
- "ax.set_ylabel(\"10-bit KeV CORRECTED\", size=20)\n",
- "plt.xticks(fontsize=20)\n",
- "plt.yticks(fontsize=20)\n",
- "ax.legend()\n",
- "pass\n",
+ "# Plot RAW data\n",
+ "plot_pulses(\n",
+ " raw_data, odd_pulse, even_pulse, title_format.format(\"RAW\"), \"12-bit ADC RAW\")\n",
+ "\n",
+ "# Plot CORRECTED data\n",
+ "plot_pulses(\n",
+ " corr_data, odd_pulse, even_pulse,\n",
+ " title_format.format(\"CORRECTED\"), \"10-bit KeV CORRECTED\")\n",
+ "\n",
+ "# Plot CORRECTED MASKED data\n",
+ "plot_pulses(\n",
+ " corr_data_masked, odd_pulse, even_pulse,\n",
+ " title_format.format(\"CORRECTED MASKED\"), \"10-bit KeV CORRECTED MASKED\")\n",
+ "\n",
"step_timer.done_step(\"Plotting RAW and CORRECTED odd/even pulses.\")"
]
}
--
GitLab