From c2d3bbab1f6dbaa3db9eece4a47c8504c4a344c5 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Wed, 18 Jan 2023 23:00:27 +0100
Subject: [PATCH] skip module with no darks or raise error for all modules with
no dark
---
...Jungfrau_Gain_Correct_and_Verify_NBC.ipynb | 212 ++++++++++--------
...retrieve_constants_precorrection_NBC.ipynb | 10 +-
2 files changed, 122 insertions(+), 100 deletions(-)
diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
index 950638548..1c78def71 100644
--- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
@@ -92,7 +92,6 @@
"from matplotlib.colors import LogNorm\n",
"\n",
"import cal_tools.restful_config as rest_cfg\n",
- "from calibration_client import CalibrationClient\n",
"from cal_tools.calcat_interface import JUNGFRAU_CalibrationData\n",
"from cal_tools.jungfraulib import JungfrauCtrl\n",
"from cal_tools.enums import BadPixels\n",
@@ -130,6 +129,7 @@
"\n",
"print(f\"Run is: {run}\")\n",
"print(f\"Instrument H5File source: {instrument_src}\")\n",
+ "karabo_da = sorted(karabo_da)\n",
"print(f\"Process modules: {karabo_da}\")\n",
"\n",
"# Run's creation time:\n",
@@ -145,42 +145,6 @@
" sys.exit(1)"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Read available sequence files to correct.\n",
- "mapped_files, num_seq_files = map_seq_files(\n",
- " run_folder, karabo_da, sequences)\n",
- "\n",
- "if not len(mapped_files):\n",
- " raise IndexError(\n",
- " \"No sequence files available to correct for the selected sequences and karabo_da.\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "print(f\"Processing a total of {num_seq_files} sequence files\")\n",
- "table = []\n",
- "fi = 0\n",
- "for kda, sfiles in mapped_files.items():\n",
- " for k, f in enumerate(sfiles):\n",
- " if k == 0:\n",
- " table.append((fi, kda, k, f))\n",
- " else:\n",
- " table.append((fi, \"\", k, f))\n",
- " fi += 1\n",
- "md = display(Latex(tabulate.tabulate(\n",
- " table, tablefmt='latex',\n",
- " headers=[\"#\", \"module\", \"# module\", \"file\"])))"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -192,6 +156,7 @@
"\n",
"if mem_cells < 0:\n",
" memory_cells, sc_start = ctrl_data.get_memory_cells()\n",
+ "\n",
" mem_cells_name = \"single cell\" if memory_cells == 1 else \"burst\"\n",
" print(f\"Run is in {mem_cells_name} mode.\\nStorage cell start: {sc_start:02d}\")\n",
"else:\n",
@@ -212,25 +177,6 @@
"print(f\"Number of memory cells are {memory_cells}\")"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "if strixel_sensor:\n",
- " from cal_tools.jfstrixel import STRIXEL_SHAPE as strixel_frame_shape, double_pixel_indices, to_strixel\n",
- " Ydouble, Xdouble = double_pixel_indices()\n",
- " print('Strixel sensor transformation enabled')"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Retrieving calibration constants ###"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -255,9 +201,9 @@
"\n",
"if const_yaml:\n",
" const_data = dict()\n",
- " for mod in karabo_da:\n",
+ " for mod, constants in const_yaml.items():\n",
" const_data[mod] = dict()\n",
- " for cname, mdata in const_yaml[mod][\"constants\"].items():\n",
+ " for cname, mdata in constants[\"constants\"].items():\n",
" const_data[mod][cname] = dict()\n",
" if mdata[\"creation-time\"]:\n",
" with h5py.File(mdata[\"path\"], \"r\") as cf:\n",
@@ -268,22 +214,38 @@
" \"Offset10Hz\", \"BadPixelsDark10Hz\",\n",
" \"BadPixelsFF10Hz\", \"RelativeGain10Hz\",\n",
" ]\n",
- " const_data = jf_cal.ndarray_map(calibrations=constant_names)\n",
- "\n",
+ " const_data = jf_cal.ndarray_map(calibrations=constant_names)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
"# Validate the constants availability and raise/warn correspondingly. \n",
- "for mod, calibrations in const_data.items():\n",
+ "for mod in karabo_da[:]:\n",
+ " calibrations = const_data.get(mod, {})\n",
+ "\n",
" missing_dark_constants = set(\n",
" c for c in [\"Offset10Hz\", \"BadPixelsDark10Hz\"] if c not in calibrations.keys())\n",
" missing_gain_constants = set(\n",
" c for c in [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"] if relative_gain and c not in calibrations.keys()) # noqa\n",
+ "\n",
" if missing_dark_constants:\n",
- " raise KeyError(\n",
- " f\"Dark constants {missing_dark_constants} are not available for correction. Module: {mod}\") # noqa\n",
+ " warning(\n",
+ " f\"Dark constants {missing_dark_constants} are not available to correct {mod}. \"\n",
+ " f\"Module {mod} wont be corrected.\")\n",
+ " karabo_da.remove(mod)\n",
+ "\n",
" if missing_gain_constants:\n",
" warning(\n",
- " f\"Gain constants {missing_gain_constants} were not retrieved. Module: {mod}\")\n",
+ " f\"Gain constants {missing_gain_constants} were not retrieved for {mod}. \"\n",
+ " \"Relative gain correction is disabled.\")\n",
" if \"RelativeGain10Hz\" in missing_gain_constants:\n",
- " relative_gain = False"
+ " relative_gain = False\n",
+ "if not karabo_da: # Dark constants are missing for all modules.\n",
+ " raise ValueError(\"Dark constants are missing for all modules.\")"
]
},
{
@@ -303,12 +265,8 @@
" module (name of module),\n",
" \"\"\"\n",
" constant_arrays = const_data[module]\n",
- " try:\n",
- " offset_map = constant_arrays[\"Offset10Hz\"]\n",
- " mask = constant_arrays[\"BadPixelsDark10Hz\"]\n",
- " except KeyError as e:\n",
- " # Fail processing if dark constants are not available.\n",
- " raise KeyError(f\"Dark constant is not available for correction for {module}. {e}\")\n",
+ " offset_map = constant_arrays[\"Offset10Hz\"]\n",
+ " mask = constant_arrays[\"BadPixelsDark10Hz\"]\n",
" \n",
" gain_map = constant_arrays.get(\"RelativeGain10Hz\")\n",
" mask_ff = constant_arrays.get(\"BadPixelsFF10Hz\")\n",
@@ -353,6 +311,61 @@
"gc.collect();"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Read available sequence files to correct.\n",
+ "mapped_files, num_seq_files = map_seq_files(\n",
+ " run_folder, karabo_da, sequences)\n",
+ "\n",
+ "if not len(mapped_files):\n",
+ " raise IndexError(\n",
+ " \"No sequence files available to correct for the selected sequences and karabo_da.\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"Processing a total of {num_seq_files} sequence files\")\n",
+ "table = []\n",
+ "fi = 0\n",
+ "for kda, sfiles in mapped_files.items():\n",
+ " for k, f in enumerate(sfiles):\n",
+ " if k == 0:\n",
+ " table.append((fi, kda, k, f))\n",
+ " else:\n",
+ " table.append((fi, \"\", k, f))\n",
+ " fi += 1\n",
+ "md = display(Latex(tabulate.tabulate(\n",
+ " table, tablefmt='latex',\n",
+ " headers=[\"#\", \"module\", \"# module\", \"file\"])))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if strixel_sensor:\n",
+ " from cal_tools.jfstrixel import STRIXEL_SHAPE as strixel_frame_shape, double_pixel_indices, to_strixel\n",
+ " Ydouble, Xdouble = double_pixel_indices()\n",
+ " print('Strixel sensor transformation enabled')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Retrieving calibration constants ###"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -667,12 +680,15 @@
" (-x_start, -y_start + (i * (mod_height + 33))) for i in range(4)\n",
" ]\n",
" orientations = [(-1, -1) for _ in range(4)] + [(1, 1) for _ in range(4)]\n",
+ " nmods = 8\n",
"elif karabo_id == \"FXE_XAD_JF1M\":\n",
" module_pos = ((-mod_width//2, 33),(-mod_width//2, -mod_height -33))\n",
" orientations = [(-1,-1), (1,1)]\n",
+ " nmods = 2\n",
"else:\n",
" module_pos = ((-mod_width//2,-mod_height//2),)\n",
" orientations = None\n",
+ " nmods = 1\n",
"\n",
"geom = JUNGFRAUGeometry.from_module_positions(module_pos, orientations=orientations, asic_gap=0)"
]
@@ -692,7 +708,7 @@
" # Reading CORR data for plotting.\n",
" jf_corr = components.JUNGFRAU(\n",
" corr_dc,\n",
- " detector_name=karabo_id,\n",
+ " detector_name=karabo_id, n_modules=nmods,\n",
" ).select_trains(np.s_[:plot_trains])\n",
" tid, jf_corr_data = next(iter(jf_corr.trains(require_all=True)))\n",
"\n",
@@ -700,31 +716,31 @@
"# TODO: Fix the case if not all modules were requested to be corrected.\n",
"# For example if only one modules was corrected. An assertion error is expected\n",
"# at `geom.plot_data_fast`, while plotting corrected images.\n",
- "corrected = jf_corr.get_array(\"data.adc\")[:, :, cell_idx_preview, ...].values\n",
+ "corrected = jf_corr.get_array(\"data.adc\")[:, :, cell_idx_preview, ...]\n",
"corrected_train = jf_corr_data[\"data.adc\"][\n",
" :, cell_idx_preview, ...\n",
- "].values # loose the train axis.\n",
+ "] # loose the train axis.\n",
"\n",
- "mask = jf_corr.get_array(\"data.mask\")[:, :, cell_idx_preview, ...].values\n",
- "mask_train = jf_corr_data[\"data.mask\"][:, cell_idx_preview, ...].values\n",
+ "mask = jf_corr.get_array(\"data.mask\")[:, :, cell_idx_preview, ...]\n",
+ "mask_train = jf_corr_data[\"data.mask\"][:, cell_idx_preview, ...]\n",
"\n",
"with RunDirectory(f\"{in_folder}/r{run:04d}/\", f\"*S{first_seq:05d}*\", _use_voview=False) as raw_dc:\n",
"\n",
" # Reading RAW data for plotting.\n",
- " jf_raw = components.JUNGFRAU(raw_dc, detector_name=karabo_id).select_trains(\n",
- " np.s_[:plot_trains]\n",
+ " jf_raw = components.JUNGFRAU(\n",
+ " raw_dc, detector_name=karabo_id, n_modules=nmods).select_trains(\n",
+ " np.s_[:plot_trains]\n",
" )\n",
"\n",
- "raw = jf_raw.get_array(\"data.adc\")[:, :, cell_idx_preview, ...].values\n",
+ "raw = jf_raw.get_array(\"data.adc\")[:, :, cell_idx_preview, ...]\n",
"raw_train = (\n",
" jf_raw.select_trains(by_id[[tid]])\n",
" .get_array(\"data.adc\")[:, 0, cell_idx_preview, ...]\n",
- " .values\n",
")\n",
"\n",
- "gain = jf_raw.get_array(\"data.gain\")[:, :, cell_idx_preview, ...].values\n",
+ "gain = jf_raw.get_array(\"data.gain\")[:, :, cell_idx_preview, ...]\n",
"gain_train_cells = (\n",
- " jf_raw.select_trains(by_id[[tid]]).get_array(\"data.gain\")[:, :, :, ...].values\n",
+ " jf_raw.select_trains(by_id[[tid]]).get_array(\"data.gain\")[:, :, :, ...]\n",
")"
]
},
@@ -745,11 +761,13 @@
"\n",
"fig, ax = plt.subplots(figsize=(18, 10))\n",
"raw_mean = np.mean(raw, axis=1)\n",
+ "raw_mean_arr = raw_mean.values\n",
+ "\n",
"geom.plot_data_fast(\n",
" raw_mean,\n",
" ax=ax,\n",
- " vmin=min(0.75*np.median(raw_mean[raw_mean > 0]), 2000),\n",
- " vmax=max(1.5*np.median(raw_mean[raw_mean > 0]), 16000),\n",
+ " vmin=min(0.75*np.median(raw_mean_arr[raw_mean_arr > 0]), 2000),\n",
+ " vmax=max(1.5*np.median(raw_mean_arr[raw_mean_arr > 0]), 16000),\n",
" cmap=\"jet\",\n",
" colorbar={'shrink': 1, 'pad': 0.01},\n",
")\n",
@@ -774,8 +792,9 @@
"\n",
"fig, ax = plt.subplots(figsize=(18, 10))\n",
"corrected_mean = np.mean(corrected, axis=1)\n",
- "_corrected_vmin = min(0.75*np.median(corrected_mean[corrected_mean > 0]), -0.5)\n",
- "_corrected_vmax = max(2.*np.median(corrected_mean[corrected_mean > 0]), 100)\n",
+ "corrected_mean_arr = corrected_mean.values\n",
+ "_corrected_vmin = min(0.75*np.median(corrected_mean_arr[corrected_mean_arr > 0]), -0.5)\n",
+ "_corrected_vmax = max(2.*np.median(corrected_mean_arr[corrected_mean_arr > 0]), 100)\n",
"\n",
"mean_plot_kwargs = dict(\n",
" vmin=_corrected_vmin, vmax=_corrected_vmax, cmap=\"jet\"\n",
@@ -804,8 +823,7 @@
"source": [
"fig, ax = plt.subplots(figsize=(18, 10))\n",
"corrected_masked = corrected.copy()\n",
- "corrected_masked[mask != 0] = np.nan\n",
- "corrected_masked_mean = np.nanmean(corrected_masked, axis=1)\n",
+ "corrected_masked_mean = corrected_masked.where(mask == 0).mean(dim=\"train\", skipna=False)\n",
"del corrected_masked\n",
"\n",
"if not strixel_sensor:\n",
@@ -832,10 +850,10 @@
"display(Markdown((f\"#### A single image from train {tid}\")))\n",
"\n",
"fig, ax = plt.subplots(figsize=(18, 10))\n",
- "\n",
+ "median_corr_train = np.median(corrected_train.values[corrected_train.values > 0])\n",
"single_plot_kwargs = dict(\n",
- " vmin=min(0.75 * np.median(corrected_train[corrected_train > 0]), -0.5),\n",
- " vmax=max(2.0 * np.median(corrected_train[corrected_train > 0]), 100),\n",
+ " vmin=min(0.75 * median_corr_train, -0.5),\n",
+ " vmax=max(2.0 * median_corr_train, 100),\n",
" cmap=\"jet\"\n",
")\n",
"\n",
@@ -896,10 +914,11 @@
"metadata": {},
"outputs": [],
"source": [
- "for i, (pdu, mod) in enumerate(da_to_pdu.items()): \n",
+ "for i, mod in enumerate(karabo_da):\n",
+ " pdu = da_to_pdu[mod]\n",
" h, ex, ey = np.histogram2d(\n",
- " raw[i].flatten(),\n",
- " gain[i].flatten(),\n",
+ " raw[i].values.flatten(),\n",
+ " gain[i].values.flatten(),\n",
" bins=[100, 4],\n",
" range=[[0, 10000], [0, 4]],\n",
" )\n",
@@ -925,9 +944,10 @@
"metadata": {},
"outputs": [],
"source": [
- "for i, (pdu, mod) in enumerate(da_to_pdu.items()): \n",
+ "for i, mod in enumerate(karabo_da):\n",
+ " pdu = da_to_pdu[mod]\n",
" fig, axs = plt.subplots(nrows=2, ncols=1, figsize=(18, 10))\n",
- " corrected_flatten = corrected[i].flatten()\n",
+ " corrected_flatten = corrected[i].values.flatten()\n",
" for ax, hist_range in zip(axs, [(-100, 1000), (-1000, 10000)]):\n",
" h = ax.hist(\n",
" corrected_flatten,\n",
diff --git a/notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb
index 3916ed75a..4895a13b6 100644
--- a/notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb
@@ -56,7 +56,6 @@
"from extra_data import RunDirectory\n",
"from pathlib import Path\n",
"\n",
- "from calibration_client import CalibrationClient\n",
"from cal_tools.calcat_interface import JUNGFRAU_CalibrationData\n",
"from cal_tools.jungfraulib import JungfrauCtrl\n",
"import cal_tools.restful_config as rest_cfg\n",
@@ -171,17 +170,20 @@
" mod_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n",
"\n",
"# Validate the constants availability and raise/warn correspondingly. \n",
+ "missing_dark_modules = set()\n",
"for mod, calibrations in jf_metadata.items():\n",
" missing_dark_constants = set(\n",
" c for c in [\"Offset10Hz\", \"BadPixelsDark10Hz\"] if c not in calibrations.keys())\n",
" missing_gain_constants = set(\n",
" c for c in [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"] if relative_gain and c not in calibrations.keys()) # noqa\n",
" if missing_dark_constants:\n",
- " raise KeyError(\n",
- " f\"Dark constants {missing_dark_constants} are not available for correction. Module: {mod}\") # noqa\n",
+ " warning(\n",
+ " f\"Dark constants {missing_dark_constants} are not available to correct {mod}\")\n",
" if missing_gain_constants:\n",
" warning(\n",
- " f\"Gain constants {missing_gain_constants} were not retrieved. Module: {mod}\")"
+ " f\"Gain constants {missing_gain_constants} were not retrieved. Module: {mod}\")\n",
+ "if missing_dark_modules == set(karabo_da):\n",
+ " raise ValueError(f\"Dark constants are missing for all modules {karabo_da}.\")"
]
},
{
--
GitLab