From 01fa30656e3cad7182d68002576b2805a19a4e91 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Tue, 13 Jun 2023 14:05:07 +0200
Subject: [PATCH] update correction notebook with func to get seq files and
better karabo_da handline
---
.../Gotthard2/Correction_Gotthard2_NBC.ipynb | 138 +++++++++---------
1 file changed, 73 insertions(+), 65 deletions(-)
diff --git a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
index f911c682e..7d2efd56c 100644
--- a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
+++ b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
@@ -1,7 +1,6 @@
{
"cells": [
{
- "attachments": {},
"cell_type": "markdown",
"id": "bed7bd15-21d9-4735-82c1-c27c1a5e3346",
"metadata": {},
@@ -30,12 +29,12 @@
"In case one of the gain constants were not retrieved `gain_correction` is switched to False and gain correction is disabled.\n",
"\n",
"The `data` datasets stored in the RECEIVER source along with the corrected image (`adc`) and `mask` are: \n",
- " - gain\n",
- " - bunchId\n",
- " - memoryCell\n",
- " - frameNumber\n",
- " - timestamp\n",
- " - trainId"
+ " - `gain`\n",
+ " - `bunchId`\n",
+ " - `memoryCell`\n",
+ " - `frameNumber`\n",
+ " - `timestamp`\n",
+ " - `trainId`"
]
},
{
@@ -114,6 +113,7 @@
"from cal_tools.tools import (\n",
" calcat_creation_time,\n",
" write_constants_fragment,\n",
+ " map_seq_files,\n",
")\n",
"from XFELDetAna.plotting.heatmap import heatmapPlot\n",
"\n",
@@ -146,34 +146,6 @@
"print(f\"Creation time: {creation_time}\")"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "b5eb816e-b5f2-44ce-9907-0273d82341b6",
- "metadata": {},
- "outputs": [],
- "source": [
- "# Select only sequence files to process for the selected detector.\n",
- "if sequences == [-1]:\n",
- " possible_patterns = list(f\"*{mod}*.h5\" for mod in karabo_da)\n",
- "else:\n",
- " possible_patterns = list(\n",
- " f\"*{mod}-S{s:05d}.h5\" for mod in karabo_da for s in sequences\n",
- " )\n",
- "\n",
- "run_folder = Path(in_folder / f\"r{run:04d}\")\n",
- "seq_files = [\n",
- " f for f in run_folder.glob(\"*.h5\") if any(f.match(p) for p in possible_patterns)\n",
- "]\n",
- "\n",
- "seq_files = sorted(seq_files)\n",
- "\n",
- "if not seq_files:\n",
- " raise ValueError(\"No sequence files available for the selected sequences.\")\n",
- "\n",
- "print(f\"Processing a total of {len(seq_files)} sequence files\")"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -211,24 +183,65 @@
"print(\"Exposure Time:\", exposure_time)\n",
"print(\"Exposure Period:\", exposure_period)\n",
"print(\"Acquisition Rate:\", acquisition_rate)\n",
- "print(\"Single Photon:\", single_photon)\n",
+ "print(\"Single Photon:\", single_photon)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "21a8953a-8c76-475e-8f4f-b201cc25c159",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "g2_cal = GOTTHARD2_CalibrationData(\n",
+ " detector_name=karabo_id,\n",
+ " sensor_bias_voltage=bias_voltage,\n",
+ " exposure_time=exposure_time,\n",
+ " exposure_period=exposure_period,\n",
+ " acquisition_rate=acquisition_rate,\n",
+ " single_photon=single_photon,\n",
+ " event_at=creation_time,\n",
+ " client=rest_cfg.calibration_client(),\n",
+ ")\n",
"\n",
+ "# Keep as long as it is essential to correct\n",
+ "# RAW data (FXE p003225) before the data mapping was added to CALCAT.\n",
+ "try: # in case local constants are used with old RAW data. This can be removed in the future.\n",
+ " da_to_pdu = g2_cal.mod_to_pdu\n",
+ "except CalCatError as e:\n",
+ " print(e)\n",
+ " db_modules = [None] * len(karabo_da)\n",
+ "\n",
+ "if karabo_da == [\"\"]:\n",
+ " karabo_da = sorted(da_to_pdu.keys())\n",
+ "else:\n",
+ " # Filter DA connected to detector in CALCAT\n",
+ " karabo_da = [da for da in karabo_da if da in da_to_pdu]\n",
+ " # Exclude non selected DA from processing.\n",
+ "\n",
+ "db_modules = [da_to_pdu[da] for da in karabo_da]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9609a142-99a4-42e5-94f0-abe9cc99cc21",
+ "metadata": {},
+ "outputs": [],
+ "source": [
"# Decide if GH2 is 25um or 50um\n",
"gh2_hostname = run_dc.get_run_value(ctrl_src, \"rxHostname\")\n",
"# gh2_hostname is a vector of bytes objects.\n",
"# GH2 25um has two hostnames unlike 50um which has one.\n",
"gh2_detector = \"25um\" if gh2_hostname[1].decode(\"utf-8\") else \"50um\"\n",
+ "print(f\"Processing {gh2_detector} Gotthard2.\")\n",
"\n",
- "# Decide if we are processing 2 25um or 1 50um Gotthard-II\n",
- "\n",
+ "# Create the correction receiver name.\n",
"receivers = sorted(list(run_dc.select(f'{karabo_id}/DET/{receiver_template}*').all_sources))\n",
- "\n",
"if gh2_detector == \"25um\": # For 25um use virtual karabo_das for CALCAT data mapping.\n",
- " karabo_da = [f\"{karabo_da[0]}/1\", f\"{karabo_da[0]}/2\"]\n",
- " print(\"Processing 25um Gotthard2.\")\n",
- " corr_karabo_da = receivers[0].split(\"/\")[-1].split(\":\")[0][:-2]\n",
+ " corr_receiver = receivers[0].split(\"/\")[-1].split(\":\")[0][:-2]\n",
"else:\n",
- " corr_karabo_da = receivers[0].split(\"/\")[-1].split(\":\")[0]"
+ " corr_receiver = receivers[0].split(\"/\")[-1].split(\":\")[0]"
]
},
{
@@ -246,30 +259,9 @@
"metadata": {},
"outputs": [],
"source": [
- "da_to_pdu = {}\n",
"# Used for old FXE (p003225) runs before adding Gotthard2 to CALCAT\n",
"const_data = dict()\n",
"\n",
- "g2_cal = GOTTHARD2_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " sensor_bias_voltage=bias_voltage,\n",
- " exposure_time=exposure_time,\n",
- " exposure_period=exposure_period,\n",
- " acquisition_rate=acquisition_rate,\n",
- " single_photon=single_photon,\n",
- " event_at=creation_time,\n",
- " client=rest_cfg.calibration_client(),\n",
- ")\n",
- "# Keep as long as it is essential to correct\n",
- "# RAW data (FXE p003225) before the data mapping was added to CALCAT.\n",
- "try: # in case local constants are used with old RAW data. This can be removed in the future.\n",
- " for mod_info in g2_cal.physical_detector_units.values():\n",
- " da_to_pdu[mod_info[\"karabo_da\"]] = mod_info[\"physical_name\"]\n",
- " db_modules = [da_to_pdu[da] for da in karabo_da]\n",
- "except CalCatError as e:\n",
- " print(e)\n",
- " db_modules = [None] * len(karabo_da)\n",
- "\n",
"if constants_file:\n",
" for mod in karabo_da:\n",
" const_data[mod] = dict()\n",
@@ -343,6 +335,22 @@
" (1280, 2, 3), dtype=np.float32)\n",
" const_data[mod][\"RelativeGainGotthard2\"] = const_data[mod][\"RelativeGainGotthard2\"].astype( # noqa\n",
" np.float32, copy=False) # Old gain constants are not float32."
+ "id": "b41f3720-2abd-43ed-b948-8d2f0815b6a3",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "raw_da = karabo_da if gh2_detector == \"50um\" else [karabo_da[0].split(\"/\")[0]]\n",
+ "mapped_files, total_files = map_seq_files(\n",
+ " run_folder,\n",
+ " raw_da,\n",
+ " sequences,\n",
+ ")\n",
+ "seq_files = mapped_files[raw_da[0]]\n",
+ "\n",
+ "if not len(seq_files):\n",
+ " raise IndexError(\n",
+ " \"No sequence files available to correct for the selected sequences and karabo_da.\")\n",
+ "print(f\"Processing a total of {total_files} sequence files\")"
]
},
{
@@ -384,7 +392,7 @@
"metadata": {},
"outputs": [],
"source": [
- "corr_data_source = f\"{karabo_id}/DET/{corr_karabo_da}:daqOutput\"\n",
+ "corr_data_source = f\"{karabo_id}/DET/{corr_receiver}:daqOutput\"\n",
"\n",
"for raw_file in seq_files:\n",
"\n",
--
GitLab