diff --git a/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
index 07f3ee0d68a4acc37dcf4ef5dafbc7ce4cdc7905..34d5ccee920627ac07c4adb07b56903292e7aeb0 100644
--- a/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
+++ b/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
@@ -30,12 +30,12 @@
"run_low = 9 # run number for G2 dark run, required\n",
"\n",
"# Parameters used to access raw data.\n",
+ "input_source_template = \"{karabo_id}/DET/RECEIVER{input_source_affixes}:daqOutput\" # data source template used to read INSTRUMENT keys.\n",
+ "ctrl_source_template = \"{karabo_id_control}/DET/{control_template}\" # template for control source name (filled with karabo_id_control)\n",
"karabo_id = \"FXE_XAD_G2XES\" # karabo prefix of Gotthard-II devices\n",
"karabo_da = [\"\"] # data aggregators\n",
- "receiver_template = \"RECEIVER{}\" # receiver template used to read INSTRUMENT keys.\n",
- "receiver_affixes = [\"\"] # The affix to format into the receiver template to be able to load the correct receiver name from the data.\n",
+ "input_source_affixes = [\"\"] # The affix to format into the input source template to be able to load the correct module's data.\n",
"control_template = \"CONTROL\" # control template used to read CONTROL keys.\n",
- "ctrl_source_template = '{}/DET/{}' # template for control source name (filled with karabo_id_control)\n",
"karabo_id_control = \"\" # Control karabo ID. Set to empty string to use the karabo-id\n",
"\n",
"# Parameters for the calibration database.\n",
@@ -110,7 +110,10 @@
"if not karabo_id_control:\n",
" karabo_id_control = karabo_id\n",
"\n",
- "ctrl_src = ctrl_source_template.format(karabo_id_control, control_template)\n",
+ "ctrl_src = ctrl_source_template.format(\n",
+ " karabo_id_control=karabo_id_control,\n",
+ " control_template=control_template,\n",
+ ")\n",
"run_nums = gotthard2lib.sort_dark_runs_by_gain(\n",
" raw_folder=in_folder,\n",
" runs=run_nums,\n",
@@ -134,9 +137,7 @@
"source": [
"run_dc = RunDirectory(in_folder / f\"r{run_nums[0]:04d}\")\n",
"proposal = list(filter(None, str(in_folder).strip('/').split('/')))[-2]\n",
- "file_loc = raw_data_location_string(proposal, run_nums)\n",
- "receiver_names = [f\"*{receiver_template.format(x)}*\" for x in receiver_affixes]\n",
- "data_sources = sorted(list(run_dc.select(receiver_names).all_sources))"
+ "file_loc = raw_data_location_string(proposal, run_nums)"
]
},
{
@@ -224,18 +225,25 @@
"# when we start storing in the CALCAT.\n",
"da_to_pdu = dict()\n",
"for i, (da, p) in enumerate(sorted(pdus_by_da.items())):\n",
- " da_to_pdu[da] = {\n",
- " \"pdu\": p['physical_name'],\n",
- " \"idx\": i,\n",
- " }\n",
+ " da_to_pdu[da] = p['physical_name']\n",
+ "\n",
+ "calcat_das = list(da_to_pdu.keys())\n",
"\n",
"if karabo_da != [\"\"]:\n",
" # Filter DA connected to detector in CALCAT\n",
- " karabo_da = [da for da in karabo_da if da in da_to_pdu]\n",
+ " karabo_da = [da for da in karabo_da if da in calcat_das]\n",
"else:\n",
- " karabo_da = list(da_to_pdu.keys())\n",
+ " karabo_da = calcat_das\n",
"\n",
- "print(f\"Processing {karabo_da}\")"
+ "print(f\"Processing {karabo_da}\")\n",
+ "\n",
+ "da_to_source = gotthard2lib.map_da_to_source(\n",
+ " run_dc,\n",
+ " calcat_das,\n",
+ " input_source_template,\n",
+ " karabo_id,\n",
+ " input_source_affixes,\n",
+ ")"
]
},
{
@@ -329,7 +337,7 @@
"for mod in karabo_da:\n",
" # For 50um idx always 0. For 25um pick the right data source\n",
" # from list of 2 sources.\n",
- " src = data_sources[da_to_pdu[mod][\"idx\"]]\n",
+ " src = da_to_source[mod]\n",
"\n",
" # Retrieve LUT constant\n",
" lut, time = get_constant_from_db_and_time(\n",
@@ -457,7 +465,7 @@
"g_name = [\"G0\", \"G1\", \"G2\"]\n",
"\n",
"for mod in karabo_da:\n",
- " pdu = da_to_pdu[mod][\"pdu\"]\n",
+ " pdu = da_to_pdu[mod]\n",
" display(Markdown(f\"### Badpixels for module {mod}:\"))\n",
"\n",
" badpixels_map[mod][\n",
@@ -495,7 +503,7 @@
"if not local_output:\n",
" for cons, cname in zip([offset_map, noise_map], [\"Offset\", \"Noise\"]):\n",
" for mod in karabo_da:\n",
- " pdu = da_to_pdu[mod][\"pdu\"]\n",
+ " pdu = da_to_pdu[mod]\n",
" display(Markdown(f\"### {cname} for module {mod}:\"))\n",
"\n",
" for cell in [0, 1]:\n",
@@ -519,7 +527,7 @@
"source": [
"step_timer.start()\n",
"for mod in karabo_da:\n",
- " db_mod = da_to_pdu[mod][\"pdu\"]\n",
+ " db_mod = da_to_pdu[mod]\n",
" constants = {\n",
" \"Offset\": offset_map[mod],\n",
" \"Noise\": noise_map[mod],\n",
@@ -585,7 +593,7 @@
" yaml.safe_dump(\n",
" {\n",
" \"module\": mod,\n",
- " \"pdu\": da_to_pdu[mod][\"pdu\"],\n",
+ " \"pdu\": da_to_pdu[mod],\n",
" },\n",
" fd,\n",
" )"
diff --git a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
index b7424cb07c6e32c83bb24a08b42d7bcb48073174..c70f1b709914371bd81f5ba3298a798fc0317b67 100644
--- a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
+++ b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
@@ -58,15 +58,14 @@
"sequences_per_node = 1 # number of sequence files per node if notebook executed through xfel-calibrate, set to 0 to not run SLURM parallel\n",
"\n",
"# Parameters used to access raw data.\n",
+ "output_source_template = \"{karabo_id}/CORR/RECEIVER:daqOutput\" # Correction data source template. filled with karabo_id and correction receiver\n",
+ "ctrl_source_template = \"{karabo_id_control}/DET/{control_template}\" # template for control source name (filled with karabo_id_control)\n",
"karabo_id = \"DETLAB_25UM_GH2\" # karabo prefix of Gotthard-II devices\n",
"karabo_da = [\"\"] # data aggregators\n",
- "receiver_template = \"RECEIVER{}\" # receiver template used to read INSTRUMENT keys.\n",
- "receiver_affixes = [\"\"] # The affix to format into the receiver template to be able to load the correct receiver name from the data.\n",
+ "input_source_affixes = [\"\"] # The affix to format into the input source template to be able to load the correct module's data.\n",
+ "input_source_template = \"{karabo_id}/DET/RECEIVER{input_source_affixes}:daqOutput\" # data source template used to read INSTRUMENT keys.\n",
"control_template = \"CONTROL\" # control template used to read CONTROL keys.\n",
- "ctrl_source_template = \"{}/DET/{}\" # template for control source name (filled with karabo_id_control)\n",
"karabo_id_control = \"\" # Control karabo ID. Set to empty string to use the karabo-id\n",
- "corr_source_template = \"{}/CORR/{}:daqOutput\" # Correction data source template. filled with karabo_id and correction receiver\n",
- "corr_receiver = \"\" # The receiver name of the corrected data. Leave empty for using the same receiver name for the 50um GH2 or the first(Master) receiver for the 25um GH2.\n",
"\n",
"# Parameters for calibration database.\n",
"cal_db_interface = \"tcp://max-exfl-cal001:8016#8025\" # the database interface to use.\n",
@@ -147,7 +146,10 @@
"if not karabo_id_control:\n",
" karabo_id_control = karabo_id\n",
"\n",
- "ctrl_src = ctrl_source_template.format(karabo_id_control, control_template)\n",
+ "ctrl_src = ctrl_source_template.format(\n",
+ " karabo_id_control=karabo_id_control,\n",
+ " control_template=control_template,\n",
+ ")\n",
"\n",
"# Run's creation time:\n",
"creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
@@ -240,17 +242,15 @@
"\n",
"print(f\"Process modules: {db_modules} for run {run}\")\n",
"\n",
- "# Create the correction receiver name.\n",
- "receiver_names = [f\"*{receiver_template.format(x)}*\" for x in receiver_affixes]\n",
- "data_sources = sorted(list(run_dc.select(receiver_names).all_sources))\n",
"\n",
- "if not corr_receiver:\n",
- " # This part assumes this data_source structure: '{karabo_id}/DET/{receiver_name}:{output_channel}'\n",
- " if gh2_detector == \"25um\": # For 25um use virtual karabo_das for CALCAT data mapping.\n",
- " corr_receiver = data_sources[0].split(\"/\")[-1].split(\":\")[0]\n",
- " else:\n",
- " corr_receiver = data_sources[0].split(\"/\")[-1].split(\":\")[0]\n",
- " print(f\"Using {corr_receiver} as a receiver name for the corrected data.\")"
+ "da_to_source = gotthard2lib.map_da_to_source(\n",
+ " run_dc,\n",
+ " karabo_da,\n",
+ " input_source_template,\n",
+ " karabo_id,\n",
+ " input_source_affixes,\n",
+ ")\n",
+ "data_sources = list(da_to_source.values())"
]
},
{
@@ -261,7 +261,8 @@
"outputs": [],
"source": [
"# Check the available trains to correct.\n",
- "total_trains = len(RunDirectory(run_folder).select(data_sources, require_all=True).train_ids)\n",
+ "total_trains = len(\n",
+ " RunDirectory(run_folder).select(data_sources, require_all=True).train_ids)\n",
"if total_trains:\n",
" print(f\"Correcting {total_trains}.\")\n",
"else:\n",
@@ -426,7 +427,7 @@
"metadata": {},
"outputs": [],
"source": [
- "corr_data_source = corr_source_template.format(karabo_id, corr_receiver)\n",
+ "corr_data_source = output_source_template.format(karabo_id=karabo_id)\n",
"\n",
"for raw_file in seq_files:\n",
"\n",
@@ -450,7 +451,8 @@
" else:\n",
" data_stored, gain_stored, mask_stored = None, None, None\n",
"\n",
- " for i, (src, mod) in enumerate(zip(data_sources, karabo_da)):\n",
+ " for i, mod in enumerate(karabo_da):\n",
+ " src = da_to_source[mod]\n",
" step_timer.start()\n",
" print(f\"Correcting {src} for {raw_file}\")\n",
"\n",
diff --git a/notebooks/Gotthard2/Summary_Darks_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Summary_Darks_Gotthard2_NBC.ipynb
index 756097845270727db4d08b130225d0f7ec89eb62..411aeeca47430d3f9738850b9efe3be71aa96c61 100644
--- a/notebooks/Gotthard2/Summary_Darks_Gotthard2_NBC.ipynb
+++ b/notebooks/Gotthard2/Summary_Darks_Gotthard2_NBC.ipynb
@@ -28,7 +28,7 @@
"karabo_id = \"DETLAB_25UM_GH2\" # detector identifier.\n",
"karabo_da = [\"DA01/1\", \"DA01/2\"] # list of data aggregators, which corresponds to different JF modules. This is only needed for the detectors of one module.\n",
"control_template = \"CONTROL\" # control template used to read CONTROL keys.\n",
- "ctrl_source_template = '{}/DET/{}' # template for control source name (filled with karabo_id_control)\n",
+ "ctrl_source_template = '{karabo_id}/DET/{control_template}' # template for control source name (filled with karabo_id_control)\n",
"karabo_id_control = \"\" # Control karabo ID. Set to empty string to use the karabo-id\n",
"\n",
"# Parameters to be used for injecting dark calibration constants.\n",
@@ -88,7 +88,10 @@
" karabo_id_control = karabo_id\n",
"g2ctrl = gotthard2lib.Gotthard2Ctrl(\n",
" run_dc=RunDirectory(Path(in_folder) / f\"r{run_high:04d}\"),\n",
- " ctrl_src=ctrl_source_template.format(karabo_id_control, control_template))\n",
+ " ctrl_src=ctrl_source_template.format(\n",
+ " karabo_id_control=karabo_id_control,\n",
+ " control_template=control_template\n",
+ " ))\n",
"gh2_detector = g2ctrl.get_det_type()\n",
"print(f\"Processing {gh2_detector} Gotthard2.\")"
]
diff --git a/src/cal_tools/gotthard2/gotthard2lib.py b/src/cal_tools/gotthard2/gotthard2lib.py
index 8b24818332c419c44a6a5a4db5abca5aaa28c6f1..e6a8267c86de2b567a1e613120f6cde3f16fd226 100644
--- a/src/cal_tools/gotthard2/gotthard2lib.py
+++ b/src/cal_tools/gotthard2/gotthard2lib.py
@@ -5,6 +5,23 @@ from typing import List, Union
import extra_data
+def map_da_to_source(dc, das, source_template, karabo_id, affixes):
+ source_names = [
+ source_template.format(
+ karabo_id=karabo_id, input_source_affixes=aff)
+ for aff in affixes
+ ]
+ if len(source_names) != len(das):
+ raise ValueError(
+ "Number of source names (and given affixes) "
+ "does not match number of DAs")
+ missing_sources = set(source_names) - dc.instrument_sources
+ if missing_sources:
+ raise ValueError(
+ f"The following sources are missing from the run: {missing_sources}")
+ return dict(zip(das, source_names))
+
+
class Gotthard2Ctrl():
def __init__(
self,
diff --git a/tests/test_reference_runs/callab_tests.py b/tests/test_reference_runs/callab_tests.py
index 2d88f7aa3c0625447aef0722517b21149a23d46b..0c3661f91a103cbce43a0b480e60af65c8d68e6f 100644
--- a/tests/test_reference_runs/callab_tests.py
+++ b/tests/test_reference_runs/callab_tests.py
@@ -869,8 +869,8 @@ automated_test_config = {
"in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
"karabo-da": ["GH200/1", "GH200/2",],
"control-template": "GOTTHARD2_CTRL",
- "receiver-affixes": ["_MASTER:daqOutput", "_SLAVE:daqOutput"],
- "receiver-template": "GOTTHARD2{}",
+ "input-source-affixes": ["MASTER", "SLAVE"],
+ "input-source-template": "{karabo_id}/DET/GOTTHARD2_{input_source_affixes}:daqOutput",
"run-high": "28", # 82
"run-med": "29", # 83
"run-low": "30", # 84
@@ -887,9 +887,9 @@ automated_test_config = {
"in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
"karabo-da": ["GH200/1", "GH200/2",],
"control-template": "GOTTHARD2_CTRL",
- "receiver-affixes": ["_MASTER:daqOutput", "_SLAVE:daqOutput"],
- "receiver-template": "GOTTHARD2{}",
- "corr-receiver": "GOTTHARD2_RECEIVER",
+ "input-source-affixes": ["MASTER", "SLAVE"],
+ "input-source-template": "{karabo_id}/DET/GOTTHARD2_{input_source_affixes}:daqOutput",
+ "output-source-template": "{karabo_id}/CORR/GOTTHARD2_RECEIVER:daqOutput",
"run": "31",
"karabo-id": "SA1_XTD9_HIREX",
"no-offset-correction": True,