From e4446ac61264ad8398d9471a6e0b76f4a25ad043 Mon Sep 17 00:00:00 2001
From: Thomas Kluyver <thomas.kluyver@xfel.eu>
Date: Thu, 27 Apr 2023 17:08:44 +0200
Subject: [PATCH] Expect karabo_da with module number, e.g. LPDMINI00/8
---
.../LPDMini/LPD_Mini_Char_Darks_NBC.ipynb | 29 +++++-----
notebooks/LPDMini/LPD_Mini_Correct.ipynb | 58 +++++++++----------
2 files changed, 41 insertions(+), 46 deletions(-)
diff --git a/notebooks/LPDMini/LPD_Mini_Char_Darks_NBC.ipynb b/notebooks/LPDMini/LPD_Mini_Char_Darks_NBC.ipynb
index 9cead9a16..179a5378c 100644
--- a/notebooks/LPDMini/LPD_Mini_Char_Darks_NBC.ipynb
+++ b/notebooks/LPDMini/LPD_Mini_Char_Darks_NBC.ipynb
@@ -30,8 +30,7 @@
"run_low = 12 # run number in which low gain data was recorded, required\n",
"\n",
"karabo_id = \"FXE_DET_LPD_MINI\" # karabo karabo_id\n",
- "karabo_da = [''] # a list of data aggregators names, Default [''] for selecting all data aggregators\n",
- "modules = [-1] # Module numbers (1-8) to process, use [-1] for all\n",
+ "karabo_da = [''] # a list of data aggregators names with module number, e.g. 'LPDMINI00/2', Default [''] for selecting all data aggregators\n",
"source_name = \"{}/DET/0CH0:xtdf\" # Source name for raw detector data\n",
"control_source_name = \"{}/FPGA/FEM\" # Source name for control device\n",
"\n",
@@ -158,15 +157,15 @@
"calcat = CalCatApi(client=calibration_client())\n",
"detector_id = calcat.detector(karabo_id)['id']\n",
"pdus_by_da = calcat.physical_detector_units(detector_id, pdu_snapshot_at=creation_time)\n",
- "mod_num_to_pdu_name = {\n",
- " int(da.split('/')[-1]): p['physical_name']\n",
- " for (da, p) in pdus_by_da.items()\n",
- "}\n",
+ "pdu_name_by_da = {da: p['physical_name'] for (da, p) in pdus_by_da.items()}\n",
"\n",
- "if modules and modules != [-1]:\n",
- " mod_num_to_pdu_name = {m: mod_num_to_pdu_name[m] for m in modules}\n",
+ "if karabo_da and karabo_da != ['']:\n",
+ " karabo_da = [da for da in karabo_da if da in pdu_name_by_da]\n",
+ " pdu_name_by_da = {da: pdu_name_by_da[da] for da in karabo_da}\n",
"else:\n",
- " modules = sorted(mod_num_to_pdu_name.keys())"
+ " karabo_da = sorted(pdu_name_by_da.keys())\n",
+ " \n",
+ "modules = [int(x.split('/')[-1]) for x in karabo_da]"
]
},
{
@@ -176,8 +175,8 @@
"outputs": [],
"source": [
"print(\"Minis in use (1-8) and PDUs:\")\n",
- "for mod_num, pdu in mod_num_to_pdu_name.items():\n",
- " print(f\"Mini {mod_num} -> {pdu}\")"
+ "for mod_num, karabo_da_m in zip(modules, karabo_da):\n",
+ " print(f\"Mini {mod_num} -> {pdu_name_by_da[karabo_da_m]}\")"
]
},
{
@@ -349,10 +348,10 @@
"print('Retrieve pre-existing constants for comparison.')\n",
"cap = capacitor_settings[0]\n",
"\n",
- "for mod_num, db_module in mod_num_to_pdu_name.items():\n",
+ "for mod_num, karabo_da_m in zip(modules, karabo_da):\n",
+ " db_module = pdu_name_by_da[karabo_da_m]\n",
" old_const[mod_num] = {}\n",
" old_mdata[mod_num] = {}\n",
- " karabo_da_m = f'{karabo_da[0]}/{mod_num}'\n",
"\n",
" if inject_cell_order:\n",
" mem_cell_order = \",\".join([str(c) for c in cellid_pattern]) + \",\"\n",
@@ -428,7 +427,8 @@
"# Save constants in the calibration DB\n",
"md = None\n",
"cap = capacitor_settings[0]\n",
- "for mod_num, db_module in mod_num_to_pdu_name.items():\n",
+ "for mod_num, karabo_da_m in zip(modules, karabo_da):\n",
+ " db_module = pdu_name_by_da[karabo_da_m]\n",
" print(f\"Storing constants for PDU {db_module}\")\n",
"\n",
" if inject_cell_order:\n",
@@ -826,7 +826,6 @@
"\n",
" for const in const_types:\n",
" data = np.copy(const_types[const][i][:, :, :, gain])\n",
- " print(\"const shape\", data.shape)\n",
"\n",
" label = 'Fraction of bad pixels'\n",
"\n",
diff --git a/notebooks/LPDMini/LPD_Mini_Correct.ipynb b/notebooks/LPDMini/LPD_Mini_Correct.ipynb
index de07bb7c9..f66d534ab 100644
--- a/notebooks/LPDMini/LPD_Mini_Correct.ipynb
+++ b/notebooks/LPDMini/LPD_Mini_Correct.ipynb
@@ -25,8 +25,7 @@
"out_folder = \"/gpfs/exfel/data/scratch/kluyvert/correct-lpdmini-p4576-r48\" # the folder to output to, required\n",
"metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate.\n",
"sequences = [-1] # Sequences to correct, use [-1] for all\n",
- "modules = [-1] # Module numbers (1-8) to correct, use [-1] for all\n",
- "karabo_da = [''] # Data aggregators names to correct, use [''] for all\n",
+ "karabo_da = [''] # Data aggregators names to correct, e.g. 'LPDMINI00/8', use [''] for all\n",
"run = 48 # run to process, required\n",
"\n",
"# Source parameters\n",
@@ -136,10 +135,6 @@
"\n",
"creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
"print(f'Using {creation_time.isoformat()} as creation time')\n",
- "\n",
- "# Pick all modules/aggregators or those selected.\n",
- "if not karabo_da or karabo_da == ['']:\n",
- " karabo_da = ['LPDMINI00']\n",
" \n",
"# Pick all sequences or those selected.\n",
"if not sequences or sequences == [-1]:\n",
@@ -176,6 +171,28 @@
"# Select data to process"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "calcat_client = calibration_client()\n",
+ "calcat = CalCatApi(client=calcat_client)\n",
+ "\n",
+ "# Look up PDUs\n",
+ "detector_id = calcat.detector(karabo_id)['id']\n",
+ "pdus_by_da = calcat.physical_detector_units(detector_id, pdu_snapshot_at=creation_time)\n",
+ "modnos_from_db = set()\n",
+ "\n",
+ "if not karabo_da or karabo_da == ['']:\n",
+ " karabo_da = sorted(pdus_by_da.keys())\n",
+ "else:\n",
+ " karabo_da = sorted(set(karabo_da) & pdus_by_da.keys())\n",
+ "\n",
+ "print(\"Modules to correct:\", karabo_da)"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -183,11 +200,12 @@
"outputs": [],
"source": [
"data_to_process = []\n",
+ "data_agg_names = {kda.split('/')[0] for kda in karabo_da}\n",
"\n",
"for inp_path in run_folder.glob('RAW-*.h5'):\n",
" match = file_re.match(inp_path.stem)\n",
" \n",
- " if match[2] not in karabo_da or not do_sequence(int(match[3])):\n",
+ " if match[2] not in data_agg_names or not do_sequence(int(match[3])):\n",
" continue\n",
" \n",
" outp_path = out_folder / 'CORR-R{run:04d}-{aggregator}-S{seq:05d}.h5'.format(\n",
@@ -207,28 +225,6 @@
"# Obtain and prepare calibration constants"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "calcat_client = calibration_client()\n",
- "calcat = CalCatApi(client=calcat_client)\n",
- "\n",
- "# Look up PDUs\n",
- "detector_id = calcat.detector(karabo_id)['id']\n",
- "pdus_by_da = calcat.physical_detector_units(detector_id, pdu_snapshot_at=creation_time)\n",
- "modnos_from_db = {int(x.split('/')[-1]) for x in pdus_by_da}\n",
- "\n",
- "if not modules or modules == [-1]:\n",
- " modules = sorted(modnos_from_db)\n",
- "else:\n",
- " modules = sorted(set(modules) & modnos_from_db)\n",
- "\n",
- "print(\"Modules to correct (of 1-8):\", modules)"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -285,11 +281,11 @@
" (dark_calibrations, dark_condition),\n",
" (illuminated_calibrations, illuminated_condition)\n",
"]:\n",
- " for mod_num in modules:\n",
+ " for karabo_da_m in karabo_da:\n",
+ " mod_num = int(karabo_da_m.split('/')[-1])\n",
" # mod_num is from 1 to 8, so b_v_0 applies to odd numbers\n",
" bias_voltage = bias_voltage_0 if mod_num % 2 == 1 else bias_voltage_1\n",
" condition_w_voltage = [dict(parameter_id=1, value=bias_voltage)] + condition\n",
- " karabo_da_m = f'{karabo_da[0]}/{mod_num}'\n",
" \n",
" resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(\n",
" calcat_client, karabo_id, list(calibrations.keys()),\n",
--
GitLab