From 1d2a0c1be63efbbd9226b1816a16814097cf3cfa Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Wed, 28 Apr 2021 16:39:28 +0200
Subject: [PATCH] LPD DARKS also save paths, don't copy old constant files
Same thing as for AGIPD. Proposing to save this alongside the module mapping (if we're creating
YAML files per module anyway). Note that I had to add "cap" and "qm" layers to old_const and
old_mdata; there's probably only one element in each, so the dictionary is overkill, but as we loop
over both, this would otherwise break if we ever have more than one of either...
---
notebooks/LPD/LPDChar_Darks_NBC.ipynb | 82 +++++++++++++++++----------
1 file changed, 51 insertions(+), 31 deletions(-)
diff --git a/notebooks/LPD/LPDChar_Darks_NBC.ipynb b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
index d6ff625f1..e4050bcba 100644
--- a/notebooks/LPD/LPDChar_Darks_NBC.ipynb
+++ b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
@@ -386,8 +386,11 @@
"detinst = getattr(Detectors, dinstance)\n",
"print('Retrieve pre-existing constants for comparison.')\n",
"for cap in capacitor_settings:\n",
+ " old_const[cap] = {}\n",
+ " old_mdata[cap] = {}\n",
" for qm in offset_g[cap].keys():\n",
- "\n",
+ " old_const[cap][qm] = {}\n",
+ " old_mdata[cap][qm] = {}\n",
" qm_db = qm_dict[qm]\n",
" karabo_da = qm_db[\"karabo_da\"]\n",
"\n",
@@ -411,22 +414,34 @@
" creation_time=creation_time,\n",
" verbosity=2, timeout=cal_db_timeout)\n",
"\n",
- " old_const[const] = data\n",
- "\n",
- " # TODO: Save old constant file paths (meta_only)\n",
- " # instead of retrieving the whole data\n",
- " if mdata is not None and data is not None:\n",
- " time = mdata.calibration_constant_version.begin_at\n",
- " old_mdata[const] = time.isoformat()\n",
- " os.makedirs('{}/old/'.format(out_folder), exist_ok=True)\n",
- " save_const_to_h5(qm_db[\"db_module\"], karabo_id,\n",
- " constant, condition, data,\n",
- " file_loc, report, creation_time,\n",
- " f'{out_folder}/old/')\n",
+ " old_const[cap][qm][const] = data\n",
+ "\n",
+ " if mdata is None or data is None:\n",
+ " old_mdata[cap][qm][const] = {\n",
+ " \"timestamp\": \"Not found\",\n",
+ " \"filepath\": None,\n",
+ " \"h5path\": None\n",
+ " }\n",
" else:\n",
- " old_mdata[const] = \"Not found\"\n",
- " with open(f\"{out_folder}/module_mapping_{qm}.yml\",\"w\") as fd:\n",
- " yaml.safe_dump({\"module_mapping\": {qm: qm_db[\"db_module\"]}}, fd)"
+ " timestamp = mdata.calibration_constant_version.begin_at.isoformat()\n",
+ " filepath = os.path.join(\n",
+ " mdata.calibration_constant_version.hdf5path,\n",
+ " mdata.calibration_constant_version.filename\n",
+ " )\n",
+ " h5path = mdata.calibration_constant_version.h5path\n",
+ " old_mdata[cap][qm][const] = {\n",
+ " \"timestamp\": timestamp,\n",
+ " \"filepath\": filepath,\n",
+ " \"h5path\": h5path\n",
+ " }\n",
+ "\n",
+ " with open(f\"{out_folder}/module_metadata_{qm}.yml\",\"w\") as fd:\n",
+ " yaml.safe_dump(\n",
+ " {\n",
+ " \"module\": qm,\n",
+ " \"pdu\": qm_db[\"db_module\"],\n",
+ " \"old-constants\": old_mdata[cap][qm]\n",
+ " }, fd)"
]
},
{
@@ -836,9 +851,14 @@
"metadata": {},
"outputs": [],
"source": [
- "display(Markdown('The following pre-existing constants are used for comparison: \\n'))\n",
- "for key in old_mdata:\n",
- " display(Markdown('**{}** at {}'.format(key, old_mdata[key])))"
+ "time_summary = []\n",
+ "for cap, cap_data in old_mdata.items():\n",
+ " time_summary.append(f\"The following pre-existing constants are used for comparison for capacitor setting **{cap}**:\")\n",
+ " for qm, qm_data in cap_data.items():\n",
+ " time_summary.append(f\"- Module {qm}\")\n",
+ " for const, const_data in qm_data.items():\n",
+ " time_summary.append(f\" - {const} created at {const_data['timestamp']}\")\n",
+ "display(Markdown(\"\\n\".join(time_summary)))"
]
},
{
@@ -884,16 +904,16 @@
" x_ticklabels=np.arange(16)+1,\n",
" x_ticks=np.arange(16)+0.5)\n",
"\n",
- " if old_const[const] is not None:\n",
+ " if old_const[cap][qm][const] is not None:\n",
" ax = fig.add_subplot(122)\n",
"\n",
- " dataold = np.copy(old_const[const][:, :, :, gain])\n",
+ " dataold = np.copy(old_const[cap][qm][const][:, :, :, gain])\n",
" \n",
" label = '$\\Delta$ {}'.format(label)\n",
"\n",
" if const != 'BadPixelsDark':\n",
- " if old_const['BadPixelsDark'] is not None:\n",
- " dataold[old_const['BadPixelsDark'][:, :, :, gain] > 0] = np.nan\n",
+ " if old_const[cap][qm]['BadPixelsDark'] is not None:\n",
+ " dataold[old_const[cap][qm]['BadPixelsDark'][:, :, :, gain] > 0] = np.nan\n",
" else:\n",
" dataold[:] = np.nan\n",
" else:\n",
@@ -1135,12 +1155,12 @@
" for bit in bits:\n",
" l_data.append(np.count_nonzero(badpix_g[cap][qm][:,:,:,gain].astype(np.uint32) & bit.value))\n",
" \n",
- " if old_const['BadPixelsDark'] is not None:\n",
- " dataold = np.copy(old_const['BadPixelsDark'][:, :, :, gain])\n",
+ " if old_const[cap][qm]['BadPixelsDark'] is not None:\n",
+ " dataold = np.copy(old_const[cap][qm]['BadPixelsDark'][:, :, :, gain])\n",
" datau32old = dataold.astype(np.uint32)\n",
" l_data_old.append(len(datau32old[datau32old>0].flatten()))\n",
" for bit in bits:\n",
- " l_data_old.append(np.count_nonzero(old_const['BadPixelsDark'][:, :, :, gain].astype(np.uint32) & bit.value))\n",
+ " l_data_old.append(np.count_nonzero(old_const[cap][qm]['BadPixelsDark'][:, :, :, gain].astype(np.uint32) & bit.value))\n",
"\n",
" l_data_name = ['All bad pixels', 'NOISE_OUT_OF_THRESHOLD', \n",
" 'OFFSET_OUT_OF_THRESHOLD', 'OFFSET_NOISE_EVAL_ERROR']\n",
@@ -1151,7 +1171,7 @@
" for i in range(len(l_data)):\n",
" line = [f'{l_data_name[i]}, gain {gain_names[gain]}', l_threshold[i], l_data[i]]\n",
" \n",
- " if old_const['BadPixelsDark'] is not None:\n",
+ " if old_const[cap][qm]['BadPixelsDark'] is not None:\n",
" line += [l_data_old[i]]\n",
" else:\n",
" line += ['-']\n",
@@ -1191,9 +1211,9 @@
" data = np.copy(res[cap][qm][const])\n",
" data[res[cap][qm]['BadPixelsDark']>0] = np.nan\n",
" \n",
- " if old_const[const] is not None and old_const['BadPixelsDark'] is not None :\n",
- " dataold = np.copy(old_const[const])\n",
- " dataold[old_const['BadPixelsDark']>0] = np.nan\n",
+ " if old_const[cap][qm][const] is not None and old_const[cap][qm]['BadPixelsDark'] is not None :\n",
+ " dataold = np.copy(old_const[cap][qm][const])\n",
+ " dataold[old_const[cap][qm]['BadPixelsDark']>0] = np.nan\n",
"\n",
" f_list = [np.nanmedian, np.nanmean, np.nanstd, np.nanmin, np.nanmax]\n",
" n_list = ['Median', 'Mean', 'Std', 'Min', 'Max']\n",
@@ -1202,7 +1222,7 @@
" line = [n_list[i]]\n",
" for gain in range(3):\n",
" line.append('{:6.1f}'.format(f(data[...,gain])))\n",
- " if old_const[const] is not None and old_const['BadPixelsDark'] is not None:\n",
+ " if old_const[cap][qm][const] is not None and old_const[cap][qm]['BadPixelsDark'] is not None:\n",
" line.append('{:6.1f}'.format(f(dataold[...,gain])))\n",
" else:\n",
" line.append('-')\n",
--
GitLab