From a7215c50fc16bcc966cebf03c5a23e320339137d Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Mon, 19 Apr 2021 11:25:07 +0200
Subject: [PATCH] Old const background retrieval, handle qm, minor cleanup
This commit does too many things, sorry, but stuff was all tangled up:
- clean up a bit (imports, unused variables)
- map to PDU with one query instead of in loop (each query gets all anyway)
- start fetching old constants (for comparison) in the background
- add qm layer to dict holding old constants (see issue #49)
---
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 161 ++++++++++++------
1 file changed, 108 insertions(+), 53 deletions(-)
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 27cdbc0e6..fe1a1ae1e 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -116,7 +116,6 @@
"from cal_tools.tools import (\n",
" get_dir_creation_date,\n",
" get_from_db,\n",
- " get_pdu_from_db,\n",
" get_random_db_interface,\n",
" get_report,\n",
" map_gain_stages,\n",
@@ -125,7 +124,7 @@
" save_const_to_h5,\n",
" send_to_db,\n",
")\n",
- "from iCalibrationDB import Conditions, Constants, Detectors"
+ "import iCalibrationDB"
]
},
{
@@ -454,13 +453,11 @@
"\n",
"inp = []\n",
"for gg, (gain, mapped_files) in enumerate(gain_mapped_files.items()):\n",
- " dones = []\n",
" for i in modules:\n",
" qm = module_index_to_qm(i)\n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
" fname_in = mapped_files[qm].get()\n",
- " print(\"Process file: \", fname_in)\n",
- " dones.append(mapped_files[qm].empty())\n",
+ " print(f\"Process file: {fname_in} for {qm}\")\n",
" else:\n",
" continue\n",
" inp.append((fname_in, i, gg))\n",
@@ -578,7 +575,36 @@
" qm_dict[qm] = {\n",
" \"karabo_da\": k_da,\n",
" \"db_module\": \"\"\n",
- " }"
+ " }\n",
+ "\n",
+ "# going through tools.get_pdu_from_db seems wasteful\n",
+ "all_pdus = iCalibrationDB.ConstantMetaData().retrieve_pdus_for_detector(\n",
+ " karabo_id=karabo_id,\n",
+ " receiver=cal_db_interface,\n",
+ " snapshot_at=creation_time.isoformat(),\n",
+ " timeout=cal_db_timeout\n",
+ ")\n",
+ "karabo_da_to_pdu = {d[\"karabo_da\"]: d[\"pdu_physical_name\"] for d in all_pdus}\n",
+ "for qm_attr in qm_dict.values():\n",
+ " qm_attr[\"db_module\"] = karabo_da_to_pdu[qm_attr[\"karabo_da\"]]\n",
+ "for pdu_dict in all_pdus:\n",
+ " this_karabo_da = pdu_dict[\"karabo_da\"]\n",
+ " if this_karabo_da in qm_dict:\n",
+ " qm_dict[this_karabo_da][\"db_module\"] = pdu_dict[\"pdu_physical_name\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with open(f\"{out_folder}/module_mapping.yml\",\"w\") as fd:\n",
+ " yaml.safe_dump({\n",
+ " \"module_mapping\": {\n",
+ " qm: qm_dict[qm][\"db_module\"] for qm in qm_dict\n",
+ " }\n",
+ " }, fd)"
]
},
{
@@ -603,44 +629,48 @@
"outputs": [],
"source": [
"# Retrieve existing constants for comparison\n",
- "old_const = {}\n",
- "old_mdata = {}\n",
- "\n",
+ "qm_x_const = [(qm, const) for const in res[qm] for qm in res]\n",
"print('Retrieve pre-existing constants for comparison.')\n",
- "for qm in res:\n",
+ "def boom(qm, const):\n",
" qm_db = qm_dict[qm]\n",
- " karabo_da = qm_db[\"karabo_da\"]\n",
- " for const in res[qm]:\n",
- " dconst = getattr(Constants.AGIPD, const)()\n",
- "\n",
- " # This should be used in case of running notebook\n",
- " # by a different method other than myMDC which already\n",
- " # sends CalCat info.\n",
- " # TODO: Set db_module to \"\" by default in the first cell\n",
- " if not qm_db[\"db_module\"]:\n",
- " qm_db[\"db_module\"] = get_pdu_from_db(karabo_id, karabo_da, dconst,\n",
- " condition, cal_db_interface,\n",
- " snapshot_at=creation_time)[0]\n",
- "\n",
- " data, mdata = get_from_db(karabo_id, karabo_da,\n",
- " dconst, condition,\n",
- " None, cal_db_interface,\n",
- " creation_time=creation_time,\n",
- " verbosity=2, timeout=cal_db_timeout)\n",
- "\n",
- " old_const[const] = data\n",
- " if mdata is not None and data is not None:\n",
- " time = mdata.calibration_constant_version.begin_at\n",
- " old_mdata[const] = time.isoformat()\n",
- " os.makedirs('{}/old/'.format(out_folder), exist_ok=True)\n",
- " save_const_to_h5(qm_db[\"db_module\"], karabo_id,\n",
- " dconst, condition, data,\n",
- " file_loc, report, creation_time,\n",
- " f'{out_folder}/old/')\n",
- " else:\n",
- " old_mdata[const] = \"Not found\"\n",
- " with open(f\"{out_folder}/module_mapping_{qm}.yml\",\"w\") as fd:\n",
- " yaml.safe_dump({\"module_mapping\": {qm: qm_db[\"db_module\"]}}, fd)"
+ " this_karabo_da = qm_db[\"karabo_da\"]\n",
+ " dconst = getattr(iCalibrationDB.Constants.AGIPD, const)()\n",
+ "\n",
+ " # This should be used in case of running notebook\n",
+ " # by a different method other than myMDC which already\n",
+ " # sends CalCat info.\n",
+ " # TODO: Set db_module to \"\" by default in the first cell\n",
+ "\n",
+ " data, mdata = get_from_db(\n",
+ " karabo_id, \n",
+ " this_karabo_da,\n",
+ " constant=dconst,\n",
+ " condition=condition,\n",
+ " empty_constant=None,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " verbosity=2,\n",
+ " timeout=cal_db_timeout\n",
+ " )\n",
+ "\n",
+ " if mdata is None or data is None:\n",
+ " timestamp = \"Not found\"\n",
+ " else:\n",
+ " timestamp = mdata.calibration_constant_version.begin_at.isoformat()\n",
+ " \n",
+ " return data, timestamp\n",
+ "\n",
+ "old_retrieval_pool = multiprocessing.Pool()\n",
+ "old_retrieval_res = old_retrieval_pool.starmap_async(boom, qm_x_const)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "old_retrieval_pool.close()"
]
},
{
@@ -652,10 +682,9 @@
"md = None\n",
"\n",
"for qm in res:\n",
- " karabo_da = qm_dict[qm][\"karabo_da\"]\n",
" db_module = qm_dict[qm][\"db_module\"]\n",
" for const in res[qm]:\n",
- " dconst = getattr(Constants.AGIPD, const)()\n",
+ " dconst = getattr(iCalibrationDB.Constants.AGIPD, const)()\n",
" dconst.data = res[qm][const]\n",
"\n",
" if db_output:\n",
@@ -666,7 +695,7 @@
" if local_output:\n",
" md = save_const_to_h5(db_module, karabo_id, dconst, condition, dconst.data,\n",
" file_loc, report, creation_time, out_folder)\n",
- " print(f\"Calibration constant {const} is stored locally.\\n\")\n",
+ " print(f\"Calibration constant {const} for {qm} is stored locally in {file_loc}.\\n\")\n",
"\n",
" print(\"Constants parameter conditions are:\\n\")\n",
" print(f\"• memory_cells: {max_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
@@ -865,9 +894,34 @@
"metadata": {},
"outputs": [],
"source": [
- "display(Markdown('The following pre-existing constants are used for comparison: \\n'))\n",
- "for key in old_mdata:\n",
- " display(Markdown('**{}** at {}'.format(key, old_mdata[key])))"
+ "# now we need the old constants\n",
+ "old_const = {}\n",
+ "old_mdata = {}\n",
+ "old_retrieval_res.wait()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for (qm, const), (data, timestamp) in zip(qm_x_const, old_retrieval_res.get()):\n",
+ " old_const.setdefault(qm, {})[const] = data\n",
+ " old_mdata.setdefault(qm, {})[const] = timestamp"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(Markdown(\"The following pre-existing constants are used for comparison:\"))\n",
+ "for qm, consts in old_mdata.items():\n",
+ " display(Markdown(f\"- {qm}\"))\n",
+ " for const in consts:\n",
+ " display(Markdown(f\" - {const} at {consts[const]}\"))"
]
},
{
@@ -954,7 +1008,8 @@
" else:\n",
" table = [['','High gain', 'High gain', 'Medium gain', 'Medium gain', 'Low gain', 'Low gain']]\n",
"\n",
- " compare_with_old_constant = old_const[const] is not None and old_const['BadPixelsDark'] is not None\n",
+ " compare_with_old_constant = old_const[qm][const] is not None and \\\n",
+ " old_const[qm]['BadPixelsDark'] is not None\n",
"\n",
" data = np.copy(res[qm][const])\n",
"\n",
@@ -965,12 +1020,12 @@
" data[res[qm]['BadPixelsDark']>0] = np.nan\n",
"\n",
" if compare_with_old_constant:\n",
- " data_old = np.copy(old_const[const])\n",
+ " data_old = np.copy(old_const[qm][const])\n",
" if const == 'ThresholdsDark':\n",
- " data_old[...,0][old_const['BadPixelsDark'][...,0]>0] = np.nan\n",
- " data_old[...,1][old_const['BadPixelsDark'][...,1]>0] = np.nan\n",
+ " data_old[...,0][old_const[qm]['BadPixelsDark'][...,0]>0] = np.nan\n",
+ " data_old[...,1][old_const[qm]['BadPixelsDark'][...,1]>0] = np.nan\n",
" else:\n",
- " data_old[old_const['BadPixelsDark']>0] = np.nan\n",
+ " data_old[old_const[qm]['BadPixelsDark']>0] = np.nan\n",
"\n",
" f_list = [np.nanmedian, np.nanmean, np.nanstd, np.nanmin, np.nanmax]\n",
" n_list = ['Median', 'Mean', 'Std', 'Min', 'Max']\n",
--
GitLab