From b5534379204ac544e1338f9f8b74d449670acce2 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Mon, 1 Feb 2021 17:51:55 +0100
Subject: [PATCH] Update use of time-summary by correction and summary
notebooks
---
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 28 +++----
...AGIPD_Correct_and_Verify_Summary_NBC.ipynb | 82 ++++++-------------
...IPD_Retrieve_Constants_Precorrection.ipynb | 7 +-
3 files changed, 39 insertions(+), 78 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index c4ba5cda0..62a4c16f6 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -582,9 +582,8 @@
"# if the yml file contains \"retrieved-constants\", that means a leading\n",
"# notebook got processed and the reporting would be generated from it.\n",
"fst_print = True\n",
+ "timestamps = {}\n",
"\n",
- "to_store = []\n",
- "line = []\n",
"for i, (error, modno, when, mod_dev) in enumerate(const_out):\n",
" qm = mod_name(modno)\n",
" # expose errors while applying correction\n",
@@ -596,7 +595,7 @@
" print(\"Constants are retrieved with creation time: \")\n",
" fst_print = False\n",
" \n",
- " line = [qm]\n",
+ " module_timestamps = {}\n",
"\n",
" # If correction is crashed\n",
" if not error:\n",
@@ -611,32 +610,25 @@
" # Add NA to keep array structure\n",
" for key in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
" if when and key in when and when[key]:\n",
- " line.append(when[key])\n",
+ " module_timestamps[key] = when[key]\n",
" else:\n",
" if error is not None:\n",
- " line.append('Err')\n",
+ " module_timestamps[key] = \"Err\"\n",
" else:\n",
- " line.append('NA')\n",
- "\n",
- " if len(line) > 0:\n",
- " to_store.append(line)\n",
+ " module_timestamps[key] = \"NA\"\n",
+ " timestamps[qm] = module_timestamps\n",
"\n",
"seq = sequences[0] if sequences else 0\n",
"\n",
- "if len(to_store) > 0:\n",
- " with open(f\"{out_folder}/retrieved_constants_s{seq}.yml\",\"w\") as fyml:\n",
- " yaml.safe_dump({\"time-summary\": {f\"S{seq}\":to_store}}, fyml)"
+ "if len(timestamps) > 0:\n",
+ " with open(f\"{out_folder}/retrieved_constants_s{seq}.yml\",\"w\") as fd:\n",
+ " yaml.safe_dump({\"time-summary\": {f\"S{seq}\": timestamps}}, fd)"
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-18T17:28:51.765030Z",
- "start_time": "2019-02-18T17:28:51.714783Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"def do_3d_plot(data, edges, x_axis, y_axis):\n",
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
index e7f079c6b..96d3c965b 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
@@ -50,8 +50,8 @@
"source": [
"yaml_fn = f'{out_folder}/metadata.yml'\n",
"if os.path.isfile(yaml_fn):\n",
- " with open(yaml_fn, \"r\") as fyml:\n",
- " metadata = yaml.safe_load(fyml)\n",
+ " with open(yaml_fn, \"r\") as fd:\n",
+ " metadata = yaml.safe_load(fd)\n",
"else:\n",
" metadata = {}\n",
"const_dict = metadata.setdefault(\"retrieved-constants\", {})\n",
@@ -79,16 +79,16 @@
"\n",
"# This is needed only if AGIPD Correction notebook had no precorrection notebooks for retrieving constants\n",
"# gather all generated sequence yml files for time summary of retrieved constant under retrieved-constants in metadata.yml\n",
- "fnames = sorted(glob.glob(f'{out_folder}/retrieved_constants_*yml')) \n",
+ "fnames = sorted(glob.glob(f'{out_folder}/retrieved_constants_*.yml')) \n",
"for f in fnames:\n",
- " with open(f, \"r\") as fyml:\n",
- " fdict = yaml.safe_load(fyml)\n",
- " # append different sequences's time summary to the main yaml\n",
+ " with open(f, \"r\") as fd:\n",
+ " fdict = yaml.safe_load(fd)\n",
+ " # append different sequences' time summary to the main yaml\n",
" time_dict.update(fdict[\"time-summary\"])\n",
" os.remove(f)\n",
"\n",
- "with open(yaml_fn, \"w\") as fyml:\n",
- " yaml.safe_dump(metadata, fyml)"
+ "with open(yaml_fn, \"w\") as fd:\n",
+ " yaml.safe_dump(metadata, fd)"
]
},
{
@@ -97,57 +97,27 @@
"metadata": {},
"outputs": [],
"source": [
- "# check if pre-notebook has retrieved constants for all modules.\n",
- "const_times = []\n",
- "seq = []\n",
- "for k, v in sorted(time_dict.items()):\n",
- " arr = np.array(v)\n",
- " arr = arr.reshape(arr.shape[0]//len(modules), len(modules), arr.shape[1])\n",
- " const_times = const_times + list(arr)\n",
- " seq.append(k)\n",
- " \n",
- "const_times = np.array(const_times)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Function print summary of constant injection time\n",
- "# To reduce printouts only unique entries are shown.\n",
- "def const_table(const, pos):\n",
- " \"\"\"\n",
- " Create a summary table for the creation time differences for\n",
- " the retrieved constants (Offset, SlopesPC, SlopesFF).\n",
- " \"\"\"\n",
- " print(f\"{const} were injected on: \")\n",
- "\n",
- " # catch timing difference in retrieve constants\n",
- " unique, idx, counts = np.unique(const_times[:,:,pos], return_inverse=True, return_counts=True)\n",
- " idx = idx.reshape((const_times.shape[0], len(modules)))\n",
- " \n",
+ "def print_const_table(const):\n",
+ " print(f\"{const} constants were injected on:\")\n",
+ " table_data = {}\n",
+ " for seq, mod_data in time_dict.items():\n",
+ " for mod, const_data in mod_data.items():\n",
+ " timestamp = const_data[const]\n",
+ " table_data.setdefault(timestamp, []).append(f\"{seq}:{mod}\")\n",
" table = []\n",
- " for i in range(0, counts.shape[0]):\n",
- " line = [ const_times[:,:,pos][idx==i][0] ]\n",
- " mods = ''\n",
- " for i_s, s in enumerate(seq):\n",
- " if(const_times[i_s,:,0][idx[i_s]==i].shape[0] > 0):\n",
- " mods = mods+ '{}: {}, '.format(s, const_times[i_s,:,0][idx[i_s]==i])\n",
- " line.append(mods)\n",
- " table.append(line)\n",
- "\n",
- " if counts.shape[0] == 1:\n",
- " table[0][1] = 'All modules'\n",
+ " if len(table_data) == 1:\n",
+ " table.append([[*table_data][0], \"All modules\"])\n",
" else:\n",
- " table[np.argmax(counts)][1] = 'Rest of the modules'\n",
+ " for timestamp, seqmod in table_data.items():\n",
+ " table.append([timestamp, seqmod[0]])\n",
+ " for further_module in seqmod[1:]:\n",
+ " table.append([\"\", further_module])\n",
+ " display(Latex(tabulate.tabulate(table,\n",
+ " tablefmt=\"latex\",\n",
+ " headers=[\"Timestamps\", \"Modules and sequences\"])))\n",
"\n",
- " md = display(Latex(tabulate.tabulate(table, tablefmt='latex',\n",
- " headers=[\"Time stamps\", \"Modules and sequences\"])))\n",
- "for i_key, key in enumerate(['Offset', 'SlopesPC', 'SlopesFF']):\n",
- " if const_times.shape[2] > i_key+1:\n",
- " const_table(key, i_key+1)"
+ "for const in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
+ " print_const_table(const)"
]
},
{
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 281b3eaa4..e65ae1788 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -380,13 +380,13 @@
" yaml_fn = f\"{out_folder}/metadata.yml\"\n",
" if os.path.exists(yaml_fn):\n",
" with open(yaml_fn, \"r\") as fd:\n",
- " metadata = yaml.safe_load(yaml_fn)\n",
+ " metadata = yaml.safe_load(fd)\n",
" else:\n",
" metadata = {}\n",
" \n",
" metadata.update({\"retrieved-constants\": mdata_dict})\n",
- " with open(yaml_fn, \"w\") as outfile:\n",
- " yaml.safe_dump(metadata, outfile)\n",
+ " with open(yaml_fn, \"w\") as fd:\n",
+ " yaml.safe_dump(metadata, fd)\n",
" \n",
" print(\"\\nRetrieved constants for modules: \",\n",
" f\"{[', '.join([f'Q{x//4+1}M{x%4+1}' for x in modules])]}\")\n",
@@ -438,7 +438,6 @@
" # if sequences[0] changed to None as it was -1\n",
" seq_num = 0\n",
"\n",
- "yaml_fn = f\"{out_folder}/metadata.yml\"\n",
"if os.path.isfile(yaml_fn):\n",
" with open(yaml_fn, \"r\") as fd:\n",
" metadata = yaml.safe_load(fd)\n",
--
GitLab