diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 22ee77702d3b9e17891e95e63686c2a7eb083f21..722eb0a113265f456e3044a81ff7ff7dd31c30d3 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -87,7 +87,7 @@
"import matplotlib\n",
"import numpy as np\n",
"import tabulate\n",
- "from cal_tools.enums import BadPixels\n",
+ "import yaml\n",
"\n",
"matplotlib.use('agg')\n",
"import matplotlib.pyplot as plt\n",
@@ -104,7 +104,9 @@
" get_gain_setting,\n",
" get_num_cells,\n",
")\n",
- "from cal_tools.enums import AgipdGainMode\n",
+ "\n",
+ "from cal_tools.enums import AgipdGainMode, BadPixels\n",
+ "\n",
"from cal_tools.plotting import (\n",
" create_constant_overview,\n",
" plot_badpix_3d,\n",
@@ -592,7 +594,6 @@
"# Retrieve existing constants for comparison\n",
"old_const = {}\n",
"old_mdata = {}\n",
- "detinst = getattr(Detectors, dinstance)\n",
"\n",
"print('Retrieve pre-existing constants for comparison.')\n",
"for qm in res:\n",
@@ -600,7 +601,6 @@
" karabo_da = qm_db[\"karabo_da\"]\n",
" for const in res[qm]:\n",
" dconst = getattr(Constants.AGIPD, const)()\n",
- " dconst.data = res[qm][const]\n",
"\n",
" # This should be used in case of running notebook\n",
" # by a different method other than myMDC which already\n",
@@ -627,7 +627,9 @@
" file_loc, report, creation_time,\n",
" f'{out_folder}/old/')\n",
" else:\n",
- " old_mdata[const] = \"Not found\""
+ " old_mdata[const] = \"Not found\"\n",
+ " with open(f\"{out_folder}/module_mapping_{qm}.yml\",\"w\") as fd:\n",
+ " yaml.safe_dump({\"module_mapping\": {qm: qm_db[\"db_module\"]}}, fd)"
]
},
{
@@ -658,6 +660,7 @@
" print(\"Constants parameter conditions are:\\n\")\n",
" print(f\"• memory_cells: {max_cells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
" f\"• acquisition_rate: {acq_rate}\\n• gain_setting: {gain_setting}\\n\"\n",
+ " f\"• gain_mode: {fixed_gain_mode}\\n\"\n",
" f\"• creation_time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\")"
]
},
@@ -845,6 +848,17 @@
"The following tables show summary information for the evaluated module. Values for currently evaluated constants are compared with values for pre-existing constants retrieved from the calibration database."
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(Markdown('The following pre-existing constants are used for comparison: \\n'))\n",
+ "for key in old_mdata:\n",
+ " display(Markdown('**{}** at {}'.format(key, old_mdata[key])))"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -988,4 +1002,4 @@
},
"nbformat": 4,
"nbformat_minor": 4
-}
+}
\ No newline at end of file
diff --git a/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb b/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
index e3381c13eebacc4b44b644edb19bc1e55e47c847..9492f7d9bf16e73fc0b4b9ebef560bca0ad962fa 100644
--- a/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
+++ b/notebooks/DSSC/Characterize_DSSC_Darks_NBC.ipynb
@@ -16,12 +16,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-20T12:42:51.255184Z",
- "start_time": "2019-02-20T12:42:51.225500Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
@@ -66,12 +61,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-20T12:42:52.599660Z",
- "start_time": "2019-02-20T12:42:51.472138Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"import os\n",
@@ -94,6 +84,8 @@
"%matplotlib inline\n",
"import numpy as np\n",
"import tabulate\n",
+ "import yaml\n",
+ "\n",
"from cal_tools.dssclib import get_dssc_ctrl_data, get_pulseid_checksum\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import (\n",
@@ -155,12 +147,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-20T12:42:52.608214Z",
- "start_time": "2019-02-20T12:42:52.601257Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"print(\"Parameters are:\")\n",
@@ -187,12 +174,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-20T12:42:54.024731Z",
- "start_time": "2019-02-20T12:42:53.901555Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"# set everything up filewise\n",
@@ -214,12 +196,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-20T10:50:55.839958Z",
- "start_time": "2019-02-20T10:50:55.468134Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"import copy\n",
@@ -452,18 +429,15 @@
" creation_time,\n",
" f'{out_folder}/old/')\n",
" else:\n",
- " old_mdata[const] = \"Not found\""
+ " old_mdata[const] = \"Not found\"\n",
+ " with open(f\"{out_folder}/module_mapping_{qm}.yml\",\"w\") as fd:\n",
+ " yaml.safe_dump({\"module_mapping\": {qm: qm_db[\"db_module\"]}}, fd)"
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2018-12-06T09:38:18.234582Z",
- "start_time": "2018-12-06T09:38:18.222838Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"res = OrderedDict()\n",
@@ -480,12 +454,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2018-12-06T09:49:32.449330Z",
- "start_time": "2018-12-06T09:49:20.231607Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
"# Push the same constant two different times.\n",
@@ -558,13 +527,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2018-12-06T09:49:14.540552Z",
- "start_time": "2018-12-06T09:49:13.009674Z"
- },
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"cell = 9\n",
@@ -576,9 +539,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"cols = {BadPixels.NOISE_OUT_OF_THRESHOLD.value: (BadPixels.NOISE_OUT_OF_THRESHOLD.name, '#FF000080'),\n",
@@ -627,9 +588,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"create_constant_overview(noise_g, \"Noise (ADU)\", max_cells, 0, 100, entries=1)"
@@ -656,6 +615,17 @@
"The following tables show summary information for the evaluated module. Values for currently evaluated constants are compared with values for pre-existing constants retrieved from the calibration database."
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(Markdown('The following pre-existing constants are used for comparison: \\n'))\n",
+ "for key in old_mdata:\n",
+ " display(Markdown('**{}** at {}'.format(key, old_mdata[key])))"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/notebooks/LPD/LPDChar_Darks_NBC.ipynb b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
index cf833d99f362bfe84f22e780b505deae9b85cedc..d6ff625f12155701a5c84355fdc2530560a828c3 100644
--- a/notebooks/LPD/LPDChar_Darks_NBC.ipynb
+++ b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
@@ -88,6 +88,8 @@
"%matplotlib inline\n",
"import numpy as np\n",
"import tabulate\n",
+ "import yaml\n",
+ "\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import (\n",
" create_constant_overview,\n",
@@ -422,7 +424,9 @@
" file_loc, report, creation_time,\n",
" f'{out_folder}/old/')\n",
" else:\n",
- " old_mdata[const] = \"Not found\""
+ " old_mdata[const] = \"Not found\"\n",
+ " with open(f\"{out_folder}/module_mapping_{qm}.yml\",\"w\") as fd:\n",
+ " yaml.safe_dump({\"module_mapping\": {qm: qm_db[\"db_module\"]}}, fd)"
]
},
{
@@ -678,9 +682,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": true
- },
+ "metadata": {},
"outputs": [],
"source": [
"cell = 12\n",
diff --git a/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb b/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
index d64e6f485787f0be3cd0f28d9e9b119e97a2dc73..e36698c9c7f52317ec26a0863cebe3c5f81b2500 100644
--- a/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
+++ b/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
@@ -9,8 +9,8 @@
"#Author: K. Ahmed, M. Karnevsky, Version: 0.1\n",
"#The following is a summary for the processing of dark images and calibration constants production.\n",
"\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/miniHalfAGIPD\" # path to output to, required\n",
- "karabo_id = \"HED_DET_AGIPD500K2G\" # detector instance\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/fixed_gain/SPB_summary_fix2\" # path to output to, required\n",
+ "karabo_id = \"SPB_DET_AGIPD1M-1\" # detector instance\n",
"gain_names = ['High gain', 'Medium gain', 'Low gain'] # a list of gain names to be used in plotting\n",
"threshold_names = ['HG-MG threshold', 'MG_LG threshold'] # a list of gain names to be used in plotting"
]
@@ -26,14 +26,15 @@
"import warnings\n",
"from collections import OrderedDict\n",
"from datetime import datetime\n",
+ "from pathlib import Path\n",
"\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"import glob\n",
- "\n",
"import h5py\n",
"import matplotlib\n",
"import numpy as np\n",
+ "import yaml\n",
"from IPython.display import Latex, Markdown, display\n",
"\n",
"matplotlib.use(\"agg\")\n",
@@ -46,6 +47,7 @@
"import tabulate\n",
"from cal_tools.ana_tools import get_range\n",
"from cal_tools.plotting import show_processed_modules\n",
+ "from cal_tools.tools import CalibrationMetadata, module_index_to_qm\n",
"from iCalibrationDB import Detectors\n",
"from XFELDetAna.plotting.heatmap import heatmapPlot\n",
"from XFELDetAna.plotting.simpleplot import simplePlot"
@@ -170,6 +172,24 @@
" \"\"\"))"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "out_folder = Path(out_folder)\n",
+ "metadata = CalibrationMetadata(out_folder)\n",
+ "mod_mapping = metadata.setdefault(\"modules-mapping\", {})\n",
+ "for fn in out_folder.glob(\"module_mapping_*.yml\"):\n",
+ " with fn.open(\"r\") as fd:\n",
+ " fdict = yaml.safe_load(fd)\n",
+ " mod_mapping.update(fdict[\"module_mapping\"])\n",
+ " fn.unlink()\n",
+ "\n",
+ "metadata.save()"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -188,16 +208,15 @@
"data = OrderedDict()\n",
"old_cons = OrderedDict()\n",
"mod_names = []\n",
+ "\n",
"# Loop over modules\n",
"for i in range(nmods):\n",
- " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ " qm = module_index_to_qm(i)\n",
+ " if qm not in metadata[\"modules_mapping\"].keys() and metadata[\"modules_mapping\"][qm]:\n",
+ " continue\n",
+ " det_name = metadata[\"modules_mapping\"][qm]\n",
" # loop over constants\n",
- " detinst = getattr(Detectors, dinstance)\n",
" for const in ['Offset', 'Noise', 'ThresholdsDark', 'BadPixelsDark']:\n",
- " det = getattr(detinst, qm)\n",
- " if det is None:\n",
- " continue\n",
- " det_name = det.device_name\n",
" fpath = '{}/const_{}_{}.h5'.format(out_folder, const, det_name)\n",
" oldfpath = '{}/old/const_{}_{}.h5'.format(out_folder, const, det_name)\n",
" if not os.path.isfile(fpath):\n",
@@ -332,7 +351,7 @@
" else:\n",
" glabel = gain_names\n",
" for i in range(nmods):\n",
- " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ " qm = module_index_to_qm(i)\n",
" if qm in mod_names:\n",
" m_idx = mod_names.index(qm)\n",
" # Check if constant shape of 5 indices e.g. AGIPD, LPD\n",
@@ -564,20 +583,6 @@
" md = display(Latex(tabulate.tabulate(\n",
" table, tablefmt='latex', headers=header)))"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {