diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index f13e878510f33e398a0b124c56ea588ab5f5492f..c66a23a15ec2e8e3d2e5c68d46d483481116caad 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -131,6 +131,7 @@
"sns.set_context(\"paper\", font_scale=1.4)\n",
"sns.set_style(\"ticks\")\n",
"\n",
+ "import cal_tools\n",
"import seaborn as sns\n",
"from cal_tools.agipdlib import (AgipdCorrections, get_acq_rate, get_gain_mode,\n",
" get_gain_setting, get_num_cells)\n",
@@ -138,7 +139,6 @@
"from cal_tools.cython import agipdalgs as calgs\n",
"from cal_tools.enums import AgipdGainMode, BadPixels\n",
"from cal_tools.step_timing import StepTimer\n",
- "from cal_tools.tools import (get_dir_creation_date, map_modules_from_folder, module_index_to_qm)\n",
"\n",
"sns.set()\n",
"sns.set_context(\"paper\", font_scale=1.4)\n",
@@ -170,7 +170,7 @@
"source": [
"# Fill dictionaries comprising bools and arguments for correction and data analysis\n",
"\n",
- "# Here the herarichy and dependability for correction booleans are defined\n",
+ "# Here the hierarchy and dependability for correction booleans are defined\n",
"corr_bools = {}\n",
"\n",
"# offset is at the bottom of AGIPD correction pyramid.\n",
@@ -195,8 +195,16 @@
" corr_bools[\"common_mode\"] = common_mode\n",
" corr_bools[\"melt_snow\"] = melt_snow\n",
" corr_bools[\"mask_zero_std\"] = mask_zero_std\n",
- " corr_bools[\"low_medium_gap\"] = low_medium_gap \n",
- " "
+ " corr_bools[\"low_medium_gap\"] = low_medium_gap\n",
+ "\n",
+ "# Many corrections don't apply to fixed gain mode; will explicitly disable later if detected\n",
+ "disable_for_fixed_gain = [\n",
+ " \"adjust_mg_baseline\",\n",
+ " \"blc_set_min\",\n",
+ " \"force_hg_if_below\",\n",
+ " \"force_mg_if_below\",\n",
+ " \"low_medium_gap\"\n",
+ "]"
]
},
{
@@ -245,8 +253,7 @@
"else:\n",
" modules = [int(x[-2:]) for x in karabo_da]\n",
"\n",
- "print(\"Process modules: \", ', '.join(\n",
- " [module_index_to_qm(x) for x in modules]))\n",
+ "print(\"Process modules:\", ', '.join(cal_tools.tools.module_index_to_qm(x) for x in modules))\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"print(f\"Instrument {instrument}\")\n",
"print(f\"Detector instance {dinstance}\")"
@@ -279,9 +286,9 @@
"outputs": [],
"source": [
"# set everything up filewise\n",
- "mmf = map_modules_from_folder(str(in_folder), run, path_template,\n",
- " karabo_da, sequences)\n",
- "mapped_files, mod_ids, total_sequences, sequences_qm, _ = mmf\n",
+ "mapped_files, _, total_sequences, _, _ = cal_tools.tools.map_modules_from_folder(\n",
+ " str(in_folder), run, path_template, karabo_da, sequences\n",
+ ")\n",
"file_list = []\n",
"\n",
"# ToDo: Split table over pages\n",
@@ -336,10 +343,9 @@
"# Evaluate creation time\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(str(in_folder), run)\n",
+ " creation_time = cal_tools.tools.get_dir_creation_date(str(in_folder), run)\n",
" offset = parser.parse(creation_date_offset)\n",
- " delta = timedelta(hours=offset.hour,\n",
- " minutes=offset.minute, seconds=offset.second)\n",
+ " delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
" creation_time += delta\n",
"\n",
"# Evaluate gain setting\n",
@@ -376,6 +382,19 @@
"print(f\"• Photon Energy: {photon_energy}\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if gain_mode:\n",
+ " for to_disable in disable_for_fixed_gain:\n",
+ " if corr_bools.get(to_disable, False):\n",
+ " print(f\"Warning: {to_disable} correction was requested, but does not apply to fixed gain mode\")\n",
+ " corr_bools[to_disable] = False"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -389,10 +408,13 @@
"metadata": {},
"outputs": [],
"source": [
- "agipd_corr = AgipdCorrections(max_cells, max_pulses,\n",
- " h5_data_path=h5path,\n",
- " h5_index_path=h5path_idx,\n",
- " corr_bools=corr_bools)\n",
+ "agipd_corr = AgipdCorrections(\n",
+ " max_cells,\n",
+ " max_pulses,\n",
+ " h5_data_path=h5path,\n",
+ " h5_index_path=h5path_idx,\n",
+ " corr_bools=corr_bools\n",
+ ")\n",
"\n",
"agipd_corr.baseline_corr_noise_threshold = -blc_noise_threshold\n",
"agipd_corr.hg_hard_threshold = hg_hard_threshold\n",
@@ -406,6 +428,15 @@
"agipd_corr.ff_gain = ff_gain"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "module_index_to_karabo_da = {mod: da for (mod, da) in zip(modules, karabo_da)}"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -415,11 +446,10 @@
"# Retrieve calibration constants to RAM\n",
"agipd_corr.allocate_constants(modules, (3, mem_cells_db, 512, 128))\n",
"\n",
- "metadata = CalibrationMetadata(out_folder)\n",
+ "metadata = cal_tools.tools.CalibrationMetadata(out_folder)\n",
"# NOTE: this notebook will not overwrite calibration metadata file\n",
"const_yaml = metadata.get(\"retrieved-constants\", {})\n",
"\n",
- "# retrive constants\n",
"def retrieve_constants(mod):\n",
" \"\"\"\n",
" Retrieve calibration constants and load them to shared memory\n",
@@ -427,13 +457,15 @@
" Metadata for constants is taken from yml file or retrieved from the DB\n",
" \"\"\"\n",
" err = \"\"\n",
- " k_da = karabo_da[mod]\n",
+ " k_da = module_index_to_karabo_da[mod]\n",
" try:\n",
" # check if there is a yaml file in out_folder that has the device constants.\n",
" if k_da in const_yaml:\n",
+ " print(f\"Pre-correction notebook already found constants for {k_da}\")\n",
" when = agipd_corr.initialize_from_yaml(k_da, const_yaml, mod)\n",
" else:\n",
- " # TODO: should we save what is found here in metadata?\n",
+ " print(f\"Have to query database for constants for {k_da}\")\n",
+ " # TODO: replace with proper retrieval (as done in pre-correction)\n",
" when = agipd_corr.initialize_from_db(\n",
" karabo_id,\n",
" k_da,\n",
@@ -466,7 +498,7 @@
"outputs": [],
"source": [
"# allocate memory for images and hists\n",
- "n_images_max = max_cells*256\n",
+ "n_images_max = max_cells * 256\n",
"data_shape = (n_images_max, 512, 128)\n",
"agipd_corr.allocate_images(data_shape, n_cores_files)"
]
@@ -590,7 +622,7 @@
"timestamps = {}\n",
"\n",
"for i, (error, modno, when, k_da) in enumerate(const_out):\n",
- " qm = module_index_to_qm(modno)\n",
+ " qm = cal_tools.tools.module_index_to_qm(modno)\n",
" # expose errors while applying correction\n",
" if error:\n",
" print(\"Error: {}\".format(error) )\n",
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 5a6300766a164bce6df7e247b44f1b9db58a28fa..0a2583b174e1f5d718c55468da8d9988daa102d5 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -259,41 +259,54 @@
" # Retrieve multiple constants through an input dictionary\n",
" # to return a dict of useful metadata.\n",
" mdata_dict = dict()\n",
- " mdata_dict['constants'] = dict()\n",
- " mdata_dict['physical-detector-unit'] = None # initialization\n",
- "\n",
- " for cname, cval in const_dict.items():\n",
- " print(cname)\n",
- " print(cval)\n",
+ " mdata_dict[\"constants\"] = dict()\n",
+ " mdata_dict[\"physical-detector-unit\"] = None # initialization\n",
+ "\n",
+ " for const_name, (const_init_fun, const_shape, (cond_type, cond_param)) in const_dict.items():\n",
+ " if gain_mode and const_name in (\"ThresholdsDark\",):\n",
+ " print(f\"Note: skipping {const_name} for fixed gain mode\")\n",
+ " continue\n",
+ " \n",
" # saving metadata in a dict\n",
- " mdata_dict['constants'][cname] = dict()\n",
- "\n",
- " if slopes_ff_from_files and cname in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = \"00:00:00\"\n",
+ " const_mdata = dict()\n",
+ " mdata_dict[\"constants\"][const_name] = const_mdata\n",
+ "\n",
+ " if slopes_ff_from_files and const_name in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
+ " const_mdata[\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
+ " const_mdata[\"creation-time\"] = \"00:00:00\"\n",
+ " continue\n",
+ " \n",
+ " if gain_mode and const_name in (\"BadPixelsPC\", \"SlopesPC\", \"BadPixelsFF\", \"SlopesFF\"):\n",
+ " param_copy = cond_param.copy()\n",
+ " del param_copy[\"gain_mode\"]\n",
+ " condition = getattr(Conditions, cond_type).AGIPD(**param_copy)\n",
+ " print(f\"Note: {const_name} based on adaptive gain mode constants will be retrieved\")\n",
" else:\n",
- " condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
- " co, mdata = tools.get_from_db(\n",
- " karabo_id,\n",
- " karabo_da,\n",
- " getattr(Constants.AGIPD, cname)(),\n",
- " condition,\n",
- " getattr(np, cval[0])(cval[1]),\n",
- " cal_db_interface,\n",
- " creation_time,\n",
- " meta_only=True,\n",
- " verbosity=1,\n",
+ " condition = getattr(Conditions, cond_type).AGIPD(**cond_param)\n",
+ "\n",
+ " _, mdata = tools.get_from_db(\n",
+ " karabo_id,\n",
+ " karabo_da,\n",
+ " getattr(Constants.AGIPD, const_name)(),\n",
+ " condition,\n",
+ " getattr(np, const_init_fun)(const_shape),\n",
+ " cal_db_interface,\n",
+ " creation_time,\n",
+ " meta_only=True,\n",
+ " verbosity=1,\n",
+ " )\n",
+ " mdata_const = mdata.calibration_constant_version\n",
+ " # check if constant was sucessfully retrieved.\n",
+ " if mdata.comm_db_success:\n",
+ " const_mdata[\"file-path\"] = (\n",
+ " f\"{mdata_const.hdf5path}\" f\"{mdata_const.filename}\"\n",
" )\n",
- " mdata_const = mdata.calibration_constant_version\n",
- " # check if constant was sucessfully retrieved.\n",
- " if mdata.comm_db_success:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
- " f\"{mdata_const.filename}\"\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
- " mdata_dict['physical-detector-unit'] = mdata_const.device_name\n",
- " else:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = const_dict[cname][:2]\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = None\n",
+ " const_mdata[\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
+ " mdata_dict[\"physical-detector-unit\"] = mdata_const.device_name\n",
+ " else:\n",
+ " const_mdata[\"file-path\"] = const_dict[const_name][:2]\n",
+ " const_mdata[\"creation-time\"] = None\n",
+ " print(const_mdata)\n",
"\n",
" return qm, mdata_dict, karabo_da, acq_rate, local_max_cells, err"
]