diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 6ce5f20bfc96f50caf3a6f6dd72a89d38c7bf467..677fe4d1f701518b0e3b05a8bc85e2adfe3b0234 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -14,15 +14,9 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-21T11:30:06.730220Z",
- "start_time": "2019-02-21T11:30:06.658286Z"
- }
- },
+ "metadata": {},
"outputs": [],
"source": [
- "cluster_profile = \"noDB\"\n",
"in_folder = \"/gpfs/exfel/exp/SPB/202030/p900119/raw\" # the folder to read data from, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_\" # the folder to output to, required\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
@@ -91,12 +85,8 @@
"metadata": {},
"outputs": [],
"source": [
- "import sys\n",
- "from collections import OrderedDict\n",
- "from functools import partial\n",
"from typing import List, Tuple\n",
"\n",
- "import h5py\n",
"import matplotlib\n",
"import numpy as np\n",
"\n",
@@ -105,10 +95,7 @@
"from datetime import timedelta\n",
"from pathlib import Path\n",
"\n",
- "import matplotlib.pyplot as plt\n",
- "from cal_tools.agipdlib import get_gain_setting\n",
- "from cal_tools.tools import (CalibrationMetadata, get_dir_creation_date,\n",
- " map_modules_from_folder)\n",
+ "from cal_tools import agipdlib, tools\n",
"from dateutil import parser\n",
"from iCalibrationDB import Conditions, Constants, Detectors"
]
@@ -122,7 +109,7 @@
"# slopes_ff_from_files left as str for now\n",
"in_folder = Path(in_folder)\n",
"out_folder = Path(out_folder)\n",
- "metadata = CalibrationMetadata(out_folder)"
+ "metadata = tools.CalibrationMetadata(out_folder)"
]
},
{
@@ -135,7 +122,7 @@
"\n",
"creation_time = None\n",
"if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(str(in_folder), run)\n",
+ " creation_time = tools.get_dir_creation_date(str(in_folder), run)\n",
" offset = parser.parse(creation_date_offset)\n",
" delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
" creation_time += delta\n",
@@ -171,7 +158,7 @@
" gain_setting = None\n",
" else:\n",
" try:\n",
- " gain_setting = get_gain_setting(str(control_fn), h5path_ctrl)\n",
+ " gain_setting = agipdlib.get_gain_setting(str(control_fn), h5path_ctrl)\n",
" except Exception as e:\n",
" print(f'ERROR: while reading gain setting from: \\n{control_fn}')\n",
" print(e)\n",
@@ -207,24 +194,6 @@
" modules = [int(x[-2:]) for x in karabo_da]"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2019-02-21T11:30:07.974174Z",
- "start_time": "2019-02-21T11:30:07.914832Z"
- }
- },
- "outputs": [],
- "source": [
- "# set everything up filewise\n",
- "print(f\"Checking the files before retrieving constants\")\n",
- "mmf = map_modules_from_folder(str(in_folder), run, path_template, karabo_da, sequences)\n",
- "\n",
- "mapped_files, mod_ids, total_sequences, sequences_qm, _ = mmf"
- ]
- },
{
"cell_type": "markdown",
"metadata": {},
@@ -238,30 +207,15 @@
"metadata": {},
"outputs": [],
"source": [
- "def retrieve_constants(karabo_id: str, bias_voltage: int, max_cells: float,\n",
- " acq_rate: float, gain_setting: float, photon_energy: float,\n",
- " only_dark: bool, nodb_with_dark: bool, \n",
- " cal_db_interface: str, creation_time: str, \n",
- " corr_bools: dict, pc_bools: List[bool],\n",
- " inp: Tuple[str, str, str, int]\n",
- " ) -> Tuple[str, str, float, float, str, dict]:\n",
+ "def retrieve_constants(\n",
+ " qm_files: List[Path],\n",
+ " qm: str,\n",
+ " karabo_da: str,\n",
+ " idx: int\n",
+ ") -> Tuple[str, str, float, float, str, dict]:\n",
" \"\"\"\n",
- " Retreive constant for each module in parallel and produce a dictionary\n",
- " with the creation-time and constant file path.\n",
+ " Retreive constants for a module.\n",
" \n",
- " :param karabo_id: (STR) Karabo ID\n",
- " :param bias_voltage: (FLOAT) Bias Voltage\n",
- " :param max_cells: (INT) Memory cells\n",
- " :param acq_rate: (FLOAT) Acquisition Rate\n",
- " :param gain_setting: (FLOAT) Gain setting\n",
- " :param photon_energy: (FLOAT) Photon Energy\n",
- " :param only_dark: (BOOL) only retrieve dark constants\n",
- " :param nodb_with_dark: (BOOL) no constant retrieval even for dark\n",
- " :param cal_db_interface: (STR) the database interface port\n",
- " :param creation_time: (STR) raw data creation time\n",
- " :param corr_bools: (DICT) A dictionary with bools for applying requested corrections\n",
- " :param pc_bools: (LIST) list of bools to retrieve pulse capacitor constants\n",
- " :param inp: (LIST) input for the parallel cluster of the partial function\n",
" :return:\n",
" qm: module virtual name i.e. Q1M1.\n",
" karabo_da: karabo data aggregator.\n",
@@ -271,41 +225,38 @@
" mdata_dict: (DICT) dictionary with the metadata for the retrieved constants.\n",
" \"\"\"\n",
"\n",
- " import sys\n",
- " import traceback\n",
- "\n",
- " import numpy as np\n",
- " from cal_tools.agipdlib import get_acq_rate, get_num_cells\n",
- " from cal_tools.agipdutils import assemble_constant_dict\n",
- " from cal_tools.tools import get_from_db\n",
- " from iCalibrationDB import Conditions, Constants, Detectors\n",
- "\n",
" err = None\n",
- "\n",
- " qm_files, qm, karabo_da, idx = inp\n",
- " # get number of memory cells from a sequence file with image data\n",
- " for f in qm_files:\n",
- " if not max_cells:\n",
- " max_cells = get_num_cells(f, karabo_id, idx)\n",
- " if max_cells is None:\n",
- " if f != qm_files[-1]:\n",
- " continue\n",
- " else:\n",
- " raise ValueError(f\"No raw images found for {qm} for all sequences\")\n",
- " else:\n",
- " cells = np.arange(max_cells)\n",
- " # get out of the loop,\n",
- " # if max_cells is successfully calculated. \n",
+ " if max_cells != 0:\n",
+ " # either use overriding notebook parameter\n",
+ " local_max_cells = max_cells\n",
+ " else:\n",
+ " # or look around in sequence files\n",
+ " for f in qm_files:\n",
+ " local_max_cells = agipdlib.get_num_cells(f, karabo_id, idx)\n",
+ " if local_max_cells is not None:\n",
" break\n",
+ " # maybe we never found this in a sequence file...\n",
+ " if local_max_cells is None:\n",
+ " raise ValueError(f\"No raw images found for {qm} for all sequences\")\n",
"\n",
- " if acq_rate == 0.:\n",
- " acq_rate = get_acq_rate((f, karabo_id, idx))\n",
+ " if acq_rate == 0: \n",
+ " local_acq_rate = agipdlib.get_acq_rate(fast_paths=(f, karabo_id, idx))\n",
+ " else:\n",
+ " local_acq_rate = acq_rate\n",
"\n",
" # avoid creating retireving constant, if requested.\n",
" if not nodb_with_dark:\n",
- " const_dict = assemble_constant_dict(corr_bools, pc_bools, max_cells, bias_voltage,\n",
- " gain_setting, acq_rate, photon_energy,\n",
- " beam_energy=None, only_dark=only_dark)\n",
+ " const_dict = agipdlib.assemble_constant_dict(\n",
+ " corr_bools,\n",
+ " pc_bools,\n",
+ " local_max_cells,\n",
+ " bias_voltage,\n",
+ " gain_setting,\n",
+ " local_acq_rate,\n",
+ " photon_energy,\n",
+ " beam_energy=None,\n",
+ " only_dark=only_dark,\n",
+ " )\n",
"\n",
" # Retrieve multiple constants through an input dictionary\n",
" # to return a dict of useful metadata.\n",
@@ -321,28 +272,42 @@
" mdata_dict['constants'][cname][\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
" mdata_dict['constants'][cname][\"creation-time\"] = \"00:00:00\"\n",
" else:\n",
- " try:\n",
- " condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
- " co, mdata = \\\n",
- " get_from_db(karabo_id, karabo_da, getattr(Constants.AGIPD, cname)(),\n",
- " condition, getattr(np, cval[0])(cval[1]),\n",
- " cal_db_interface, creation_time, meta_only=True, verbosity=0)\n",
- " mdata_const = mdata.calibration_constant_version\n",
- " device_name = mdata.calibration_constant_version.device_name\n",
- " # check if constant was sucessfully retrieved.\n",
- " if mdata.comm_db_success:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
- " f\"{mdata_const.filename}\"\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
- " mdata_dict['physical-detector-unit'] = mdata_const.device_name\n",
- " else:\n",
- " mdata_dict['constants'][cname][\"file-path\"] = const_dict[cname][:2]\n",
- " mdata_dict['constants'][cname][\"creation-time\"] = None\n",
- " except Exception as e:\n",
- " err = f\"Error: {e}, Traceback: {traceback.format_exc()}\"\n",
- " print(err)\n",
- "\n",
- " return qm, mdata_dict, karabo_da, acq_rate, max_cells, err\n",
+ " condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
+ " co, mdata = tools.get_from_db(\n",
+ " karabo_id,\n",
+ " karabo_da,\n",
+ " getattr(Constants.AGIPD, cname)(),\n",
+ " condition,\n",
+ " getattr(np, cval[0])(cval[1]),\n",
+ " cal_db_interface,\n",
+ " creation_time,\n",
+ " meta_only=True,\n",
+ " verbosity=1,\n",
+ " )\n",
+ " mdata_const = mdata.calibration_constant_version\n",
+ " # check if constant was sucessfully retrieved.\n",
+ " if mdata.comm_db_success:\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
+ " f\"{mdata_const.filename}\"\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
+ " mdata_dict['physical-detector-unit'] = mdata_const.device_name\n",
+ " else:\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = const_dict[cname][:2]\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = None\n",
+ "\n",
+ " return qm, mdata_dict, karabo_da, acq_rate, local_max_cells, err"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# set everything up filewise\n",
+ "mapped_files, _, _, _, _ = tools.map_modules_from_folder(\n",
+ " str(in_folder), run, path_template, karabo_da, sequences\n",
+ ")\n",
"\n",
"pc_bools = [corr_bools.get(\"rel_gain\"),\n",
" corr_bools.get(\"adjust_mg_baseline\"),\n",
@@ -361,26 +326,35 @@
"\n",
"# A dict to connect virtual device\n",
"# to actual device name.\n",
- "for i, k_da in zip(modules, karabo_da):\n",
- " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ "for module_index, k_da in zip(modules, karabo_da):\n",
+ " qm = tools.module_index_to_qm(module_index)\n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
" device = getattr(getattr(Detectors, dinstance), qm)\n",
- " qm_files = [str(mapped_files[qm].get()) for _ in range(mapped_files[qm].qsize())]\n",
- "\n",
+ " # TODO: make map_modules_from_folder just return list(s)\n",
+ " qm_files = [Path(mapped_files[qm].get()) for _ in range(mapped_files[qm].qsize())]\n",
" else:\n",
" print(f\"Skipping {qm}\")\n",
" continue\n",
"\n",
- " inp.append((qm_files, qm, k_da, i))\n",
- "\n",
- "p = partial(retrieve_constants, karabo_id, bias_voltage, max_cells, \n",
- " acq_rate, gain_setting, photon_energy, only_dark, nodb_with_dark, \n",
- " cal_db_interface, creation_time, \n",
- " corr_bools, pc_bools)\n",
- "\n",
- "with mp.Pool(processes=nmods) as pool:\n",
- " results = pool.map(p, inp)\n",
- "\n",
+ " inp.append((qm_files, qm, k_da, module_index))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "with multiprocessing.Pool(processes=nmods) as pool:\n",
+ " results = pool.starmap(p, inp)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
"mod_dev = dict()\n",
"mdata_dict = dict()\n",
"for r in results:\n",
@@ -393,12 +367,12 @@
"# check if it is requested not to retrieve any constants from the database\n",
"if not nodb_with_dark:\n",
" metadata.update({\"retrieved-constants\": mdata_dict})\n",
- " \n",
+ "\n",
" print(\"\\nRetrieved constants for modules: \",\n",
" f\"{[', '.join([f'Q{x//4+1}M{x%4+1}' for x in modules])]}\")\n",
" print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Memory cells: {max_cells}\\n\"\n",
" f\"• Acquisition rate: {acq_rate}\\n• Gain setting: {gain_setting}\\n• Photon Energy: {photon_energy}\\n\")\n",
- " print(f\"Constant metadata is saved under \\\"retrieved-constants\\\" in calibration_metadata.yml\\n\")\n",
+ " print(\"Constant metadata is saved under \\\"retrieved-constants\\\" in calibration_metadata.yml\\n\")\n",
"else:\n",
" print(\"No constants were retrieved as calibrated files will be used.\")"
]
@@ -425,7 +399,7 @@
" # Store few time stamps if exists\n",
" # Add NA to keep array structure\n",
" for cname in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
- " if not k_da in mdata_dict or dinfo[\"err\"]:\n",
+ " if k_da not in mdata_dict or dinfo[\"err\"]:\n",
" module_timestamps[cname] = \"Err\"\n",
" else:\n",
" if cname in mdata_dict[k_da]:\n",