diff --git a/notebooks/LPD/Inject_calibration_constants_from_h5files.ipynb b/notebooks/LPD/Inject_calibration_constants_from_h5files.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..463bf868cf6884a092f72407d700864ef8dbb4cf
--- /dev/null
+++ b/notebooks/LPD/Inject_calibration_constants_from_h5files.ipynb
@@ -0,0 +1,245 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Injecting calibration constant data to the database #\n",
+ "\n",
+ "Author: European XFEL Detector Group, Version: 1.0\n",
+ "\n",
+ "Reading h5files of calibration constants to inject them to the database. Used for LPD"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# calibration constant parameters:\n",
+ "constant_names = [\"\"] # calibration constant name, required.\n",
+ "in_folder = \"\" # calibration constants folder, required.\n",
+ "out_folder = \"\" # output folder to store report path in case the notebook is executed by CLI, required.\n",
+ "proposal = \"\" # Add proposal number to be sent to the database as a part of Raw data location.\n",
+ "runs = [\"\"] # Add list of runs to be sent to the database as a part of Raw data location.\n",
+ "\n",
+ "# detector parameters:\n",
+ "karabo_id = \"FXE_DET_LPD1M-1\" # detector identifier.\n",
+ "karabo_da = [\"all\"] # karabo data aggregators. default \"all\" for all 16 karabo data aggregator names.\n",
+ "\n",
+ "# calibration database parameters:\n",
+ "cal_db_interface = \"tcp://max-exfl016:8015#8045\" # calibration DB zmq address.\n",
+ "\n",
+ "# calibration constant conditions:\n",
+ "memory_cells = 512 # Number of memory cells. Used for constant conditions.\n",
+ "bias_voltage = 250 # bias voltage value. Used for constant conditions.\n",
+ "capacitor = 5 # capacitor value. Used for constant conditions.\n",
+ "category = 0 # calibration constant source category, 0 for European XFEl and 1 for RAL. Used for constant conditions.\n",
+ "photon_energy = 9.2 # calibration constant photon energy. Used for constant conditions.\n",
+ "creation_time = '2020-01-20T14:12:06' # creation time for the injected constants. required format '2019-01-20T14:12:06'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import multiprocessing # noqa\n",
+ "from datetime import datetime # noqa\n",
+ "from pathlib import Path # noqa\n",
+ "from typing import List, Tuple # noqa\n",
+ "\n",
+ "import h5py # noqa \n",
+ "from cal_tools.tools import get_pdu_from_db, get_report, send_to_db # noqa\n",
+ "from iCalibrationDB import Conditions, Constants # noqa"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "pixels_x = pixels_y = 256\n",
+ "\n",
+ "if karabo_da[0].lower() == \"all\":\n",
+ " karabo_da = [f\"LPD{i:02d}\" for i in range(16)]\n",
+ "\n",
+ "# if proposal or runs are given assign file_loc\n",
+ "# for calibration constant versions metadata.\n",
+ "file_loc = \"\"\n",
+ "if proposal:\n",
+ " file_loc += f\"proposal:{proposal}\"\n",
+ "if runs[0] != \"\":\n",
+ " file_loc += f\"runs: {runs}\"\n",
+ "\n",
+ "if file_loc == \"\":\n",
+ " print(\n",
+ " \"No proposal or runs were given for constant source.\"\n",
+ " \" No \\\"Raw data location\\\" will be injected with the constants.\\n\"\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def validate_input_paths(\n",
+ " in_folder: str,\n",
+ " creation_time: str,\n",
+ " constant_names: List[str],\n",
+ ") -> Tuple[str, datetime]:\n",
+ "\n",
+ " # Validate input parameters:\n",
+ " if not (in_folder):\n",
+ " raise ValueError(\n",
+ " \"ERROR: \\\"in_folder\\\" is not given.\"\n",
+ " \" Please provide the constants input folder.\"\n",
+ " )\n",
+ "\n",
+ " c_folder = Path(in_folder)\n",
+ "\n",
+ " if not c_folder.is_dir():\n",
+ " raise ValueError(\n",
+ " f\"ERROR: in_folder {in_folder} directory doesn't exist.\"\n",
+ " )\n",
+ "\n",
+ " try:\n",
+ " creation_time = datetime.strptime(creation_time, '%Y-%m-%dT%H:%M:%S')\n",
+ " except ValueError:\n",
+ " raise ValueError(\n",
+ " \"Incorrect data format, \"\n",
+ " \"should be YYYY-MM-DDTHH:MM:SS i.e. 2019-01-20T14:12:06\"\n",
+ " )\n",
+ "\n",
+ " for constant in constant_names:\n",
+ " if not hasattr(Constants.LPD, constant):\n",
+ " raise ValueError(\n",
+ " f\"ERROR: Constant name \\\"{constant}\\\" is not a known LPD constant. \"\n",
+ " f\"Available LPD Constants are {[c for c in dir(Constants.LPD) if c[0] != '_']}\"\n",
+ " )\n",
+ " return c_folder, creation_time"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def inject_constants(\n",
+ " constant_name: str,\n",
+ " karabo_da: str,\n",
+ " physical_unit: str,\n",
+ "):\n",
+ " constant = getattr(Constants.LPD, constant_name)()\n",
+ "\n",
+ " cfile = c_folder / f\"{constant_name}_{karabo_da}.h5\"\n",
+ "\n",
+ " if not cfile.exists():\n",
+ " raise ValueError(f\"ERROR: constant file {cfile} doesn't exists.\\n\")\n",
+ "\n",
+ " # load constant data.\n",
+ " with h5py.File(cfile, \"r\") as f:\n",
+ " cdata = f[constant_name][()]\n",
+ "\n",
+ " # Validate for only LPD at the moment.\n",
+ " if not cdata.shape == (pixels_x, pixels_y, memory_cells, 3):\n",
+ " raise ValueError(\n",
+ " f\"ERROR: {const} constant data shape is not as expected.\"\n",
+ " f\" {cdata.shape} != ({pixels_x}, {pixels_y}, {memory_cells}, 3).\\n\"\n",
+ " )\n",
+ "\n",
+ " constant.data = cdata\n",
+ "\n",
+ " send_to_db(\n",
+ " db_module=physical_unit,\n",
+ " karabo_id=karabo_id,\n",
+ " constant=constant,\n",
+ " condition=condition,\n",
+ " file_loc=file_loc,\n",
+ " report_path=report_path,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "c_folder, creation_time = validate_input_paths(\n",
+ " in_folder,\n",
+ " creation_time,\n",
+ " constant_names,\n",
+ ")\n",
+ "\n",
+ "# create a report path for calibration constant versions metadata.\n",
+ "report_name = f\"No_report/LPD_{datetime.now().strftime('%Y-%m-%dT%H:%M:%S')}\"\n",
+ "report_path = get_report(\n",
+ " out_folder=in_folder,\n",
+ " default_path=report_name\n",
+ ")\n",
+ "\n",
+ "# Calibration constants condition object.\n",
+ "condition = Conditions.Illuminated.LPD(\n",
+ " memory_cells=memory_cells,\n",
+ " bias_voltage=bias_voltage,\n",
+ " photon_energy=photon_energy,\n",
+ " pixels_x=pixels_x,\n",
+ " pixels_y=pixels_y,\n",
+ " capacitor=capacitor,\n",
+ " category=category,\n",
+ ")\n",
+ "\n",
+ "# Retrieve all physical detector units for the given karabo_da list.\n",
+ "physical_units = get_pdu_from_db(\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=karabo_da,\n",
+ " constant=getattr(Constants.LPD, constant_names[0])(),\n",
+ " condition=condition,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " snapshot_at=creation_time\n",
+ ")\n",
+ "\n",
+ "mod_mapping = dict(zip(karabo_da, physical_units))\n",
+ "print(\"Physical detector units retrieved are: \", mod_mapping, \"\\n\")\n",
+ "\n",
+ "inp = []\n",
+ "for const in constant_names:\n",
+ " for k_da, pdu in mod_mapping.items():\n",
+ " inp.append((const, k_da, pdu))\n",
+ "\n",
+ "with multiprocessing.Pool(processes=5) as pool:\n",
+ " results = pool.starmap(inject_constants, inp)"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/setup.py b/setup.py
index 3ad381ac0a111e758849cbc05e2ad3a9819e4b8a..7a65bbfa5b7dc6d52ebce011678d601677284a85 100644
--- a/setup.py
+++ b/setup.py
@@ -73,7 +73,7 @@ setup(
},
ext_modules=cythonize(ext_modules),
install_requires=[
- "iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.0.5", # noqa
+ "iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.0.7", # noqa
"XFELDetectorAnalysis @ git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git@2.5.6-2.10.0#subdirectory=lib", # noqa
"pasha @ git+https://github.com/European-XFEL/pasha.git@0.1.0",
"Cython==0.29.21",
diff --git a/src/cal_tools/tools.py b/src/cal_tools/tools.py
index 587bb8970a0a645d8bfbe6113669e4c42f1e4953..f8cd5279a060407d065f8d166015d59b77de9a49 100644
--- a/src/cal_tools/tools.py
+++ b/src/cal_tools/tools.py
@@ -373,13 +373,13 @@ def get_random_db_interface(cal_db_interface):
return cal_db_interface
-def get_report(out_folder: str):
+def get_report(out_folder: str, default_path: str = ""):
"""Get the report path from calibration_metadata.yml
stored in the out_folder.
"""
metadata = CalibrationMetadata(out_folder)
- report_path = metadata.get("report-path", "")
+ report_path = metadata.get("report-path", default_path)
if not report_path:
print("WARNING: No report path will be injected "
"with the constants.\n")
@@ -413,11 +413,13 @@ def get_pdu_from_db(karabo_id: str, karabo_da: Union[str, list],
metadata = _init_metadata(constant, condition, None)
+ # A random interface is chosen if there is # for address range.
+ this_interface = get_random_db_interface(cal_db_interface)
# CalibrationDBRemote expects a string.
if snapshot_at is not None and hasattr(snapshot_at, 'isoformat'):
snapshot_at = snapshot_at.isoformat()
- pdu_dicts = metadata.retrieve_pdus_for_detector(receiver=cal_db_interface,
+ pdu_dicts = metadata.retrieve_pdus_for_detector(receiver=this_interface,
karabo_id=karabo_id,
snapshot_at=snapshot_at,
timeout=timeout)
@@ -622,9 +624,11 @@ def send_to_db(db_module: str, karabo_id: str, constant, condition,
except Exception as e:
# TODO: refactor to use custom exception class
if "has already been taken" in str(e):
- print(f"WARNING: {constant.name} has already been "
- "injected with the same parameter "
- "conditions\n")
+ print(
+ f"WARNING: {constant.name} for {db_module}"
+ " has already been injected with the same "
+ "parameter conditions."
+ )
else:
print(f"{e}\n")
@@ -636,8 +640,11 @@ def send_to_db(db_module: str, karabo_id: str, constant, condition,
raise RuntimeError(f'{e}')
if success:
- print(f"{constant.name} is injected with creation-time: "
- f"{metadata.calibration_constant_version.begin_at}\n")
+ print(
+ f"{constant.name} for {db_module} "
+ "is injected with creation-time: "
+ f"{metadata.calibration_constant_version.begin_at}."
+ )
return metadata
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index 1b4e4cd965172286162c83590c30175bf17041a8..a98aeef536ec49d0a51cccb2abe789203c1c935a 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -98,6 +98,12 @@ notebooks = {
"default concurrency": [-1],
"cluster cores": 2},
},
+ "INJECT_CONSTANTS": {
+ "notebook": "notebooks/LPD/Inject_calibration_constants_from_h5files.ipynb",
+ "concurrency": {"parameter": None,
+ "default concurrency": None,
+ "cluster cores": 1},
+ }
},
"PNCCD": {
"DARK": {