From b5ef7995052aa21db51deabacb195249b7759e9d Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Tue, 20 Apr 2021 15:42:41 +0200
Subject: [PATCH] New notebook for injecting constants from stored h5files
---
...t_calibration_constants_from_h5files.ipynb | 218 ++++++++++++++++++
src/cal_tools/tools.py | 4 +-
2 files changed, 220 insertions(+), 2 deletions(-)
create mode 100644 notebooks/LPD/inject_calibration_constants_from_h5files.ipynb
diff --git a/notebooks/LPD/inject_calibration_constants_from_h5files.ipynb b/notebooks/LPD/inject_calibration_constants_from_h5files.ipynb
new file mode 100644
index 000000000..6b93ca8b4
--- /dev/null
+++ b/notebooks/LPD/inject_calibration_constants_from_h5files.ipynb
@@ -0,0 +1,218 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Injecting calibration constant data to the database #\n",
+ "\n",
+ "Author: European XFEL Detector Group, Version: 1.0\n",
+ "\n",
+ "Reading h5files of calibration constants to inject them to the database. Used for LPD"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# calibration constant parameters:\n",
+ "constant_names = [\"GainMap\", \"GainAmpMap\", \"FFMap\", \"BadPixelsFF\"] # calibration constant name, required.\n",
+ "in_folder = \"/gpfs/exfel/data/scratch/yousefh/LPD_CalinConst_Provisional/\" # calibration constant file, required.\n",
+ "proposal = \"\" # Add proposal number to be sent to the database as a part of Raw data location.\n",
+ "runs = [] # Add list of runs to be sent to the database as a part of Raw data location.\n",
+ "\n",
+ "# detector parameters:\n",
+ "karabo_id = \"FXE_DET_LPD1M-1\" # detector identifier, required.\n",
+ "karabo_da = [\"LPD00\"] # karabo data aggregators, required.\n",
+ "\n",
+ "# calibration database parameters:\n",
+ "cal_db_interface = \"tcp://max-exfl017:8020\"\n",
+ "\n",
+ "# calibration constant conditions:\n",
+ "memory_cells = 512\n",
+ "bias_voltage = 250\n",
+ "capacitor = 5\n",
+ "creation_time = '2020-01-20T14:12:06' # creation time for the injected constants. required format '2019-01-20T14:12:06'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 36,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import datetime\n",
+ "import h5py\n",
+ "from pathlib import Path\n",
+ "\n",
+ "from iCalibrationDB import (\n",
+ " Constants,\n",
+ " Conditions,\n",
+ ")\n",
+ "from cal_tools.tools import (\n",
+ " get_from_db,\n",
+ " get_pdu_from_db,\n",
+ " send_to_db,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "pixels_x = pixels_y = 256\n",
+ "\n",
+ "# if proposal or runs are given assign file_loc \n",
+ "# for calibration constant versions metadata.\n",
+ "file_loc = \"\"\n",
+ "if proposal:\n",
+ " file_loc += f\"proposal:{proposal}\"\n",
+ "if len(runs) > 0:\n",
+ " file_loc += f\"runs: {runs}\"\n",
+ "\n",
+ "if file_loc == \"\"\n",
+ " print(\n",
+ " \"No proposal or runs were given for constant source.\"\n",
+ " \" No \\\"Raw data location\\\" will be injected with the constants\"\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def validate_input_paths(in_folder):\n",
+ " \n",
+ " # Validate input parameters:\n",
+ " if not (in_folder):\n",
+ " raise ValueError(\n",
+ " \"ERROR: \\\"in_folder\\\" is not given.\"\n",
+ " \" Please provide the constants input folder.\"\n",
+ " )\n",
+ "\n",
+ " c_folder = Path(in_folder)\n",
+ " \n",
+ " if not c_folder.isdir():\n",
+ " raise ValueError(\n",
+ " f\"ERROR: in_folder {in_folder} directory doesn't exist.\"\n",
+ " )\n",
+ "\n",
+ " try:\n",
+ " creation_time = datetime.datetime.strptime(creation_time, '%Y-%m-%dT%H:%M:%S')\n",
+ " except ValueError:\n",
+ " raise ValueError(\"Incorrect data format, should be YYYY-MM-DDTHH:MM:SS i.e. 2019-01-20T14:12:06\")\n",
+ " \n",
+ " return c_folder, creation_time"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "No proposal or runs were given for constant source. No \"Raw data location\" will be injected with the constants\n",
+ "Physical detector units retrieved are: {'LPD00': 'LPD_SIV1_LPDV2_SM015'}\n",
+ "BadPixelsFF is injected with creation-time: 2020-01-20 14:12:06\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "c_folder, creation_time = validate_input_paths(in_folder)\n",
+ "\n",
+ "# create a report path for calibration constant versions metadata.\n",
+ "report_path = get_report(\n",
+ " out_folder=in_folder,\n",
+ " default_path=f\"No_report/LPD_{datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S')}\"\n",
+ ")\n",
+ "\n",
+ "# Calibration constants condition object.\n",
+ "condition = Conditions.Dark.LPD(\n",
+ " memory_cells=memory_cells,\n",
+ " bias_voltage=bias_voltage,\n",
+ " pixels_x=pixels_x,\n",
+ " pixels_y=pixels_y,\n",
+ " capacitor=capacitor,\n",
+ ")\n",
+ "\n",
+ "mod_mapping = {}\n",
+ "for const in constant_names:\n",
+ "\n",
+ " # Calibration constant object.\n",
+ " constant = getattr(Constants.LPD, const)()\n",
+ " \n",
+ " if len(mod_mapping.keys()) == 0\n",
+ " # Retrieve the detector modules if it is not known.\n",
+ " physical_units = get_pdu_from_db(\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=karabo_da,\n",
+ " constant=constant,\n",
+ " condition=condition,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " snapshot_at=creation_time\n",
+ " )\n",
+ " mod_mapping = dict(zip(karabo_da, physical_units))\n",
+ " print(\"Physical detector units retrieved are: \", mod_mapping)\n",
+ "\n",
+ " for pdu, k_da in zip(physical_units, karabo_da): \n",
+ "\n",
+ " cfile = c_folder / f\"{const}_{k_da}\"\n",
+ "\n",
+ " # load constant data.\n",
+ " with h5py.File(cfile, \"r\") as f:\n",
+ " cdata = f[const][()]\n",
+ "\n",
+ " # Validate for only LPD at the moment.\n",
+ " if not cdata.shape == (pixels_x, pixels_y, memory_cells, 3):\n",
+ " raise ValueError(\n",
+ " f\"ERROR: {const} constant data shape is not as expected.\"\n",
+ " f\" {cdata.shape} != ({pixels_x}, {pixels_y}, {memory_cells}, 3).\"\n",
+ " )\n",
+ "\n",
+ " constant.data = cdata\n",
+ "\n",
+ " md = send_to_db(\n",
+ " db_module=pdu,\n",
+ " karabo_id=karabo_id,\n",
+ " constant=constant,\n",
+ " condition=condition,\n",
+ " file_loc=file_loc,\n",
+ " report_path=report_path,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " )"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "cal_venv",
+ "language": "python",
+ "name": "cal_venv"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.6.7"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/src/cal_tools/tools.py b/src/cal_tools/tools.py
index 587bb8970..4f6a3fe6c 100644
--- a/src/cal_tools/tools.py
+++ b/src/cal_tools/tools.py
@@ -373,13 +373,13 @@ def get_random_db_interface(cal_db_interface):
return cal_db_interface
-def get_report(out_folder: str):
+def get_report(out_folder: str, default_path: str = ""):
"""Get the report path from calibration_metadata.yml
stored in the out_folder.
"""
metadata = CalibrationMetadata(out_folder)
- report_path = metadata.get("report-path", "")
+ report_path = metadata.get("report-path", default_path)
if not report_path:
print("WARNING: No report path will be injected "
"with the constants.\n")
--
GitLab