{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Injecting calibration constant data to the database #\n",
"\n",
"Author: European XFEL Detector Group, Version: 1.0\n",
"\n",
"Reading h5files of calibration constants to inject them to the database. Used for LPD"
]
},
{
"cell_type": "code",
"execution_count": 35,
"metadata": {},
"outputs": [],
"source": [
"# calibration constant parameters:\n",
"constant_names = [\"GainMap\", \"GainAmpMap\", \"FFMap\", \"BadPixelsFF\"] # calibration constant name, required.\n",
"in_folder = \"/gpfs/exfel/data/scratch/yousefh/LPD_CalinConst_Provisional/\" # calibration constant file, required.\n",
"proposal = \"\" # Add proposal number to be sent to the database as a part of Raw data location.\n",
"runs = [] # Add list of runs to be sent to the database as a part of Raw data location.\n",
"\n",
"# detector parameters:\n",
"karabo_id = \"FXE_DET_LPD1M-1\" # detector identifier, required.\n",
"karabo_da = [\"LPD00\"] # karabo data aggregators, required.\n",
"\n",
"# calibration database parameters:\n",
"cal_db_interface = \"tcp://max-exfl017:8020\"\n",
"\n",
"# calibration constant conditions:\n",
"memory_cells = 512\n",
"bias_voltage = 250\n",
"capacitor = 5\n",
"creation_time = '2020-01-20T14:12:06' # creation time for the injected constants. required format '2019-01-20T14:12:06'"
]
},
{
"cell_type": "code",
"execution_count": 36,
"metadata": {},
"outputs": [],
"source": [
"import datetime\n",
"import h5py\n",
"from pathlib import Path\n",
"\n",
"from iCalibrationDB import (\n",
" Constants,\n",
" Conditions,\n",
")\n",
"from cal_tools.tools import (\n",
" get_from_db,\n",
" get_pdu_from_db,\n",
" send_to_db,\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"pixels_x = pixels_y = 256\n",
"\n",
"# if proposal or runs are given assign file_loc \n",
"# for calibration constant versions metadata.\n",
"file_loc = \"\"\n",
"if proposal:\n",
" file_loc += f\"proposal:{proposal}\"\n",
"if len(runs) > 0:\n",
" file_loc += f\"runs: {runs}\"\n",
"\n",
"if file_loc == \"\"\n",
" print(\n",
" \"No proposal or runs were given for constant source.\"\n",
" \" No \\\"Raw data location\\\" will be injected with the constants\"\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def validate_input_paths(in_folder):\n",
" \n",
" # Validate input parameters:\n",
" if not (in_folder):\n",
" raise ValueError(\n",
" \"ERROR: \\\"in_folder\\\" is not given.\"\n",
" \" Please provide the constants input folder.\"\n",
" )\n",
"\n",
" c_folder = Path(in_folder)\n",
" \n",
" if not c_folder.isdir():\n",
" raise ValueError(\n",
" f\"ERROR: in_folder {in_folder} directory doesn't exist.\"\n",
" )\n",
"\n",
" try:\n",
" creation_time = datetime.datetime.strptime(creation_time, '%Y-%m-%dT%H:%M:%S')\n",
" except ValueError:\n",
" raise ValueError(\"Incorrect data format, should be YYYY-MM-DDTHH:MM:SS i.e. 2019-01-20T14:12:06\")\n",
" \n",
" return c_folder, creation_time"
]
},
{
"cell_type": "code",
"execution_count": 37,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"No proposal or runs were given for constant source. No \"Raw data location\" will be injected with the constants\n",
"Physical detector units retrieved are: {'LPD00': 'LPD_SIV1_LPDV2_SM015'}\n",
"BadPixelsFF is injected with creation-time: 2020-01-20 14:12:06\n",
"\n"
]
}
],
"source": [
"c_folder, creation_time = validate_input_paths(in_folder)\n",
"\n",
"# create a report path for calibration constant versions metadata.\n",
"report_path = get_report(\n",
" out_folder=in_folder,\n",
" default_path=f\"No_report/LPD_{datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S')}\"\n",
")\n",
"\n",
"# Calibration constants condition object.\n",
"condition = Conditions.Dark.LPD(\n",
" memory_cells=memory_cells,\n",
" bias_voltage=bias_voltage,\n",
" pixels_x=pixels_x,\n",
" pixels_y=pixels_y,\n",
" capacitor=capacitor,\n",
")\n",
"\n",
"mod_mapping = {}\n",
"for const in constant_names:\n",
"\n",
" # Calibration constant object.\n",
" constant = getattr(Constants.LPD, const)()\n",
" \n",
" if len(mod_mapping.keys()) == 0\n",
" # Retrieve the detector modules if it is not known.\n",
" physical_units = get_pdu_from_db(\n",
" karabo_id=karabo_id,\n",
" karabo_da=karabo_da,\n",
" constant=constant,\n",
" condition=condition,\n",
" cal_db_interface=cal_db_interface,\n",
" snapshot_at=creation_time\n",
" )\n",
" mod_mapping = dict(zip(karabo_da, physical_units))\n",
" print(\"Physical detector units retrieved are: \", mod_mapping)\n",
"\n",
" for pdu, k_da in zip(physical_units, karabo_da): \n",
"\n",
" cfile = c_folder / f\"{const}_{k_da}\"\n",
"\n",
" # load constant data.\n",
" with h5py.File(cfile, \"r\") as f:\n",
" cdata = f[const][()]\n",
"\n",
" # Validate for only LPD at the moment.\n",
" if not cdata.shape == (pixels_x, pixels_y, memory_cells, 3):\n",
" raise ValueError(\n",
" f\"ERROR: {const} constant data shape is not as expected.\"\n",
" f\" {cdata.shape} != ({pixels_x}, {pixels_y}, {memory_cells}, 3).\"\n",
" )\n",
"\n",
" constant.data = cdata\n",
"\n",
" md = send_to_db(\n",
" db_module=pdu,\n",
" karabo_id=karabo_id,\n",
" constant=constant,\n",
" condition=condition,\n",
" file_loc=file_loc,\n",
" report_path=report_path,\n",
" cal_db_interface=cal_db_interface,\n",
" creation_time=creation_time,\n",
" )"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "cal_venv",
"language": "python",
"name": "cal_venv"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
}
},
"nbformat": 4,
"nbformat_minor": 4
}