{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# LPD Retrieving Constants Pre-correction #\n",
"\n",
"Author: European XFEL Detector Group, Version: 1.0\n",
"\n",
"The following notebook provides a constants metadata in a YAML file to use while correcting LPD images."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Input parameters\n",
"in_folder = \"/gpfs/exfel/exp/FXE/202201/p003073/raw/\" # the folder to read data from, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/LPD_test\" # the folder to output to, required\n",
"metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate.\n",
"modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty\n",
"karabo_da = [''] # Data aggregators names to correct, use [''] for all\n",
"run = 10 # run to process, required\n",
"\n",
"# Source parameters\n",
"karabo_id = 'FXE_DET_LPD1M-1' # Karabo domain for detector.\n",
"\n",
"# CalCat parameters\n",
"creation_time = \"\" # The timestamp to use with Calibration DB. Required Format: \"YYYY-MM-DD hh:mm:ss\" e.g. 2019-07-04 11:02:41\n",
"\n",
"# Operating conditions\n",
"mem_cells = 512 # Memory cells, LPD constants are always taken with 512 cells.\n",
"bias_voltage = 250.0 # Detector bias voltage.\n",
"capacitor = '5pF' # Capacitor setting: 5pF or 50pF\n",
"photon_energy = 9.2 # Photon energy in keV.\n",
"category = 0 # Whom to blame."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pathlib import Path\n",
"from time import perf_counter\n",
"\n",
"from calibration_client import CalibrationClient\n",
"from calibration_client.modules import CalibrationConstantVersion\n",
"\n",
"from cal_tools.tools import (\n",
" CalibrationMetadata,\n",
" calcat_creation_time,\n",
" save_constant_metadata,\n",
")\n",
"from cal_tools.restful_config import restful_config"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"out_folder = Path(out_folder)\n",
"out_folder.mkdir(exist_ok=True)\n",
"\n",
"metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
"# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
"retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
"\n",
"creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
"print(f'Using {creation_time.isoformat()} as creation time')\n",
"\n",
"# Pick all modules/aggregators or those selected.\n",
"if not karabo_da or karabo_da == ['']:\n",
" if not modules or modules == [-1]:\n",
" modules = list(range(16))\n",
"\n",
" karabo_da = [f'LPD{i:02d}' for i in modules] "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Connect to CalCat.\n",
"calcat_config = restful_config['calcat']\n",
"client = CalibrationClient(\n",
" base_api_url=calcat_config['base-api-url'],\n",
" use_oauth2=calcat_config['use-oauth2'],\n",
" client_id=calcat_config['user-id'],\n",
" client_secret=calcat_config['user-secret'],\n",
" user_email=calcat_config['user-email'],\n",
" token_url=calcat_config['token-url'],\n",
" refresh_url=calcat_config['refresh-url'],\n",
" auth_url=calcat_config['auth-url'],\n",
" scope='')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"dark_calibrations = {\n",
" 1: 'Offset',\n",
" 14: 'BadPixelsDark',\n",
"}\n",
"\n",
"dark_condition = [\n",
" dict(parameter_id=1, value=bias_voltage), # Sensor bias voltage\n",
" dict(parameter_id=7, value=mem_cells), # Memory cells\n",
" dict(parameter_id=15, value=capacitor), # Feedback capacitor\n",
" dict(parameter_id=13, value=256), # Pixels X\n",
" dict(parameter_id=14, value=256), # Pixels Y\n",
"]\n",
"\n",
"print('Querying calibration database', end='', flush=True)\n",
"start = perf_counter()\n",
"for k_da in karabo_da:\n",
" pdu = None\n",
" if k_da in retrieved_constants:\n",
" print(f\"Constant for {k_da} already in {metadata.filename}, won't query again.\") # noqa\n",
" continue\n",
" retrieved_constants[k_da] = dict()\n",
" const_mdata = retrieved_constants[k_da][\"constants\"] = dict()\n",
" for calibrations, condition in [\n",
" (dark_calibrations, dark_condition),\n",
" (illuminated_calibrations, illuminated_condition)\n",
" ]:\n",
" resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(\n",
" client, karabo_id, list(calibrations.keys()),\n",
" {'parameters_conditions_attributes': condition},\n",
" karabo_da=k_da, event_at=creation_time.isoformat())\n",
"\n",
"lpd_cal_metadata = lpd_cal.metadata()\n",
"\n",
"# Validate the constants availability and raise/warn correspondingly. \n",
"for mod, ccv_dict in lpd_cal_metadata.items():\n",
" missing_dark_constants = set(\n",
" c for c in [\"Offset\", \"BadPixelsDark\"] if c not in ccv_dict.keys())\n",
" missing_gain_constants = set(\n",
" c for c in [\"BadPixelsFF\", \"GainAmpMap\", \"FFMap\", \"RelativeGain\"] if c not in ccv_dict.keys()) # noqa\n",
" if missing_dark_constants:\n",
" raise KeyError(\n",
" f\"Dark constants {missing_dark_constants} are not available for correction. Module: {mod}\") # noqa\n",
" if missing_gain_constants:\n",
" warning(\n",
" f\"Gain constants {missing_gain_constants} were not retrieved. Module: {mod}\")\n",
"\n",
"for mod, ccv_dict in lpd_cal_metadata.items():\n",
" mdata_dict = {\"constants\": dict()}\n",
" for cname, ccv_metadata in ccv_dict.items():\n",
" mdata_dict[\"constants\"][cname] = {\n",
" \"path\": str(lpd_cal.caldb_root / ccv_metadata[\"path\"]),\n",
" \"dataset\": ccv_metadata[\"dataset\"],\n",
" \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
" }\n",
" mdata_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n",
" retrieved_constants[mod] = mdata_dict\n",
"metadata.save()\n",
"\n",
"total_time = perf_counter() - start\n",
"print(f'{total_time:.1f}s')\n",
"print(f\"Stored retrieved constants in {metadata.filename}\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "cal4_venv",
"language": "python",
"name": "cal4_venv"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.11 (default, Jul 2 2021, 14:23:46) \n[GCC 4.8.5 20150623 (Red Hat 4.8.5-44)]"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "ccde353e8822f411c1c49844e1cbe3edf63293a69efd975d1b44f5e852832668"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}