{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# LPD Retrieving Constants Pre-correction #\n",
"\n",
"Author: European XFEL Detector Group, Version: 1.0\n",
"\n",
"The following notebook provides a constants metadata in a YAML file to use while correcting LPD images."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# Input parameters\n",
"in_folder = \"/gpfs/exfel/exp/FXE/202201/p003073/raw/\" # the folder to read data from, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/LPD_test\" # the folder to output to, required\n",
"metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate.\n",
"modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty\n",
"karabo_da = [''] # Data aggregators names to correct, use [''] for all\n",
"run = 10 # run to process, required\n",
"\n",
"# Source parameters\n",
"karabo_id = 'FXE_DET_LPD1M-1' # Karabo domain for detector.\n",
"input_source = '{karabo_id}/DET/{module_index}CH0:xtdf' # Input fast data source.\n",
"\n",
"# CalCat parameters\n",
"creation_time = \"\" # The timestamp to use with Calibration DB. Required Format: \"YYYY-MM-DD hh:mm:ss\" e.g. 2019-07-04 11:02:41\n",
"\n",
"# Operating conditions\n",
"mem_cells = 512 # Memory cells, LPD constants are always taken with 512 cells.\n",
"bias_voltage = 250.0 # Detector bias voltage.\n",
"capacitor = '5pF' # Capacitor setting: 5pF or 50pF\n",
"photon_energy = 9.2 # Photon energy in keV.\n",
"category = 0 # Whom to blame.\n",
"use_cell_order = 'auto' # Whether to use memory cell order as a detector condition (not stored for older constants)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from logging import warning\n",
"from pathlib import Path\n",
"\n",
"import numpy as np\n",
"\n",
"from calibration_client import CalibrationClient\n",
"from calibration_client.modules import CalibrationConstantVersion\n",
"import extra_data as xd\n",
"\n",
"from cal_tools.lpdlib import get_mem_cell_pattern, make_cell_order_condition\n",
"import cal_tools.restful_config as rest_cfg\n",
"from cal_tools.calcat_interface import CalCatError, LPD_CalibrationData\n",
"from cal_tools.tools import (\n",
" CalibrationMetadata,\n",
" calcat_creation_time,\n",
")\n",
"from cal_tools.step_timing import StepTimer"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"out_folder = Path(out_folder)\n",
"out_folder.mkdir(exist_ok=True)\n",
"\n",
"metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
"# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
"retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
"\n",
"creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
"print(f'Using {creation_time.isoformat()} as creation time')\n",
"\n",
"# Pick all modules/aggregators or those selected.\n",
"if not karabo_da or karabo_da == ['']:\n",
" if not modules or modules == [-1]:\n",
" modules = list(range(16))\n",
"\n",
" karabo_da = [f'LPD{i:02d}' for i in modules]\n",
"else:\n",
" modules = [int(x[-2:]) for x in karabo_da]\n",
"\n",
"# List of detector sources.\n",
"det_inp_sources = [input_source.format(karabo_id=karabo_id, module_index=int(da[-2:])) for da in karabo_da]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"step_timer = StepTimer()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"step_timer.start()\n",
"\n",
"cell_ids_pattern_s = None\n",
"if use_cell_order != 'never':\n",
" # Read the order of memory cells used\n",
" raw_data = xd.RunDirectory(Path(in_folder, f'r{run:04d}'))\n",
" cell_ids_pattern_s = make_cell_order_condition(\n",
" use_cell_order, get_mem_cell_pattern(raw_data, det_inp_sources)\n",
" )\n",
" print(\"Memory cells order:\", cell_ids_pattern_s)\n",
"\n",
"lpd_cal = LPD_CalibrationData(\n",
" detector_name=karabo_id,\n",
" modules=karabo_da,\n",
" sensor_bias_voltage=bias_voltage,\n",
" memory_cells=mem_cells,\n",
" feedback_capacitor=capacitor,\n",
" source_energy=photon_energy,\n",
" memory_cell_order=cell_ids_pattern_s,\n",
" category=category,\n",
" event_at=creation_time,\n",
" client=rest_cfg.calibration_client(),\n",
")\n",
"\n",
"# Initialize const_data in case no constants were retrieved.\n",
"lpd_cal_metadata = {mod: {\"no-constants-retrieved\": None} for mod in karabo_da}\n",
"\n",
"constant_data = lpd_cal.metadata([\"Offset\", \"BadPixelsDark\"])\n",
"try:\n",
" illum_const_data = lpd_cal.metadata(lpd_cal.illuminated_calibrations)\n",
" for key, value in illum_const_data.items():\n",
" const_data.setdefault(key, {}).update(value)\n",
"except CalCatError as e: # TODO: replace when API errors are improved.\n",
" warning(f\"CalCatError: {e}\")\n",
"\n",
"# Validate the constants availability and raise/warn correspondingly. \n",
"for mod, ccv_dict in lpd_cal_metadata.items():\n",
" missing_offset = {\"Offset\"} - set(ccv_dict)\n",
" warn_missing_constants = {\n",
" \"BadPixelsDark\", \"BadPixelsFF\", \"GainAmpMap\", \"FFMap\", \"RelativeGain\"} - set(ccv_dict)\n",
" if missing_offset:\n",
" warning(f\"Offset constant is not available to correct {mod}\")\n",
" karabo_da.remove(mod)\n",
" if warn_missing_constants:\n",
" warning(f\"Constants {warn_missing_constants} were not retrieved for {mod}.\")\n",
"if not karabo_da: # Offsets are missing for all modules.\n",
" raise Exception(\"Could not find offset constants for all modules, will not correct data.\")\n",
"\n",
"for mod, ccv_dict in lpd_cal_metadata.items():\n",
" mdata_dict = {\"constants\": dict()}\n",
" for cname, ccv_metadata in ccv_dict.items():\n",
" mdata_dict[\"constants\"][cname] = {\n",
" \"path\": str(lpd_cal.caldb_root / ccv_metadata[\"path\"]),\n",
" \"dataset\": ccv_metadata[\"dataset\"],\n",
" \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
" \"ccv_id\": ccv_metadata[\"ccv_id\"],\n",
" }\n",
" mdata_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n",
" retrieved_constants[mod] = mdata_dict\n",
"metadata.save()\n",
"\n",
"step_timer.done_step(f\"Stored retrieved constants in {metadata.filename}.\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.8.11 ('.cal4_venv')",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.11"
},
"orig_nbformat": 4,
"vscode": {
"interpreter": {
"hash": "ccde353e8822f411c1c49844e1cbe3edf63293a69efd975d1b44f5e852832668"
}
}
},
"nbformat": 4,
"nbformat_minor": 2
}