From e8b98ddd212ff54d7e47550dac87069efb468083 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Tue, 13 Dec 2022 14:18:16 +0100
Subject: [PATCH] Agipd correction using calcat_interface
---
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 105 ++++--
...IPD_Retrieve_Constants_Precorrection.ipynb | 300 ++++++++----------
src/cal_tools/agipdlib.py | 174 +++++-----
3 files changed, 315 insertions(+), 264 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 03f09957c..f54860b29 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -123,10 +123,9 @@
"outputs": [],
"source": [
"import itertools\n",
- "import os\n",
"import math\n",
"import multiprocessing\n",
- "import re\n",
+ "import os\n",
"import warnings\n",
"from datetime import timedelta\n",
"from logging import warning\n",
@@ -155,6 +154,7 @@
"sns.set_context(\"paper\", font_scale=1.4)\n",
"sns.set_style(\"ticks\")\n",
"\n",
+ "from calibration_client import CalibrationClient\n",
"from cal_tools import agipdalgs as calgs\n",
"from cal_tools.agipdlib import (\n",
" AgipdCorrections,\n",
@@ -164,6 +164,7 @@
")\n",
"from cal_tools.ana_tools import get_range\n",
"from cal_tools.enums import AgipdGainMode, BadPixels\n",
+ "from cal_tools.restful_config import restful_config\n",
"from cal_tools.step_timing import StepTimer\n",
"from cal_tools.tools import (\n",
" CalibrationMetadata,\n",
@@ -368,7 +369,6 @@
"\n",
"instrument_src_mod = [\n",
" s for s in list(dc.all_sources) if f\"{first_mod_channel}CH\" in s][0]\n",
- "mod_channel = int(re.findall(rf\".*{first_mod_channel}CH([0-9]+):.*\", instrument_src_mod)[0])\n",
"\n",
"agipd_cond = AgipdCtrl(\n",
" run_dc=dc,\n",
@@ -559,6 +559,26 @@
"module_index_to_karabo_da = {mod: da for (mod, da) in zip(modules, karabo_da)}"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Connect to CalCat.\n",
+ "calcat_config = restful_config['calcat']\n",
+ "client = CalibrationClient(\n",
+ " base_api_url=calcat_config['base-api-url'],\n",
+ " use_oauth2=calcat_config['use-oauth2'],\n",
+ " client_id=calcat_config['user-id'],\n",
+ " client_secret=calcat_config['user-secret'],\n",
+ " user_email=calcat_config['user-email'],\n",
+ " token_url=calcat_config['token-url'],\n",
+ " refresh_url=calcat_config['refresh-url'],\n",
+ " auth_url=calcat_config['auth-url'],\n",
+ " scope='')"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -584,26 +604,21 @@
" when = agipd_corr.initialize_from_yaml(k_da, const_yaml, mod)\n",
" print(f\"Found constants for {k_da} in calibration_metadata.yml\")\n",
" else:\n",
- " try:\n",
- " # TODO: replace with proper retrieval (as done in pre-correction)\n",
- " when = agipd_corr.initialize_from_db(\n",
- " karabo_id=karabo_id,\n",
- " karabo_da=k_da,\n",
- " cal_db_interface=cal_db_interface,\n",
- " creation_time=creation_time,\n",
- " memory_cells=mem_cells_db,\n",
- " bias_voltage=bias_voltage,\n",
- " photon_energy=9.2,\n",
- " gain_setting=gain_setting,\n",
- " acquisition_rate=acq_rate,\n",
- " integration_time=integration_time,\n",
- " module_idx=mod,\n",
- " only_dark=False,\n",
- " )\n",
- " print(f\"Queried CalCat for {k_da}\")\n",
- " except Exception as e:\n",
- " warning(f\"Module: {k_da}, {e}\")\n",
- " when = None\n",
+ " when = agipd_corr.initialize_from_db(\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=k_da,\n",
+ " creation_time=creation_time,\n",
+ " memory_cells=mem_cells_db,\n",
+ " bias_voltage=bias_voltage,\n",
+ " photon_energy=9.2,\n",
+ " gain_setting=gain_setting,\n",
+ " acquisition_rate=acq_rate,\n",
+ " integration_time=integration_time,\n",
+ " gain_mode=gain_mode,\n",
+ " module_idx=mod,\n",
+ " client=client,\n",
+ " )\n",
+ " print(f\"Queried CalCat for {k_da}\")\n",
" return mod, when, k_da\n",
"\n",
"\n",
@@ -615,6 +630,50 @@
"print(f\"Constants were loaded in {perf_counter()-ts:.01f}s\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# if the yml file contains \"retrieved-constants\", that means a leading\n",
+ "# notebook got processed and the reporting would be generated from it.\n",
+ "fst_print = True\n",
+ "timestamps = {}\n",
+ "\n",
+ "for i, (modno, when, k_da) in enumerate(const_out):\n",
+ " qm = module_index_to_qm(modno)\n",
+ "\n",
+ " if k_da not in const_yaml:\n",
+ " if fst_print:\n",
+ " print(\"Constants are retrieved with creation time: \")\n",
+ " fst_print = False\n",
+ "\n",
+ " module_timestamps = {}\n",
+ "\n",
+ " print(f\"{qm}:\")\n",
+ " for key, item in when.items():\n",
+ " if hasattr(item, 'strftime'):\n",
+ " item = item.strftime('%y-%m-%d %H:%M')\n",
+ " when[key] = item\n",
+ " print('{:.<12s}'.format(key), item)\n",
+ "\n",
+ " # Store few time stamps if exists\n",
+ " # Add NA to keep array structure\n",
+ " for key in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
+ " if when and key in when and when[key]:\n",
+ " module_timestamps[key] = when[key]\n",
+ " else:\n",
+ " module_timestamps[key] = \"NA\"\n",
+ " timestamps[qm] = module_timestamps\n",
+ "\n",
+ "seq = sequences[0] if sequences else 0\n",
+ "\n",
+ "if timestamps:\n",
+ " with open(f\"{out_folder}/retrieved_constants_s{seq}.yml\",\"w\") as fd:\n",
+ " yaml.safe_dump({\"time-summary\": {f\"S{seq}\": timestamps}}, fd)"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index d0b20cadf..ee782ac7f 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -81,29 +81,21 @@
"metadata": {},
"outputs": [],
"source": [
- "import numpy as np\n",
"from logging import warning\n",
"from pathlib import Path\n",
- "from typing import Tuple\n",
"\n",
"import multiprocessing\n",
"from datetime import timedelta\n",
"from dateutil import parser\n",
"from extra_data import RunDirectory\n",
"\n",
- "from cal_tools.agipdlib import (\n",
- " AgipdCtrl,\n",
- " SnowResolution,\n",
- " assemble_constant_dict,\n",
- ")\n",
- "from cal_tools.enums import AgipdGainMode\n",
- "from cal_tools.tools import (\n",
- " calcat_creation_time,\n",
- " get_from_db,\n",
- " module_index_to_qm,\n",
- " CalibrationMetadata,\n",
- ")\n",
- "from iCalibrationDB import Conditions, Constants, Detectors"
+ "from calibration_client import CalibrationClient\n",
+ "from cal_tools.agipdlib import AgipdCtrl, SnowResolution\n",
+ "from cal_tools.calcat_interface import AGIPD_CalibrationData, CalCatError\n",
+ "from cal_tools.restful_config import restful_config\n",
+ "from cal_tools.step_timing import StepTimer\n",
+ "from cal_tools.tools import CalibrationMetadata, calcat_creation_time, module_index_to_qm\n",
+ "from cal_tools.enums import AgipdGainMode"
]
},
{
@@ -233,7 +225,11 @@
" corr_bools.get('blc_hmatch'),\n",
" corr_bools.get('blc_stripes'),\n",
" melt_snow,\n",
- "]"
+ "]\n",
+ "\n",
+ "da_to_qm = dict()\n",
+ "for module_index, k_da in zip(modules, karabo_da):\n",
+ " da_to_qm[k_da] = module_index_to_qm(module_index)"
]
},
{
@@ -242,124 +238,36 @@
"metadata": {},
"outputs": [],
"source": [
- "def retrieve_constants(\n",
- " k_da: str, idx: int\n",
- ") -> Tuple[str, str, float, float, str, dict]:\n",
- " \"\"\"\n",
- " Retrieve constants for a module.\n",
- "\n",
- " :return:\n",
- " k_da: karabo data aggregator.\n",
- " acq_rate: acquisition rate parameter.\n",
- " mem_cells: number of memory cells.\n",
- " mdata_dict: (DICT) dictionary with the metadata for the retrieved constants.\n",
- " \"\"\"\n",
- " # check if this module has images to process.\n",
- " if instrument_src.format(idx) not in instr_dc.all_sources:\n",
- " print(\"ERROR: No raw images found for \"\n",
- " f\"{module_index_to_qm(idx)}({k_da}).\")\n",
- "\n",
- " return None, k_da, None, None\n",
- "\n",
+ "def find_cells_acq_rate_module(idx):\n",
+ " \"\"\"Find memory cells and acquisition rate for a module.\"\"\"\n",
" agipd_cond.image_src = instrument_src.format(idx)\n",
"\n",
- " if mem_cells == -1:\n",
- " # Read value from fast data.\n",
- " local_mem_cells = agipd_cond.get_num_cells()\n",
- " else:\n",
- " # or use overriding notebook parameter.\n",
- " local_mem_cells = mem_cells\n",
- "\n",
- " if acq_rate == -1.:\n",
- " local_acq_rate = agipd_cond.get_acq_rate()\n",
- " else:\n",
- " local_acq_rate = acq_rate\n",
- "\n",
- " const_dict = assemble_constant_dict(\n",
- " corr_bools,\n",
- " pc_bools,\n",
- " local_mem_cells,\n",
- " bias_voltage,\n",
- " gain_setting,\n",
- " local_acq_rate,\n",
- " photon_energy=9.2,\n",
- " gain_mode=gain_mode,\n",
- " beam_energy=None,\n",
- " only_dark=False,\n",
- " integration_time=integration_time\n",
- " )\n",
- "\n",
- " # Retrieve multiple constants through an input dictionary\n",
- " # to return a dict of useful metadata.\n",
- " mdata_dict = dict()\n",
- " mdata_dict[\"constants\"] = dict()\n",
- " mdata_dict[\"physical-detector-unit\"] = None # initialization\n",
- "\n",
- " for const_name, (const_init_fun, const_shape, (cond_type, cond_param)) in const_dict.items(): # noqa\n",
- " if gain_mode and const_name in (\"ThresholdsDark\",):\n",
- " continue\n",
- " \n",
- " # saving metadata in a dict\n",
- " const_mdata = dict()\n",
- " mdata_dict[\"constants\"][const_name] = const_mdata\n",
- "\n",
- " if slopes_ff_from_files and const_name in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
- " const_mdata[\"file-path\"] = (\n",
- " f\"{slopes_ff_from_files}/slopesff_bpmask_module_{module_index_to_qm(idx)}.h5\") # noqa\n",
- " const_mdata[\"creation-time\"] = \"00:00:00\"\n",
- " continue\n",
- " \n",
- " if gain_mode and const_name in (\n",
- " \"BadPixelsPC\", \"SlopesPC\", \"BadPixelsFF\", \"SlopesFF\"\n",
- " ):\n",
- " param_copy = cond_param.copy()\n",
- " del param_copy[\"gain_mode\"]\n",
- " condition = getattr(Conditions, cond_type).AGIPD(**param_copy)\n",
- " else:\n",
- " condition = getattr(Conditions, cond_type).AGIPD(**cond_param)\n",
- "\n",
- " _, mdata = get_from_db(\n",
- " karabo_id,\n",
- " k_da,\n",
- " getattr(Constants.AGIPD, const_name)(),\n",
- " condition,\n",
- " getattr(np, const_init_fun)(const_shape),\n",
- " cal_db_interface,\n",
- " creation_time,\n",
- " meta_only=True,\n",
- " verbosity=0,\n",
- " )\n",
- " mdata_const = mdata.calibration_constant_version\n",
- " # check if constant was sucessfully retrieved.\n",
- " if mdata.comm_db_success:\n",
- " const_mdata[\"file-path\"] = (\n",
- " f\"{mdata_const.hdf5path}\" f\"{mdata_const.filename}\"\n",
- " )\n",
- " const_mdata[\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
- " mdata_dict[\"physical-detector-unit\"] = mdata_const.device_name\n",
- " else:\n",
- " const_mdata[\"file-path\"] = const_dict[const_name][:2]\n",
- " const_mdata[\"creation-time\"] = None\n",
+ " if agipd_cond.image_src not in instr_dc.all_sources:\n",
+ " warning(f\"No raw images found for {module_index_to_qm(idx)}({k_da}).\")\n",
+ " return None, None\n",
+ " \n",
+ " _mem_cells = agipd_cond.get_num_cells() if mem_cells == -1 else mem_cells\n",
+ " _acq_rate = agipd_cond.get_acq_rate() if acq_rate == -1 else acq_rate\n",
"\n",
- " return mdata_dict, k_da, local_acq_rate, local_mem_cells"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "inp = []\n",
- "da_to_qm = dict()\n",
- "for module_index, k_da in zip(modules, karabo_da):\n",
- " da_to_qm[k_da] = module_index_to_qm(module_index)\n",
- " if k_da in retrieved_constants:\n",
- " print(\n",
- " f\"Constant for {k_da} already in calibration_metadata.yml, won't query again.\")\n",
- " continue\n",
+ " return _mem_cells, _acq_rate\n",
+ "\n",
+ "\n",
+ "all_mem_cells = set()\n",
+ "all_acq_rates = set()\n",
"\n",
- " inp.append((k_da, module_index))"
+ "with multiprocessing.Pool(processes=nmods) as pool:\n",
+ " results = pool.map(find_cells_acq_rate_module, modules)\n",
+ "\n",
+ "for (_mem_cells, _acq_rate) in results:\n",
+ " all_mem_cells.add(_mem_cells)\n",
+ " all_acq_rates.add(_acq_rate)\n",
+ "# Validate that mem_cells and acq_rate are the same for all modules.\n",
+ "if len(all_mem_cells) != 1 or len(all_acq_rates) != 1:\n",
+ " warning(\n",
+ " \"Number of memory cells or acquisition rate are not identical for all modules.\\n\"\n",
+ " f\"different mem_cells values: {all_mem_cells}.\\ndifferent acq_rate values: {all_acq_rates}.\")\n",
+ "mem_cells = all_mem_cells.pop()\n",
+ "acq_rate = all_acq_rates.pop()"
]
},
{
@@ -368,8 +276,18 @@
"metadata": {},
"outputs": [],
"source": [
- "with multiprocessing.Pool(processes=nmods) as pool:\n",
- " results = pool.starmap(retrieve_constants, inp)"
+ "# Connect to CalCat.\n",
+ "calcat_config = restful_config['calcat']\n",
+ "client = CalibrationClient(\n",
+ " base_api_url=calcat_config['base-api-url'],\n",
+ " use_oauth2=calcat_config['use-oauth2'],\n",
+ " client_id=calcat_config['user-id'],\n",
+ " client_secret=calcat_config['user-secret'],\n",
+ " user_email=calcat_config['user-email'],\n",
+ " token_url=calcat_config['token-url'],\n",
+ " refresh_url=calcat_config['refresh-url'],\n",
+ " auth_url=calcat_config['auth-url'],\n",
+ " scope='')"
]
},
{
@@ -378,36 +296,77 @@
"metadata": {},
"outputs": [],
"source": [
- "acq_rate_mods = []\n",
- "mem_cells_mods = []\n",
- "for md_dict, k_da, acq_rate, mem_cells in results:\n",
- " if acq_rate is None and mem_cells is None:\n",
- " continue\n",
- " md_dict, k_da, acq_rate, mem_cells\n",
- " retrieved_constants[k_da] = md_dict\n",
- " mem_cells_mods.append(mem_cells)\n",
- " acq_rate_mods.append(acq_rate)\n",
+ "step_timer = StepTimer()\n",
+ "step_timer.start()\n",
+ "agipd_cal = AGIPD_CalibrationData(\n",
+ " detector_name=karabo_id,\n",
+ " modules=karabo_da,\n",
+ " sensor_bias_voltage=bias_voltage,\n",
+ " memory_cells=mem_cells,\n",
+ " acquisition_rate=acq_rate,\n",
+ " integration_time=integration_time,\n",
+ " source_energy=9.2,\n",
+ " gain_mode=gain_mode,\n",
+ " gain_setting=gain_setting,\n",
+ " event_at=creation_time,\n",
+ " client=client,\n",
+ ")\n",
"\n",
- "# Validate that mem_cells and acq_rate are the same for all modules.\n",
- "# TODO: Should a warning be enough?\n",
- "if len(set(mem_cells_mods)) != 1 or len(set(acq_rate_mods)) != 1:\n",
- " print(\n",
- " \"WARNING: Number of memory cells or \"\n",
- " \"acquisition rate are not identical for all modules.\\n\"\n",
- " f\"mem_cells: {mem_cells_mods}.\\nacq_rate: {acq_rate_mods}.\")\n",
- "\n",
- "# check if it is requested not to retrieve any constants from the database\n",
- "print(\"\\nRetrieved constants for modules:\",\n",
- " ', '.join([module_index_to_qm(x) for x in modules]))\n",
- "print(f\"Operating conditions are:\")\n",
- "print(f\"• Bias voltage: {bias_voltage}\")\n",
- "print(f\"• Memory cells: {mem_cells}\")\n",
- "print(f\"• Acquisition rate: {acq_rate}\")\n",
- "print(f\"• Gain mode: {gain_mode.name}\")\n",
- "print(f\"• Gain setting: {gain_setting}\")\n",
- "print(f\"• Integration time: {integration_time}\")\n",
- "print(f\"• Photon Energy: 9.2\")\n",
- "print(\"Constant metadata is saved under \\\"retrieved-constants\\\" in calibration_metadata.yml\\n\")"
+ "dark_constants = [\"Offset\", \"Noise\", \"BadPixelsDark\"] if gain_mode else [\"ThresholdsDark\", \"Offset\", \"Noise\", \"BadPixelsDark\"]\n",
+ "\n",
+ "gain_constants = []\n",
+ "if pc_bools:\n",
+ " gain_constants += [\"SlopesPC\", \"BadPixelsPC\"]\n",
+ "if xray_gain:\n",
+ " gain_constants += [\"SlopesFF\", \"BadPixelsFF\"]\n",
+ "\n",
+ "constants = dark_constants\n",
+ "constants += gain_constants\n",
+ "\n",
+ "# The main metadata dict for all modules and retrieved CCVs.\n",
+ "agipd_metadata = {mod:{} for mod in agipd_cal.modules}\n",
+ "\n",
+ "# This is done to alter the condition for gain constants and avoid using gain_mode.\n",
+ "for cname in constants:\n",
+ " if cname in gain_constants:\n",
+ " agipd_cal.gain_mode = None\n",
+ " try:\n",
+ " a = agipd_cal.metadata([cname])\n",
+ " for mod, ccv_entry in a.items():\n",
+ " agipd_metadata[mod].update(a[mod])\n",
+ " except CalCatError as e:\n",
+ " pass # a validation of missing constants is done later.\n",
+ "\n",
+ "# Validate the constants availability and raise/warn correspondingly. \n",
+ "for mod, ccv_dict in agipd_metadata.items():\n",
+ " missing_dark_constants = set(\n",
+ " c for c in dark_constants if c not in ccv_dict.keys())\n",
+ " missing_gain_constants = set(\n",
+ " c for c in gain_constants if c not in ccv_dict.keys())\n",
+ " if missing_dark_constants:\n",
+ " raise KeyError(\n",
+ " f\"Dark constants {missing_dark_constants} are not available for correction. Module: {mod}\") # noqa\n",
+ " if missing_gain_constants:\n",
+ " warning(\n",
+ " f\"Gain constants {missing_gain_constants} were not retrieved. Module: {mod}\")\n",
+ "\n",
+ "for mod, ccv_dict in agipd_metadata.items():\n",
+ " mdata_dict = {\"constants\": dict()}\n",
+ " for cname, ccv_metadata in ccv_dict.items():\n",
+ " if slopes_ff_from_files and cname in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
+ " mdata_dict[\"constants\"][cname] = {\n",
+ " \"path\": f\"{slopes_ff_from_files}/slopesff_bpmask_module_{module_index_to_qm(int(mod[-2:]))}.h5\", # noqa\n",
+ " \"dataset\": \"\",\n",
+ " \"creation-time\": \"00:00:00\",\n",
+ " }\n",
+ " else:\n",
+ " mdata_dict[\"constants\"][cname] = {\n",
+ " \"path\": str(agipd_cal.caldb_root / ccv_metadata[\"path\"]),\n",
+ " \"dataset\": ccv_metadata[\"dataset\"],\n",
+ " \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
+ " }\n",
+ " mdata_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n",
+ " retrieved_constants[mod] = mdata_dict"
]
},
{
@@ -442,6 +401,25 @@
"\n",
"metadata.save()"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(\"\\nRetrieved constants for modules:\",\n",
+ " ', '.join([module_index_to_qm(x) for x in modules]))\n",
+ "print(f\"Operating conditions are:\")\n",
+ "print(f\"• Bias voltage: {bias_voltage}\")\n",
+ "print(f\"• Memory cells: {mem_cells}\")\n",
+ "print(f\"• Acquisition rate: {acq_rate}\")\n",
+ "print(f\"• Gain mode: {gain_mode.name}\")\n",
+ "print(f\"• Gain setting: {gain_setting}\")\n",
+ "print(f\"• Integration time: {integration_time}\")\n",
+ "print(f\"• Photon Energy: 9.2\")\n",
+ "step_timer.done_step(f\"Constant metadata is saved under \\\"retrieved-constants\\\" in {metadata.filename}\")"
+ ]
}
],
"metadata": {
diff --git a/src/cal_tools/agipdlib.py b/src/cal_tools/agipdlib.py
index 038ca342b..5d5aa54a2 100644
--- a/src/cal_tools/agipdlib.py
+++ b/src/cal_tools/agipdlib.py
@@ -2,20 +2,19 @@ import os
import posixpath
import zlib
from datetime import datetime
+from logging import warning
from multiprocessing import Manager
from multiprocessing.pool import ThreadPool
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional
import h5py
import numpy as np
import sharedmem
from dateutil import parser
from extra_data import DataCollection, H5File, by_id, components
-from iCalibrationDB import Conditions, Constants
from cal_tools import agipdalgs as calgs
from cal_tools.agipdutils import (
- assemble_constant_dict,
baseline_correct_via_noise,
baseline_correct_via_stripe,
correct_baseline_via_hist,
@@ -27,7 +26,6 @@ from cal_tools.agipdutils import (
)
from cal_tools.enums import AgipdGainMode, BadPixels, SnowResolution
from cal_tools.h5_copy_except import h5_copy_except_paths
-from cal_tools.tools import get_from_db
class AgipdCtrl:
@@ -1357,12 +1355,12 @@ class AgipdCorrections:
cons_data = dict()
when = dict()
variant = dict()
- db_module = const_yaml[karabo_da]["physical-detector-unit"]
+ db_module = const_yaml[karabo_da]["physical-name"]
for cname, mdata in const_yaml[karabo_da]["constants"].items():
base_key = f"{db_module}/{cname}/0"
when[cname] = mdata["creation-time"]
if when[cname]:
- with h5py.File(mdata["file-path"], "r") as cf:
+ with h5py.File(mdata["path"], "r") as cf:
cons_data[cname] = np.copy(cf[f"{base_key}/data"])
# Set variant to 0 if the attribute is missing
# as for old constants.
@@ -1372,54 +1370,43 @@ class AgipdCorrections:
variant[cname] = 0
else:
# Create empty constant using the list elements
- cons_data[cname] = getattr(np, mdata["file-path"][0])(mdata["file-path"][1]) # noqa
+ cons_data[cname] = getattr(np, mdata["path"][0])(mdata["path"][1]) # noqa
self.init_constants(cons_data, when, module_idx, variant)
return when
- def initialize_from_db(self, karabo_id: str, karabo_da: str,
- cal_db_interface: str,
- creation_time: datetime,
- memory_cells: float, bias_voltage: int,
- photon_energy: float, gain_setting: float,
- acquisition_rate: float, integration_time: int,
- module_idx: int, only_dark: bool = False):
+ def initialize_from_db(
+ self, karabo_id: str,
+ karabo_da: str,
+ module_idx: int,
+ creation_time: datetime,
+ memory_cells: int,
+ bias_voltage: int,
+ gain_setting: float,
+ gain_mode: int,
+ acquisition_rate: float,
+ integration_time: int,
+ photon_energy: float = 9.2,
+ client=None,
+ ):
""" Initialize calibration constants from the calibration database
- :param karabo_id: karabo identifier
- :param karabo_da: karabo data aggregator
- :param cal_db_interface: database interaface port
- :param creation_time: time for desired calibration constant version
- :param memory_cells: number of memory cells used for CCV conditions
- :param bias_voltage: bias voltage used for CCV conditions
- :param photon_energy: photon energy used for CCV conditions
- :param gain_setting: gain setting used for CCV conditions
- :param acquisition_rate: acquistion rate used for CCV conditions
- :param integration_time: integration time used for CCV conditions
- :param module_idx: module index to save retrieved CCV in sharedmem
- :param only_dark: load only dark image derived constants. This
- implies that a `calfile` is used to load the remaining
- constants. Useful to reduce DB traffic and interactions
- for non-frequently changing constants, i.e. such which are
- not usually updated during a beamtime.
-
- The `cal_db_interface` parameter in the `dbparms` tuple may be in
- one of the following notations:
- * tcp://host:port to directly identify the host and port to
- connect to
- * tcp://host:port_low#port_high to specify a port range from
- which a random port will be picked. E.g. specifying
-
- tcp://max-exfl-cal001:8015#8025
-
- will randomly pick an address in the range max-exfl-cal001:8015 and
- max-exfl-cal001:8025.
-
- The latter notation allows for load-balancing.
-
- This routine loads the following constants as given in
- `iCalibrationDB`:
+ :param karabo_id: karabo detector identifier.
+ :param karabo_da: karabo module data aggregator name.
+ :param module_idx: module index to save retrieved CCV in sharedmem.
+ :param creation_time: time for desired calibration constant version.
+ :param memory_cells: number of memory cells used for CCV conditions.
+ :param bias_voltage: bias voltage used for CCV conditions.
+ :param gain_setting: gain setting used for CCV conditions.
+ :param gain_mode: gain mode used for CCV conditions.
+ :param acquisition_rate: acquistion rate used for CCV conditions.
+ :param integration_time: integration time used for CCV conditions.
+ :param photon_energy: photon energy value used for CCV conditions.
+ :param client: calibration_client object for CalibrationData objects.
+
+ Loading the following constants based on the
+ correction configuration:
Dark Image Derived
------------------
@@ -1433,50 +1420,77 @@ class AgipdCorrections:
-----------------------
* Constants.AGIPD.SlopesPC
+ * Constants.AGIPD.BadPixelsPC
Flat-Field Derived
* Constants.AGIPD.SlopesFF
-
+ * Constants.AGIPD.BadPixelsFF
"""
-
- const_dict = assemble_constant_dict(
- self.corr_bools,
- self.pc_bools,
- memory_cells,
- bias_voltage,
- gain_setting,
- acquisition_rate,
- photon_energy,
- beam_energy=None,
- only_dark=only_dark,
- integration_time=integration_time
- )
+ from cal_tools.calcat_interface import AGIPD_CalibrationData
when = {}
cons_data = {}
variant = {}
- for cname, cval in const_dict.items():
- condition = getattr(
- Conditions, cval[2][0]).AGIPD(**cval[2][1])
- cdata, md = get_from_db(
- karabo_id=karabo_id,
- karabo_da=karabo_da,
- constant=getattr(Constants.AGIPD, cname)(),
- condition=condition,
- empty_constant=getattr(np, cval[0])(cval[1]),
- cal_db_interface=cal_db_interface,
- creation_time=creation_time,
- verbosity=0,
- )
- cons_data[cname] = cdata
- variant[cname] = md.calibration_constant_version.variant
+ agipd_cal = AGIPD_CalibrationData(
+ detector_name=karabo_id,
+ modules=[karabo_da],
+ sensor_bias_voltage=bias_voltage,
+ memory_cells=memory_cells,
+ acquisition_rate=acquisition_rate,
+ integration_time=integration_time,
+ source_energy=photon_energy,
+ gain_mode=gain_mode,
+ gain_setting=gain_setting,
+ event_at=creation_time,
+ client=client,
+ )
- when[cname] = None
- # Read the CCV begin at if constant was retrieved successfully.
- if md and md.comm_db_success:
- when[cname] = md.calibration_constant_version.begin_at
+ dark_constants = [
+ "Offset",
+ "Noise",
+ "BadPixelsDark"
+ ] if self.gain_mode else [
+ "ThresholdsDark",
+ "Offset",
+ "Noise",
+ "BadPixelsDark"
+ ]
+
+ gain_constants = []
+ if any(self.pc_bools):
+ gain_constants += ["SlopesPC", "BadPixelsPC"]
+ if self.corr_bools.get("xray_corr"):
+ gain_constants += ["SlopesFF", "BadPixelsFF"]
+
+ constants = dark_constants
+ constants += gain_constants
+ metadata = agipd_cal.metadata(constants)
+ # Validate the constants availability and raise/warn correspondingly.
+ for mod, ccv_dict in metadata.items():
+ missing_dark_constants = set(
+ c for c in dark_constants if c not in ccv_dict.keys())
+ missing_gain_constants = set(
+ c for c in gain_constants if c not in ccv_dict.keys())
+ if missing_dark_constants:
+ raise KeyError(
+ f"Dark constants {missing_dark_constants} are not available"
+ f" for correction. Module: {mod}")
+ if missing_gain_constants:
+ warning(
+ f"Gain constants {missing_gain_constants} were"
+ f" not retrieved. Module: {mod}")
+
+ for cname in constants:
+ if metadata[karabo_da].get(cname):
+ when[cname] = metadata[karabo_da][cname]["begin_validity_at"]
+ dataset = metadata[karabo_da][cname]["dataset"]
+ with h5py.File(agipd_cal.caldb_root / metadata[karabo_da][cname]["path"], "r") as cf: # noqa
+ cons_data[cname] = np.copy(cf[f"{dataset}/data"])
+ variant[cname] = cf[dataset].attrs["variant"] if cf[dataset].attrs.keys() else 0 # noqa
+ else:
+ when[cname] = None
self.init_constants(cons_data, when, module_idx, variant)
--
GitLab