diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index e38e3245e24c1af6486941f4c6120fcb6efa8dfd..6336a6513bc238d5152fb86fd0d584cc901122db 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -18,7 +18,7 @@
"outputs": [],
"source": [
"in_folder = \"/gpfs/exfel/exp/MID/202201/p002834/raw\" # the folder to read data from, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/esobolev/pycal_litfrm/p002834/r0225\" # the folder to output to, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_\" # the folder to output to, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
@@ -551,6 +551,17 @@
"module_index_to_karabo_da = {mod: da for (mod, da) in zip(modules, karabo_da)}"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "# NOTE: this notebook will not overwrite calibration metadata file\n",
+ "const_yaml = metadata.get(\"retrieved-constants\", {})"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -607,6 +618,50 @@
"print(f\"Constants were loaded in {perf_counter()-ts:.01f}s\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# if the yml file contains \"retrieved-constants\", that means a leading\n",
+ "# notebook got processed and the reporting would be generated from it.\n",
+ "fst_print = True\n",
+ "timestamps = {}\n",
+ "\n",
+ "for i, (modno, when, k_da) in enumerate(const_out):\n",
+ " qm = module_index_to_qm(modno)\n",
+ "\n",
+ " if k_da not in const_yaml:\n",
+ " if fst_print:\n",
+ " print(\"Constants are retrieved with creation time: \")\n",
+ " fst_print = False\n",
+ "\n",
+ " module_timestamps = {}\n",
+ "\n",
+ " print(f\"{qm}:\")\n",
+ " for key, item in when.items():\n",
+ " if hasattr(item, 'strftime'):\n",
+ " item = item.strftime('%y-%m-%d %H:%M')\n",
+ " when[key] = item\n",
+ " print('{:.<12s}'.format(key), item)\n",
+ "\n",
+ " # Store few time stamps if exists\n",
+ " # Add NA to keep array structure\n",
+ " for key in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
+ " if when and key in when and when[key]:\n",
+ " module_timestamps[key] = when[key]\n",
+ " else:\n",
+ " module_timestamps[key] = \"NA\"\n",
+ " timestamps[qm] = module_timestamps\n",
+ "\n",
+ "seq = sequences[0] if sequences else 0\n",
+ "\n",
+ "if timestamps:\n",
+ " with open(f\"{out_folder}/retrieved_constants_s{seq}.yml\",\"w\") as fd:\n",
+ " yaml.safe_dump({\"time-summary\": {f\"S{seq}\": timestamps}}, fd)"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index bbb32e928c2784a85ea2c1b28b58a1c39cb8388f..870a43c413734c32a24c803c88334b0c49723417 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -364,8 +364,36 @@
"metadata": {},
"outputs": [],
"source": [
+ "def find_cells_acq_rate_module(idx):\n",
+ " \"\"\"Find memory cells and acquisition rate for a module.\"\"\"\n",
+ " agipd_cond.image_src = instrument_src.format(idx)\n",
+ "\n",
+ " if agipd_cond.image_src not in instr_dc.all_sources:\n",
+ " warning(f\"No raw images found for {module_index_to_qm(idx)}({k_da}).\")\n",
+ " return None, None\n",
+ " \n",
+ " _mem_cells = agipd_cond.get_num_cells() if mem_cells == -1 else mem_cells\n",
+ " _acq_rate = agipd_cond.get_acq_rate() if acq_rate == -1 else acq_rate\n",
+ "\n",
+ " return _mem_cells, _acq_rate\n",
+ "\n",
+ "\n",
+ "all_mem_cells = set()\n",
+ "all_acq_rates = set()\n",
+ "\n",
"with multiprocessing.Pool(processes=nmods) as pool:\n",
- " results = pool.starmap(retrieve_constants, inp)"
+ " results = pool.map(find_cells_acq_rate_module, modules)\n",
+ "\n",
+ "for (_mem_cells, _acq_rate) in results:\n",
+ " all_mem_cells.add(_mem_cells)\n",
+ " all_acq_rates.add(_acq_rate)\n",
+ "# Validate that mem_cells and acq_rate are the same for all modules.\n",
+ "if len(all_mem_cells) != 1 or len(all_acq_rates) != 1:\n",
+ " warning(\n",
+ " \"Number of memory cells or acquisition rate are not identical for all modules.\\n\"\n",
+ " f\"different mem_cells values: {all_mem_cells}.\\ndifferent acq_rate values: {all_acq_rates}.\")\n",
+ "mem_cells = all_mem_cells.pop()\n",
+ "acq_rate = all_acq_rates.pop()"
]
},
{
diff --git a/src/cal_tools/agipdlib.py b/src/cal_tools/agipdlib.py
index fe7e1079d9e4d278a6c1156411bead1397a9b0c5..ca8175ea63c50e3fde2bd659d8b58f77975c262b 100644
--- a/src/cal_tools/agipdlib.py
+++ b/src/cal_tools/agipdlib.py
@@ -2,20 +2,19 @@ import os
import posixpath
import zlib
from datetime import datetime
+from logging import warning
from multiprocessing import Manager
from multiprocessing.pool import ThreadPool
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Dict, List, Optional
import h5py
import numpy as np
import sharedmem
from dateutil import parser
from extra_data import DataCollection, H5File, by_id, components
-from iCalibrationDB import Conditions, Constants
from cal_tools import agipdalgs as calgs
from cal_tools.agipdutils import (
- assemble_constant_dict,
baseline_correct_via_noise,
baseline_correct_via_stripe,
correct_baseline_via_hist,
@@ -27,7 +26,6 @@ from cal_tools.agipdutils import (
)
from cal_tools.enums import AgipdGainMode, BadPixels, SnowResolution
from cal_tools.h5_copy_except import h5_copy_except_paths
-from cal_tools.tools import get_from_db
class AgipdCtrl:
@@ -1339,12 +1337,12 @@ class AgipdCorrections:
cons_data = dict()
when = dict()
variant = dict()
- db_module = const_yaml[karabo_da]["physical-detector-unit"]
+ db_module = const_yaml[karabo_da]["physical-name"]
for cname, mdata in const_yaml[karabo_da]["constants"].items():
base_key = f"{db_module}/{cname}/0"
when[cname] = mdata["creation-time"]
if when[cname]:
- with h5py.File(mdata["file-path"], "r") as cf:
+ with h5py.File(mdata["path"], "r") as cf:
cons_data[cname] = np.copy(cf[f"{base_key}/data"])
# Set variant to 0 if the attribute is missing
# as for old constants.
@@ -1354,54 +1352,38 @@ class AgipdCorrections:
variant[cname] = 0
else:
# Create empty constant using the list elements
- cons_data[cname] = getattr(np, mdata["file-path"][0])(mdata["file-path"][1]) # noqa
+ cons_data[cname] = getattr(np, mdata["path"][0])(mdata["path"][1]) # noqa
self.init_constants(cons_data, when, module_idx, variant)
return when
- def initialize_from_db(self, karabo_id: str, karabo_da: str,
- cal_db_interface: str,
- creation_time: datetime,
- memory_cells: float, bias_voltage: int,
- photon_energy: float, gain_setting: float,
- acquisition_rate: float, integration_time: int,
- module_idx: int, only_dark: bool = False):
+ def initialize_from_db(
+ self, karabo_id: str,
+ karabo_da: str,
+ creation_time: datetime,
+ memory_cells: int,
+ bias_voltage: int,
+ gain_setting: float,
+ gain_mode: int,
+ acquisition_rate: float,
+ integration_time: int,
+ module_idx: int,
+ ):
""" Initialize calibration constants from the calibration database
- :param karabo_id: karabo identifier
- :param karabo_da: karabo data aggregator
- :param cal_db_interface: database interaface port
+ :param karabo_id: karabo detector identifier
+ :param karabo_da: karabo module data aggregator name
:param creation_time: time for desired calibration constant version
:param memory_cells: number of memory cells used for CCV conditions
:param bias_voltage: bias voltage used for CCV conditions
- :param photon_energy: photon energy used for CCV conditions
:param gain_setting: gain setting used for CCV conditions
:param acquisition_rate: acquistion rate used for CCV conditions
:param integration_time: integration time used for CCV conditions
:param module_idx: module index to save retrieved CCV in sharedmem
- :param only_dark: load only dark image derived constants. This
- implies that a `calfile` is used to load the remaining
- constants. Useful to reduce DB traffic and interactions
- for non-frequently changing constants, i.e. such which are
- not usually updated during a beamtime.
-
- The `cal_db_interface` parameter in the `dbparms` tuple may be in
- one of the following notations:
- * tcp://host:port to directly identify the host and port to
- connect to
- * tcp://host:port_low#port_high to specify a port range from
- which a random port will be picked. E.g. specifying
-
- tcp://max-exfl016:8015#8025
-
- will randomly pick an address in the range max-exfl016:8015 and
- max-exfl016:8025.
- The latter notation allows for load-balancing.
-
- This routine loads the following constants as given in
- `iCalibrationDB`:
+ Loading the following constants based on the
+ correction configuration:
Dark Image Derived
------------------
@@ -1415,50 +1397,76 @@ class AgipdCorrections:
-----------------------
* Constants.AGIPD.SlopesPC
+ * Constants.AGIPD.BadPixelsPC
Flat-Field Derived
* Constants.AGIPD.SlopesFF
-
+ * Constants.AGIPD.BadPixelsFF
"""
-
- const_dict = assemble_constant_dict(
- self.corr_bools,
- self.pc_bools,
- memory_cells,
- bias_voltage,
- gain_setting,
- acquisition_rate,
- photon_energy,
- beam_energy=None,
- only_dark=only_dark,
- integration_time=integration_time
- )
+ from cal_tools.calcat_interface import AGIPD_CalibrationData
when = {}
cons_data = {}
variant = {}
- for cname, cval in const_dict.items():
- condition = getattr(
- Conditions, cval[2][0]).AGIPD(**cval[2][1])
- cdata, md = get_from_db(
- karabo_id=karabo_id,
- karabo_da=karabo_da,
- constant=getattr(Constants.AGIPD, cname)(),
- condition=condition,
- empty_constant=getattr(np, cval[0])(cval[1]),
- cal_db_interface=cal_db_interface,
- creation_time=creation_time,
- verbosity=0,
- )
- cons_data[cname] = cdata
- variant[cname] = md.calibration_constant_version.variant
+ agipd_cal = AGIPD_CalibrationData(
+ detector_name=karabo_id,
+ modules=[karabo_da],
+ sensor_bias_voltage=bias_voltage,
+ memory_cells=memory_cells,
+ acquisition_rate=acquisition_rate,
+ integration_time=integration_time,
+ source_energy=9.2,
+ gain_mode=gain_mode,
+ gain_setting=gain_setting,
+ event_at=creation_time,
+ )
- when[cname] = None
- # Read the CCV begin at if constant was retrieved successfully.
- if md and md.comm_db_success:
- when[cname] = md.calibration_constant_version.begin_at
+ dark_constants = [
+ "Offset",
+ "Noise",
+ "BadPixelsDark"
+ ] if self.gain_mode else [
+ "ThresholdsDark",
+ "Offset",
+ "Noise",
+ "BadPixelsDark"
+ ]
+
+ gain_constants = []
+ if self.pc_bools:
+ gain_constants += ["SlopesPC", "BadPixelsPC"]
+ if self.corr_bools.get("xray_corr"):
+ gain_constants += ["SlopesFF", "BadPixelsFF"]
+
+ constants = dark_constants
+ constants += gain_constants
+ metadata = agipd_cal.metadata(constants)
+ # Validate the constants availability and raise/warn correspondingly.
+ for mod, ccv_dict in metadata.items():
+ missing_dark_constants = set(
+ c for c in dark_constants if c not in ccv_dict.keys())
+ missing_gain_constants = set(
+ c for c in gain_constants if c not in ccv_dict.keys())
+ if missing_dark_constants:
+ raise KeyError(
+ f"Dark constants {missing_dark_constants} are not available"
+ f" for correction. Module: {mod}")
+ if missing_gain_constants:
+ warning(
+ f"Gain constants {missing_gain_constants} were"
+ f" not retrieved. Module: {mod}")
+
+ for cname in constants:
+ if metadata[karabo_da].get(cname):
+ when[cname] = metadata[karabo_da][cname]["begin_validity_at"]
+ dataset = metadata[karabo_da][cname]["dataset"]
+ with h5py.File(agipd_cal.caldb_root / metadata[karabo_da][cname]["path"], "r") as cf: # noqa
+ cons_data[cname] = np.copy(cf[f"{dataset}/data"])
+ variant[cname] = cf[dataset].attrs["variant"] if cf[dataset].attrs.keys() else 0 # noqa
+ else:
+ when[cname] = None
self.init_constants(cons_data, when, module_idx, variant)