diff --git a/notebooks/ePix100/Correction_ePix100_NBC.ipynb b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
index f850adfdd6c506fdc152be638678e6bde86dfef8..0d28fa8e63b0a44a053bfec25421d0dd79c00fae 100644
--- a/notebooks/ePix100/Correction_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
@@ -100,10 +100,9 @@
"from cal_tools.calcat_interface import EPIX100_CalibrationData\n",
"from cal_tools.epix100 import epix100lib\n",
"from cal_tools.files import DataFile\n",
- "from cal_tools.restful_config import restful_config\n",
"from cal_tools.tools import (\n",
+ " record_constant_details,\n",
" calcat_creation_time,\n",
- " CalibrationMetadata,\n",
")\n",
"from cal_tools.step_timing import StepTimer\n",
"\n",
@@ -304,22 +303,12 @@
"outputs": [],
"source": [
"# Record constant details in YAML metadata\n",
- "epix_metadata = const_metadata[karabo_da]\n",
- "CalibrationMetadata(metadata_folder or out_folder).add_fragment({\n",
- " \"retrieved-constants\": {\n",
- " karabo_da: {\n",
- " \"constants\": {\n",
- " cname: {\n",
- " \"path\": str(epix_cal.caldb_root / ccv_metadata[\"path\"]),\n",
- " \"dataset\": ccv_metadata[\"dataset\"],\n",
- " \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
- " \"ccv_id\": ccv_metadata[\"ccv_id\"],\n",
- " } for cname, ccv_metadata in epix_metadata.items()\n",
- " },\n",
- " \"physical-name\": list(epix_metadata.values())[0][\"physical_name\"],\n",
- " }\n",
- " }\n",
- "})"
+ "record_constant_details(\n",
+ " out_folder=(metadata_folder or out_folder),\n",
+ " karabo_da=karabo_da,\n",
+ " det_metadata=const_metadata[karabo_da],\n",
+ " caldb_root=epix_cal.caldb_root,\n",
+ " )"
]
},
{
diff --git a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
index 0d671b3f1e6041555f1e50333b243d9c685cd89b..c5b771faa4b98e17653c647ef39eae50123d96fd 100644
--- a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
+++ b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
@@ -102,7 +102,6 @@
"import numpy as np\n",
"import pasha as psh\n",
"from IPython.display import Markdown, display\n",
- "from calibration_client import CalibrationClient\n",
"from extra_data import H5File, RunDirectory\n",
"from prettytable import PrettyTable\n",
"\n",
@@ -116,10 +115,9 @@
"from cal_tools.calcat_interface import PNCCD_CalibrationData\n",
"from cal_tools.tools import (\n",
" calcat_creation_time,\n",
- " CalibrationMetadata,\n",
+ " record_constant_details,\n",
")\n",
- "from cal_tools.step_timing import StepTimer\n",
- "from cal_tools import h5_copy_except"
+ "from cal_tools.step_timing import StepTimer"
]
},
{
@@ -171,11 +169,6 @@
"# Output Folder Creation:\n",
"os.makedirs(out_folder, exist_ok=True)\n",
"\n",
- "# NOTE: this notebook shouldn't overwrite calibration metadata file.\n",
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# Constant paths are saved under retrieved-constants in calibration_metadata.yml\n",
- "const_yaml = metadata.get(\"retrieved-constants\", {})\n",
- "\n",
"# extract control data\n",
"step_timer.start()\n",
"\n",
@@ -318,27 +311,19 @@
"if relgain:\n",
" constant_names += [\"RelativeGainCCD\"]\n",
"\n",
- "if const_yaml: # Used while reproducing corrected data.\n",
- " print(f\"Using stored constants in {metadata.filename}\")\n",
- " constants = dict()\n",
- " for cname, mdata in const_yaml[karabo_da][\"constants\"].items():\n",
- " constants[cname] = dict()\n",
- " if mdata[\"creation-time\"]:\n",
- " with h5py.File(mdata[\"path\"], \"r\") as cf:\n",
- " constants[cname] = np.copy(\n",
- " cf[f\"{mdata['dataset']}/data\"])\n",
- "else:\n",
- " pnccd_cal = PNCCD_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " sensor_bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " sensor_temperature=fix_temperature_top,\n",
- " gain_setting=gain,\n",
- " event_at=creation_time,\n",
- " source_energy=photon_energy,\n",
- " client=rest_cfg.calibration_client(),\n",
- " )\n",
- " constants = pnccd_cal.ndarray_map(calibrations=constant_names)[karabo_da]\n",
+ "pnccd_cal = PNCCD_CalibrationData(\n",
+ " detector_name=karabo_id,\n",
+ " sensor_bias_voltage=bias_voltage,\n",
+ " integration_time=integration_time,\n",
+ " sensor_temperature=fix_temperature_top,\n",
+ " gain_setting=gain,\n",
+ " event_at=creation_time,\n",
+ " source_energy=photon_energy,\n",
+ " client=rest_cfg.calibration_client(),\n",
+ ")\n",
+ "\n",
+ "\n",
+ "constants = pnccd_cal.ndarray_map(calibrations=constant_names)[karabo_da]\n",
"\n",
"# Validate the constants availability and raise/warn correspondingly. \n",
"missing_dark_constants = set(\n",
@@ -354,6 +339,23 @@
"step_timer.done_step(\"Constants retrieval\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Record constant details in YAML metadata\n",
+ "pnccd_metadata = pnccd_cal.metadata(calibrations=constant_names)[karabo_da]\n",
+ "\n",
+ "record_constant_details(\n",
+ " out_folder=(metadata_folder or out_folder),\n",
+ " karabo_da=karabo_da,\n",
+ " det_metadata=pnccd_metadata,\n",
+ " caldb_root=pnccd_cal.caldb_root,\n",
+ " )"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb b/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb
deleted file mode 100644
index ee52c8d278e0e8e1d39dd2496e348235d3419a0a..0000000000000000000000000000000000000000
--- a/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb
+++ /dev/null
@@ -1,220 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# pnCCD retrieve constants precorrection\n",
- "\n",
- "Author: European XFEL Detector Group, Version: 1.0\n",
- "\n",
- "The following notebook provides constants for the selected pnCCD modules before executing correction on the selected sequence files."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "in_folder = \"/gpfs/exfel/exp/SQS/202031/p900166/raw\" # input folder\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/pnccd_correct\" # output folder\n",
- "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
- "run = 347 # which run to read data from\n",
- "\n",
- "karabo_da = 'PNCCD01' # data aggregators\n",
- "karabo_id = \"SQS_NQS_PNCCD1MP\" # detector Karabo_ID\n",
- "\n",
- "# Conditions for retrieving calibration constants\n",
- "fix_temperature_top = 0. # fix temperature for top sensor in K, set to 0. to use value from slow data.\n",
- "gain = -1 # the detector's gain setting. Set to -1 to use the value from the slow data.\n",
- "bias_voltage = 0. # the detector's bias voltage. set to 0. to use value from slow data.\n",
- "integration_time = 70 # detector's integration time\n",
- "photon_energy = 1.6 # Al fluorescence in keV\n",
- "\n",
- "# Parameters for the calibration database.\n",
- "cal_db_interface = \"tcp://max-exfl016:8015\" # calibration DB interface to use\n",
- "cal_db_timeout = 300000 # timeout on CalibrationDB requests\n",
- "creation_time = \"\" # The timestamp to use with Calibration DBe. Required Format: \"YYYY-MM-DD hh:mm:ss\" e.g. 2019-07-04 11:02:41\n",
- "\n",
- "# Booleans for selecting corrections to apply.\n",
- "only_offset = False # Only, apply offset.\n",
- "relgain = True # Apply relative gain correction\n",
- "\n",
- "# parameters affecting stored output data.\n",
- "overwrite = True # keep this as True to not overwrite the output "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from logging import warning\n",
- "from pathlib import Path\n",
- "\n",
- "from IPython.display import Markdown, display\n",
- "from calibration_client import CalibrationClient\n",
- "from extra_data import RunDirectory\n",
- "\n",
- "import cal_tools.restful_config as rest_cfg\n",
- "from cal_tools import pnccdlib\n",
- "from cal_tools.calcat_interface import PNCCD_CalibrationData\n",
- "from cal_tools.tools import (\n",
- " calcat_creation_time,\n",
- " CalibrationMetadata,\n",
- ")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "in_folder = Path(in_folder)\n",
- "out_folder = Path(out_folder)\n",
- "\n",
- "out_folder.mkdir(parents=True, exist_ok=True)\n",
- "\n",
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# NOTE: this notebook will not overwrite calibration metadata file,\n",
- "# if it already contains details about which constants to use.\n",
- "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Here the correction booleans dictionary is defined\n",
- "corr_bools = {}\n",
- "\n",
- "corr_bools[\"only_offset\"] = only_offset\n",
- "\n",
- "# Apply offset only.\n",
- "if not only_offset:\n",
- " corr_bools[\"relgain\"] = relgain"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "print(f\"Calibration database interface selected: {cal_db_interface}\")\n",
- "\n",
- "# Run's creation time:\n",
- "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
- "print(f\"Creation time: {creation_time}\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "run_dc = RunDirectory(in_folder / f\"r{run:04d}\", _use_voview=False)\n",
- "ctrl_data = pnccdlib.PnccdCtrl(run_dc, karabo_id)\n",
- "\n",
- "# extract control data\n",
- "if bias_voltage == 0.0:\n",
- " bias_voltage = ctrl_data.get_bias_voltage()\n",
- "if gain == -1:\n",
- " gain = ctrl_data.get_gain()\n",
- "if fix_temperature_top == 0:\n",
- " fix_temperature_top = ctrl_data.get_fix_temperature_top()\n",
- "\n",
- "# Printing the Parameters Read from the Data File:\n",
- "display(Markdown(\"### Detector Parameters\"))\n",
- "print(f\"Bias voltage: {bias_voltage:0.1f} V.\")\n",
- "print(f\"Detector gain: {int(gain)}.\")\n",
- "print(f\"Detector integration time: {integration_time} ms\")\n",
- "print(f\"Top pnCCD sensor temperature: {fix_temperature_top:0.2f} K\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "display(Markdown(\"### Constants retrieval\"))\n",
- "pnccd_cal = PNCCD_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " sensor_bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " sensor_temperature=fix_temperature_top,\n",
- " gain_setting=gain,\n",
- " source_energy=photon_energy,\n",
- " event_at=creation_time,\n",
- " client=rest_cfg.calibration_client(),\n",
- ")\n",
- "\n",
- "mdata_dict = {\"constants\": dict()}\n",
- "\n",
- "constant_names = [\"OffsetCCD\", \"NoiseCCD\", \"BadPixelsDarkCCD\"]\n",
- "if relgain:\n",
- " constant_names += [\"RelativeGainCCD\"]\n",
- "\n",
- "# Retrieve metadata for all pnccd constants.\n",
- "\n",
- "pnccd_metadata = pnccd_cal.metadata(constant_names)\n",
- "\n",
- "# Validate the constants availability and raise/warn correspondingly. \n",
- "missing_dark_constants = set(\n",
- " c for c in [\"OffsetCCD\", \"NoiseCCD\", \"BadPixelsDarkCCD\"] if c not in pnccd_metadata[karabo_da].keys())\n",
- "if missing_dark_constants:\n",
- " raise KeyError(\n",
- " f\"Dark constants {missing_dark_constants} are not available for correction.\")\n",
- "if relgain and \"RelativeGainCCD\" not in list(pnccd_metadata.values())[0].keys():\n",
- " warning(\"RelativeGainCCD is not found in CALCAT.\")\n",
- "\n",
- "for cname, ccv_metadata in list(pnccd_metadata.values())[0].items():\n",
- " mdata_dict[\"constants\"][cname] = {\n",
- " \"path\": str(pnccd_cal.caldb_root / ccv_metadata[\"path\"]),\n",
- " \"dataset\": ccv_metadata[\"dataset\"],\n",
- " \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
- " }\n",
- " print(f\"Retrieved {cname} with creation-time: {ccv_metadata['begin_validity_at']}\")\n",
- "\n",
- "mdata_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n",
- "retrieved_constants[karabo_da] = mdata_dict\n",
- "metadata.save()\n",
- "print(f\"Stored retrieved constants in {metadata.filename}\")"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "cal4_venv",
- "language": "python",
- "name": "cal4_venv"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.8.11"
- },
- "orig_nbformat": 4,
- "vscode": {
- "interpreter": {
- "hash": "ccde353e8822f411c1c49844e1cbe3edf63293a69efd975d1b44f5e852832668"
- }
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/src/cal_tools/tools.py b/src/cal_tools/tools.py
index b69502a037a9af5bc254f0a6440f6f463e4b99e6..89a6b711fb3b0dc50ea4861576ebe591d60dd5b5 100644
--- a/src/cal_tools/tools.py
+++ b/src/cal_tools/tools.py
@@ -962,6 +962,38 @@ def load_specified_constants(
return const_data, when
+def record_constant_details(
+ out_folder: Path,
+ karabo_da: str,
+ det_metadata: dict,
+ caldb_root: Path,
+):
+ """Record constant details to a separate fragment file.
+
+ Args:
+ out_folder (Path): The output folder to store the fragment file.
+ karabo_da (str): The detector karabo DA.
+ det_metadata (dict): A dictionary with the desired detector metadata.
+ caldb_root (Path): The calibration database root path for constant files.
+ """
+ CalibrationMetadata(out_folder).add_fragment({
+ "retrieved-constants": {
+ karabo_da: {
+ "constants": {
+ cname: {
+ "path": str(caldb_root / ccv_metadata["path"]),
+ "dataset": ccv_metadata["dataset"],
+ "creation-time": ccv_metadata["begin_validity_at"],
+ "ccv_id": ccv_metadata["ccv_id"],
+ } for cname, ccv_metadata in det_metadata.items()
+ },
+ "physical-name": list(
+ det_metadata.values())[0]["physical_name"],
+ }
+ }
+ })
+
+
def write_compressed_frames(
arr: np.ndarray,
ofile: h5py.File,
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index 9a5f29c2ffb68a605cc5c0164505f64a91adbe2b..ab00e99a87d5eac38ec9ee16649d07c3a6a4e0fc 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -112,7 +112,6 @@ notebooks = {
"cluster cores": 32},
},
"CORRECT": {
- "pre_notebooks": ["notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb"],
"notebook": "notebooks/pnCCD/Correct_pnCCD_NBC.ipynb",
"concurrency": {"parameter": "sequences",
"default concurrency": [-1],
diff --git a/tests/test_cal_tools.py b/tests/test_cal_tools.py
index edf08193a951c8372a67287fc8f4cb9545a2b9da..f1b583ddd99d37367c7a417fa1e126e385bd9f2a 100644
--- a/tests/test_cal_tools.py
+++ b/tests/test_cal_tools.py
@@ -4,6 +4,7 @@ from unittest.mock import patch
import numpy as np
import pytest
+import yaml
import zmq
from extra_data import open_run
from iCalibrationDB import Conditions, ConstantMetaData, Constants
@@ -18,8 +19,8 @@ from cal_tools.tools import (
get_pdu_from_db,
map_seq_files,
module_index_to_qm,
- send_to_db,
recursive_update,
+ send_to_db,
)
# AGIPD operating conditions.
@@ -484,3 +485,60 @@ def test_recursive_update():
src = {"a": {"b": 3}, "e": 4}
assert recursive_update(tgt, src) is True
assert tgt == {"a": {"b": 1}, "c": 2, "e": 4}
+
+from cal_tools.tools import record_constant_details
+
+
+def test_record_constant_details():
+ pnccd_metadata = {
+ "OffsetCCD": {
+ 'cc_id': 8540,
+ 'cc_name': 'pnCCD-Type_OffsetCCD_CCD Def9/wVu81XPSIX',
+ 'condition_id': 2478,
+ 'ccv_id': 51443,
+ 'ccv_name': '20200820_184154_sIdx=0',
+ 'path': Path('xfel/cal/pnccd-type/pnccd_m205_m206/cal.1597948913.0951352.h5'), # noqa
+ 'dataset': '/PnCCD_M205_M206/OffsetCCD/0',
+ 'begin_validity_at': '2020-08-20T20:34:28.000+02:00',
+ 'end_validity_at': None,
+ 'raw_data_location': 'Proposal: p900166, Run: 338',
+ 'start_idx': 0,
+ 'end_idx': 0,
+ 'physical_name': 'pnCCD_M205_M206',
+ },
+ }
+ test_folder = Path("test_record_constant_details")
+ test_folder.mkdir(exist_ok=True)
+
+ record_constant_details(
+ test_folder,
+ "PNCCD01",
+ pnccd_metadata,
+ Path("/gpfs/exfel/d/cal/caldb_store")
+ )
+ fragments = list(test_folder.glob("metadata_frag*yml"))
+ assert len(fragments) == 1
+
+ try:
+ # Open YAML file
+ with open(fragments[0], 'r') as file:
+ # Load YAML content into dictionary
+ yaml_dict = yaml.safe_load(file)
+ assert yaml_dict == {
+ "retrieved-constants": {
+ "PNCCD01": {
+ "constants": {
+ "OffsetCCD": {
+ "path": "/gpfs/exfel/d/cal/caldb_store/xfel/cal/pnccd-type/pnccd_m205_m206/cal.1597948913.0951352.h5", # noqa
+ "dataset": "/PnCCD_M205_M206/OffsetCCD/0",
+ "creation-time": "2020-08-20T20:34:28.000+02:00",
+ "ccv_id": 51443,
+ },
+ },
+ "physical-name": "pnCCD_M205_M206",
+ }
+ }
+ }
+ finally: # remove yml file.
+ for file_path in test_folder.glob('*'):
+ file_path.unlink()