diff --git a/notebooks/ePix100/Correction_ePix100_NBC.ipynb b/notebooks/ePix100/Correction_ePix100_NBC.ipynb index f850adfdd6c506fdc152be638678e6bde86dfef8..baa4bc08c982e90ef187d5ae802cc34a6af0762c 100644 --- a/notebooks/ePix100/Correction_ePix100_NBC.ipynb +++ b/notebooks/ePix100/Correction_ePix100_NBC.ipynb @@ -100,10 +100,9 @@ "from cal_tools.calcat_interface import EPIX100_CalibrationData\n", "from cal_tools.epix100 import epix100lib\n", "from cal_tools.files import DataFile\n", - "from cal_tools.restful_config import restful_config\n", "from cal_tools.tools import (\n", " calcat_creation_time,\n", - " CalibrationMetadata,\n", + " write_constants_fragment,\n", ")\n", "from cal_tools.step_timing import StepTimer\n", "\n", @@ -304,22 +303,11 @@ "outputs": [], "source": [ "# Record constant details in YAML metadata\n", - "epix_metadata = const_metadata[karabo_da]\n", - "CalibrationMetadata(metadata_folder or out_folder).add_fragment({\n", - " \"retrieved-constants\": {\n", - " karabo_da: {\n", - " \"constants\": {\n", - " cname: {\n", - " \"path\": str(epix_cal.caldb_root / ccv_metadata[\"path\"]),\n", - " \"dataset\": ccv_metadata[\"dataset\"],\n", - " \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n", - " \"ccv_id\": ccv_metadata[\"ccv_id\"],\n", - " } for cname, ccv_metadata in epix_metadata.items()\n", - " },\n", - " \"physical-name\": list(epix_metadata.values())[0][\"physical_name\"],\n", - " }\n", - " }\n", - "})" + "write_constants_fragment(\n", + " out_folder=(metadata_folder or out_folder),\n", + " det_metadata=const_metadata,\n", + " caldb_root=epix_cal.caldb_root,\n", + " )" ] }, { diff --git a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb index fe3c9eacb650af9d250d8feb5f6565a223ff9b4f..fe6db973a6b7004a4fa5cb953391f392cbe7e965 100644 --- a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb +++ b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb @@ -107,22 +107,17 @@ "\n", "%matplotlib inline\n", "\n", + "import cal_tools.restful_config as rest_cfg\n", "from XFELDetAna import xfelpyanatools as xana\n", "from XFELDetAna import xfelpycaltools as xcal\n", "from cal_tools import pnccdlib\n", "from cal_tools.files import DataFile\n", + "from cal_tools.calcat_interface import PNCCD_CalibrationData\n", "from cal_tools.tools import (\n", " calcat_creation_time,\n", - " get_dir_creation_date,\n", - " get_constant_from_db_and_time,\n", - " get_random_db_interface,\n", - " load_specified_constants,\n", - " CalibrationMetadata,\n", + " write_constants_fragment,\n", ")\n", - "from cal_tools.step_timing import StepTimer\n", - "from cal_tools import h5_copy_except\n", - "from iCalibrationDB import Conditions, Constants\n", - "from iCalibrationDB.detectors import DetectorTypes" + "from cal_tools.step_timing import StepTimer" ] }, { @@ -174,11 +169,6 @@ "# Output Folder Creation:\n", "os.makedirs(out_folder, exist_ok=True)\n", "\n", - "# NOTE: this notebook shouldn't overwrite calibration metadata file.\n", - "metadata = CalibrationMetadata(metadata_folder or out_folder)\n", - "# Constant paths are saved under retrieved-constants in calibration_metadata.yml\n", - "const_yaml = metadata.get(\"retrieved-constants\", {})\n", - "\n", "# extract control data\n", "step_timer.start()\n", "\n", @@ -317,50 +307,36 @@ "source": [ "display(Markdown(\"### Constants retrieval\"))\n", "step_timer.start()\n", + "constant_names = [\"OffsetCCD\", \"NoiseCCD\", \"BadPixelsDarkCCD\"]\n", + "if relgain:\n", + " constant_names += [\"RelativeGainCCD\"]\n", + "\n", + "pnccd_cal = PNCCD_CalibrationData(\n", + " detector_name=karabo_id,\n", + " sensor_bias_voltage=bias_voltage,\n", + " integration_time=integration_time,\n", + " sensor_temperature=fix_temperature_top,\n", + " gain_setting=gain,\n", + " event_at=creation_time,\n", + " source_energy=photon_energy,\n", + " client=rest_cfg.calibration_client(),\n", + ")\n", "\n", - "conditions_dict = {\n", - " \"bias_voltage\": bias_voltage,\n", - " \"integration_time\": integration_time,\n", - " \"gain_setting\": gain,\n", - " \"temperature\": fix_temperature_top,\n", - " \"pixels_x\": pixels_x,\n", - " \"pixels_y\": pixels_y,\n", - "}\n", - "# Dark condition\n", - "dark_condition = Conditions.Dark.CCD(**conditions_dict)\n", - "# Add photon energy.\n", - "conditions_dict.update({\"photon_energy\": photon_energy})\n", - "illum_condition = Conditions.Illuminated.CCD(**conditions_dict)\n", - "\n", - "# A dictionary for initializing constants. {cname: empty constant array}\n", - "empty_constants = {\n", - " \"Offset\": np.zeros((pixels_x, pixels_y, 1), dtype=np.float32),\n", - " \"Noise\": np.zeros((pixels_x, pixels_y, 1), dtype=np.float32),\n", - " \"BadPixelsDark\": np.zeros((pixels_x, pixels_y, 1), dtype=np.uint32),\n", - " \"RelativeGain\": np.zeros((pixels_x, pixels_y), dtype=np.float32),\n", - "}\n", - "\n", - "if const_yaml: # Used while reproducing corrected data.\n", - " print(f\"Using stored constants in {metadata.filename}\")\n", - " constants, when = load_specified_constants(\n", - " const_yaml[karabo_da][\"constants\"], empty_constants\n", - " )\n", - "else:\n", - " constants = dict()\n", - " when = dict()\n", - " for cname, cempty in empty_constants.items():\n", - " # No need for retrieving RelativeGain, if not used for correction.\n", - " if not corr_bools.get(\"relgain\") and cname == \"RelativeGain\":\n", - " continue\n", - " constants[cname], when[cname] = get_constant_from_db_and_time(\n", - " karabo_id,\n", - " karabo_da,\n", - " constant=getattr(Constants.CCD(DetectorTypes.pnCCD), cname)(),\n", - " condition=illum_condition if cname == \"RelativeGain\" else dark_condition,\n", - " empty_constant=cempty,\n", - " cal_db_interface=get_random_db_interface(cal_db_interface),\n", - " creation_time=creation_time,\n", - " )" + "pnccd_metadata = pnccd_cal.metadata(calibrations=constant_names)\n", + "constants = pnccd_cal.ndarray_map(metadata=pnccd_metadata)[karabo_da]\n", + "\n", + "# Validate the constants availability and raise/warn correspondingly. \n", + "missing_dark_constants = set(\n", + " c for c in [\"OffsetCCD\", \"NoiseCCD\", \"BadPixelsDarkCCD\"] if c not in constants.keys())\n", + "\n", + "if missing_dark_constants:\n", + " raise KeyError(\n", + " f\"Dark constants {missing_dark_constants} are not available for correction.\")\n", + "\n", + "if corr_bools.get('relgain') and \"RelativeGainCCD\" not in constants.keys():\n", + " warning(\"RelativeGainEPix100 is not found in the calibration database.\")\n", + " corr_bools['relgain'] = False\n", + "step_timer.done_step(\"Constants retrieval\")" ] }, { @@ -369,33 +345,46 @@ "metadata": {}, "outputs": [], "source": [ - "fig = xana.heatmapPlot(constants[\"Offset\"][:,:,0], x_label='Columns', y_label='Rows', lut_label='Offset (ADU)', \n", + "# Record constant details in YAML metadata\n", + "write_constants_fragment(\n", + " out_folder=(metadata_folder or out_folder),\n", + " det_metadata=pnccd_metadata,\n", + " caldb_root=pnccd_cal.caldb_root,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fig = xana.heatmapPlot(constants[\"OffsetCCD\"][:,:,0], x_label='Columns', y_label='Rows', lut_label='Offset (ADU)', \n", " aspect=1, \n", " x_range=(0, pixels_y), y_range=(0, pixels_x), vmax=16000, \n", " panel_x_label='Row Stat (ADU)', panel_y_label='Column Stat (ADU)', \n", " title = 'Dark Offset Map')\n", "\n", - "fig = xana.heatmapPlot(constants[\"Noise\"][:,:,0], x_label='Columns', y_label='Rows', \n", + "fig = xana.heatmapPlot(constants[\"NoiseCCD\"][:,:,0], x_label='Columns', y_label='Rows', \n", " lut_label='Corrected Noise (ADU)', \n", " aspect=1, x_range=(0, pixels_y), y_range=(0, pixels_x), \n", " panel_x_label='Row Stat (ADU)', panel_y_label='Column Stat (ADU)', \n", " title = 'Dark Noise Map')\n", "\n", - "fig = xana.heatmapPlot(np.log2(constants[\"BadPixelsDark\"][:,:,0]), x_label='Columns', y_label='Rows', \n", + "fig = xana.heatmapPlot(np.log2(constants[\"BadPixelsDarkCCD\"][:,:,0]), x_label='Columns', y_label='Rows', \n", " lut_label='Bad Pixel Value (ADU)', \n", " aspect=1, x_range=(0, pixels_y), y_range=(0, pixels_x), \n", " panel_x_label='Row Stat (ADU)', panel_y_label='Column Stat (ADU)', \n", " title = 'Dark Bad Pixels Map')\n", "\n", "if corr_bools.get('relgain'):\n", - " fig = xana.heatmapPlot(constants[\"RelativeGain\"], figsize=(8, 8), x_label='Columns', y_label='Rows', \n", + " fig = xana.heatmapPlot(constants[\"RelativeGainCCD\"], figsize=(8, 8), x_label='Columns', y_label='Rows', \n", " lut_label='Relative Gain', \n", " aspect=1, x_range=(0, pixels_y), y_range=(0, pixels_x), vmin=0.8, vmax=1.2, \n", " panel_x_label='Row Stat (ADU)', panel_y_label='Column Stat (ADU)', \n", " panel_top_low_lim = 0.5, panel_top_high_lim = 1.5, panel_side_low_lim = 0.5, \n", " panel_side_high_lim = 1.5, \n", - " title = f'Relative Gain Map for pnCCD (Gain = 1/{int(gain)})')\n", - "step_timer.done_step(\"Constants retrieval\")" + " title = f'Relative Gain Map for pnCCD (Gain = 1/{int(gain)})')" ] }, { @@ -411,13 +400,13 @@ " commonModeBlockSize,\n", " commonModeAxis,\n", " parallel=False, dType=np.float32, stride=1,\n", - " noiseMap=constants[\"Noise\"].astype(np.float32), minFrac=0.25)\n", + " noiseMap=constants[\"NoiseCCD\"].astype(np.float32), minFrac=0.25)\n", "\n", "if corr_bools.get('pattern_class'):\n", " # Pattern Classifier Calculator:\n", " # Left Hemisphere:\n", " patternClassifierLH = xcal.PatternClassifier([pixels_x, pixels_y//2],\n", - " constants[\"Noise\"][:, :pixels_y//2],\n", + " constants[\"NoiseCCD\"][:, :pixels_y//2],\n", " split_evt_primary_threshold,\n", " split_evt_secondary_threshold,\n", " split_evt_mip_threshold,\n", @@ -429,7 +418,7 @@ "\n", " # Right Hemisphere:\n", " patternClassifierRH = xcal.PatternClassifier([pixels_x, pixels_y//2],\n", - " constants[\"Noise\"][:, pixels_y//2:],\n", + " constants[\"NoiseCCD\"][:, pixels_y//2:],\n", " split_evt_primary_threshold,\n", " split_evt_secondary_threshold,\n", " split_evt_mip_threshold,\n", @@ -442,11 +431,11 @@ " patternClassifierLH._imagesPerChunk = 1\n", " patternClassifierRH._imagesPerChunk = 1\n", "\n", - " patternClassifierLH._noisemap = constants[\"Noise\"][:, :pixels_x//2]\n", - " patternClassifierRH._noisemap = constants[\"Noise\"][:, pixels_x//2:]\n", + " patternClassifierLH._noisemap = constants[\"NoiseCCD\"][:, :pixels_x//2]\n", + " patternClassifierRH._noisemap = constants[\"NoiseCCD\"][:, pixels_x//2:]\n", " # Setting bad pixels:\n", - " patternClassifierLH.setBadPixelMask(constants[\"BadPixelsDark\"][:, :pixels_x//2] != 0)\n", - " patternClassifierRH.setBadPixelMask(constants[\"BadPixelsDark\"][:, pixels_x//2:] != 0)" + " patternClassifierLH.setBadPixelMask(constants[\"BadPixelsDarkCCD\"][:, :pixels_x//2] != 0)\n", + " patternClassifierRH.setBadPixelMask(constants[\"BadPixelsDarkCCD\"][:, pixels_x//2:] != 0)" ] }, { @@ -586,10 +575,10 @@ "\n", "data_path = \"INSTRUMENT/\"+instrument_src+\"/data/\"\n", "\n", - "offset = np.squeeze(constants[\"Offset\"])\n", - "noise = np.squeeze(constants[\"Noise\"])\n", - "bpix = np.squeeze(constants[\"BadPixelsDark\"])\n", - "relativegain = constants.get(\"RelativeGain\")" + "offset = np.squeeze(constants[\"OffsetCCD\"])\n", + "noise = np.squeeze(constants[\"NoiseCCD\"])\n", + "bpix = np.squeeze(constants[\"BadPixelsDarkCCD\"])\n", + "relativegain = constants.get(\"RelativeGainCCD\")" ] }, { diff --git a/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb b/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb deleted file mode 100644 index eeac52fe3e033eb3bfef5a82fc3376ca1e8aecf4..0000000000000000000000000000000000000000 --- a/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb +++ /dev/null @@ -1,223 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# pnCCD retrieve constants precorrection\n", - "\n", - "Author: European XFEL Detector Group, Version: 1.0\n", - "\n", - "The following notebook provides constants for the selected pnCCD modules before executing correction on the selected sequence files." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "in_folder = \"/gpfs/exfel/exp/SQS/202031/p900166/raw\" # input folder\n", - "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/pnccd_correct\" # output folder\n", - "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n", - "run = 347 # which run to read data from\n", - "sequences = [0] # sequences to correct, set to -1 for all, range allowed\n", - "\n", - "karabo_da = 'PNCCD01' # data aggregators\n", - "karabo_id = \"SQS_NQS_PNCCD1MP\" # detector Karabo_ID\n", - "\n", - "# Conditions for retrieving calibration constants\n", - "fix_temperature_top = 0. # fix temperature for top sensor in K, set to 0. to use value from slow data.\n", - "fix_temperature_bot = 0. # fix temperature for bottom sensor in K, set to 0. to use value from slow data.\n", - "gain = -1 # the detector's gain setting. Set to -1 to use the value from the slow data.\n", - "bias_voltage = 0. # the detector's bias voltage. set to 0. to use value from slow data.\n", - "integration_time = 70 # detector's integration time\n", - "photon_energy = 1.6 # Al fluorescence in keV\n", - "\n", - "# Parameters for the calibration database.\n", - "cal_db_interface = \"tcp://max-exfl016:8015\" # calibration DB interface to use\n", - "cal_db_timeout = 300000 # timeout on CalibrationDB requests\n", - "creation_time = \"\" # The timestamp to use with Calibration DBe. Required Format: \"YYYY-MM-DD hh:mm:ss\" e.g. 2019-07-04 11:02:41\n", - "\n", - "# Booleans for selecting corrections to apply.\n", - "only_offset = False # Only, apply offset.\n", - "relgain = True # Apply relative gain correction\n", - "\n", - "# parameters affecting stored output data.\n", - "overwrite = True # keep this as True to not overwrite the output " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import datetime\n", - "from pathlib import Path\n", - "\n", - "from IPython.display import Markdown, display\n", - "from extra_data import RunDirectory\n", - "\n", - "from cal_tools import pnccdlib\n", - "from cal_tools.tools import (\n", - " calcat_creation_time,\n", - " get_dir_creation_date,\n", - " get_from_db,\n", - " get_random_db_interface,\n", - " save_constant_metadata,\n", - " CalibrationMetadata,\n", - ")\n", - "from iCalibrationDB import Conditions, Constants\n", - "from iCalibrationDB.detectors import DetectorTypes" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "metadata = CalibrationMetadata(metadata_folder or out_folder)\n", - "# NOTE: this notebook will not overwrite calibration metadata file,\n", - "# if it already contains details about which constants to use.\n", - "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n", - "if karabo_da in retrieved_constants:\n", - " print(\n", - " f\"Constant for {karabo_da} already in {metadata.filename}, won't query again.\"\n", - " ) # noqa\n", - " import sys\n", - "\n", - " sys.exit(0)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Here the correction booleans dictionary is defined\n", - "corr_bools = {}\n", - "\n", - "corr_bools[\"only_offset\"] = only_offset\n", - "\n", - "# Apply offset only.\n", - "if not only_offset:\n", - " corr_bools[\"relgain\"] = relgain" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "print(f\"Calibration database interface selected: {cal_db_interface}\")\n", - "\n", - "# Run's creation time:\n", - "creation_time = calcat_creation_time(in_folder, run, creation_time)\n", - "print(f\"Creation time: {creation_time}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "run_dc = RunDirectory(Path(in_folder) / f\"r{run:04d}\", _use_voview=False)\n", - "ctrl_data = pnccdlib.PnccdCtrl(run_dc, karabo_id)\n", - "\n", - "# extract control data\n", - "if bias_voltage == 0.0:\n", - " bias_voltage = ctrl_data.get_bias_voltage()\n", - "if gain == -1:\n", - " gain = ctrl_data.get_gain()\n", - "if fix_temperature_top == 0:\n", - " fix_temperature_top = ctrl_data.get_fix_temperature_top()\n", - "\n", - "# Printing the Parameters Read from the Data File:\n", - "display(Markdown(\"### Detector Parameters\"))\n", - "print(f\"Bias voltage: {bias_voltage:0.1f} V.\")\n", - "print(f\"Detector gain: {int(gain)}.\")\n", - "print(f\"Detector integration time: {integration_time} ms\")\n", - "print(f\"Top pnCCD sensor temperature: {fix_temperature_top:0.2f} K\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "display(Markdown(\"### Constants retrieval\"))\n", - "\n", - "conditions_dict = {\n", - " \"bias_voltage\": bias_voltage,\n", - " \"integration_time\": integration_time,\n", - " \"gain_setting\": gain,\n", - " \"temperature\": fix_temperature_top,\n", - " \"pixels_x\": 1024,\n", - " \"pixels_y\": 1024,\n", - "}\n", - "# Dark condition\n", - "dark_condition = Conditions.Dark.CCD(**conditions_dict)\n", - "# Add photon energy.\n", - "conditions_dict.update({\"photon_energy\": photon_energy})\n", - "illum_condition = Conditions.Illuminated.CCD(**conditions_dict)\n", - "\n", - "mdata_dict = dict()\n", - "mdata_dict[\"constants\"] = dict()\n", - "for cname in [\"Offset\", \"Noise\", \"BadPixelsDark\", \"RelativeGain\"]:\n", - " # No need for retrieving RelativeGain, if not used for correction.\n", - " if not corr_bools.get(\"relgain\") and cname == \"RelativeGain\":\n", - " continue\n", - " _, mdata = get_from_db(\n", - " karabo_id=karabo_id,\n", - " karabo_da=karabo_da,\n", - " constant=getattr(Constants.CCD(DetectorTypes.pnCCD), cname)(),\n", - " condition=illum_condition if cname == \"RelativeGain\" else dark_condition,\n", - " empty_constant=None,\n", - " cal_db_interface=get_random_db_interface(cal_db_interface),\n", - " creation_time=creation_time,\n", - " verbosity=1,\n", - " load_data=False,\n", - " )\n", - " save_constant_metadata(mdata_dict[\"constants\"], mdata, cname)\n", - "\n", - "mdata_dict[\"physical-detector-unit\"] = mdata.calibration_constant_version.device_name\n", - "retrieved_constants[karabo_da] = mdata_dict\n", - "metadata.save()\n", - "print(f\"Stored retrieved constants in {metadata.filename}\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3.8.11 ('.cal4_venv')", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.11" - }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "ccde353e8822f411c1c49844e1cbe3edf63293a69efd975d1b44f5e852832668" - } - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/cal_tools/tools.py b/src/cal_tools/tools.py index b69502a037a9af5bc254f0a6440f6f463e4b99e6..de53ba77e02ca2879d2502299124c8988d91eb3e 100644 --- a/src/cal_tools/tools.py +++ b/src/cal_tools/tools.py @@ -962,6 +962,36 @@ def load_specified_constants( return const_data, when +def write_constants_fragment( + out_folder: Path, + det_metadata: dict, + caldb_root: Path, +): + """Record calibration constants metadata to a fragment file. + + Args: + out_folder (Path): The output folder to store the fragment file. + det_metadata (dict): A dictionary with the desired detector metadata. + {karabo_da: {constant_name: metadata}} + caldb_root (Path): The calibration database root path for constant files. + """ + metadata = {"retrieved-constants": {}} + for karabo_da, const_metadata in det_metadata.items(): + mod_metadata = {} + mod_metadata["constants"] = { + cname: { + "path": str(caldb_root / ccv_metadata["path"]), + "dataset": ccv_metadata["dataset"], + "creation-time": ccv_metadata["begin_validity_at"], + "ccv_id": ccv_metadata["ccv_id"], + } for cname, ccv_metadata in const_metadata.items() + } + mod_metadata["physical-name"] = list( + const_metadata.values())[0]["physical_name"] + metadata["retrieved-constants"][karabo_da] = mod_metadata + CalibrationMetadata(out_folder).add_fragment(metadata) + + def write_compressed_frames( arr: np.ndarray, ofile: h5py.File, diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py index 9a5f29c2ffb68a605cc5c0164505f64a91adbe2b..ab00e99a87d5eac38ec9ee16649d07c3a6a4e0fc 100644 --- a/src/xfel_calibrate/notebooks.py +++ b/src/xfel_calibrate/notebooks.py @@ -112,7 +112,6 @@ notebooks = { "cluster cores": 32}, }, "CORRECT": { - "pre_notebooks": ["notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb"], "notebook": "notebooks/pnCCD/Correct_pnCCD_NBC.ipynb", "concurrency": {"parameter": "sequences", "default concurrency": [-1], diff --git a/tests/test_cal_tools.py b/tests/test_cal_tools.py index edf08193a951c8372a67287fc8f4cb9545a2b9da..36d9d324021b3ce650434d3d651ca1834207b972 100644 --- a/tests/test_cal_tools.py +++ b/tests/test_cal_tools.py @@ -4,6 +4,7 @@ from unittest.mock import patch import numpy as np import pytest +import yaml import zmq from extra_data import open_run from iCalibrationDB import Conditions, ConstantMetaData, Constants @@ -18,8 +19,9 @@ from cal_tools.tools import ( get_pdu_from_db, map_seq_files, module_index_to_qm, - send_to_db, recursive_update, + send_to_db, + write_constants_fragment, ) # AGIPD operating conditions. @@ -484,3 +486,131 @@ def test_recursive_update(): src = {"a": {"b": 3}, "e": 4} assert recursive_update(tgt, src) is True assert tgt == {"a": {"b": 1}, "c": 2, "e": 4} + + +def test_write_constants_fragment(tmp_path: Path): + """Test `write_constants_fragment` with jungfrau. + This metadata is from constants used to correct FXE_XAD_JF1M + detector from proposal 900226, run 106. + + tmp_path: + tmp_path (pathlib.Path): Temporary directory for file tests. + https://docs.pytest.org/en/7.1.x/how-to/tmp_path.html + """ + + jf_metadata = { + "JNGFR01": { + "Offset10Hz": { + "cc_id": 7064, + "cc_name": "jungfrau-Type_Offset10Hz_Jungfrau DefiFE6iJX", + "condition_id": 2060, + "ccv_id": 41876, + "ccv_name": "20200304_152733_sIdx=0", + "path": Path("xfel/cal/jungfrau-type/jungfrau_m233/cal.1583335651.8084984.h5"), + "dataset": "/Jungfrau_M233/Offset10Hz/0", + "begin_validity_at": "2020-03-04T15:16:34.000+01:00", + "end_validity_at": None, + "raw_data_location": "proposal:p900121 runs:136 137 138", + "start_idx": 0, + "end_idx": 0, + "physical_name": "Jungfrau_M233"}, + "BadPixelsDark10Hz": { + "cc_id": 7066, + "cc_name": "jungfrau-Type_BadPixelsDark10Hz_Jungfrau DefiFE6iJX", + "condition_id": 2060, + "ccv_id": 41878, + "ccv_name": "20200304_152740_sIdx=0", + "path": Path("xfel/cal/jungfrau-type/jungfrau_m233/cal.1583335658.6813955.h5"), + "dataset": "/Jungfrau_M233/BadPixelsDark10Hz/0", + "begin_validity_at": "2020-03-04T15:16:34.000+01:00", + "end_validity_at": None, + "raw_data_location": "proposal:p900121 runs:136 137 138", + "start_idx": 0, + "end_idx": 0, + "physical_name": "Jungfrau_M233" + } + }, + "JNGFR02": { + "Offset10Hz": { + "cc_id": 7067, + "cc_name": "jungfrau-Type_Offset10Hz_Jungfrau DefzgIVHz1", + "condition_id": 2061, + "ccv_id": 41889, + "ccv_name": "20200304_154434_sIdx=0", + "path": Path("xfel/cal/jungfrau-type/jungfrau_m125/cal.1583336672.760199.h5"), + "dataset": "/Jungfrau_M125/Offset10Hz/0", + "begin_validity_at": "2020-03-04T15:16:34.000+01:00", + "end_validity_at": None, + "raw_data_location": "proposal:p900121 runs:136 137 138", + "start_idx": 0, + "end_idx": 0, + "physical_name": "Jungfrau_M125", + }, + "BadPixelsDark10Hz": { + "cc_id": 7069, + "cc_name": "jungfrau-Type_BadPixelsDark10Hz_Jungfrau DefzgIVHz1", + "condition_id": 2061, + "ccv_id": 41893, + "ccv_name": "20200304_154441_sIdx=0", + "path": Path("xfel/cal/jungfrau-type/jungfrau_m125/cal.1583336679.5835564.h5"), + "dataset": "/Jungfrau_M125/BadPixelsDark10Hz/0", + "begin_validity_at": "2020-03-04T15:16:34.000+01:00", + "end_validity_at": None, + "raw_data_location": "proposal:p900121 runs:136 137 138", + "start_idx": 0, + "end_idx": 0, + "physical_name": "Jungfrau_M125", + } + } + } + + write_constants_fragment( + tmp_path, + jf_metadata, + Path("/gpfs/exfel/d/cal/caldb_store") + ) + fragments = list(tmp_path.glob("metadata_frag*yml")) + assert len(fragments) == 1 + + # Open YAML file + with open(fragments[0], "r") as file: + # Load YAML content into dictionary + yaml_dict = yaml.safe_load(file) + assert yaml_dict == { + "retrieved-constants":{ + "JNGFR01": { + "constants": { + "BadPixelsDark10Hz": { + "ccv_id": 41878, + "creation-time": "2020-03-04T15:16:34.000+01:00", + "dataset": "/Jungfrau_M233/BadPixelsDark10Hz/0", + "path": "/gpfs/exfel/d/cal/caldb_store/xfel/cal/jungfrau-type/jungfrau_m233/cal.1583335658.6813955.h5", # noqa + }, + "Offset10Hz": { + "ccv_id": 41876, + "creation-time": "2020-03-04T15:16:34.000+01:00", + "dataset": "/Jungfrau_M233/Offset10Hz/0", + "path": "/gpfs/exfel/d/cal/caldb_store/xfel/cal/jungfrau-type/jungfrau_m233/cal.1583335651.8084984.h5", # noqa + }, + }, + "physical-name": "Jungfrau_M233", + }, + "JNGFR02": { + "constants": { + "BadPixelsDark10Hz": { + "ccv_id": 41893, + "creation-time": "2020-03-04T15:16:34.000+01:00", + "dataset": "/Jungfrau_M125/BadPixelsDark10Hz/0", + "path": "/gpfs/exfel/d/cal/caldb_store/xfel/cal/jungfrau-type/jungfrau_m125/cal.1583336679.5835564.h5", # noqa + }, + "Offset10Hz": { + "ccv_id": 41889, + "creation-time": "2020-03-04T15:16:34.000+01:00", + "dataset": "/Jungfrau_M125/Offset10Hz/0", + "path": "/gpfs/exfel/d/cal/caldb_store/xfel/cal/jungfrau-type/jungfrau_m125/cal.1583336672.760199.h5", # noqa + }, + }, + "physical-name": "Jungfrau_M125", + }, + } + }