diff --git a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
index af1f2532e5253bb7077ebafc8a7de7ed1e2e823c..8c84f0b3aa5fa833b9cfa542264daf24c77d3eb4 100644
--- a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
+++ b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
@@ -88,7 +88,7 @@
"from cal_tools.step_timing import StepTimer\n",
"from cal_tools.tools import (\n",
" calcat_creation_time,\n",
- " CalibrationMetadata,\n",
+ " write_constants_fragment,\n",
")\n",
"from XFELDetAna.plotting.heatmap import heatmapPlot\n",
"\n",
@@ -109,10 +109,6 @@
"out_folder = Path(out_folder)\n",
"out_folder.mkdir(parents=True, exist_ok=True)\n",
"\n",
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# NOTE: this notebook will not overwrite calibration metadata file\n",
- "const_yaml = metadata.get(\"retrieved-constants\", {})\n",
- "\n",
"if not karabo_id_control:\n",
" karabo_id_control = karabo_id\n",
"\n",
@@ -242,25 +238,13 @@
" const_data[mod][\"RelativeGainGotthard2\"] = cfile[\"gain_map\"][()].astype(np.float32)\n",
" const_data[mod][\"Mask\"] = cfile[\"bpix_ff\"][()].astype(np.uint32)\n",
"else:\n",
- " if const_yaml:\n",
- " const_data = dict()\n",
- " for mod in karabo_da:\n",
- " const_data[mod] = dict()\n",
- " for cname, mdata in const_yaml[mod][\"constants\"].items():\n",
- " const_data[mod][cname] = dict()\n",
- " if mdata[\"creation-time\"]:\n",
- " with h5py.File(mdata[\"path\"], \"r\") as cf:\n",
- " const_data[mod][cname] = np.copy(\n",
- " cf[f\"{mdata['dataset']}/data\"])\n",
- " else:\n",
- " mdata_dict = {\"constants\": dict()}\n",
- "\n",
- " constant_names = [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"]\n",
- " if gain_correction:\n",
- " constant_names += [\"RelativeGainGotthard2\", \"BadPixelsFFGotthard2\"]\n",
- "\n",
- " # Retrieve metadata for all pnccd constants.\n",
- " const_data = g2_cal.ndarray_map(constant_names)\n",
+ " constant_names = [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"]\n",
+ " if gain_correction:\n",
+ " constant_names += [\"RelativeGainGotthard2\", \"BadPixelsFFGotthard2\"]\n",
+ "\n",
+ " g2_metadata = g2_cal.metadata(calibrations=constant_names)\n",
+ " # Retrieve metadata for all pnccd constants.\n",
+ " const_data = g2_cal.ndarray_map(metadata=g2_metadata)\n",
"\n",
" # Validate the constants availability and raise/warn correspondingly.\n",
" for mod, calibrations in const_data.items():\n",
@@ -279,32 +263,45 @@
" if gain_correction and missing_gain_constants:\n",
" warning(f\"Gain constants {missing_gain_constants} are not retrieved for mod {mod}.\")\n",
"\n",
- " # Create the mask array.\n",
- " bpix = const_data[mod].get(\"BadPixelsDarkGotthard2\")\n",
- " if bpix is None:\n",
- " bpix = np.zeros((1280, 2, 3), dtype=np.uint32)\n",
- " if const_data[mod].get(\"BadPixelsFFGotthard2\") is not None:\n",
- " bpix |= const_data[mod][\"BadPixelsFFGotthard2\"]\n",
- " const_data[mod][\"Mask\"] = bpix\n",
- "\n",
- " # Prepare empty arrays for missing constants.\n",
- " if const_data[mod].get(\"OffsetGotthard2\") is None:\n",
- " const_data[mod][\"OffsetGotthard2\"] = np.zeros(\n",
- " (1280, 2, 3), dtype=np.float32)\n",
- "\n",
- " if const_data[mod].get(\"RelativeGainGotthard2\") is None:\n",
- " const_data[mod][\"RelativeGainGotthard2\"] = np.ones(\n",
- " (1280, 2, 3), dtype=np.float32)\n",
- " if gain_correction:\n",
- " gain_correction = False\n",
- " warning(\"Gain correction is disabled for this module.\")\n",
- " const_data[mod][\"RelativeGainGotthard2\"] = const_data[mod][\"RelativeGainGotthard2\"].astype( # noqa\n",
- " np.float32, copy=False) # Old gain constants are not float32.\n",
- "\n",
"if not karabo_da:\n",
" raise ValueError(\"Dark constants are not available for all modules.\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ac1cdec5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Record constant details in YAML metadata.\n",
+ "write_constants_fragment(\n",
+ " out_folder=(metadata_folder or out_folder),\n",
+ " det_metadata=g2_metadata,\n",
+ " caldb_root=g2_cal.caldb_root)\n",
+ "\n",
+ "# Prepare constant arrays.\n",
+ "if not constants_file:\n",
+ " # Create the mask array.\n",
+ " bpix = const_data[mod].get(\"BadPixelsDarkGotthard2\")\n",
+ " if bpix is None:\n",
+ " bpix = np.zeros((1280, 2, 3), dtype=np.uint32)\n",
+ " if const_data[mod].get(\"BadPixelsFFGotthard2\") is not None:\n",
+ " bpix |= const_data[mod][\"BadPixelsFFGotthard2\"]\n",
+ " const_data[mod][\"Mask\"] = bpix\n",
+ "\n",
+ " # Prepare empty arrays for missing constants.\n",
+ " if const_data[mod].get(\"OffsetGotthard2\") is None:\n",
+ " const_data[mod][\"OffsetGotthard2\"] = np.zeros(\n",
+ " (1280, 2, 3), dtype=np.float32)\n",
+ "\n",
+ " if const_data[mod].get(\"RelativeGainGotthard2\") is None:\n",
+ " const_data[mod][\"RelativeGainGotthard2\"] = np.ones(\n",
+ " (1280, 2, 3), dtype=np.float32)\n",
+ " const_data[mod][\"RelativeGainGotthard2\"] = const_data[mod][\"RelativeGainGotthard2\"].astype( # noqa\n",
+ " np.float32, copy=False) # Old gain constants are not float32."
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb b/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
deleted file mode 100644
index 679bc8506b03f887e077be4993ae63aeee29b0c3..0000000000000000000000000000000000000000
--- a/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
+++ /dev/null
@@ -1,249 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# GOTTHARD2 Retrieving Constants Pre-correction #\n",
- "\n",
- "Author: European XFEL Detector Group, Version: 1.0\n",
- "\n",
- "Retrieving Required Constants for Offline Calibration of the Gotthard2 Detector"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "in_folder = \"/gpfs/exfel/exp/FXE/202221/p003225/raw\" # the folder to read data from, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/gotthard2\" # the folder to output to, required\n",
- "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
- "run = 50 # run to process, required\n",
- "\n",
- "# Parameters used to access raw data.\n",
- "karabo_id = \"FXE_XAD_G2XES\" # karabo prefix of Gotthard-II devices\n",
- "karabo_da = [\"GH201\"] # data aggregators\n",
- "receiver_template = \"RECEIVER\" # receiver template used to read INSTRUMENT keys.\n",
- "control_template = \"CONTROL\" # control template used to read CONTROL keys.\n",
- "instrument_source_template = \"{}/DET/{}:daqOutput\" # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput'\n",
- "ctrl_source_template = \"{}/DET/{}\" # template for control source name (filled with karabo_id_control)\n",
- "karabo_id_control = \"\" # Control karabo ID. Set to empty string to use the karabo-id\n",
- "\n",
- "# Parameters for calibration database.\n",
- "cal_db_interface = \"tcp://max-exfl016:8017#8025\" # the database interface to use.\n",
- "cal_db_timeout = 180000 # timeout on caldb requests.\n",
- "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
- "\n",
- "# Parameters affecting corrected data.\n",
- "constants_file = \"\" # /gpfs/exfel/data/scratch/ahmedk/dont_remove/gotthard2/constants/calibration_constants_GH2.h5\" # Retrieve constants from local.\n",
- "offset_correction = True # apply offset correction. This can be disabled to only apply LUT or apply LUT and gain correction for non-linear differential results.\n",
- "gain_correction = True # apply gain correction.\n",
- "\n",
- "# Parameter conditions.\n",
- "bias_voltage = -1 # Detector bias voltage, set to -1 to use value in raw file.\n",
- "exposure_time = -1. # Detector exposure time, set to -1 to use value in raw file.\n",
- "exposure_period = -1. # Detector exposure period, set to -1 to use value in raw file.\n",
- "acquisition_rate = -1. # Detector acquisition rate (1.1/4.5), set to -1 to use value in raw file.\n",
- "single_photon = -1 # Detector single photon mode (High/Low CDS), set to -1 to use value in raw file.\n",
- "\n",
- "if constants_file:\n",
- " print(\"Skipping constant retrieval. Specified constants_file is used.\")\n",
- " import sys\n",
- "\n",
- " sys.exit(0)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from logging import warning\n",
- "\n",
- "from extra_data import RunDirectory\n",
- "from pathlib import Path\n",
- "\n",
- "import cal_tools.restful_config as rest_cfg\n",
- "from cal_tools.calcat_interface import GOTTHARD2_CalibrationData\n",
- "from cal_tools.gotthard2 import gotthard2lib\n",
- "from cal_tools.tools import (\n",
- " calcat_creation_time,\n",
- " CalibrationMetadata,\n",
- ")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "in_folder = Path(in_folder)\n",
- "run_folder = in_folder / f\"r{run:04d}\"\n",
- "out_folder = Path(out_folder)\n",
- "out_folder.mkdir(parents=True, exist_ok=True)\n",
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# Constant paths are saved under retrieved-constants in calibration_metadata.yml\n",
- "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
- "\n",
- "if not karabo_id_control:\n",
- " karabo_id_control = karabo_id\n",
- "\n",
- "instrument_src = instrument_source_template.format(karabo_id, receiver_template)\n",
- "ctrl_src = ctrl_source_template.format(karabo_id_control, control_template)\n",
- "\n",
- "print(f\"Retrieve constants for modules: {karabo_da} for run {run}\")\n",
- "\n",
- "# Run's creation time:\n",
- "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
- "print(f\"Creation time: {creation_time}\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Read slow data\n",
- "run_dc = RunDirectory(run_folder)\n",
- "g2ctrl = gotthard2lib.Gotthard2Ctrl(run_dc=run_dc, ctrl_src=ctrl_src)\n",
- "\n",
- "if bias_voltage == -1:\n",
- " bias_voltage = g2ctrl.get_bias_voltage()\n",
- "if exposure_time == -1:\n",
- " exposure_time = g2ctrl.get_exposure_time()\n",
- "if exposure_period == -1:\n",
- " exposure_period = g2ctrl.get_exposure_period()\n",
- "if acquisition_rate == -1:\n",
- " acquisition_rate = g2ctrl.get_acquisition_rate()\n",
- "if single_photon == -1:\n",
- " single_photon = g2ctrl.get_single_photon()\n",
- "\n",
- "print(\"Bias Voltage:\", bias_voltage)\n",
- "print(\"Exposure Time:\", exposure_time)\n",
- "print(\"Exposure Period:\", exposure_period)\n",
- "print(\"Acquisition Rate:\", acquisition_rate)\n",
- "print(\"Single Photon:\", single_photon)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "g2_cal = GOTTHARD2_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " sensor_bias_voltage=bias_voltage,\n",
- " exposure_time=exposure_time,\n",
- " exposure_period=exposure_period,\n",
- " acquisition_rate=acquisition_rate,\n",
- " single_photon=single_photon,\n",
- " event_at=creation_time,\n",
- " client=rest_cfg.calibration_client(),\n",
- ")\n",
- "\n",
- "mdata_dict = {\"constants\": dict()}\n",
- "\n",
- "constant_names = [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"]\n",
- "if gain_correction:\n",
- " constant_names += [\"RelativeGainGotthard2\", \"BadPixelsFFGotthard2\"]\n",
- "\n",
- "# Retrieve metadata for all pnccd constants.\n",
- "g2_metadata = g2_cal.metadata(constant_names)\n",
- "\n",
- "missing_dark_modules = set()\n",
- "# Validate the constants availability and raise/warn correspondingly.\n",
- "for mod, ccv_dict in g2_metadata.items():\n",
- "\n",
- " dark_constants = {\"LUTGotthard2\"}\n",
- " if offset_correction:\n",
- " dark_constants |= {\"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"}\n",
- " missing_dark_constants = dark_constants - set(ccv_dict)\n",
- "\n",
- " if missing_dark_constants:\n",
- " warning(f\"Dark constants {missing_dark_constants} are not available to correct {mod}\")\n",
- " missing_dark_modules.add(mod)\n",
- "\n",
- " missing_gain_constants = {\"BadPixelsFFGotthard2\", \"RelativeGainGotthard2\"} - set(ccv_dict)\n",
- " if gain_correction and missing_gain_constants:\n",
- " warning(f\"Gain constants {missing_gain_constants} are not retrieved for {mod}\")\n",
- "\n",
- "if missing_dark_modules == set(karabo_da):\n",
- " raise ValueError(f\"{missing_dark_constants} constants are not available for all modules.\")\n",
- "\n",
- "# Add constants metadata in retrieved_constants dict.\n",
- "for mod, ccv_dict in g2_metadata.items():\n",
- " mod_dict = retrieved_constants.setdefault(mod, dict())\n",
- " const_dict = mod_dict.setdefault(\"constants\", dict())\n",
- " for cname, ccv_metadata in ccv_dict.items():\n",
- " const_dict[cname] = {\n",
- " \"path\": str(g2_cal.caldb_root / ccv_metadata[\"path\"]),\n",
- " \"dataset\": ccv_metadata[\"dataset\"],\n",
- " \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
- " \"ccv_id\": ccv_metadata[\"ccv_id\"],\n",
- " }\n",
- " mod_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n",
- " \n",
- "print(f\"Stored retrieved constants in {metadata.filename}\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "timestamps = dict()\n",
- "\n",
- "for mod in karabo_da:\n",
- " module_timestamps = timestamps[mod] = dict()\n",
- " module_constants = retrieved_constants[mod]\n",
- " print(f\"Module: {mod}:\")\n",
- " for cname, mdata in module_constants[\"constants\"].items():\n",
- " print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
- "\n",
- " for cname in [\"OffsetGotthard2\", \"BadPixelsDarkGotthard2\", \"RelativeGainGotthard2\", \"BadPixelsFFGotthard2\"]:\n",
- " if cname in module_constants[\"constants\"]:\n",
- " module_timestamps[cname] = module_constants[\"constants\"][cname][\"creation-time\"]\n",
- " else:\n",
- " module_timestamps[cname] = \"NA\"\n",
- "\n",
- "retrieved_constants[\"time-summary\"] = timestamps\n",
- "\n",
- "metadata.save()"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3.8.11 ('.cal4_venv')",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.8.11"
- },
- "orig_nbformat": 4,
- "vscode": {
- "interpreter": {
- "hash": "ccde353e8822f411c1c49844e1cbe3edf63293a69efd975d1b44f5e852832668"
- }
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index 1c847c8b939da4392f8d86ccdd72574afb6b9b26..9e2ce6c836b391f055c6f643d72586f13d607afc 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -205,8 +205,6 @@ notebooks = {
},
"GOTTHARD2": {
"CORRECT": {
- "pre_notebooks": [
- "notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb"], # noqa
"notebook":
"notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb",
"concurrency": {"parameter": "sequences",