diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
index c85e59e2c11f2269aa6a702d745b001d7f6df70d..4ec341a90520c82522fd09e3c6b3f94528de477c 100644
--- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
@@ -79,7 +79,6 @@
"from logging import warning\n",
"from pathlib import Path\n",
"\n",
- "import h5py\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
@@ -100,7 +99,7 @@
"from cal_tools.tools import (\n",
" calcat_creation_time,\n",
" map_seq_files,\n",
- " CalibrationMetadata,\n",
+ " write_constants_fragment,\n",
")\n",
"\n",
"warnings.filterwarnings('ignore')\n",
@@ -121,10 +120,6 @@
"run_dc = RunDirectory(run_folder)\n",
"instrument_src = instrument_source_template.format(karabo_id, receiver_template)\n",
"\n",
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# NOTE: this notebook will not overwrite calibration metadata file\n",
- "const_yaml = metadata.get(\"retrieved-constants\", {})\n",
- "\n",
"out_folder.mkdir(parents=True, exist_ok=True)\n",
"\n",
"print(f\"Run is: {run}\")\n",
@@ -178,7 +173,6 @@
]
},
{
- "attachments": {},
"cell_type": "markdown",
"metadata": {},
"source": [
@@ -207,24 +201,12 @@
"for mod_info in jf_cal.physical_detector_units.values():\n",
" da_to_pdu[mod_info[\"karabo_da\"]] = mod_info[\"physical_name\"]\n",
"\n",
- "if const_yaml:\n",
- " const_data = dict()\n",
- " for mod, constants in const_yaml.items():\n",
- " if mod not in karabo_da:\n",
- " continue # skip other keys like time-summary\n",
- " const_data[mod] = dict()\n",
- " for cname, mdata in constants[\"constants\"].items():\n",
- " const_data[mod][cname] = dict()\n",
- " if mdata[\"creation-time\"]:\n",
- " with h5py.File(mdata[\"path\"], \"r\") as cf:\n",
- " const_data[mod][cname] = np.copy(\n",
- " cf[f\"{mdata['dataset']}/data\"])\n",
- "else:\n",
- " constant_names = [\"Offset10Hz\", \"BadPixelsDark10Hz\"]\n",
- " if relative_gain:\n",
- " constant_names += [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"]\n",
"\n",
- " const_data = jf_cal.ndarray_map(calibrations=constant_names)"
+ "constant_names = [\"Offset10Hz\", \"BadPixelsDark10Hz\"]\n",
+ "if relative_gain:\n",
+ " constant_names += [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"]\n",
+ "jf_metadata = jf_cal.metadata(calibrations=constant_names)\n",
+ "const_data = jf_cal.ndarray_map(metadata=jf_metadata)"
]
},
{
@@ -253,6 +235,18 @@
" raise ValueError(\"Dark constants are missing for all modules.\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "write_constants_fragment(\n",
+ " out_folder=(metadata_folder or out_folder),\n",
+ " det_metadata=jf_metadata,\n",
+ " caldb_root=jf_cal.caldb_root)"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -1012,9 +1006,9 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3",
+ "display_name": "cal2_venv",
"language": "python",
- "name": "python3"
+ "name": "cal2_venv"
},
"language_info": {
"codemirror_mode": {
@@ -1026,7 +1020,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.10"
+ "version": "3.8.11"
}
},
"nbformat": 4,
diff --git a/notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb
deleted file mode 100644
index 166c4c0a2d108d7f037dc7aa997f9811473a575d..0000000000000000000000000000000000000000
--- a/notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb
+++ /dev/null
@@ -1,245 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# JUNGFRAU Retrieving Constants Pre-correction #\n",
- "\n",
- "Author: European XFEL Detector Group, Version: 1.0\n",
- "\n",
- "Retrieving Required Constants for Offline Calibration of the JUNGFRAU Detector"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "in_folder = \"/gpfs/exfel/exp/SPB/202130/p900204/raw\" # the folder to read data from, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove\" # the folder to output to, required\n",
- "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
- "run = 95 # run to process, required\n",
- "\n",
- "# Parameters used to access raw data.\n",
- "karabo_id = \"SPB_IRDA_JF4M\" # karabo prefix of Jungfrau devices\n",
- "karabo_da = ['JNGFR01', 'JNGFR02', 'JNGFR03', 'JNGFR04', 'JNGFR05', 'JNGFR06', 'JNGFR07', 'JNGFR08'] # data aggregators\n",
- "ctrl_source_template = '{}/DET/CONTROL' # template for control source name (filled with karabo_id_control)\n",
- "karabo_id_control = \"\" # if control is on a different ID, set to empty string if it is the same a karabo-id\n",
- "\n",
- "# Parameters for calibration database.\n",
- "cal_db_interface = \"tcp://max-exfl016:8017#8025\" # the database interface to use\n",
- "cal_db_timeout = 180000 # timeout on cal db requests\n",
- "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
- "\n",
- "# Parameters affecting corrected data.\n",
- "relative_gain = True # do relative gain correction\n",
- "\n",
- "# Parameters for retrieving calibration constants\n",
- "manual_slow_data = False # if true, use manually entered bias_voltage, integration_time, gain_setting, and gain_mode values\n",
- "integration_time = 4.96 # integration time in us, will be overwritten by value in file\n",
- "gain_setting = 0 # 0 for dynamic gain, 1 for dynamic HG0, will be overwritten by value in file\n",
- "gain_mode = 0 # 0 for runs with dynamic gain setting, 1 for fixgain. It will be overwritten by value in file, if manual_slow_data is set to True.\n",
- "mem_cells = -1 # Set mem_cells to -1 to automatically use the value stored in RAW data.#\n",
- "bias_voltage = 180 # will be overwritten by value in file"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from logging import warning\n",
- "\n",
- "from extra_data import RunDirectory\n",
- "from pathlib import Path\n",
- "\n",
- "from cal_tools.calcat_interface import JUNGFRAU_CalibrationData\n",
- "from cal_tools.jungfraulib import JungfrauCtrl\n",
- "import cal_tools.restful_config as rest_cfg\n",
- "from cal_tools.step_timing import StepTimer\n",
- "from cal_tools.tools import (\n",
- " calcat_creation_time,\n",
- " CalibrationMetadata,\n",
- ")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "in_folder = Path(in_folder)\n",
- "out_folder = Path(out_folder)\n",
- "\n",
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
- "retrieved_constants = metadata[\"retrieved-constants\"] = dict()\n",
- "\n",
- "run_folder = in_folder / f'r{run:04d}'\n",
- "run_dc = RunDirectory(run_folder)\n",
- "\n",
- "out_folder.mkdir(parents=True, exist_ok=True)\n",
- "\n",
- "# Run's creation time:\n",
- "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
- "print(f\"Creation time: {creation_time}\")\n",
- "\n",
- "if karabo_id_control == \"\":\n",
- " karabo_id_control = karabo_id"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "ctrl_src = ctrl_source_template.format(karabo_id_control)\n",
- "ctrl_data = JungfrauCtrl(run_dc, ctrl_src)\n",
- "\n",
- "if mem_cells < 0:\n",
- " memory_cells, sc_start = ctrl_data.get_memory_cells()\n",
- " mem_cells_name = \"single cell\" if memory_cells == 1 else \"burst\"\n",
- " print(f\"Run is in {mem_cells_name} mode.\\nStorage cell start: {sc_start:02d}\")\n",
- "else:\n",
- " memory_cells = mem_cells\n",
- " mem_cells_name = \"single cell\" if memory_cells == 1 else \"burst\"\n",
- " print(f\"Run is in manually set to {mem_cells_name} mode. With {memory_cells} memory cells\")\n",
- "\n",
- "if not manual_slow_data:\n",
- " integration_time = ctrl_data.get_integration_time()\n",
- " bias_voltage = ctrl_data.get_bias_voltage()\n",
- " gain_setting = ctrl_data.get_gain_setting()\n",
- " gain_mode = ctrl_data.get_gain_mode()\n",
- "\n",
- "print(f\"Integration time is {integration_time} us\")\n",
- "print(f\"Gain setting is {gain_setting} (run settings: {ctrl_data.run_settings}\")\n",
- "print(f\"Gain mode is {gain_mode} ({ctrl_data.run_mode})\")\n",
- "print(f\"Bias voltage is {bias_voltage} V\")\n",
- "print(f\"Number of memory cells are {memory_cells}\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "step_timer = StepTimer()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "step_timer.start()\n",
- "jf_cal = JUNGFRAU_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " sensor_bias_voltage=bias_voltage,\n",
- " event_at=creation_time,\n",
- " modules=karabo_da,\n",
- " memory_cells=memory_cells,\n",
- " integration_time=integration_time,\n",
- " gain_setting=gain_setting,\n",
- " gain_mode=gain_mode,\n",
- " client=rest_cfg.calibration_client(),\n",
- ")\n",
- "constant_names = [\"Offset10Hz\", \"BadPixelsDark10Hz\"]\n",
- "\n",
- "if relative_gain:\n",
- " constant_names += [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"]\n",
- "\n",
- "jf_metadata = jf_cal.metadata(constant_names)\n",
- "\n",
- "# Add constants metadata in retrieved_constants dict.\n",
- "for mod, ccv_dict in jf_metadata.items():\n",
- " mod_dict = retrieved_constants.setdefault(mod, dict())\n",
- " const_dict = mod_dict.setdefault(\"constants\", dict())\n",
- " for cname, ccv_metadata in ccv_dict.items():\n",
- " const_dict[cname] = {\n",
- " \"path\": str(jf_cal.caldb_root / ccv_metadata[\"path\"]),\n",
- " \"dataset\": ccv_metadata[\"dataset\"],\n",
- " \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
- " \"ccv_id\": ccv_metadata[\"ccv_id\"],\n",
- " }\n",
- " mod_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n",
- "\n",
- "# Validate the constants availability and raise/warn correspondingly. \n",
- "missing_dark_modules = set()\n",
- "for mod, calibrations in jf_metadata.items():\n",
- " missing_dark_constants = {\"Offset10Hz\", \"BadPixelsDark10Hz\"} - set(calibrations)\n",
- " missing_gain_constants = {\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"} - set(calibrations)\n",
- " if missing_dark_constants:\n",
- " warning(\n",
- " f\"Dark constants {missing_dark_constants} are not available to correct {mod}\")\n",
- " missing_dark_modules.add(mod)\n",
- " if missing_gain_constants:\n",
- " warning(\n",
- " f\"Gain constants {missing_gain_constants} were not retrieved. Module: {mod}\")\n",
- "if missing_dark_modules == set(karabo_da):\n",
- " raise ValueError(f\"Dark constants are missing for all modules {karabo_da}.\")\n",
- "step_timer.done_step(f'Retrieving calibration constants.')"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "timestamps = dict()\n",
- "\n",
- "for mod in karabo_da:\n",
- " module_timestamps = timestamps[mod] = dict()\n",
- " module_constants = retrieved_constants[mod]\n",
- "\n",
- " print(f\"Module: {mod}:\")\n",
- " for cname, mdata in module_constants[\"constants\"].items():\n",
- " print(f\"\\t{cname:.<12s}\", mdata[\"creation-time\"])\n",
- "\n",
- " for cname in constant_names:\n",
- " if cname in module_constants[\"constants\"]:\n",
- " module_timestamps[cname] = module_constants[\"constants\"][cname][\"creation-time\"]\n",
- " else:\n",
- " module_timestamps[cname] = \"NA\"\n",
- "\n",
- "retrieved_constants[\"time-summary\"] = timestamps\n",
- "\n",
- "metadata.save()"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3.8.11 ('.cal4_venv')",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.8.11"
- },
- "orig_nbformat": 4,
- "vscode": {
- "interpreter": {
- "hash": "ccde353e8822f411c1c49844e1cbe3edf63293a69efd975d1b44f5e852832668"
- }
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index b505e7530d4daf3fe438be191fc0e1743dfca0b9..1c847c8b939da4392f8d86ccdd72574afb6b9b26 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -195,8 +195,6 @@ notebooks = {
"cluster cores": 4},
},
"CORRECT": {
- "pre_notebooks": [
- "notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb"], # noqa
"notebook":
"notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb",
"concurrency": {"parameter": "sequences",