diff --git a/notebooks/ePix100/Correction_ePix100_NBC.ipynb b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
index 499a0d2cf135bcb39648862220f5ad4fd7ba0c8b..f850adfdd6c506fdc152be638678e6bde86dfef8 100644
--- a/notebooks/ePix100/Correction_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
@@ -157,12 +157,7 @@
"outputs": [],
"source": [
"creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
- "print(f\"Using {creation_time.isoformat()} as creation time\")\n",
- "\n",
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# Constant paths are saved under retrieved-constants in calibration_metadata.yml.\n",
- "# NOTE: this notebook shouldn't overwrite calibration metadata file.\n",
- "const_yaml = metadata.get(\"retrieved-constants\", {})"
+ "print(f\"Using {creation_time.isoformat()} as creation time\")"
]
},
{
@@ -274,30 +269,20 @@
"if relative_gain:\n",
" constant_names += [\"RelativeGainEPix100\"]\n",
"\n",
- "const_data = dict()\n",
- "\n",
- "if const_yaml: # Used while reproducing corrected data.\n",
- " print(f\"Using stored constants in {metadata.filename}\")\n",
- " when = dict()\n",
- " for cname, mdata in const_yaml[karabo_da][\"constants\"].items():\n",
- " const_data[cname] = dict()\n",
- " when[cname] = mdata[\"creation-time\"]\n",
- " if when[cname]:\n",
- " with h5py.File(mdata[\"path\"], \"r\") as cf:\n",
- " const_data[cname] = np.copy(\n",
- " cf[f\"{mdata['dataset']}/data\"])\n",
- "else:\n",
- " epix_cal = EPIX100_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " sensor_bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " sensor_temperature=temperature_k,\n",
- " in_vacuum=in_vacuum,\n",
- " source_energy=gain_photon_energy,\n",
- " event_at=creation_time,\n",
- " client=rest_cfg.calibration_client(),\n",
- " )\n",
- " const_data = epix_cal.ndarray_map(calibrations=constant_names)[karabo_da]\n",
+ "epix_cal = EPIX100_CalibrationData(\n",
+ " detector_name=karabo_id,\n",
+ " sensor_bias_voltage=bias_voltage,\n",
+ " integration_time=integration_time,\n",
+ " sensor_temperature=temperature_k,\n",
+ " in_vacuum=in_vacuum,\n",
+ " source_energy=gain_photon_energy,\n",
+ " event_at=creation_time,\n",
+ " client=rest_cfg.calibration_client(),\n",
+ ")\n",
+ "const_metadata = epix_cal.metadata(calibrations=constant_names)\n",
+ "\n",
+ "# Load the constant data from files\n",
+ "const_data = epix_cal.ndarray_map(metadata=const_metadata)[karabo_da]\n",
"\n",
"# Validate the constants availability and raise/warn correspondingly. \n",
"missing_dark_constants = {\"OffsetEPix100\", \"NoiseEPix100\"} - set(const_data)\n",
@@ -312,6 +297,31 @@
" absolute_gain = False"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Record constant details in YAML metadata\n",
+ "epix_metadata = const_metadata[karabo_da]\n",
+ "CalibrationMetadata(metadata_folder or out_folder).add_fragment({\n",
+ " \"retrieved-constants\": {\n",
+ " karabo_da: {\n",
+ " \"constants\": {\n",
+ " cname: {\n",
+ " \"path\": str(epix_cal.caldb_root / ccv_metadata[\"path\"]),\n",
+ " \"dataset\": ccv_metadata[\"dataset\"],\n",
+ " \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
+ " \"ccv_id\": ccv_metadata[\"ccv_id\"],\n",
+ " } for cname, ccv_metadata in epix_metadata.items()\n",
+ " },\n",
+ " \"physical-name\": list(epix_metadata.values())[0][\"physical_name\"],\n",
+ " }\n",
+ " }\n",
+ "})"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -892,7 +902,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3.8.11 ('.cal3_venv')",
+ "display_name": "Python 3",
"language": "python",
"name": "python3"
},
@@ -906,7 +916,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.11"
+ "version": "3.8.12"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
diff --git a/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb b/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
deleted file mode 100644
index cf506f0e4a1bf79d355cf7e241871730f01bc4b4..0000000000000000000000000000000000000000
--- a/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
+++ /dev/null
@@ -1,214 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# ePix100 retrieve constants precorrection\n",
- "\n",
- "Author: European XFEL Detector Group, Version: 1.0\n",
- "\n",
- "The following notebook provides constants for the selected ePix100 modules before executing correction on the selected sequence files."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "in_folder = \"/gpfs/exfel/exp/CALLAB/202031/p900113/raw\" # input folder, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/epix_correct\" # output folder, required\n",
- "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
- "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
- "run = 9988 # which run to read data from, required\n",
- "\n",
- "# Parameters for accessing the raw data.\n",
- "karabo_id = \"MID_EXP_EPIX-1\" # Detector Karabo_ID\n",
- "karabo_da = \"EPIX01\" # data aggregators\n",
- "receiver_template = \"RECEIVER\" # detector receiver template for accessing raw data files\n",
- "instrument_source_template = '{}/DET/{}:daqOutput' # instrument detector data source in h5files\n",
- "\n",
- "# Parameters for the calibration database.\n",
- "creation_time = \"\" # The timestamp to use with Calibration DB. Required Format: \"YYYY-MM-DD hh:mm:ss\" e.g. 2019-07-04 11:02:41\n",
- "cal_db_interface = \"tcp://max-exfl016:8015#8025\" # calibration DB interface to use\n",
- "cal_db_timeout = 300000 # timeout on CalibrationDB requests\n",
- "\n",
- "# Conditions for retrieving calibration constants.\n",
- "bias_voltage = 200 # bias voltage\n",
- "in_vacuum = False # detector operated in vacuum\n",
- "fix_temperature = 290 # fixed temperature value in Kelvin. Default value -1 to use the value from files.\n",
- "integration_time = -1 # Detector integration time, Default value -1 to use the value from the slow data.\n",
- "gain_photon_energy = 9.0 # Photon energy used for gain calibration\n",
- "\n",
- "# Flags to select type of applied corrections.\n",
- "relative_gain = True # Apply relative gain correction."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from logging import warning\n",
- "\n",
- "import numpy as np\n",
- "from extra_data import RunDirectory\n",
- "from pathlib import Path\n",
- "\n",
- "import cal_tools.restful_config as rest_cfg\n",
- "from cal_tools.calcat_interface import EPIX100_CalibrationData\n",
- "from cal_tools.epix100 import epix100lib\n",
- "from cal_tools.tools import (\n",
- " calcat_creation_time,\n",
- " CalibrationMetadata,\n",
- ")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "in_folder = Path(in_folder)\n",
- "out_folder = Path(out_folder)\n",
- "\n",
- "out_folder.mkdir(parents=True, exist_ok=True)\n",
- "\n",
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# NOTE: this notebook will not overwrite calibration metadata file,\n",
- "# if it already contains details about which constants to use.\n",
- "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
- "print(f\"Using {creation_time.isoformat()} as creation time\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Read control data.\n",
- "run_dc = RunDirectory(in_folder / f\"r{run:04d}\")\n",
- "\n",
- "ctrl_data = epix100lib.epix100Ctrl(\n",
- " run_dc=run_dc,\n",
- " instrument_src=f\"{karabo_id}/DET/{receiver_template}:daqOutput\",\n",
- " ctrl_src=f\"{karabo_id}/DET/CONTROL\",\n",
- " )\n",
- "\n",
- "if integration_time < 0:\n",
- " integration_time = ctrl_data.get_integration_time()\n",
- " integration_time_str_add = \"\"\n",
- "else:\n",
- " integration_time_str_add = \"(manual input)\"\n",
- "\n",
- "if fix_temperature < 0:\n",
- " temperature = ctrl_data.get_temprature()\n",
- " temperature_k = temperature + 273.15\n",
- " temp_str_add = \"\"\n",
- "else:\n",
- " temperature_k = fix_temperature\n",
- " temperature = fix_temperature - 273.15\n",
- " temp_str_add = \"(manual input)\"\n",
- "\n",
- "\n",
- "print(f\"Bias voltage is {bias_voltage} V\")\n",
- "print(f\"Detector integration time is set to {integration_time} \\u03BCs {integration_time_str_add}\")\n",
- "print(f\"Mean temperature: {temperature:0.2f}°C / {temperature_k:0.2f} K {temp_str_add}\")\n",
- "print(f\"Operated in vacuum: {in_vacuum}\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "tags": []
- },
- "outputs": [],
- "source": [
- "epix_cal = EPIX100_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " sensor_bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " sensor_temperature=temperature_k,\n",
- " in_vacuum=in_vacuum,\n",
- " source_energy=gain_photon_energy,\n",
- " event_at=creation_time,\n",
- " client=rest_cfg.calibration_client(),\n",
- " )\n",
- "\n",
- "mdata_dict = {\"constants\": dict()}\n",
- "\n",
- "constant_names = [\"OffsetEPix100\", \"NoiseEPix100\"]\n",
- "if relative_gain:\n",
- " constant_names += [\"RelativeGainEPix100\"]\n",
- "\n",
- "# Retrieve metadata for all epix100 constants.\n",
- "\n",
- "epix_metadata = epix_cal.metadata(constant_names)[karabo_da]\n",
- "\n",
- "# Validate the constants availability and raise/warn correspondingly.\n",
- "missing_dark_constants = {\"OffsetEPix100\", \"NoiseEPix100\"} - set(epix_metadata)\n",
- "if missing_dark_constants:\n",
- " raise ValueError(\n",
- " f\"Dark constants {missing_dark_constants} are not available to correct {karabo_da}.\")\n",
- "\n",
- "if relative_gain and \"RelativeGainEPix100\" not in epix_metadata.keys():\n",
- " warning(\"RelativeGainEPix100 is not found in CALCAT.\")\n",
- "\n",
- "for cname, ccv_metadata in epix_metadata.items():\n",
- " mdata_dict[\"constants\"][cname] = {\n",
- " \"path\": str(epix_cal.caldb_root / ccv_metadata[\"path\"]),\n",
- " \"dataset\": ccv_metadata[\"dataset\"],\n",
- " \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
- " \"ccv_id\": ccv_metadata[\"ccv_id\"],\n",
- " }\n",
- " print(f\"Retrieved {cname} with creation-time: {ccv_metadata['begin_validity_at']}\")\n",
- "\n",
- "mdata_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n",
- "retrieved_constants[karabo_da] = mdata_dict\n",
- "metadata.save()\n",
- "print(f\"Stored retrieved constants in {metadata.filename}\")"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3.8.11 ('.cal4_venv')",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.8.11"
- },
- "orig_nbformat": 4,
- "vscode": {
- "interpreter": {
- "hash": "ccde353e8822f411c1c49844e1cbe3edf63293a69efd975d1b44f5e852832668"
- }
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/src/cal_tools/epix100/__init__.py b/src/cal_tools/epix100/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index 55f1113a4cbb2f25178c60cea6cf67c5d4bc3b36..9a5f29c2ffb68a605cc5c0164505f64a91adbe2b 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -217,7 +217,6 @@ notebooks = {
},
"CORRECT": {
- "pre_notebooks": ["notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb"],
"notebook": "notebooks/ePix100/Correction_ePix100_NBC.ipynb",
"concurrency": {"parameter": "sequences",
"default concurrency": [-1],