diff --git a/notebooks/ePix100/Correction_ePix100_NBC.ipynb b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
index f0cd9cc588607237ff49165055e0d017dca9d3f3..26bd24e9d3389467247b47a8aa92e54f85dcedc1 100644
--- a/notebooks/ePix100/Correction_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
@@ -18,7 +18,7 @@
"outputs": [],
"source": [
"in_folder = \"/gpfs/exfel/exp/CALLAB/202031/p900113/raw\" # input folder, required\n",
- "out_folder = \"\" # output folder, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/epix_correct\" # output folder, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"sequences_per_node = 1 # number of sequence files per cluster node if run as slurm job, set to 0 to not run SLURM parallel\n",
@@ -93,7 +93,8 @@
"from cal_tools import h5_copy_except\n",
"from cal_tools.tools import (\n",
" get_dir_creation_date,\n",
- " get_from_db,\n",
+ " get_constant_from_db,\n",
+ " read_const_yaml,\n",
" CalibrationMetadata,\n",
")\n",
"from cal_tools.step_timing import StepTimer\n",
@@ -136,37 +137,8 @@
"ped_dir = in_folder / f\"r{run:04d}\"\n",
"run_dc = RunDirectory(ped_dir, _use_voview=False)\n",
"metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# NOTE: this notebook will not overwrite calibration metadata file\n",
- "const_yaml = metadata.get(\"retrieved-constants\", {})\n",
- "\n",
- "fp_name = path_template.format(run, karabo_da)\n",
- "\n",
- "print(f\"Reading from: {ped_dir / fp_name}\")\n",
- "print(f\"Run is: {run}\")\n",
- "print(f\"Instrument H5File source: {instrument_src}\")\n",
- "print(f\"Data corrected files are stored at: {out_folder}\")\n",
- "\n",
- "creation_time = None\n",
- "if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run)\n",
- "if creation_time:\n",
- " print(f\"Using {creation_time.isoformat()} as creation time\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "x = 708 # rows of the ePix100\n",
- "y = 768 # columns of the ePix100\n",
- "\n",
- "in_folder = Path(in_folder)\n",
- "ped_dir = in_folder / f\"r{run:04d}\"\n",
- "run_dc = RunDirectory(ped_dir, _use_voview=False)\n",
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# NOTE: this notebook will not overwrite calibration metadata file\n",
+ "# Constant paths are saved under retrieved-constants in calibration_metadata.yml.\n",
+ "# NOTE: this notebook shouldn't overwrite calibration metadata file.\n",
"const_yaml = metadata.get(\"retrieved-constants\", {})\n",
"\n",
"fp_name = path_template.format(run, karabo_da)\n",
@@ -321,32 +293,17 @@
"metadata": {},
"outputs": [],
"source": [
- "const_data = dict()\n",
- "when = dict()\n",
- "# Constant paths are saved under retrieved-constants in calibration_metadata.yml\n",
- "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
- "\n",
"if const_yaml: # Used while reproducing corrected data.\n",
- " print(f\"Using stored constants in {metadata.yamlfile}\")\n",
- " for cname, mdata in const_yaml[karabo_da][\"constants\"].items():\n",
- " const_data[cname] = dict()\n",
- " when[cname] = mdata[\"creation-time\"]\n",
- " if when[cname]:\n",
- " with h5py.File(mdata[\"file-path\"], \"r\") as cf:\n",
- " const_data[cname] = np.copy(\n",
- " cf[f\"{mdata['dataset-name']}/data\"])\n",
- " else:\n",
- " const_data[cname] = None\n",
+ " print(f\"Using stored constants in {metadata.filename}\")\n",
+ " const_data, _ = read_const_yaml(const_yaml[karabo_da][\"constants\"])\n",
"else: # First correction attempt.\n",
- " mdata_dict = dict()\n",
- " mdata_dict[\"constants\"] = dict()\n",
+ " const_data = dict()\n",
" for cname, condition in const_cond.items():\n",
- " mdata_dict[\"constants\"][cname] = dict()\n",
" # Avoid retrieving RelativeGain, if not needed for correction.\n",
" if cname == \"RelativeGain\" and not relative_gain:\n",
" const_data[cname] = None\n",
" else:\n",
- " const_data[cname], mdata = get_from_db(\n",
+ " const_data[cname] = get_constant_from_db(\n",
" karabo_id=karabo_id,\n",
" karabo_da=karabo_da,\n",
" constant=getattr(Constants.ePix100, cname)(),\n",
@@ -354,54 +311,9 @@
" empty_constant=None,\n",
" cal_db_interface=cal_db_interface,\n",
" creation_time=creation_time,\n",
- " verbosity=2,\n",
- " timeout=cal_db_timeout,\n",
- " meta_only=True,\n",
- " )\n",
- " mdata_const = mdata.calibration_constant_version\n",
- " const_mdata = mdata_dict[\"constants\"][cname]\n",
- " # check if constant was successfully retrieved.\n",
- " if mdata.comm_db_success:\n",
- " const_mdata[\"file-path\"] = (\n",
- " f\"{mdata_const.hdf5path}\" f\"{mdata_const.filename}\"\n",
- " )\n",
- " const_mdata[\"dataset-name\"] = mdata_const.h5path\n",
- " const_mdata[\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
- " mdata_dict[\"physical-detector-unit\"] = mdata_const.device_name\n",
- " else:\n",
- " const_mdata[\"file-path\"] = None\n",
- " const_mdata[\"creation-time\"] = None"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Store constants metadata in the corresponding calibration_metadata.yml file.\n",
- "if not const_yaml:\n",
- " retrieved_constants[karabo_da] = mdata_dict\n",
- "\n",
- " timestamps = dict()\n",
- "\n",
- " module_timestamps = timestamps[karabo_da] = dict()\n",
- " module_constants = retrieved_constants[karabo_da]\n",
- "\n",
- " for cname, mdata in module_constants[\"constants\"].items():\n",
- " if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
- " mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
- "\n",
- " for cname in [\"Offset\", \"BadPixelsDark\", \"RelativeGain\"]:\n",
- " if cname in module_constants[\"constants\"]:\n",
- " module_timestamps[cname] = module_constants[\"constants\"][cname][\"creation-time\"]\n",
- " else:\n",
- " module_timestamps[cname] = \"NA\"\n",
- "\n",
- " time_summary = retrieved_constants.setdefault(\"time-summary\", {})\n",
- " time_summary[\"SAll\"] = timestamps\n",
- " metadata.save()\n",
- " print(f\"Stored retrieved constants in {metadata.yamlfile}\")"
+ " print_once=2,\n",
+ " timeout=cal_db_timeout\n",
+ " )"
]
},
{
diff --git a/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb b/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..b9634c18e5462af0e12b9af3100ee889e62d6d93
--- /dev/null
+++ b/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
@@ -0,0 +1,239 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# ePix100 retrieve constants precorrection\n",
+ "\n",
+ "Author: European XFEL Detector Group, Version: 1.0\n",
+ "\n",
+ "The following notebook provides constants for the selected ePix100 modules before executing correction on the selected sequence files."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = \"/gpfs/exfel/exp/CALLAB/202031/p900113/raw\" # input folder, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/epix_correct\" # output folder, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
+ "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
+ "run = 9988 # which run to read data from, required\n",
+ "\n",
+ "# Parameters for accessing the raw data.\n",
+ "karabo_id = \"MID_EXP_EPIX-1\" # Detector Karabo_ID\n",
+ "karabo_da = \"EPIX01\" # data aggregators\n",
+ "receiver_template = \"RECEIVER\" # detector receiver template for accessing raw data files\n",
+ "instrument_source_template = '{}/DET/{}:daqOutput' # instrument detector data source in h5files\n",
+ "\n",
+ "# Parameters for the calibration database.\n",
+ "use_dir_creation_date = True # date constants injected before directory creation time\n",
+ "cal_db_interface = \"tcp://max-exfl016:8015#8025\" # calibration DB interface to use\n",
+ "cal_db_timeout = 300000 # timeout on CalibrationDB requests\n",
+ "\n",
+ "# Parameters affecting writing corrected data.\n",
+ "overwrite = True # overwrite output folder\n",
+ "\n",
+ "# Conditions for retrieving calibration constants.\n",
+ "bias_voltage = 200 # bias voltage\n",
+ "in_vacuum = False # detector operated in vacuum\n",
+ "fix_temperature = 290. # fix temperature to this value\n",
+ "temp_deviations = 5. # temperature deviation for the constant operating conditions\n",
+ "gain_photon_energy = 9.0 # Photon energy used for gain calibration\n",
+ "\n",
+ "# Flags to select type of applied corrections.\n",
+ "relative_gain = True # Apply relative gain correction."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "from extra_data import RunDirectory\n",
+ "from pathlib import Path\n",
+ "\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " get_from_db,\n",
+ " save_constant_metadata,\n",
+ " CalibrationMetadata,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "# NOTE: this notebook will not overwrite calibration metadata file,\n",
+ "# if it already contains details about which constants to use.\n",
+ "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
+ "if karabo_da in retrieved_constants:\n",
+ " print(\n",
+ " f\"Constant for {karabo_da} already in {metadata.filename}, won't query again.\"\n",
+ " ) \n",
+ " import sys\n",
+ "\n",
+ " sys.exit(0)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = Path(in_folder)\n",
+ "ped_dir = in_folder / f\"r{run:04d}\"\n",
+ "run_dc = RunDirectory(ped_dir)\n",
+ "\n",
+ "creation_time = None\n",
+ "if use_dir_creation_date:\n",
+ " creation_time = get_dir_creation_date(in_folder, run)\n",
+ "if creation_time:\n",
+ " print(f\"Using {creation_time.isoformat()} as creation time\")\n",
+ "\n",
+ "out_folder = Path(out_folder)\n",
+ "if out_folder.is_dir() and not overwrite:\n",
+ " raise AttributeError(\n",
+ " \"Output path exists! Exiting as overwriting corrected data is prohibited.\"\n",
+ " )\n",
+ "out_folder.mkdir(parents=True, exist_ok=True)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Read control data.\n",
+ "integration_time = int(\n",
+ " run_dc.get_run_value(f\"{karabo_id}/DET/CONTROL\", \"expTime.value\")\n",
+ ")\n",
+ "temperature = (\n",
+ " np.mean(\n",
+ " run_dc.get_array(\n",
+ " f\"{karabo_id}/DET/{receiver_template}:daqOutput\", f\"data.backTemp\"\n",
+ " ).values\n",
+ " )\n",
+ " / 100.0\n",
+ ")\n",
+ "\n",
+ "if fix_temperature != 0:\n",
+ " temperature_k = fix_temperature\n",
+ " print(\"Temperature is fixed!\")\n",
+ "else:\n",
+ " temperature_k = temperature + 273.15\n",
+ "\n",
+ "print(f\"Bias voltage is {bias_voltage} V\")\n",
+ "print(f\"Detector integration time is set to {integration_time}\")\n",
+ "print(\n",
+ " f\"Mean temperature was {temperature:0.2f} °C \"\n",
+ " f\"/ {temperature_k:0.2f} K at beginning of the run.\"\n",
+ ")\n",
+ "print(f\"Operated in vacuum: {in_vacuum}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "cond_dict = {\n",
+ " \"bias_voltage\": bias_voltage,\n",
+ " \"integration_time\": integration_time,\n",
+ " \"temperature\": temperature_k,\n",
+ " \"in_vacuum\": in_vacuum,\n",
+ "}\n",
+ "\n",
+ "dark_condition = Conditions.Dark.ePix100(**cond_dict)\n",
+ "for parm in dark_condition.parameters:\n",
+ " if parm.name == \"Sensor Temperature\":\n",
+ " parm.lower_deviation = temp_deviations\n",
+ " parm.upper_deviation = temp_deviations\n",
+ "\n",
+ "# update conditions with illuminated conditions.\n",
+ "cond_dict.update({\"photon_energy\": gain_photon_energy})\n",
+ "\n",
+ "illum_condition = Conditions.Illuminated.ePix100(**cond_dict)\n",
+ "\n",
+ "const_cond = {\n",
+ " \"Offset\": dark_condition,\n",
+ " \"Noise\": dark_condition,\n",
+ " \"RelativeGain\": illum_condition,\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "const_data = dict()\n",
+ "mdata_dict = dict()\n",
+ "mdata_dict[\"constants\"] = dict()\n",
+ "for cname, condition in const_cond.items():\n",
+ " # Avoid retrieving RelativeGain, if not needed for correction.\n",
+ " if cname == \"RelativeGain\" and not relative_gain:\n",
+ " const_data[cname] = None\n",
+ " else:\n",
+ " const_data[cname], mdata = get_from_db(\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=karabo_da,\n",
+ " constant=getattr(Constants.ePix100, cname)(),\n",
+ " condition=condition,\n",
+ " empty_constant=None,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " verbosity=2,\n",
+ " timeout=cal_db_timeout,\n",
+ " meta_only=True,\n",
+ " )\n",
+ " save_constant_metadata(mdata_dict[\"constants\"], mdata, cname)\n",
+ "mdata_dict[\"physical-detector-unit\"] = mdata.calibration_constant_version.device_name\n",
+ "retrieved_constants[karabo_da] = mdata_dict\n",
+ "metadata.save()\n",
+ "print(f\"Stored retrieved constants in {metadata.filename}\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.8.11 ('.cal4_venv')",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.11"
+ },
+ "orig_nbformat": 4,
+ "vscode": {
+ "interpreter": {
+ "hash": "ccde353e8822f411c1c49844e1cbe3edf63293a69efd975d1b44f5e852832668"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
index cedfdd1c953be178391534e0d04c53ff342c22f1..31ed6f5ba3b89535f1a5f7b58c5ce88a5f7bbc8c 100644
--- a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
+++ b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
@@ -77,7 +77,7 @@
"metadata": {},
"outputs": [],
"source": [
- "# Here the herarichy and dependability for correction booleans are defined \n",
+ "# Here the herarichy and dependability for correction booleans are defined\n",
"corr_bools = {}\n",
"\n",
"corr_bools[\"only_offset\"] = only_offset\n",
@@ -116,8 +116,9 @@
"from cal_tools import pnccdlib\n",
"from cal_tools.tools import (\n",
" get_dir_creation_date,\n",
- " get_from_db,\n",
+ " get_constant_from_db_and_time,\n",
" get_random_db_interface,\n",
+ " read_const_yaml,\n",
" CalibrationMetadata,\n",
")\n",
"from cal_tools.step_timing import StepTimer\n",
@@ -185,7 +186,8 @@
"run_dc = RunDirectory(Path(in_folder) / f\"r{run:04d}\", _use_voview=False)\n",
"ctrl_data = pnccdlib.PnccdCtrl(run_dc, karabo_id)\n",
"metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# NOTE: this notebook will not overwrite calibration metadata file\n",
+ "# Constant paths are saved under retrieved-constants in calibration_metadata.yml\n",
+ "# NOTE: this notebook shouldn't overwrite calibration metadata file.\n",
"const_yaml = metadata.get(\"retrieved-constants\", {})\n",
"\n",
"# extract control data\n",
@@ -317,10 +319,7 @@
"outputs": [],
"source": [
"# Output Folder Creation:\n",
- "os.makedirs(out_folder, exist_ok=True if overwrite else False)\n",
- "\n",
- "# Constant paths are saved under retrieved-constants in calibration_metadata.yml\n",
- "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})"
+ "os.makedirs(out_folder, exist_ok=True if overwrite else False)"
]
},
{
@@ -336,7 +335,7 @@
"metadata": {},
"outputs": [],
"source": [
- "display(Markdown('### Constants retrieval'))\n",
+ "display(Markdown(\"### Constants retrieval\"))\n",
"step_timer.start()\n",
"\n",
"conditions_dict = {\n",
@@ -346,95 +345,42 @@
" \"temperature\": fix_temperature_top,\n",
" \"pixels_x\": pixels_x,\n",
" \"pixels_y\": pixels_y,\n",
- " }\n",
+ "}\n",
"# Dark condition\n",
"dark_condition = Conditions.Dark.CCD(**conditions_dict)\n",
"# Add photon energy.\n",
"conditions_dict.update({\"photon_energy\": photon_energy})\n",
"illum_condition = Conditions.Illuminated.CCD(**conditions_dict)\n",
"\n",
- "constants = dict()\n",
- "when = dict()\n",
- "init_constants = { # A dictionary for initializing constants. {cname: (empty_constants, condition)} # noqa\n",
- " \"Offset\": (np.zeros((pixels_x, pixels_y, 1), dtype=np.float32), dark_condition), # noqa\n",
- " \"Noise\": (np.zeros((pixels_x, pixels_y, 1), dtype=np.float32), dark_condition), # noqa\n",
- " \"BadPixelsDark\": (np.zeros((pixels_x, pixels_y, 1), dtype=np.uint32), dark_condition), # noqa\n",
- " \"RelativeGain\": (np.zeros((pixels_x, pixels_y), dtype=np.float32), illum_condition), # noqa\n",
+ "# A dictionary for initializing constants. {cname: empty constant array}\n",
+ "empty_constants = {\n",
+ " \"Offset\": np.zeros((pixels_x, pixels_y, 1), dtype=np.float32),\n",
+ " \"Noise\": np.zeros((pixels_x, pixels_y, 1), dtype=np.float32),\n",
+ " \"BadPixelsDark\": np.zeros((pixels_x, pixels_y, 1), dtype=np.uint32),\n",
+ " \"RelativeGain\": np.zeros((pixels_x, pixels_y), dtype=np.float32),\n",
"}\n",
+ "\n",
"if const_yaml: # Used while reproducing corrected data.\n",
" print(f\"Using stored constants in {metadata.filename}\")\n",
- " for cname, mdata in const_yaml[karabo_da][\"constants\"].items():\n",
- " constants[cname] = dict()\n",
- " when[cname] = mdata[\"creation-time\"]\n",
- " if when[cname]:\n",
- " with h5py.File(mdata[\"file-path\"], \"r\") as cf:\n",
- " constants[cname] = np.copy(\n",
- " cf[f\"{mdata['dataset-name']}/data\"])\n",
- " else:\n",
- " constants[cname] = init_constants[cname][0]\n",
+ " constants, when = read_const_yaml(\n",
+ " const_yaml[karabo_da][\"constants\"], empty_constants\n",
+ " )\n",
"else:\n",
- " mdata_dict = dict()\n",
- " mdata_dict[\"constants\"] = dict()\n",
- " for cname, cval in init_constants.items():\n",
- " mdata_dict[\"constants\"][cname] = dict()\n",
+ " constants = dict()\n",
+ " when = dict()\n",
+ " for cname, cempty in empty_constants.items():\n",
" # No need for retrieving RelativeGain, if not used for correction.\n",
" if not corr_bools.get(\"relgain\") and cname == \"RelativeGain\":\n",
" continue\n",
- " constants[cname], mdata = \\\n",
- " get_from_db(\n",
- " karabo_id=karabo_id,\n",
- " karabo_da=karabo_da,\n",
- " constant=getattr(Constants.CCD(DetectorTypes.pnCCD), cname)(),\n",
- " condition=cval[1],\n",
- " empty_constant=cval[0],\n",
- " cal_db_interface=get_random_db_interface(cal_db_interface),\n",
- " creation_time=creation_time,\n",
- " verbosity=1,\n",
- " )\n",
- " mdata_const = mdata.calibration_constant_version\n",
- " const_mdata = mdata_dict[\"constants\"][cname]\n",
- " # check if constant was successfully retrieved.\n",
- " if mdata.comm_db_success:\n",
- " const_mdata[\"file-path\"] = (\n",
- " f\"{mdata_const.hdf5path}\" f\"{mdata_const.filename}\"\n",
- " )\n",
- " const_mdata[\"dataset-name\"] = mdata_const.h5path\n",
- " const_mdata[\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
- " mdata_dict[\"physical-detector-unit\"] = mdata_const.device_name\n",
- " else:\n",
- " const_mdata[\"file-path\"] = None\n",
- " const_mdata[\"creation-time\"] = None\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Store constants metadata in the corresponding calibration_metadata.yml file.\n",
- "if not const_yaml:\n",
- " retrieved_constants[karabo_da] = mdata_dict\n",
- "\n",
- " timestamps = dict()\n",
- "\n",
- " module_timestamps = timestamps[karabo_da] = dict()\n",
- " module_constants = retrieved_constants[karabo_da]\n",
- "\n",
- " for cname, mdata in module_constants[\"constants\"].items():\n",
- " if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
- " mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
- "\n",
- " for cname in [\"Offset\", \"BadPixelsDark\", \"RelativeGain\"]:\n",
- " if cname in module_constants[\"constants\"]:\n",
- " module_timestamps[cname] = module_constants[\"constants\"][cname][\"creation-time\"]\n",
- " else:\n",
- " module_timestamps[cname] = \"NA\"\n",
- "\n",
- " time_summary = retrieved_constants.setdefault(\"time-summary\", {})\n",
- " time_summary[\"SAll\"] = timestamps\n",
- " metadata.save()\n",
- " print(f\"Stored retrieved constants in {metadata.filename}\")"
+ " constants[cname], when[cname] = get_constant_from_db_and_time(\n",
+ " karabo_id,\n",
+ " karabo_da,\n",
+ " constant=getattr(Constants.CCD(DetectorTypes.pnCCD), cname)(),\n",
+ " condition=illum_condition if cname == \"RelativeGain\" else dark_condition,\n",
+ " empty_constant=cempty,\n",
+ " cal_db_interface=get_random_db_interface(cal_db_interface),\n",
+ " creation_time=creation_time,\n",
+ " )"
]
},
{
diff --git a/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb b/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..6d7b5c6926618209e0a4b1dceeeb15ce955597e2
--- /dev/null
+++ b/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb
@@ -0,0 +1,241 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# pnCCD retrieve constants precorrection\n",
+ "\n",
+ "Author: European XFEL Detector Group, Version: 1.0\n",
+ "\n",
+ "The following notebook provides constants for the selected pnCCD modules before executing correction on the selected sequence files."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = \"/gpfs/exfel/exp/SQS/202031/p900166/raw\" # input folder\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/pnccd_correct\" # output folder\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
+ "run = 347 # which run to read data from\n",
+ "sequences = [0] # sequences to correct, set to -1 for all, range allowed\n",
+ "\n",
+ "karabo_da = 'PNCCD01' # data aggregators\n",
+ "karabo_id = \"SQS_NQS_PNCCD1MP\" # detector Karabo_ID\n",
+ "\n",
+ "# Conditions for retrieving calibration constants\n",
+ "fix_temperature_top = 0. # fix temperature for top sensor in K, set to 0. to use value from slow data.\n",
+ "fix_temperature_bot = 0. # fix temperature for bottom sensor in K, set to 0. to use value from slow data.\n",
+ "gain = -1 # the detector's gain setting. Set to -1 to use the value from the slow data.\n",
+ "bias_voltage = 0. # the detector's bias voltage. set to 0. to use value from slow data.\n",
+ "integration_time = 70 # detector's integration time\n",
+ "photon_energy = 1.6 # Al fluorescence in keV\n",
+ "\n",
+ "cal_db_interface = \"tcp://max-exfl016:8015\" # calibration DB interface to use\n",
+ "cal_db_timeout = 300000 # timeout on CalibrationDB requests\n",
+ "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.ms e.g. 2019-07-04 11:02:41.00\n",
+ "use_dir_creation_date = True # To obtain creation time of the run\n",
+ "\n",
+ "# Booleans for selecting corrections to apply.\n",
+ "only_offset = False # Only, apply offset.\n",
+ "relgain = True # Apply relative gain correction\n",
+ "\n",
+ "# parameters affecting stored output data.\n",
+ "overwrite = True # keep this as True to not overwrite the output "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import datetime\n",
+ "from pathlib import Path\n",
+ "\n",
+ "from IPython.display import Markdown, display\n",
+ "from extra_data import RunDirectory\n",
+ "\n",
+ "from cal_tools import pnccdlib\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " get_from_db,\n",
+ " get_random_db_interface,\n",
+ " save_constant_metadata,\n",
+ " CalibrationMetadata,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants\n",
+ "from iCalibrationDB.detectors import DetectorTypes"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "# NOTE: this notebook will not overwrite calibration metadata file,\n",
+ "# if it already contains details about which constants to use.\n",
+ "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
+ "if karabo_da in retrieved_constants:\n",
+ " print(\n",
+ " f\"Constant for {karabo_da} already in {metadata.filename}, won't query again.\"\n",
+ " ) # noqa\n",
+ " import sys\n",
+ "\n",
+ " sys.exit(0)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Here the correction booleans dictionary is defined\n",
+ "corr_bools = {}\n",
+ "\n",
+ "corr_bools[\"only_offset\"] = only_offset\n",
+ "\n",
+ "# Dont apply any corrections if only_offset is requested.\n",
+ "if not only_offset:\n",
+ " corr_bools[\"relgain\"] = relgain"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"Calibration database interface selected: {cal_db_interface}\")\n",
+ "\n",
+ "# Run's creation time:\n",
+ "if creation_time:\n",
+ " try:\n",
+ " creation_time = datetime.datetime.strptime(\n",
+ " creation_time, \"%Y-%m-%d %H:%M:%S.%f\"\n",
+ " )\n",
+ " except Exception as e:\n",
+ " print(\n",
+ " f\"creation_time value error: {e}.\"\n",
+ " \"Use same format as YYYY-MM-DD HR:MN:SC.ms e.g. 2019-07-04 11:02:41.00/n\"\n",
+ " )\n",
+ " creation_time = None\n",
+ " print(\"Given creation time will not be used.\")\n",
+ "else:\n",
+ " creation_time = None\n",
+ "\n",
+ "if not creation_time and use_dir_creation_date:\n",
+ " creation_time = get_dir_creation_date(in_folder, run)\n",
+ "\n",
+ "print(f\"Creation time: {creation_time}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_dc = RunDirectory(Path(in_folder) / f\"r{run:04d}\", _use_voview=False)\n",
+ "ctrl_data = pnccdlib.PnccdCtrl(run_dc, karabo_id)\n",
+ "\n",
+ "# extract control data\n",
+ "if bias_voltage == 0.0:\n",
+ " bias_voltage = ctrl_data.get_bias_voltage()\n",
+ "if gain == -1:\n",
+ " gain = ctrl_data.get_gain()\n",
+ "if fix_temperature_top == 0:\n",
+ " fix_temperature_top = ctrl_data.get_fix_temperature_top()\n",
+ "if fix_temperature_bot == 0:\n",
+ " fix_temperature_bot = ctrl_data.get_fix_temperature_bot()\n",
+ "\n",
+ "# Printing the Parameters Read from the Data File:\n",
+ "display(Markdown(\"### Detector Parameters\"))\n",
+ "print(f\"Bias voltage: {bias_voltage:0.1f} V.\")\n",
+ "print(f\"Detector gain: {int(gain)}.\")\n",
+ "print(f\"Detector integration time: {integration_time} ms\")\n",
+ "print(f\"Top pnCCD sensor temperature: {fix_temperature_top:0.2f} K\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(Markdown(\"### Constants retrieval\"))\n",
+ "\n",
+ "conditions_dict = {\n",
+ " \"bias_voltage\": bias_voltage,\n",
+ " \"integration_time\": integration_time,\n",
+ " \"gain_setting\": gain,\n",
+ " \"temperature\": fix_temperature_top,\n",
+ " \"pixels_x\": 1024,\n",
+ " \"pixels_y\": 1024,\n",
+ "}\n",
+ "# Dark condition\n",
+ "dark_condition = Conditions.Dark.CCD(**conditions_dict)\n",
+ "# Add photon energy.\n",
+ "conditions_dict.update({\"photon_energy\": photon_energy})\n",
+ "illum_condition = Conditions.Illuminated.CCD(**conditions_dict)\n",
+ "\n",
+ "mdata_dict = dict()\n",
+ "mdata_dict[\"constants\"] = dict()\n",
+ "for cname in [\"Offset\", \"Noise\", \"BadPixelsDark\", \"RelativeGain\"]:\n",
+ " # No need for retrieving RelativeGain, if not used for correction.\n",
+ " if not corr_bools.get(\"relgain\") and cname == \"RelativeGain\":\n",
+ " continue\n",
+ " _, mdata = get_from_db(\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=karabo_da,\n",
+ " constant=getattr(Constants.CCD(DetectorTypes.pnCCD), cname)(),\n",
+ " condition=illum_condition if cname == \"RelativeGain\" else dark_condition,\n",
+ " empty_constant=None,\n",
+ " cal_db_interface=get_random_db_interface(cal_db_interface),\n",
+ " creation_time=creation_time,\n",
+ " verbosity=1,\n",
+ " load_data=False,\n",
+ " )\n",
+ " save_constant_metadata(mdata_dict[\"constants\"], mdata, cname)\n",
+ "\n",
+ "mdata_dict[\"physical-detector-unit\"] = mdata.calibration_constant_version.device_name\n",
+ "retrieved_constants[karabo_da] = mdata_dict\n",
+ "metadata.save()\n",
+ "print(f\"Stored retrieved constants in {metadata.filename}\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.8.11 ('.cal4_venv')",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.11"
+ },
+ "orig_nbformat": 4,
+ "vscode": {
+ "interpreter": {
+ "hash": "ccde353e8822f411c1c49844e1cbe3edf63293a69efd975d1b44f5e852832668"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/src/cal_tools/tools.py b/src/cal_tools/tools.py
index c489f48b38993aa4ae28e13fb4a576981ad59a6e..6766d93f8809dfbc825776c44f26b2ef5f44f0c3 100644
--- a/src/cal_tools/tools.py
+++ b/src/cal_tools/tools.py
@@ -885,6 +885,72 @@ class CalibrationMetadata(dict):
yaml.safe_dump(dict(self), fd)
+def save_constant_metadata(
+ retrieved_constants: dict,
+ mdata: ConstantMetaData,
+ constant_name: str,
+ ) -> dict :
+ """Save constant metadata to the input meta data dictionary.
+ The constant's metadata stored are file path, dataset name,
+ creation time, and physical detector unit name.
+
+ :param retrieved_constants: A dictionary to store the metadata for
+ the retrieved constant.
+ :param mdata: A ConstantMetaData object after retrieving trying
+ to retrieve a constant with get_from_db().
+ :param constant_name: String for constant name to be used as a key.
+ :param constants_key: The key name when all constants metadata
+ will be stored.
+ """
+
+ mdata_const = mdata.calibration_constant_version
+ const_mdata = retrieved_constants[constant_name] = dict()
+ # check if constant was successfully retrieved.
+ if mdata.comm_db_success:
+ const_mdata["file-path"] = (
+ f"{mdata_const.hdf5path}" f"{mdata_const.filename}"
+ )
+ const_mdata["dataset-name"] = mdata_const.h5path
+ const_mdata["creation-time"] = mdata_const.begin_at
+ else:
+ const_mdata["file-path"] = None
+ const_mdata["creation-time"] = None
+
+
+def read_const_yaml(
+ retrieved_constants: dict,
+ empty_constants: Optional[dict] = None,
+ ) -> dict:
+ """Read constant data from stored metadata in the
+ retrieved_constants dictionary.
+
+ :param retrieved_constants: A dict. with the constant filepaths and
+ dataset-name to read the constant data arrays.
+ {
+ 'Constant Name':
+ {'file-path': /gpfs/.../*.h5},
+ {'dataset-name': /module_name/...},
+ {'creation-time': str(datetime)},
+ }
+ :param empty_constants: A dict of constant names keys and
+ the empty constant array to use in case of not non-retrieved constants.
+ :return constant_data: A dict of constant names keys and their data.
+ """
+ const_data = dict()
+ when = dict()
+
+ for cname, mdata in retrieved_constants.items():
+ const_data[cname] = dict()
+ when[cname] = mdata["creation-time"]
+ if when[cname]:
+ with h5py.File(mdata["file-path"], "r") as cf:
+ const_data[cname] = np.copy(
+ cf[f"{mdata['dataset-name']}/data"])
+ else:
+ const_data[cname] = empty_constants[cname] if empty_constants else None # noqa
+ return const_data, when
+
+
def write_compressed_frames(
arr: np.ndarray,
ofile: h5py.File,
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index aaee10cacb262c48448b60be3bcbb5c60ff4761b..c6a85445bed4507d82f0f235d629d2f20bf4acf5 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -110,6 +110,7 @@ notebooks = {
"cluster cores": 32},
},
"CORRECT": {
+ "pre_notebooks": ["notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb"],
"notebook": "notebooks/pnCCD/Correct_pnCCD_NBC.ipynb",
"concurrency": {"parameter": "sequences",
"default concurrency": [-1],
@@ -210,6 +211,7 @@ notebooks = {
},
"CORRECT": {
+ "pre_notebooks": ["notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb"],
"notebook": "notebooks/ePix100/Correction_ePix100_NBC.ipynb",
"concurrency": {"parameter": "sequences",
"default concurrency": [-1],