From 1defc617e988398b9826882c8eb6b7a539ed970d Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Wed, 30 Nov 2022 16:35:06 +0100
Subject: [PATCH] update gotthard2
---
...retrieve_constants_precorrection_NBC.ipynb | 140 ++++++++----------
1 file changed, 60 insertions(+), 80 deletions(-)
diff --git a/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb b/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
index 18aa31302..e3a81c929 100644
--- a/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
+++ b/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
@@ -32,13 +32,12 @@
"karabo_id_control = \"\" # Control karabo ID. Set to empty string to use the karabo-id\n",
"\n",
"# Parameters for calibration database.\n",
- "use_dir_creation_date = True # use the creation data of the input dir for database queries.\n",
- "cal_db_interface = \"tcp://max-exfl017:8017#8025\" # the database interface to use.\n",
+ "cal_db_interface = \"tcp://max-exfl016:8017#8025\" # the database interface to use.\n",
"cal_db_timeout = 180000 # timeout on caldb requests.\n",
- "overwrite_creation_time = \"2022-06-28 13:00:00.00\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. \"2022-06-28 13:00:00.00\"\n",
+ "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
"\n",
"# Parameters affecting corrected data.\n",
- "constants_file = \"\"#/gpfs/exfel/data/scratch/ahmedk/dont_remove/gotthard2/constants/calibration_constants_GH2.h5\" # Retrieve constants from local.\n",
+ "constants_file = \"\" # /gpfs/exfel/data/scratch/ahmedk/dont_remove/gotthard2/constants/calibration_constants_GH2.h5\" # Retrieve constants from local.\n",
"offset_correction = True # apply offset correction. This can be disabled to only apply LUT or apply LUT and gain correction for non-linear differential results.\n",
"gain_correction = True # apply gain correction.\n",
"\n",
@@ -46,8 +45,8 @@
"bias_voltage = -1 # Detector bias voltage, set to -1 to use value in raw file.\n",
"exposure_time = -1. # Detector exposure time, set to -1 to use value in raw file.\n",
"exposure_period = -1. # Detector exposure period, set to -1 to use value in raw file.\n",
- "acquisition_rate = 1.1 # Detector acquisition rate (1.1/4.5), set to -1 to use value in raw file.\n",
- "single_photon = 0 # Detector single photon mode (High/Low CDS), set to -1 to use value in raw file.\n",
+ "acquisition_rate = -1. # Detector acquisition rate (1.1/4.5), set to -1 to use value in raw file.\n",
+ "single_photon = -1 # Detector single photon mode (High/Low CDS), set to -1 to use value in raw file.\n",
"\n",
"if constants_file:\n",
" print(\"Skipping constant retrieval. Specified constants_file is used.\")\n",
@@ -62,20 +61,18 @@
"metadata": {},
"outputs": [],
"source": [
- "import datetime\n",
- "from functools import partial\n",
+ "from logging import warning\n",
"\n",
- "import multiprocessing\n",
"from extra_data import RunDirectory\n",
"from pathlib import Path\n",
"\n",
+ "from cal_tools.calcat_interface import GOTTHARD2_CalibrationData\n",
"from cal_tools.gotthard2 import gotthard2lib\n",
"from cal_tools.tools import (\n",
- " get_dir_creation_date,\n",
+ " calcat_creation_time,\n",
" get_from_db,\n",
" CalibrationMetadata,\n",
- ")\n",
- "from iCalibrationDB import Conditions, Constants"
+ ")"
]
},
{
@@ -89,6 +86,8 @@
"out_folder = Path(out_folder)\n",
"out_folder.mkdir(parents=True, exist_ok=True)\n",
"metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "# Constant paths are saved under retrieved-constants in calibration_metadata.yml\n",
+ "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
"\n",
"if not karabo_id_control:\n",
" karabo_id_control = karabo_id\n",
@@ -98,14 +97,9 @@
"\n",
"print(f\"Retrieve constants for modules: {karabo_da} for run {run}\")\n",
"\n",
- "creation_time = None\n",
- "if overwrite_creation_time:\n",
- " creation_time = datetime.datetime.strptime(\n",
- " overwrite_creation_time, \"%Y-%m-%d %H:%M:%S.%f\"\n",
- " )\n",
- "elif use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run)\n",
- " print(f\"Using {creation_time} as creation time\")"
+ "# Run's creation time:\n",
+ "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
+ "print(f\"Creation time: {creation_time}\")"
]
},
{
@@ -142,60 +136,52 @@
"metadata": {},
"outputs": [],
"source": [
- "condition = Conditions.Dark.Gotthard2(\n",
- " bias_voltage=bias_voltage,\n",
+ "g2_cal = GOTTHARD2_CalibrationData(\n",
+ " detector_name=karabo_id,\n",
+ " sensor_bias_voltage=bias_voltage,\n",
" exposure_time=exposure_time,\n",
" exposure_period=exposure_period,\n",
- " single_photon=single_photon,\n",
" acquisition_rate=acquisition_rate,\n",
+ " single_photon=single_photon,\n",
+ " event_at=creation_time,\n",
+ " snapshot_at=creation_time,\n",
")\n",
"\n",
- "def get_constants_for_module(mod: str):\n",
- " \"\"\"Get calibration constants for given module for Gotthard2.\"\"\"\n",
- " retrieval_function = partial(\n",
- " get_from_db,\n",
- " karabo_id=karabo_id,\n",
- " karabo_da=mod,\n",
- " cal_db_interface=cal_db_interface,\n",
- " creation_time=creation_time,\n",
- " timeout=cal_db_timeout,\n",
- " verbosity=1,\n",
- " meta_only=True,\n",
- " load_data=False,\n",
- " empty_constant=None\n",
- " )\n",
- "\n",
- " mdata_dict = dict()\n",
- " mdata_dict[\"constants\"] = dict()\n",
- " constants = [\n",
- " \"LUT\", \"Offset\", \"BadPixelsDark\",\n",
- " \"RelativeGain\", \"BadPixelsFF\",\n",
- " ]\n",
- " for cname in constants:\n",
- " mdata_dict[\"constants\"][cname] = dict()\n",
- " if not gain_correction and cname in [\"BadPixelsFF\", \"RelativeGain\"]:\n",
- " continue\n",
- " _, mdata = retrieval_function(\n",
- " condition=condition,\n",
- " constant=getattr(Constants.Gotthard2, cname)(),\n",
- " )\n",
- " mdata_const = mdata.calibration_constant_version\n",
- " const_mdata = mdata_dict[\"constants\"][cname]\n",
- " # check if constant was successfully retrieved.\n",
- " if mdata.comm_db_success:\n",
- " const_mdata[\"file-path\"] = (\n",
- " f\"{mdata_const.hdf5path}\" f\"{mdata_const.filename}\"\n",
- " )\n",
- " const_mdata[\"dataset-name\"] = mdata_const.h5path\n",
- " const_mdata[\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
- " mdata_dict[\"physical-detector-unit\"] = mdata_const.device_name\n",
- " else:\n",
- " const_mdata[\"file-path\"] = None\n",
- " const_mdata[\"creation-time\"] = None\n",
- " return mdata_dict, mod\n",
- "\n",
- "with multiprocessing.Pool() as pool:\n",
- " results = pool.map(get_constants_for_module, karabo_da)"
+ "mdata_dict = {\"constants\": dict()}\n",
+ "\n",
+ "constant_names = [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"]\n",
+ "if gain_correction:\n",
+ " constant_names += [\"RelativeGainGotthard2\", \"BadPixelsFFGotthard2\"]\n",
+ "\n",
+ "# Retrieve metadata for all pnccd constants.\n",
+ "\n",
+ "g2_metadata = g2_cal.metadata(constant_names)\n",
+ "# Validate the constants availability and raise/warn correspondingly.\n",
+ "for mod, ccv_dict in g2_metadata.items():\n",
+ " missing_dark_constants = set(\n",
+ " c for c in [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"] if c not in ccv_dict.keys())\n",
+ " missing_gain_constants = set(\n",
+ " c for c in [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"] if gain_correction and c not in ccv_dict.keys()) # noqa\n",
+ " if missing_dark_constants:\n",
+ " raise KeyError(\n",
+ " f\"Dark constants {missing_dark_constants} are not available for correction. Mod: {mod}\")\n",
+ " \n",
+ " if gain_correction and missing_gain_constants:\n",
+ " warning(f\"Gain constants {missing_gain_constants} are not retrieved. Mod {mod}\")\n",
+ "\n",
+ "# Add constants metadata in retrieved_constants dict.\n",
+ "for mod, ccv_dict in g2_metadata.items():\n",
+ " mod_dict = retrieved_constants.setdefault(mod, dict())\n",
+ " const_dict = mod_dict.setdefault(\"constants\", dict())\n",
+ " for cname, ccv_metadata in ccv_dict.items():\n",
+ " const_dict[cname] = {\n",
+ " \"path\": str(g2_cal.caldb_root / ccv_metadata[\"path\"]),\n",
+ " \"dataset\": ccv_metadata[\"dataset\"],\n",
+ " \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
+ " }\n",
+ " mod_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n",
+ " \n",
+ "print(f\"Stored retrieved constants in {metadata.filename}\")"
]
},
{
@@ -204,26 +190,20 @@
"metadata": {},
"outputs": [],
"source": [
- "# Constant paths are saved under retrieved-constants in calibration_metadata.yml\n",
- "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
"timestamps = dict()\n",
"\n",
- "for md_dict, mod in results:\n",
- " if mod in retrieved_constants:\n",
- " print(f\"Constant for {mod} already in calibration_metadata.yml, won't query again.\") # noqa\n",
- " continue\n",
- " retrieved_constants[mod] = md_dict\n",
+ "for mod in karabo_da:\n",
" module_timestamps = timestamps[mod] = dict()\n",
- "\n",
+ " module_constants = retrieved_constants[mod]\n",
" print(f\"Module: {mod}:\")\n",
- " for cname, mdata in md_dict[\"constants\"].items():\n",
+ " for cname, mdata in module_constants[\"constants\"].items():\n",
" if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
" mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
" print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
"\n",
" for cname in [\"Offset\", \"BadPixelsDark\", \"RelativeGain\", \"BadPixelsFF\"]:\n",
- " if cname in md_dict[\"constants\"]:\n",
- " module_timestamps[cname] = md_dict[\"constants\"][cname][\"creation-time\"]\n",
+ " if cname in module_constants[\"constants\"]:\n",
+ " module_timestamps[cname] = module_constants[\"constants\"][cname][\"creation-time\"]\n",
" else:\n",
" module_timestamps[cname] = \"NA\"\n",
"\n",
--
GitLab