From bc55ab07f81f9686aaa1ac874d929870615ce078 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Mon, 25 Jul 2022 13:51:11 +0200
Subject: [PATCH] insert fixes for Gotthard2 correct before deployment and last
test for dark
---
.../Characterize_Darks_Gotthard2_NBC.ipynb | 4 +-
.../Gotthard2/Correction_Gotthard2_NBC.ipynb | 10 +-
...Retrieve_constants_precorrection_NBC.ipynb | 270 ------------------
src/xfel_calibrate/notebooks.py | 2 +-
4 files changed, 9 insertions(+), 277 deletions(-)
delete mode 100644 notebooks/Gotthard2/Retrieve_constants_precorrection_NBC.ipynb
diff --git a/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
index 7a26a8197..657b80095 100644
--- a/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
+++ b/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
@@ -34,7 +34,7 @@
"\n",
"# Parameters for the calibration database.\n",
"use_dir_creation_date = True\n",
- "cal_db_interface = \"tcp://max-exfl017:8020\" # calibration DB interface to use\n",
+ "cal_db_interface = \"tcp://max-exfl016:8020\" # calibration DB interface to use\n",
"cal_db_timeout = 300000 # timeout on caldb requests\n",
"overwrite_creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. \"2022-06-28 13:00:00.00\"\n",
"db_output = False # Output constants to the calibration database\n",
@@ -284,7 +284,7 @@
"offset_map = dict()\n",
"badpixels_map = dict()\n",
"\n",
- "context = psh.context.ProcessContext(num_workers=2)\n",
+ "context = psh.ProcessContext(num_workers=2)\n",
"\n",
"empty_lut = (np.arange(2 ** 12).astype(np.float64) * 2 ** 10 / 2 ** 12).astype(\n",
" np.uint16\n",
diff --git a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
index 8368dcb97..fc8913230 100644
--- a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
+++ b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
@@ -37,7 +37,7 @@
"\n",
"# Parameters for calibration database.\n",
"use_dir_creation_date = True # use the creation data of the input dir for database queries.\n",
- "cal_db_interface = \"tcp://max-exfl016:8017#8025\" # the database interface to use.\n",
+ "cal_db_interface = \"tcp://max-exfl016:8016#8025\" # the database interface to use.\n",
"cal_db_timeout = 180000 # timeout on caldb requests.\n",
"overwrite_creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. \"2022-06-28 13:00:00.00\"\n",
"\n",
@@ -394,7 +394,7 @@
" compression_opts=1,\n",
" shuffle=True,\n",
" )\n",
- " step_timer.done_step(\"Storing data\")"
+ " step_timer.done_step(\"Storing data\")"
]
},
{
@@ -430,13 +430,15 @@
"outputs": [],
"source": [
"mod_dcs = {}\n",
+ "first_seq_raw = seq_files[0]\n",
+ "first_seq_corr = out_folder / first_seq_raw.name.replace(\"RAW\", \"CORR\")\n",
"for mod in karabo_da:\n",
" mod_dcs[mod] = {}\n",
- " with RunDirectory(out_folder) as out_dc:\n",
+ " with H5File(first_seq_corr) as out_dc:\n",
" tid, mod_dcs[mod][\"train_corr_data\"] = next(\n",
" out_dc[instr_mod_src, \"data.adc\"].trains()\n",
" )\n",
- " with RunDirectory(run_folder) as in_dc:\n",
+ " with H5File(first_seq_raw) as in_dc:\n",
" train_dict = in_dc.train_from_id(tid)[1][instr_mod_src]\n",
" mod_dcs[mod][\"train_raw_data\"] = train_dict[\"data.adc\"]\n",
" mod_dcs[mod][\"train_raw_gain\"] = train_dict[\"data.gain\"]"
diff --git a/notebooks/Gotthard2/Retrieve_constants_precorrection_NBC.ipynb b/notebooks/Gotthard2/Retrieve_constants_precorrection_NBC.ipynb
deleted file mode 100644
index 191057994..000000000
--- a/notebooks/Gotthard2/Retrieve_constants_precorrection_NBC.ipynb
+++ /dev/null
@@ -1,270 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# GOTTHARD2 Retrieving Constants Pre-correction #\n",
- "\n",
- "Author: European XFEL Detector Group, Version: 1.0\n",
- "\n",
- "Retrieving Required Constants for Offline Calibration of the Gotthard2 Detector"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "in_folder = \"/gpfs/exfel/exp/FXE/202221/p003225/raw\" # the folder to read data from, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/gotthard2\" # the folder to output to, required\n",
- "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
- "run = 50 # run to process, required\n",
- "\n",
- "# Parameters used to access raw data.\n",
- "karabo_id = \"FXE_XAD_G2XES\" # karabo prefix of Gotthard-II devices\n",
- "karabo_da = [\"GH201\"] # data aggregators\n",
- "receiver_template = \"RECEIVER\" # receiver template used to read INSTRUMENT keys.\n",
- "control_template = \"CONTROL\" # control template used to read CONTROL keys.\n",
- "instrument_source_template = \"{}/DET/{}:daqOutput\" # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput'\n",
- "ctrl_source_template = \"{}/DET/{}\" # template for control source name (filled with karabo_id_control)\n",
- "karabo_id_control = \"\" # Control karabo ID. Set to empty string to use the karabo-id\n",
- "\n",
- "# Parameters for calibration database.\n",
- "use_dir_creation_date = True # use the creation data of the input dir for database queries.\n",
- "cal_db_interface = \"tcp://max-exfl017:8017#8025\" # the database interface to use.\n",
- "cal_db_timeout = 180000 # timeout on caldb requests.\n",
- "overwrite_creation_time = \"2022-06-28 13:00:00.00\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. \"2022-06-28 13:00:00.00\"\n",
- "\n",
- "# Parameters affecting corrected data.\n",
- "constants_file = \"\"#/gpfs/exfel/data/scratch/ahmedk/dont_remove/gotthard2/constants/calibration_constants_GH2.h5\" # Retrieve constants from local.\n",
- "offset_correction = True # apply offset correction. This can be disabled to only apply LUT or apply LUT and gain correction for non-linear differential results.\n",
- "gain_correction = True # apply gain correction.\n",
- "\n",
- "# Parameter conditions.\n",
- "bias_voltage = -1 # Detector bias voltage, set to -1 to use value in raw file.\n",
- "exposure_time = -1. # Detector exposure time, set to -1 to use value in raw file.\n",
- "exposure_period = -1. # Detector exposure period, set to -1 to use value in raw file.\n",
- "acquisition_rate = 1.1 # Detector acquisition rate (1.1/4.5), set to -1 to use value in raw file.\n",
- "single_photon = 0 # Detector single photon mode (High/Low CDS), set to -1 to use value in raw file.\n",
- "\n",
- "if constants_file:\n",
- " print(\"Skipping constant retrieval. Specified constants_file is used.\")\n",
- " import sys\n",
- "\n",
- " sys.exit(0)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import datetime\n",
- "from functools import partial\n",
- "\n",
- "import multiprocessing\n",
- "from extra_data import RunDirectory\n",
- "from pathlib import Path\n",
- "\n",
- "from cal_tools.gotthard2 import gotthard2lib\n",
- "from cal_tools.tools import (\n",
- " get_dir_creation_date,\n",
- " get_from_db,\n",
- " CalibrationMetadata,\n",
- ")\n",
- "from iCalibrationDB import Conditions, Constants"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "in_folder = Path(in_folder)\n",
- "run_folder = in_folder / f\"r{run:04d}\"\n",
- "out_folder = Path(out_folder)\n",
- "out_folder.mkdir(parents=True, exist_ok=True)\n",
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "\n",
- "if not karabo_id_control:\n",
- " karabo_id_control = karabo_id\n",
- "\n",
- "instrument_src = instrument_source_template.format(karabo_id, receiver_template)\n",
- "ctrl_src = ctrl_source_template.format(karabo_id_control, control_template)\n",
- "\n",
- "print(f\"Retrieve constants for modules: {karabo_da} for run {run}\")\n",
- "\n",
- "creation_time = None\n",
- "if overwrite_creation_time:\n",
- " creation_time = datetime.datetime.strptime(\n",
- " overwrite_creation_time, \"%Y-%m-%d %H:%M:%S.%f\"\n",
- " )\n",
- "elif use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run)\n",
- " print(f\"Using {creation_time} as creation time\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Read slow data\n",
- "run_dc = RunDirectory(run_folder)\n",
- "g2ctrl = gotthard2lib.Gotthard2Ctrl(run_dc=run_dc, ctrl_src=ctrl_src)\n",
- "\n",
- "if bias_voltage == -1:\n",
- " bias_voltage = g2ctrl.get_bias_voltage()\n",
- "if exposure_time == -1:\n",
- " exposure_time = g2ctrl.get_exposure_time()\n",
- "if exposure_period == -1:\n",
- " exposure_period = g2ctrl.get_exposure_period()\n",
- "if acquisition_rate == -1:\n",
- " acquisition_rate = g2ctrl.get_acquisition_rate()\n",
- "if single_photon == -1:\n",
- " single_photon = g2ctrl.get_single_photon()\n",
- "\n",
- "print(\"Bias Voltage:\", bias_voltage)\n",
- "print(\"Exposure Time:\", exposure_time)\n",
- "print(\"Exposure Period:\", exposure_period)\n",
- "print(\"Acquisition Rate:\", acquisition_rate)\n",
- "print(\"Single Photon:\", single_photon)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "condition = Conditions.Dark.Gotthard2(\n",
- " bias_voltage=bias_voltage,\n",
- " exposure_time=exposure_time,\n",
- " exposure_period=exposure_period,\n",
- " single_photon=single_photon,\n",
- " acquisition_rate=acquisition_rate,\n",
- ")\n",
- "\n",
- "def get_constants_for_module(mod: str):\n",
- " \"\"\"Get calibration constants for given module for Gotthard2.\"\"\"\n",
- " retrieval_function = partial(\n",
- " get_from_db,\n",
- " karabo_id=karabo_id,\n",
- " karabo_da=mod,\n",
- " cal_db_interface=cal_db_interface,\n",
- " creation_time=creation_time,\n",
- " timeout=cal_db_timeout,\n",
- " verbosity=1,\n",
- " meta_only=True,\n",
- " load_data=False,\n",
- " empty_constant=None\n",
- " )\n",
- "\n",
- " mdata_dict = dict()\n",
- " mdata_dict[\"constants\"] = dict()\n",
- " constants = [\n",
- " \"LUT\", \"Offset\", \"BadPixelsDark\",\n",
- " \"RelativeGain\", \"BadPixelsFF\",\n",
- " ]\n",
- " for cname in constants:\n",
- " mdata_dict[\"constants\"][cname] = dict()\n",
- " if not gain_correction and cname in [\"BadPixelsFF\", \"RelativeGain\"]:\n",
- " continue\n",
- " _, mdata = retrieval_function(\n",
- " condition=condition,\n",
- " constant=getattr(Constants.Gotthard2, cname)(),\n",
- " )\n",
- " mdata_const = mdata.calibration_constant_version\n",
- " const_mdata = mdata_dict[\"constants\"][cname]\n",
- " # check if constant was successfully retrieved.\n",
- " if mdata.comm_db_success:\n",
- " const_mdata[\"file-path\"] = (\n",
- " f\"{mdata_const.hdf5path}\" f\"{mdata_const.filename}\"\n",
- " )\n",
- " const_mdata[\"dataset-name\"] = mdata_const.h5path\n",
- " const_mdata[\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
- " mdata_dict[\"physical-detector-unit\"] = mdata_const.device_name\n",
- " else:\n",
- " const_mdata[\"file-path\"] = None\n",
- " const_mdata[\"creation-time\"] = None\n",
- " return mdata_dict, mod\n",
- "\n",
- "with multiprocessing.Pool() as pool:\n",
- " results = pool.map(get_constants_for_module, karabo_da)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "results"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Constant paths are saved under retrieved-constants in calibration_metadata.yml\n",
- "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
- "timestamps = dict()\n",
- "\n",
- "for md_dict, mod in results:\n",
- " retrieved_constants[mod] = md_dict\n",
- " module_timestamps = timestamps[mod] = dict()\n",
- "\n",
- " print(f\"Module: {mod}:\")\n",
- " for cname, mdata in md_dict[\"constants\"].items():\n",
- " if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
- " mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
- " print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
- "\n",
- " for cname in [\"Offset\", \"BadPixelsDark\", \"RelativeGain\", \"BadPixelsFF\"]:\n",
- " if cname in md_dict[\"constants\"]:\n",
- " module_timestamps[cname] = md_dict[\"constants\"][cname][\"creation-time\"]\n",
- " else:\n",
- " module_timestamps[cname] = \"NA\"\n",
- "\n",
- "time_summary = retrieved_constants.setdefault(\"time-summary\", {})\n",
- "time_summary[\"SAll\"] = timestamps\n",
- "\n",
- "metadata.save()"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3.8.11 ('.cal4_venv')",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.8.11"
- },
- "orig_nbformat": 4,
- "vscode": {
- "interpreter": {
- "hash": "ccde353e8822f411c1c49844e1cbe3edf63293a69efd975d1b44f5e852832668"
- }
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index 618074a95..aaee10cac 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -185,7 +185,7 @@ notebooks = {
"GOTTHARD2": {
"CORRECT": {
"pre_notebooks": [
- "notebooks/Gotthard2/Retrieve_constants_precorrection_NBC.ipynb"], # noqa
+ "notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb"], # noqa
"notebook":
"notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb",
"concurrency": {"parameter": "sequences",
--
GitLab