From b1ef743afc576dee3c4ffd18aebc70b844d3ba72 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Fri, 9 Dec 2022 17:23:03 +0100
Subject: [PATCH] add missing client input args
---
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 22 +++++++-
...IPD_Retrieve_Constants_Precorrection.ipynb | 19 +++++++
.../Gotthard2/Correction_Gotthard2_NBC.ipynb | 1 +
...retrieve_constants_precorrection_NBC.ipynb | 1 +
...Jungfrau_Gain_Correct_and_Verify_NBC.ipynb | 23 ++++++++
...retrieve_constants_precorrection_NBC.ipynb | 23 ++++++++
notebooks/LPD/LPD_Correct_Fast.ipynb | 52 +++++++++++++++++++
...100_retrieve_constants_precorrection.ipynb | 11 +---
notebooks/pnCCD/Correct_pnCCD_NBC.ipynb | 1 +
...CCD_retrieve_constants_precorrection.ipynb | 14 +----
src/cal_tools/agipdlib.py | 23 ++++----
11 files changed, 158 insertions(+), 32 deletions(-)
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 6336a6513..dae62fab7 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -122,7 +122,6 @@
"outputs": [],
"source": [
"import itertools\n",
- "import os\n",
"import math\n",
"import multiprocessing\n",
"import re\n",
@@ -163,6 +162,7 @@
")\n",
"from cal_tools.ana_tools import get_range\n",
"from cal_tools.enums import AgipdGainMode, BadPixels\n",
+ "from cal_tools.restful_config import restful_config\n",
"from cal_tools.step_timing import StepTimer\n",
"from cal_tools.tools import (\n",
" CalibrationMetadata,\n",
@@ -562,6 +562,26 @@
"const_yaml = metadata.get(\"retrieved-constants\", {})"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Connect to CalCat.\n",
+ "calcat_config = restful_config['calcat']\n",
+ "client = CalibrationClient(\n",
+ " base_api_url=calcat_config['base-api-url'],\n",
+ " use_oauth2=calcat_config['use-oauth2'],\n",
+ " client_id=calcat_config['user-id'],\n",
+ " client_secret=calcat_config['user-secret'],\n",
+ " user_email=calcat_config['user-email'],\n",
+ " token_url=calcat_config['token-url'],\n",
+ " refresh_url=calcat_config['refresh-url'],\n",
+ " auth_url=calcat_config['auth-url'],\n",
+ " scope='')"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 870a43c41..62f64d8c5 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -466,6 +466,25 @@
"\n",
"metadata.save()"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(\"\\nRetrieved constants for modules:\",\n",
+ " ', '.join([module_index_to_qm(x) for x in modules]))\n",
+ "print(f\"Operating conditions are:\")\n",
+ "print(f\"• Bias voltage: {bias_voltage}\")\n",
+ "print(f\"• Memory cells: {mem_cells}\")\n",
+ "print(f\"• Acquisition rate: {acq_rate}\")\n",
+ "print(f\"• Gain mode: {gain_mode.name}\")\n",
+ "print(f\"• Gain setting: {gain_setting}\")\n",
+ "print(f\"• Integration time: {integration_time}\")\n",
+ "print(f\"• Photon Energy: 9.2\")\n",
+ "step_timer.done_step(f\"Constant metadata is saved under \\\"retrieved-constants\\\" in {metadata.filename}\")"
+ ]
}
],
"metadata": {
diff --git a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
index 240dcaaf8..7bf1e4136 100644
--- a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
+++ b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
@@ -241,6 +241,7 @@
" acquisition_rate=acquisition_rate,\n",
" single_photon=single_photon,\n",
" event_at=creation_time,\n",
+ " client=client,\n",
")\n",
"# Keep as long as it is essential to correct\n",
"# RAW data (FXE p003225) before the data mapping was added to CALCAT.\n",
diff --git a/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb b/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
index fd5edba39..6effefce9 100644
--- a/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
+++ b/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
@@ -167,6 +167,7 @@
" single_photon=single_photon,\n",
" event_at=creation_time,\n",
" snapshot_at=creation_time,\n",
+ " client=client,\n",
")\n",
"\n",
"mdata_dict = {\"constants\": dict()}\n",
diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
index 761119c47..150ae71d6 100644
--- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
@@ -90,10 +90,12 @@
"from extra_geom import JUNGFRAUGeometry\n",
"from matplotlib.colors import LogNorm\n",
"\n",
+ "from calibration_client import CalibrationClient\n",
"from cal_tools.calcat_interface import JUNGFRAU_CalibrationData\n",
"from cal_tools.jungfraulib import JungfrauCtrl\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.files import DataFile\n",
+ "from cal_tools.restful_config import restful_config\n",
"from cal_tools.step_timing import StepTimer\n",
"from cal_tools.tools import (\n",
" calcat_creation_time,\n",
@@ -221,6 +223,26 @@
" print('Strixel sensor transformation enabled')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Connect to CalCat.\n",
+ "calcat_config = restful_config['calcat']\n",
+ "client = CalibrationClient(\n",
+ " base_api_url=calcat_config['base-api-url'],\n",
+ " use_oauth2=calcat_config['use-oauth2'],\n",
+ " client_id=calcat_config['user-id'],\n",
+ " client_secret=calcat_config['user-secret'],\n",
+ " user_email=calcat_config['user-email'],\n",
+ " token_url=calcat_config['token-url'],\n",
+ " refresh_url=calcat_config['refresh-url'],\n",
+ " auth_url=calcat_config['auth-url'],\n",
+ " scope='')"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -257,6 +279,7 @@
" integration_time=integration_time,\n",
" gain_setting=gain_setting,\n",
" gain_mode=gain_mode,\n",
+ " client=client,\n",
" )\n",
" constant_names = [\n",
" \"Offset10Hz\", \"BadPixelsDark10Hz\",\n",
diff --git a/notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb
index 7a56e49a6..9b1eb9cd5 100644
--- a/notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_retrieve_constants_precorrection_NBC.ipynb
@@ -60,8 +60,10 @@
"from extra_data import RunDirectory\n",
"from pathlib import Path\n",
"\n",
+ "from calibration_client import CalibrationClient\n",
"from cal_tools.calcat_interface import JUNGFRAU_CalibrationData\n",
"from cal_tools.jungfraulib import JungfrauCtrl\n",
+ "from cal_tools.restful_config import restful_config\n",
"from cal_tools.step_timing import StepTimer\n",
"from cal_tools.tools import (\n",
" calcat_creation_time,\n",
@@ -135,6 +137,26 @@
"step_timer = StepTimer()"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Connect to CalCat.\n",
+ "calcat_config = restful_config['calcat']\n",
+ "client = CalibrationClient(\n",
+ " base_api_url=calcat_config['base-api-url'],\n",
+ " use_oauth2=calcat_config['use-oauth2'],\n",
+ " client_id=calcat_config['user-id'],\n",
+ " client_secret=calcat_config['user-secret'],\n",
+ " user_email=calcat_config['user-email'],\n",
+ " token_url=calcat_config['token-url'],\n",
+ " refresh_url=calcat_config['refresh-url'],\n",
+ " auth_url=calcat_config['auth-url'],\n",
+ " scope='')"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -152,6 +174,7 @@
" integration_time=integration_time,\n",
" gain_setting=gain_setting,\n",
" gain_mode=gain_mode,\n",
+ " client=client,\n",
")\n",
"constant_names = [\"Offset10Hz\", \"BadPixelsDark10Hz\"]\n",
"\n",
diff --git a/notebooks/LPD/LPD_Correct_Fast.ipynb b/notebooks/LPD/LPD_Correct_Fast.ipynb
index e841269af..c7d23ccdd 100644
--- a/notebooks/LPD/LPD_Correct_Fast.ipynb
+++ b/notebooks/LPD/LPD_Correct_Fast.ipynb
@@ -209,6 +209,26 @@
"const_yaml = metadata.setdefault(\"retrieved-constants\", {})"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Connect to CalCat.\n",
+ "calcat_config = restful_config['calcat']\n",
+ "client = CalibrationClient(\n",
+ " base_api_url=calcat_config['base-api-url'],\n",
+ " use_oauth2=calcat_config['use-oauth2'],\n",
+ " client_id=calcat_config['user-id'],\n",
+ " client_secret=calcat_config['user-secret'],\n",
+ " user_email=calcat_config['user-email'],\n",
+ " token_url=calcat_config['token-url'],\n",
+ " refresh_url=calcat_config['refresh-url'],\n",
+ " auth_url=calcat_config['auth-url'],\n",
+ " scope='')"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -228,6 +248,7 @@
"\n",
" dtype = np.uint32 if calibration_name.startswith('BadPixels') else np.float32\n",
"\n",
+<<<<<<< HEAD
" const_data[(da, calibration_name)] = dict(\n",
" path=Path(ccv['path']),\n",
" dataset=ccv['dataset'],\n",
@@ -263,6 +284,37 @@
" )\n",
" print('.', end='', flush=True)\n",
" \n",
+=======
+ "else:\n",
+ " lpd_cal = LPD_CalibrationData(\n",
+ " detector_name=karabo_id,\n",
+ " modules=karabo_da,\n",
+ " sensor_bias_voltage=bias_voltage,\n",
+ " memory_cells=mem_cells,\n",
+ " feedback_capacitor=capacitor,\n",
+ " source_energy=photon_energy,\n",
+ " category=category,\n",
+ " event_at=creation_time,\n",
+ " snapshot_at=creation_time,\n",
+ " client=client,\n",
+ " )\n",
+ " const_data = lpd_cal.ndarray_map()\n",
+ "\n",
+ "# Validate the constants availability and raise/warn correspondingly. \n",
+ "for mod, calibrations in const_data.items():\n",
+ " missing_dark_constants = set(\n",
+ " c for c in [\"Offset\", \"BadPixelsDark\"] if c not in calibrations.keys())\n",
+ " missing_gain_constants = set(\n",
+ " c for c in [\"BadPixelsFF\", \"GainAmpMap\", \"FFMap\", \"RelativeGain\"] if c not in calibrations.keys()) # noqa\n",
+ " if missing_dark_constants:\n",
+ " raise KeyError(\n",
+ " f\"Dark constants {missing_dark_constants} are not available for correction. Module: {mod}\") # noqa\n",
+ " if missing_gain_constants:\n",
+ " warning(\n",
+ " f\"Gain constants {missing_gain_constants} were not retrieved. Module: {mod}\")\n",
+ " if calibrations[\"BadPixelsDark\"].dtype != np.uint32: # Old LPD constants are stored as float32.\n",
+ " calibrations[\"BadPixelsDark\"] = calibrations[\"BadPixelsDark\"].astype(np.uint32)\n",
+>>>>>>> add missing client input args
"total_time = perf_counter() - start\n",
"print(f'{total_time:.1f}s')"
]
diff --git a/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb b/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
index f77963b71..0c07518d4 100644
--- a/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
+++ b/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
@@ -81,16 +81,7 @@
"metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
"# NOTE: this notebook will not overwrite calibration metadata file,\n",
"# if it already contains details about which constants to use.\n",
- "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
- "\n",
- "# TODO decide if this is required or not!\n",
- "if karabo_da in retrieved_constants:\n",
- " print(\n",
- " f\"Constant for {karabo_da} already in {metadata.filename}, won't query again.\"\n",
- " ) \n",
- " import sys\n",
- "\n",
- " sys.exit(0)"
+ "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})"
]
},
{
diff --git a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
index 147d45a26..bf328394c 100644
--- a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
+++ b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
@@ -351,6 +351,7 @@
" gain_setting=gain,\n",
" event_at=creation_time,\n",
" snapshot_at=creation_time,\n",
+ " client=client,\n",
" )\n",
" constants = pnccd_cal.ndarray_map(calibrations=constant_names)[karabo_da]\n",
"\n",
diff --git a/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb b/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb
index 304958fad..44a5b5934 100644
--- a/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb
+++ b/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb
@@ -21,18 +21,15 @@
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/pnccd_correct\" # output folder\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"run = 347 # which run to read data from\n",
- "sequences = [0] # sequences to correct, set to -1 for all, range allowed\n",
"\n",
"karabo_da = 'PNCCD01' # data aggregators\n",
"karabo_id = \"SQS_NQS_PNCCD1MP\" # detector Karabo_ID\n",
"\n",
"# Conditions for retrieving calibration constants\n",
"fix_temperature_top = 0. # fix temperature for top sensor in K, set to 0. to use value from slow data.\n",
- "fix_temperature_bot = 0. # fix temperature for bottom sensor in K, set to 0. to use value from slow data.\n",
"gain = -1 # the detector's gain setting. Set to -1 to use the value from the slow data.\n",
"bias_voltage = 0. # the detector's bias voltage. set to 0. to use value from slow data.\n",
"integration_time = 70 # detector's integration time\n",
- "photon_energy = 1.6 # Al fluorescence in keV\n",
"\n",
"# Parameters for the calibration database.\n",
"cal_db_interface = \"tcp://max-exfl016:8015\" # calibration DB interface to use\n",
@@ -83,15 +80,7 @@
"metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
"# NOTE: this notebook will not overwrite calibration metadata file,\n",
"# if it already contains details about which constants to use.\n",
- "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
- "\n",
- "if karabo_da in retrieved_constants:\n",
- " print(\n",
- " f\"Constant for {karabo_da} already in {metadata.filename}, won't query again.\"\n",
- " )\n",
- " import sys\n",
- "\n",
- " sys.exit(0)"
+ "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})"
]
},
{
@@ -183,6 +172,7 @@
" gain_setting=gain,\n",
" event_at=creation_time,\n",
" snapshot_at=creation_time,\n",
+ " client=client,\n",
")\n",
"\n",
"mdata_dict = {\"constants\": dict()}\n",
diff --git a/src/cal_tools/agipdlib.py b/src/cal_tools/agipdlib.py
index b53229388..13781f7cd 100644
--- a/src/cal_tools/agipdlib.py
+++ b/src/cal_tools/agipdlib.py
@@ -1370,18 +1370,22 @@ class AgipdCorrections:
acquisition_rate: float,
integration_time: int,
photon_energy: float = 9.2,
+ client=None,
):
""" Initialize calibration constants from the calibration database
- :param karabo_id: karabo detector identifier
- :param karabo_da: karabo module data aggregator name
- :param creation_time: time for desired calibration constant version
- :param memory_cells: number of memory cells used for CCV conditions
- :param bias_voltage: bias voltage used for CCV conditions
- :param gain_setting: gain setting used for CCV conditions
- :param acquisition_rate: acquistion rate used for CCV conditions
- :param integration_time: integration time used for CCV conditions
- :param module_idx: module index to save retrieved CCV in sharedmem
+ :param karabo_id: karabo detector identifier.
+ :param karabo_da: karabo module data aggregator name.
+ :param module_idx: module index to save retrieved CCV in sharedmem.
+ :param creation_time: time for desired calibration constant version.
+ :param memory_cells: number of memory cells used for CCV conditions.
+ :param bias_voltage: bias voltage used for CCV conditions.
+ :param gain_setting: gain setting used for CCV conditions.
+ :param gain_mode: gain mode used for CCV conditions.
+ :param acquisition_rate: acquistion rate used for CCV conditions.
+ :param integration_time: integration time used for CCV conditions.
+ :param photon_energy: photon energy value used for CCV conditions.
+ :param client: calibration_client object for CalibrationData objects.
Loading the following constants based on the
correction configuration:
@@ -1422,6 +1426,7 @@ class AgipdCorrections:
gain_mode=gain_mode,
gain_setting=gain_setting,
event_at=creation_time,
+ client=client,
)
dark_constants = [
--
GitLab