diff --git a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
index fc8913230eacddea36f4c21b9ab1f557455bf743..240dcaaf8ae79eccadc7a90b2692ace4a49ad7ee 100644
--- a/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
+++ b/notebooks/Gotthard2/Correction_Gotthard2_NBC.ipynb
@@ -36,10 +36,9 @@
"karabo_id_control = \"\" # Control karabo ID. Set to empty string to use the karabo-id\n",
"\n",
"# Parameters for calibration database.\n",
- "use_dir_creation_date = True # use the creation data of the input dir for database queries.\n",
"cal_db_interface = \"tcp://max-exfl016:8016#8025\" # the database interface to use.\n",
"cal_db_timeout = 180000 # timeout on caldb requests.\n",
- "overwrite_creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. \"2022-06-28 13:00:00.00\"\n",
+ "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
"\n",
"# Parameters affecting corrected data.\n",
"constants_file = \"\" # Use constants in given constant file path. /gpfs/exfel/data/scratch/ahmedk/dont_remove/gotthard2/constants/calibration_constants_GH2.h5\n",
@@ -70,9 +69,8 @@
"metadata": {},
"outputs": [],
"source": [
- "import datetime\n",
"import warnings\n",
- "from functools import partial\n",
+ "from logging import warning\n",
"\n",
"import h5py\n",
"import pasha as psh\n",
@@ -82,16 +80,16 @@
"from extra_data import RunDirectory, H5File\n",
"from pathlib import Path\n",
"\n",
+ "from calibration_client import CalibrationClient\n",
"from cal_tools import h5_copy_except\n",
+ "from cal_tools.calcat_interface import CalCatError, GOTTHARD2_CalibrationData\n",
"from cal_tools.gotthard2 import gotthard2algs, gotthard2lib\n",
+ "from cal_tools.restful_config import restful_config\n",
"from cal_tools.step_timing import StepTimer\n",
"from cal_tools.tools import (\n",
- " get_constant_from_db_and_time,\n",
- " get_dir_creation_date,\n",
- " get_pdu_from_db,\n",
+ " calcat_creation_time,\n",
" CalibrationMetadata,\n",
")\n",
- "from iCalibrationDB import Conditions, Constants\n",
"from XFELDetAna.plotting.heatmap import heatmapPlot\n",
"\n",
"warnings.filterwarnings('ignore')\n",
@@ -123,14 +121,9 @@
"\n",
"print(f\"Process modules: {karabo_da} for run {run}\")\n",
"\n",
- "creation_time = None\n",
- "if overwrite_creation_time:\n",
- " creation_time = datetime.datetime.strptime(\n",
- " overwrite_creation_time, \"%Y-%m-%d %H:%M:%S.%f\"\n",
- " )\n",
- "elif use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run)\n",
- " print(f\"Using {creation_time} as creation time\")"
+ "# Run's creation time:\n",
+ "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
+ "print(f\"Creation time: {creation_time}\")"
]
},
{
@@ -200,6 +193,27 @@
"print(\"Single Photon:\", single_photon)"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b3613034",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Connect to CalCat.\n",
+ "calcat_config = restful_config['calcat']\n",
+ "client = CalibrationClient(\n",
+ " base_api_url=calcat_config['base-api-url'],\n",
+ " use_oauth2=calcat_config['use-oauth2'],\n",
+ " client_id=calcat_config['user-id'],\n",
+ " client_secret=calcat_config['user-secret'],\n",
+ " user_email=calcat_config['user-email'],\n",
+ " token_url=calcat_config['token-url'],\n",
+ " refresh_url=calcat_config['refresh-url'],\n",
+ " auth_url=calcat_config['auth-url'],\n",
+ " scope='')"
+ ]
+ },
{
"cell_type": "markdown",
"id": "8c852392-bb19-4c40-b2ce-3b787538a92d",
@@ -211,12 +225,32 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "5717d722",
+ "id": "1817d0d8",
"metadata": {},
"outputs": [],
"source": [
+ "da_to_pdu = {}\n",
"# Used for old FXE (p003225) runs before adding Gotthard2 to CALCAT\n",
"const_data = dict()\n",
+ "\n",
+ "g2_cal = GOTTHARD2_CalibrationData(\n",
+ " detector_name=karabo_id,\n",
+ " sensor_bias_voltage=bias_voltage,\n",
+ " exposure_time=exposure_time,\n",
+ " exposure_period=exposure_period,\n",
+ " acquisition_rate=acquisition_rate,\n",
+ " single_photon=single_photon,\n",
+ " event_at=creation_time,\n",
+ ")\n",
+ "# Keep as long as it is essential to correct\n",
+ "# RAW data (FXE p003225) before the data mapping was added to CALCAT.\n",
+ "try: # in case local constants are used with old RAW data. This can be removed in the future.\n",
+ " for mod_info in g2_cal.physical_detector_units.values():\n",
+ " da_to_pdu[mod_info[\"karabo_da\"]] = mod_info[\"physical_name\"]\n",
+ "except CalCatError as e:\n",
+ " print(e)\n",
+ " db_modules = [None] * len(karabo_da)\n",
+ "\n",
"if constants_file:\n",
" for mod in karabo_da:\n",
" const_data[mod] = dict()\n",
@@ -225,80 +259,59 @@
" const_data[mod][\"LUT\"] = cfile[\"LUT\"][()]\n",
" const_data[mod][\"Offset\"] = cfile[\"offset_map\"][()].astype(np.float32)\n",
" const_data[mod][\"RelativeGain\"] = cfile[\"gain_map\"][()].astype(np.float32)\n",
- " const_data[mod][\"Mask\"] = cfile[\"bpix_ff\"][()].astype(np.uint32)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "1cdbe818",
- "metadata": {},
- "outputs": [],
- "source": [
- "# Conditions iCalibrationDB object.\n",
- "condition = Conditions.Dark.Gotthard2(\n",
- " bias_voltage=bias_voltage,\n",
- " exposure_time=exposure_time,\n",
- " exposure_period=exposure_period,\n",
- " single_photon=single_photon,\n",
- " acquisition_rate=acquisition_rate,\n",
- ")\n",
- "\n",
- "# TODO: Maybe this condition and previous cell can be removed later after the initial phase.\n",
- "if not constants_file:\n",
- " # Prepare a dictionary of empty constants to loop on\n",
- " # it's keys and initiate non-retrieved constants.\n",
- " empty_lut = (np.arange(2 ** 12).astype(np.float64) * 2 ** 10 / 2 ** 12).astype(\n",
- " np.uint16\n",
- " )\n",
- " empty_lut = np.stack(1280 * [np.stack([empty_lut] * 2)], axis=0)\n",
- " empty_constants = {\n",
- " \"LUT\": empty_lut,\n",
- " \"Offset\": np.zeros((1280, 2, 3), dtype=np.float32),\n",
- " \"BadPixelsDark\": np.zeros((1280, 2, 3), dtype=np.uint32),\n",
- " \"RelativeGain\": np.ones((1280, 2, 3), dtype=np.float32),\n",
- " \"BadPixelsFF\": np.zeros((1280, 2, 3), dtype=np.uint32),\n",
- " }\n",
- "\n",
- " for mod in karabo_da:\n",
- " const_data[mod] = dict()\n",
- " # Only used for printing timestamps within the loop.\n",
- " when = dict()\n",
- " # Check YAML file for constant metadata of file path and creation-time\n",
- " if const_yaml:\n",
+ " const_data[mod][\"Mask\"] = cfile[\"bpix_ff\"][()].astype(np.uint32)\n",
+ "else:\n",
+ " if const_yaml:\n",
+ " const_data = dict()\n",
+ " for mod in karabo_da:\n",
+ " const_data[mod] = dict()\n",
" for cname, mdata in const_yaml[mod][\"constants\"].items():\n",
" const_data[mod][cname] = dict()\n",
- " when[cname] = mdata[\"creation-time\"]\n",
- " if when[cname]:\n",
- " with h5py.File(mdata[\"file-path\"], \"r\") as cf:\n",
+ " if mdata[\"creation-time\"]:\n",
+ " with h5py.File(mdata[\"path\"], \"r\") as cf:\n",
" const_data[mod][cname] = np.copy(\n",
- " cf[f\"{mdata['dataset-name']}/data\"]\n",
- " )\n",
- " else:\n",
- " const_data[mod][cname] = empty_constants[cname]\n",
- " else: # Retrieve constants from CALCAT. Missing YAML file or running notebook interactively.\n",
- " for cname, cempty in empty_constants.items():\n",
- " const_data[mod][cname] = dict()\n",
- " const_data[mod][cname], when[cname] = get_constant_from_db_and_time(\n",
- " karabo_id=karabo_id,\n",
- " karabo_da=mod,\n",
- " cal_db_interface=cal_db_interface,\n",
- " creation_time=creation_time,\n",
- " timeout=cal_db_timeout,\n",
- " print_once=False,\n",
- " condition=condition,\n",
- " constant=getattr(Constants.Gotthard2, cname)(),\n",
- " empty_constant=cempty,\n",
- " )\n",
+ " cf[f\"{mdata['dataset']}/data\"])\n",
+ " else:\n",
+ " g2_cal = GOTTHARD2_CalibrationData(\n",
+ " detector_name=karabo_id,\n",
+ " sensor_bias_voltage=bias_voltage,\n",
+ " exposure_time=exposure_time,\n",
+ " exposure_period=exposure_period,\n",
+ " acquisition_rate=acquisition_rate,\n",
+ " single_photon=single_photon,\n",
+ " event_at=creation_time,\n",
+ " snapshot_at=creation_time,\n",
+ " )\n",
+ "\n",
+ " mdata_dict = {\"constants\": dict()}\n",
+ "\n",
+ " constant_names = [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"]\n",
+ " if gain_correction:\n",
+ " constant_names += [\"RelativeGainGotthard2\", \"BadPixelsFFGotthard2\"]\n",
+ "\n",
+ " # Retrieve metadata for all pnccd constants.\n",
+ "\n",
+ " g2_metadata = g2_cal.ndarray_map(constant_names)\n",
+ "\n",
+ " # Validate the constants availability and raise/warn correspondingly.\n",
+ " for mod, calibrations in const_data.items():\n",
+ " missing_dark_constants = set(\n",
+ " c for c in [\"LUTGotthard2\", \"OffsetGotthard2\", \"BadPixelsDarkGotthard2\"] if c not in calibrations.keys())\n",
+ " missing_gain_constants = set(\n",
+ " c for c in [\"BadPixelsFF10Hz\", \"RelativeGain10Hz\"] if gain_correction and c not in calibrations.keys()) # noqa\n",
+ " if missing_dark_constants:\n",
+ " raise KeyError(\n",
+ " f\"Dark constants {missing_dark_constants} are not available for correction. Mod: {mod}\")\n",
+ "\n",
+ " if gain_correction and missing_gain_constants:\n",
+ " warning(f\"Gain constants {missing_gain_constants} are not retrieved. Mod {mod}\")\n",
+ " gain_correction = False\n",
+ "\n",
+ " # Create the mask array.\n",
" bpix = const_data[mod][\"BadPixelsDark\"]\n",
- " bpix |= const_data[mod][\"BadPixelsFF\"]\n",
- " const_data[mod][\"Mask\"] = bpix\n",
- "\n",
- " # Print timestamps for the retrieved constants.\n",
- " print(f\"Constants for module {mod}:\")\n",
- " for cname, ctime in when.items():\n",
- " print(f\" {cname} injected at {ctime}\")\n",
- " del when"
+ " if const_data[mod].get(\"BadPixelsFF\") is not None:\n",
+ " bpix |= const_data[mod][\"BadPixelsFF\"]\n",
+ " const_data[mod][\"Mask\"] = bpix"
]
},
{
diff --git a/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb b/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
index e3a81c929ea0fd109dfa81d8ec337f3ba9312279..fd5edba39f6a5574296943918d9c1ddfdaead93b 100644
--- a/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
+++ b/notebooks/Gotthard2/Gotthard2_retrieve_constants_precorrection_NBC.ipynb
@@ -66,8 +66,10 @@
"from extra_data import RunDirectory\n",
"from pathlib import Path\n",
"\n",
+ "from calibration_client import CalibrationClient\n",
"from cal_tools.calcat_interface import GOTTHARD2_CalibrationData\n",
"from cal_tools.gotthard2 import gotthard2lib\n",
+ "from cal_tools.restful_config import restful_config\n",
"from cal_tools.tools import (\n",
" calcat_creation_time,\n",
" get_from_db,\n",
@@ -130,6 +132,26 @@
"print(\"Single Photon:\", single_photon)"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Connect to CalCat.\n",
+ "calcat_config = restful_config['calcat']\n",
+ "client = CalibrationClient(\n",
+ " base_api_url=calcat_config['base-api-url'],\n",
+ " use_oauth2=calcat_config['use-oauth2'],\n",
+ " client_id=calcat_config['user-id'],\n",
+ " client_secret=calcat_config['user-secret'],\n",
+ " user_email=calcat_config['user-email'],\n",
+ " token_url=calcat_config['token-url'],\n",
+ " refresh_url=calcat_config['refresh-url'],\n",
+ " auth_url=calcat_config['auth-url'],\n",
+ " scope='')"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
index 13c7c22721d4833f0f869f54008511035dcfe625..d8b57a67147dd7588c51c8ba6750cd8c96822fe0 100644
--- a/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
+++ b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
@@ -53,6 +53,7 @@
" CalibrationMetadata,\n",
" calcat_creation_time,\n",
")\n",
+ "from calibration_client import CalibrationClient\n",
"from cal_tools.restful_config import restful_config\n",
"from cal_tools.step_timing import StepTimer"
]
@@ -91,6 +92,26 @@
"step_timer = StepTimer()"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Connect to CalCat.\n",
+ "calcat_config = restful_config['calcat']\n",
+ "client = CalibrationClient(\n",
+ " base_api_url=calcat_config['base-api-url'],\n",
+ " use_oauth2=calcat_config['use-oauth2'],\n",
+ " client_id=calcat_config['user-id'],\n",
+ " client_secret=calcat_config['user-secret'],\n",
+ " user_email=calcat_config['user-email'],\n",
+ " token_url=calcat_config['token-url'],\n",
+ " refresh_url=calcat_config['refresh-url'],\n",
+ " auth_url=calcat_config['auth-url'],\n",
+ " scope='')"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -108,6 +129,7 @@
" category=category,\n",
" event_at=creation_time,\n",
" snapshot_at=creation_time,\n",
+ " client=client,\n",
")\n",
"constant_names = [\n",
" \"Offset\", \"BadPixelsDark\",\n",
diff --git a/notebooks/ePix100/Correction_ePix100_NBC.ipynb b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
index e66a4e9050afb29b0c6ad782d7e44f26999b83c5..b05093c176064c34224a3f2a1dddbfb7b40fab3a 100644
--- a/notebooks/ePix100/Correction_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
@@ -97,7 +97,8 @@
"\n",
"from XFELDetAna import xfelpyanatools as xana\n",
"from XFELDetAna import xfelpycaltools as xcal\n",
- "from cal_tools import h5_copy_except\n",
+ "from calibration_client import CalibrationClient\n",
+ "from cal_tools import h5_copy_except, restful_config\n",
"from cal_tools.epix100 import epix100lib\n",
"from cal_tools.calcat_interface import EPIX100_CalibrationData\n",
"from cal_tools.tools import (\n",
@@ -255,6 +256,26 @@
" )))"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Connect to CalCat.\n",
+ "calcat_config = restful_config['calcat']\n",
+ "client = CalibrationClient(\n",
+ " base_api_url=calcat_config['base-api-url'],\n",
+ " use_oauth2=calcat_config['use-oauth2'],\n",
+ " client_id=calcat_config['user-id'],\n",
+ " client_secret=calcat_config['user-secret'],\n",
+ " user_email=calcat_config['user-email'],\n",
+ " token_url=calcat_config['token-url'],\n",
+ " refresh_url=calcat_config['refresh-url'],\n",
+ " auth_url=calcat_config['auth-url'],\n",
+ " scope='')"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -287,6 +308,7 @@
" source_energy=gain_photon_energy,\n",
" event_at=creation_time,\n",
" snapshot_at=creation_time,\n",
+ " client=client,\n",
" )\n",
" const_data = epix_cal.ndarray_map(calibrations=constant_names)[karabo_da]\n",
"\n",
diff --git a/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb b/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
index a3bf60258927e6c2eb119d499d291db6058c4890..f77963b7110feb135d97f95423d173be9faeab3c 100644
--- a/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
+++ b/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
@@ -57,8 +57,10 @@
"from extra_data import RunDirectory\n",
"from pathlib import Path\n",
"\n",
+ "from calibration_client import CalibrationClient\n",
"from cal_tools.calcat_interface import EPIX100_CalibrationData\n",
"from cal_tools.epix100 import epix100lib\n",
+ "from cal_tools.restful_config import restful_config\n",
"from cal_tools.tools import (\n",
" calcat_creation_time,\n",
" CalibrationMetadata,\n",
@@ -138,6 +140,26 @@
"print(f\"Operated in vacuum: {in_vacuum}\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Connect to CalCat.\n",
+ "calcat_config = restful_config['calcat']\n",
+ "client = CalibrationClient(\n",
+ " base_api_url=calcat_config['base-api-url'],\n",
+ " use_oauth2=calcat_config['use-oauth2'],\n",
+ " client_id=calcat_config['user-id'],\n",
+ " client_secret=calcat_config['user-secret'],\n",
+ " user_email=calcat_config['user-email'],\n",
+ " token_url=calcat_config['token-url'],\n",
+ " refresh_url=calcat_config['refresh-url'],\n",
+ " auth_url=calcat_config['auth-url'],\n",
+ " scope='')"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -155,6 +177,7 @@
" source_energy=gain_photon_energy,\n",
" event_at=creation_time,\n",
" snapshot_at=creation_time,\n",
+ " client=client,\n",
" )\n",
"\n",
"mdata_dict = {\"constants\": dict()}\n",
diff --git a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
index d616c4ccfab8aa38bbedd3d4f24dbc5ffa5616b8..147d45a265185f59726584f80e22a5fda04d3dbb 100644
--- a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
+++ b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
@@ -108,10 +108,12 @@
"\n",
"%matplotlib inline\n",
"\n",
+ "from calibration_client import CalibrationClient\n",
"from XFELDetAna import xfelpyanatools as xana\n",
"from XFELDetAna import xfelpycaltools as xcal\n",
"from cal_tools import pnccdlib\n",
"from cal_tools.calcat_interface import PNCCD_CalibrationData\n",
+ "from cal_tools.restful_config import restful_config\n",
"from cal_tools.tools import (\n",
" calcat_creation_time,\n",
" load_constants_dict,\n",
@@ -298,6 +300,26 @@
"b_range = Event_Bin_Dict[\"b_range\"]"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Connect to CalCat.\n",
+ "calcat_config = restful_config['calcat']\n",
+ "client = CalibrationClient(\n",
+ " base_api_url=calcat_config['base-api-url'],\n",
+ " use_oauth2=calcat_config['use-oauth2'],\n",
+ " client_id=calcat_config['user-id'],\n",
+ " client_secret=calcat_config['user-secret'],\n",
+ " user_email=calcat_config['user-email'],\n",
+ " token_url=calcat_config['token-url'],\n",
+ " refresh_url=calcat_config['refresh-url'],\n",
+ " auth_url=calcat_config['auth-url'],\n",
+ " scope='')"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
diff --git a/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb b/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb
index a80d746d23931fc9f5cfa98fc773c73c6cd7113e..304958fadef552e5c51199d1417784a084343d40 100644
--- a/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb
+++ b/notebooks/pnCCD/pnCCD_retrieve_constants_precorrection.ipynb
@@ -59,8 +59,10 @@
"from IPython.display import Markdown, display\n",
"from extra_data import RunDirectory\n",
"\n",
+ "from calibration_client import CalibrationClient\n",
"from cal_tools import pnccdlib\n",
"from cal_tools.calcat_interface import PNCCD_CalibrationData\n",
+ "from cal_tools.restful_config import restful_config\n",
"from cal_tools.tools import (\n",
" calcat_creation_time,\n",
" CalibrationMetadata,\n",
@@ -146,6 +148,26 @@
"print(f\"Top pnCCD sensor temperature: {fix_temperature_top:0.2f} K\")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Connect to CalCat.\n",
+ "calcat_config = restful_config['calcat']\n",
+ "client = CalibrationClient(\n",
+ " base_api_url=calcat_config['base-api-url'],\n",
+ " use_oauth2=calcat_config['use-oauth2'],\n",
+ " client_id=calcat_config['user-id'],\n",
+ " client_secret=calcat_config['user-secret'],\n",
+ " user_email=calcat_config['user-email'],\n",
+ " token_url=calcat_config['token-url'],\n",
+ " refresh_url=calcat_config['refresh-url'],\n",
+ " auth_url=calcat_config['auth-url'],\n",
+ " scope='')"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,