diff --git a/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
index d8b57a67147dd7588c51c8ba6750cd8c96822fe0..37468ba95bb853a8631f491c5d48cbf8968b0b4b 100644
--- a/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
+++ b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
@@ -188,9 +188,9 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Python 3.8.11 ('.cal4_venv')",
+ "display_name": "cal4_venv",
"language": "python",
- "name": "python3"
+ "name": "cal4_venv"
},
"language_info": {
"codemirror_mode": {
@@ -202,7 +202,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.11"
+ "version": "3.8.11 (default, Jul 2 2021, 14:23:46) \n[GCC 4.8.5 20150623 (Red Hat 4.8.5-44)]"
},
"orig_nbformat": 4,
"vscode": {
diff --git a/notebooks/ePix100/Correction_ePix100_NBC.ipynb b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
index e413ac9afaa7c4ee38a9830033b146d6d55472f6..a18f7c655b17f6e643761fc8300a35eaf6579e11 100644
--- a/notebooks/ePix100/Correction_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
@@ -85,7 +85,6 @@
"source": [
"import tabulate\n",
"import warnings\n",
- "from logging import warning\n",
"\n",
"import h5py\n",
"import pasha as psh\n",
@@ -97,17 +96,20 @@
"\n",
"from XFELDetAna import xfelpyanatools as xana\n",
"from XFELDetAna import xfelpycaltools as xcal\n",
- "from calibration_client import CalibrationClient\n",
"from cal_tools import h5_copy_except\n",
- "from cal_tools.calcat_interface import EPIX100_CalibrationData\n",
"from cal_tools.epix100 import epix100lib\n",
- "from cal_tools.restful_config import restful_config\n",
"from cal_tools.tools import (\n",
" calcat_creation_time,\n",
- " load_constants_dict,\n",
+ " get_dir_creation_date,\n",
+ " get_constant_from_db,\n",
+ " load_specified_constants,\n",
" CalibrationMetadata,\n",
")\n",
"from cal_tools.step_timing import StepTimer\n",
+ "from iCalibrationDB import (\n",
+ " Conditions,\n",
+ " Constants,\n",
+ ")\n",
"\n",
"warnings.filterwarnings('ignore')\n",
"\n",
@@ -258,32 +260,41 @@
]
},
{
- "cell_type": "code",
- "execution_count": null,
+ "cell_type": "markdown",
"metadata": {},
- "outputs": [],
"source": [
- "# Connect to CalCat.\n",
- "calcat_config = restful_config['calcat']\n",
- "client = CalibrationClient(\n",
- " base_api_url=calcat_config['base-api-url'],\n",
- " use_oauth2=calcat_config['use-oauth2'],\n",
- " client_id=calcat_config['user-id'],\n",
- " client_secret=calcat_config['user-secret'],\n",
- " user_email=calcat_config['user-email'],\n",
- " token_url=calcat_config['token-url'],\n",
- " refresh_url=calcat_config['refresh-url'],\n",
- " auth_url=calcat_config['auth-url'],\n",
- " scope='')"
+ "## Retrieving calibration constants\n",
+ "\n",
+ "As a first step, dark maps have to be loaded."
]
},
{
- "cell_type": "markdown",
+ "cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
- "## Retrieving calibration constants\n",
+ "cond_dict = {\n",
+ " \"bias_voltage\": bias_voltage,\n",
+ " \"integration_time\": integration_time,\n",
+ " \"temperature\": temperature_k,\n",
+ " \"in_vacuum\": in_vacuum,\n",
+ "}\n",
+ "\n",
+ "dark_condition = Conditions.Dark.ePix100(**cond_dict)\n",
+ "\n",
+ "# update conditions with illuminated conditins.\n",
+ "cond_dict.update({\n",
+ " \"photon_energy\": gain_photon_energy\n",
+ " })\n",
"\n",
- "As a first step, dark maps have to be loaded."
+ "illum_condition = Conditions.Illuminated.ePix100(**cond_dict)\n",
+ "\n",
+ "const_cond = {\n",
+ " \"Offset\": dark_condition,\n",
+ " \"Noise\": dark_condition,\n",
+ " \"RelativeGain\": illum_condition,\n",
+ "}"
]
},
{
@@ -292,38 +303,31 @@
"metadata": {},
"outputs": [],
"source": [
- "constant_names = [\"OffsetEPix100\", \"NoiseEPix100\"]\n",
- "if relative_gain:\n",
- " constant_names += [\"RelativeGainEPix100\"]\n",
- "\n",
+ "empty_constant = np.zeros((708, 768, 1), dtype=np.float32)\n",
"if const_yaml: # Used while reproducing corrected data.\n",
" print(f\"Using stored constants in {metadata.filename}\")\n",
- " const_data, _ = load_constants_dict(const_yaml[karabo_da][\"constants\"])\n",
- "else:\n",
- " epix_cal = EPIX100_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " sensor_bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " sensor_temperature=temperature_k,\n",
- " in_vacuum=in_vacuum,\n",
- " source_energy=gain_photon_energy,\n",
- " event_at=creation_time,\n",
- " snapshot_at=creation_time,\n",
- " client=client,\n",
- " )\n",
- " const_data = epix_cal.ndarray_map(calibrations=constant_names)[karabo_da]\n",
- "\n",
- "# Validate the constants availability and raise/warn correspondingly. \n",
- "missing_dark_constants = set(\n",
- " c for c in [\"OffsetEPix100\", \"NoiseEPix100\"] if c not in const_data.keys())\n",
- "if missing_dark_constants:\n",
- " raise KeyError(\n",
- " f\"Dark constants {missing_dark_constants} are not available for correction.\")\n",
- "\n",
- "if relative_gain and \"RelativeGainEPix100\" not in const_data.keys():\n",
- " warning(\"RelativeGainEPix100 is not found in the calibration database.\")\n",
- " relative_gain = False\n",
- " absolute_gain = False"
+ " const_data, _ = load_specified_constants(const_yaml[karabo_da][\"constants\"])\n",
+ " for cname, cval in const_data.items():\n",
+ " if cval is None and cname != \"RelativeGain\":\n",
+ " const_data[cname] = empty_constant\n",
+ "else: # First correction attempt.\n",
+ " const_data = dict()\n",
+ " for cname, condition in const_cond.items():\n",
+ " # Avoid retrieving RelativeGain, if not needed for correction.\n",
+ " if cname == \"RelativeGain\" and not relative_gain:\n",
+ " const_data[cname] = None\n",
+ " else:\n",
+ " const_data[cname] = get_constant_from_db(\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=karabo_da,\n",
+ " constant=getattr(Constants.ePix100, cname)(),\n",
+ " condition=condition,\n",
+ " empty_constant=None if cname == \"RelativeGain\" else empty_constant,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " print_once=2,\n",
+ " timeout=cal_db_timeout\n",
+ " )"
]
},
{
@@ -332,6 +336,14 @@
"metadata": {},
"outputs": [],
"source": [
+ "if relative_gain and const_data.get(\"RelativeGain\", None) is None:\n",
+ " print(\n",
+ " \"WARNING: RelativeGain map is requested, but not found.\\n\"\n",
+ " \"No gain correction will be applied\"\n",
+ " )\n",
+ " relative_gain = False\n",
+ " absolute_gain = False\n",
+ "\n",
"# Initializing some parameters.\n",
"hscale = 1\n",
"stats = True\n",
@@ -386,7 +398,7 @@
" blockSize=commonModeBlockSize, \n",
" orientation='block',\n",
" nCells=memoryCells, \n",
- " noiseMap=const_data['NoiseEPix100'],\n",
+ " noiseMap=const_data['Noise'],\n",
" runParallel=run_parallel,\n",
" parallel=run_parallel,\n",
" stats=stats,\n",
@@ -398,7 +410,7 @@
" blockSize=commonModeBlockSize, \n",
" orientation='row',\n",
" nCells=memoryCells, \n",
- " noiseMap=const_data['NoiseEPix100'],\n",
+ " noiseMap=const_data['Noise'],\n",
" runParallel=run_parallel,\n",
" parallel=run_parallel,\n",
" stats=stats,\n",
@@ -410,7 +422,7 @@
" blockSize=commonModeBlockSize, \n",
" orientation='col',\n",
" nCells=memoryCells, \n",
- " noiseMap=const_data['NoiseEPix100'],\n",
+ " noiseMap=const_data['Noise'],\n",
" runParallel=run_parallel,\n",
" parallel=run_parallel,\n",
" stats=stats,\n",
@@ -426,7 +438,7 @@
"outputs": [],
"source": [
"if relative_gain:\n",
- " gain_cnst = np.median(const_data[\"RelativeGainEPix100\"])\n",
+ " gain_cnst = np.median(const_data[\"RelativeGain\"])\n",
" hscale = gain_cnst\n",
" plot_unit = 'keV'\n",
" if photon_energy > 0:\n",
@@ -435,7 +447,7 @@
" \n",
" gainCorrection = xcal.RelativeGainCorrection(\n",
" sensorSize,\n",
- " gain_cnst/const_data[\"RelativeGainEPix100\"][..., None],\n",
+ " gain_cnst/const_data[\"RelativeGain\"][..., None],\n",
" nCells=memoryCells,\n",
" parallel=run_parallel,\n",
" blockSize=blockSize,\n",
@@ -471,7 +483,7 @@
"if pattern_classification :\n",
" patternClassifier = xcal.PatternClassifier(\n",
" [x, y],\n",
- " const_data[\"NoiseEPix100\"],\n",
+ " const_data[\"Noise\"],\n",
" split_evt_primary_threshold,\n",
" split_evt_secondary_threshold,\n",
" split_evt_mip_threshold,\n",
@@ -527,7 +539,7 @@
" np.any(d > 0, axis=(0, 1)), d, axis=2)\n",
" \n",
" # Offset correction.\n",
- " d -= const_data[\"OffsetEPix100\"]\n",
+ " d -= const_data[\"Offset\"]\n",
"\n",
" histCalOffsetCor.fill(d)\n",
" # Common Mode correction.\n",
@@ -551,7 +563,7 @@
" it changes the scale (the unit of measurement)\n",
" of the data from ADU to either keV or n_of_photons.\n",
" But the pattern classification relies on comparing\n",
- " data with the NoiseEPix100 map, which is still in ADU.\n",
+ " data with the noise map, which is still in ADU.\n",
"\n",
" The best solution is to do a relative gain\n",
" correction first and apply the global absolute\n",
diff --git a/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb b/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
index 0c07518d4d37f5993025b49890eda887382330e7..9c19e81d0e87cd690b1bf81250b83e7bf1d43995 100644
--- a/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
+++ b/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
@@ -51,20 +51,19 @@
"metadata": {},
"outputs": [],
"source": [
- "from logging import warning\n",
- "\n",
"import numpy as np\n",
"from extra_data import RunDirectory\n",
"from pathlib import Path\n",
"\n",
- "from calibration_client import CalibrationClient\n",
- "from cal_tools.calcat_interface import EPIX100_CalibrationData\n",
"from cal_tools.epix100 import epix100lib\n",
- "from cal_tools.restful_config import restful_config\n",
"from cal_tools.tools import (\n",
" calcat_creation_time,\n",
+ " get_dir_creation_date,\n",
+ " get_from_db,\n",
+ " save_constant_metadata,\n",
" CalibrationMetadata,\n",
- ")"
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants"
]
},
{
@@ -81,7 +80,15 @@
"metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
"# NOTE: this notebook will not overwrite calibration metadata file,\n",
"# if it already contains details about which constants to use.\n",
- "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})"
+ "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
+ "\n",
+ "if karabo_da in retrieved_constants:\n",
+ " print(\n",
+ " f\"Constant for {karabo_da} already in {metadata.filename}, won't query again.\"\n",
+ " ) \n",
+ " import sys\n",
+ "\n",
+ " sys.exit(0)"
]
},
{
@@ -137,68 +144,55 @@
"metadata": {},
"outputs": [],
"source": [
- "# Connect to CalCat.\n",
- "calcat_config = restful_config['calcat']\n",
- "client = CalibrationClient(\n",
- " base_api_url=calcat_config['base-api-url'],\n",
- " use_oauth2=calcat_config['use-oauth2'],\n",
- " client_id=calcat_config['user-id'],\n",
- " client_secret=calcat_config['user-secret'],\n",
- " user_email=calcat_config['user-email'],\n",
- " token_url=calcat_config['token-url'],\n",
- " refresh_url=calcat_config['refresh-url'],\n",
- " auth_url=calcat_config['auth-url'],\n",
- " scope='')"
+ "cond_dict = {\n",
+ " \"bias_voltage\": bias_voltage,\n",
+ " \"integration_time\": integration_time,\n",
+ " \"temperature\": temperature_k,\n",
+ " \"in_vacuum\": in_vacuum,\n",
+ "}\n",
+ "\n",
+ "dark_condition = Conditions.Dark.ePix100(**cond_dict)\n",
+ "\n",
+ "# update conditions with illuminated conditions.\n",
+ "cond_dict.update({\"photon_energy\": gain_photon_energy})\n",
+ "\n",
+ "illum_condition = Conditions.Illuminated.ePix100(**cond_dict)\n",
+ "\n",
+ "const_cond = {\n",
+ " \"Offset\": dark_condition,\n",
+ " \"Noise\": dark_condition,\n",
+ " \"RelativeGain\": illum_condition,\n",
+ "}"
]
},
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "tags": []
- },
+ "metadata": {},
"outputs": [],
"source": [
- "epix_cal = EPIX100_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " sensor_bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " sensor_temperature=temperature_k,\n",
- " in_vacuum=in_vacuum,\n",
- " source_energy=gain_photon_energy,\n",
- " event_at=creation_time,\n",
- " snapshot_at=creation_time,\n",
- " client=client,\n",
- " )\n",
- "\n",
- "mdata_dict = {\"constants\": dict()}\n",
- "\n",
- "constant_names = [\"OffsetEPix100\", \"NoiseEPix100\"]\n",
- "if relative_gain:\n",
- " constant_names += [\"RelativeGainEPix100\"]\n",
- "\n",
- "# Retrieve metadata for all epix100 constants.\n",
- "\n",
- "epix_metadata = epix_cal.metadata(constant_names)\n",
- "\n",
- "# Validate the constants availability and raise/warn correspondingly. \n",
- "missing_dark_constants = set(\n",
- " c for c in [\"OffsetEPix100\", \"NoiseEPix100\"] if c not in epix_metadata[karabo_da].keys())\n",
- "if missing_dark_constants:\n",
- " raise KeyError(\n",
- " f\"Dark constants {missing_dark_constants} are not available for correction.\")\n",
- "if relative_gain and \"RelativeGainEPix100\" not in list(epix_metadata.values())[0].keys():\n",
- " warning(\"RelativeGainEPix100 is not found in CALCAT.\")\n",
- "\n",
- "for cname, ccv_metadata in list(epix_metadata.values())[0].items():\n",
- " mdata_dict[\"constants\"][cname] = {\n",
- " \"path\": str(epix_cal.caldb_root / ccv_metadata[\"path\"]),\n",
- " \"dataset\": ccv_metadata[\"dataset\"],\n",
- " \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
- " }\n",
- " print(f\"Retrieved {cname} with creation-time: {ccv_metadata['begin_validity_at']}\")\n",
- "\n",
- "mdata_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n",
+ "const_data = dict()\n",
+ "mdata_dict = dict()\n",
+ "mdata_dict[\"constants\"] = dict()\n",
+ "for cname, condition in const_cond.items():\n",
+ " # Avoid retrieving RelativeGain, if not needed for correction.\n",
+ " if cname == \"RelativeGain\" and not relative_gain:\n",
+ " const_data[cname] = None\n",
+ " else:\n",
+ " const_data[cname], mdata = get_from_db(\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=karabo_da,\n",
+ " constant=getattr(Constants.ePix100, cname)(),\n",
+ " condition=condition,\n",
+ " empty_constant=None,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " verbosity=2,\n",
+ " timeout=cal_db_timeout,\n",
+ " meta_only=True,\n",
+ " )\n",
+ " save_constant_metadata(mdata_dict[\"constants\"], mdata, cname)\n",
+ "mdata_dict[\"physical-detector-unit\"] = mdata.calibration_constant_version.device_name\n",
"retrieved_constants[karabo_da] = mdata_dict\n",
"metadata.save()\n",
"print(f\"Stored retrieved constants in {metadata.filename}\")"