From f0f765c1815873e47478d105a5cd1c4239004598 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Sun, 1 Dec 2024 22:56:51 +0100
Subject: [PATCH] feat: Move post methods into an indvidual class in constants
---
...rk_analysis_all_gains_burst_mode_NBC.ipynb | 6 +-
src/cal_tools/calcat_interface2.py | 21 +++
src/cal_tools/constants.py | 146 ++++++++++++------
src/cal_tools/restful_config.py | 36 +++--
4 files changed, 144 insertions(+), 65 deletions(-)
diff --git a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
index 17d7c74f4..f6a0e9322 100644
--- a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
@@ -92,7 +92,6 @@
"from cal_tools import step_timing\n",
"from cal_tools.calcat_interface2 import (\n",
" CalibrationData,\n",
- " CCVAlreadyInjectedError,\n",
" JUNGFRAUConditions,\n",
")\n",
"from cal_tools.constants import (\n",
@@ -613,7 +612,6 @@
" constants['Noise10Hz'] = np.moveaxis(noise_map[mod], 0, 1)\n",
" constants['BadPixelsDark10Hz'] = np.moveaxis(bad_pixels_map[mod], 0, 1)\n",
"\n",
- " md = None\n",
" for const_name, const_data in constants.items():\n",
" with NamedTemporaryFile(dir=out_folder) as tempf:\n",
" ccv_root = write_ccv(\n",
@@ -657,7 +655,7 @@
" f\"• Exposure timeout: {exposure_timeout}\\n\"\n",
" f\"• Gain setting: {gain_setting}\\n\"\n",
" f\"• Gain mode: {gain_mode}\\n\"\n",
- " f\"• Creation time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\") # noqa\n",
+ " f\"• Creation time: {creation_time}\\n\") # noqa\n",
"step_timer.done_step(\"Injecting constants.\")"
]
},
@@ -696,7 +694,7 @@
"jf_caldata = CalibrationData.from_condition(\n",
" conditions,\n",
" karabo_id,\n",
- " event_at=creation_time-timedelta(seconds=60) if creation_time else None,\n",
+ " event_at=creation_time-timedelta(seconds=60),\n",
" begin_at_strategy=\"prior\",\n",
")\n",
"\n",
diff --git a/src/cal_tools/calcat_interface2.py b/src/cal_tools/calcat_interface2.py
index 53b01f843..dd159e011 100644
--- a/src/cal_tools/calcat_interface2.py
+++ b/src/cal_tools/calcat_interface2.py
@@ -29,6 +29,27 @@ class ModuleNameError(KeyError):
class CalCatAPIError(requests.HTTPError):
"""Used when the response includes error details as JSON"""
+ ...
+
+class CalibrationConstantNotFound(CalCatAPIError):
+ ...
+
+
+def _get_failed_response(resp):
+ # TODO: Add more errors if needed
+ if "calibration_constant" in resp.url.lstrip("/"):
+ if resp.status_code == 404:
+ raise CalibrationNotFound("Calibration Constant was not found in CALCAT.")
+ if resp.status_code >= 400:
+ try:
+ d = json.loads(resp.content.decode("utf-8"))
+ except Exception:
+ resp.raise_for_status()
+ else:
+ raise CalCatAPIError(
+ f"Error {resp.status_code} from API: "
+ f"{d.get('info', 'missing details')}"
+ )
class CalCatAPIClient:
def __init__(self, base_api_url, oauth_client=None, user_email=""):
diff --git a/src/cal_tools/constants.py b/src/cal_tools/constants.py
index b7d4f8fc9..e62940280 100644
--- a/src/cal_tools/constants.py
+++ b/src/cal_tools/constants.py
@@ -19,7 +19,9 @@ from cal_tools.calcat_interface2 import (
CalCatAPIClient,
CalCatAPIError,
get_default_caldb_root,
+ CalibrationConstantNotFound
)
+from oauth2_xfel_client import Oauth2ClientBackend
CONDITION_NAME_MAX_LENGTH = 60
@@ -32,12 +34,12 @@ class CCVAlreadyInjectedError(InjectAPIError):
...
-def _failed_response(resp):
+def _post_failed_response(resp):
# TODO: Add more errors if needed
if "calibration_constant_versions" in resp.url.lstrip("/"):
if resp.status_code == 422:
raise CCVAlreadyInjectedError
- elif resp.status_code >= 400:
+ if resp.status_code >= 400:
try:
d = json.loads(resp.content.decode("utf-8"))
except Exception:
@@ -70,7 +72,7 @@ class InjectAPI(CalCatAPIClient):
def _parse_post_response(resp: requests.Response):
if resp.status_code >= 400:
- _failed_response(resp)
+ _post_failed_response(resp)
if resp.content == b"":
return None
@@ -107,8 +109,29 @@ class ParameterConditionAttribute:
description: str = ''
-def generate_unique_cond_name(detector_type, pdu_name, pdu_uuid, cond_params):
- # Generate condition name.
+def generate_unique_condition_name(
+ detector_type: str,
+ pdu_name: str,
+ pdu_uuid: float,
+ cond_params: List[dict],
+):
+ """Generate a unique condition using UUID and timestamp.
+
+ Args:
+ detector_type (str): detector type.
+ pdu_name (str): Physical detector unit db name.
+ pdu_uuid (float): Physical detector unit db id.
+ cond_params (List[dict]): A list of dictionary with each condition
+ e.g. [{
+ "parameter_name": "Memory Cells",
+ "value": 352.0,
+ "lower-deviation": 0.0,
+ "upper-deviation": 0.0
+ }]
+
+ Returns:
+ str: A unique name used for the table of conditions.
+ """
unique_name = detector_type[:detector_type.index('-Type')] + ' Def'
cond_hash = md5(pdu_name.encode())
cond_hash.update(int(pdu_uuid).to_bytes(
@@ -119,7 +142,7 @@ def generate_unique_cond_name(detector_type, pdu_name, pdu_uuid, cond_params):
cond_hash.update(str(pattrs.value).encode())
unique_name += binascii.b2a_base64(cond_hash.digest()).decode()
- return unique_name[:60]
+ return unique_name[:CONDITION_NAME_MAX_LENGTH]
def create_unique_cc_name(det_type, calibration, condition_name):
@@ -270,44 +293,8 @@ def get_condition_dict(
}
-def generate_unique_condition_name(
- detector_type: str,
- pdu_name: str,
- pdu_uuid: float,
- cond_params: List[dict],
-):
- """Generate a unique condition using UUID and timestamp.
-
- Args:
- detector_type (str): detector type.
- pdu_name (str): Physical detector unit db name.
- pdu_uuid (float): Physical detector unit db id.
- cond_params (List[dict]): A list of dictionary with each condition
- e.g. [{
- "parameter_name": "Memory Cells",
- "value": 352.0,
- "lower-deviation": 0.0,
- "upper-deviation": 0.0
- }]
-
- Returns:
- str: A unique name used for the table of conditions.
- """
- unique_name = detector_type[:detector_type.index('-Type')] + ' Def'
- cond_hash = md5(pdu_name.encode())
- cond_hash.update(int(pdu_uuid).to_bytes(
- length=8, byteorder='little', signed=False))
-
- for param_dict in cond_params:
- cond_hash.update(str(param_dict['parameter_name']).encode())
- cond_hash.update(str(param_dict['value']).encode())
-
- unique_name += binascii.b2a_base64(cond_hash.digest()).decode()
- return unique_name[:CONDITION_NAME_MAX_LENGTH]
-
-
def get_or_create_calibration_constant(
- client: CalCatAPIClient,
+ client: InjectAPI,
calibration: str,
detector_type: str,
condition_id: int,
@@ -329,9 +316,11 @@ def get_or_create_calibration_constant(
flg_available = 'true',
description = "",
)
- resp = client.get_calibration_constant(calibration_constant)
- return resp['id'] if resp else client.create_calibration_constant(
- calibration_constant)['id'] # calibration constant id
+ try:
+ cc_id = client.get_calibration_constant(calibration_constant)
+ except CalibrationConstantNotFound:
+ cc_id = client.create_calibration_constant(calibration_constant)['id']
+ return cc_id
def create_condition(
@@ -342,7 +331,7 @@ def create_condition(
cond_params: dict,
) -> Tuple[int, str]:
# Create condition unique name
- cond_name = generate_unique_cond_name(
+ cond_name = generate_unique_condition_name(
detector_type, pdu_name, pdu_uuid, cond_params)
# Add the missing parameter_id value in `ParameterConditionAttribute`s.
@@ -398,6 +387,68 @@ def get_ccv_info_from_file(
return cond_params, begin_at, pdu_uuid, detector_type, raw_data_location
+global_client = None
+
+
+def get_client():
+ """Get the global CalCat API client.
+
+ The default assumes we're running in the DESY network; this is used unless
+ `setup_client()` has been called to specify otherwise.
+ """
+ global global_client
+ if global_client is None:
+ setup_client(CALCAT_PROXY_URL, None, None, None)
+ return global_client
+
+
+def setup_client(
+ base_url,
+ client_id,
+ client_secret,
+ user_email,
+ scope="",
+ session_token=None,
+ oauth_retries=3,
+ oauth_timeout=12,
+ ssl_verify=True,
+):
+ """Configure the global CalCat API client."""
+ global global_client
+ if client_id is not None:
+ oauth_client = Oauth2ClientBackend(
+ client_id=client_id,
+ client_secret=client_secret,
+ scope=scope,
+ token_url=f"{base_url}/oauth/token",
+ session_token=session_token,
+ max_retries=oauth_retries,
+ timeout=oauth_timeout,
+ ssl_verify=ssl_verify,
+ )
+ else:
+ oauth_client = None
+ global_client = InjectAPI(
+ f"{base_url}/api/",
+ oauth_client=oauth_client,
+ user_email=user_email,
+ )
+
+ # Check we can connect to exflcalproxy
+ if oauth_client is None and base_url == CALCAT_PROXY_URL:
+ try:
+ # timeout=(connect_timeout, read_timeout)
+ global_client.get_request("me", timeout=(1, 5))
+ except requests.ConnectionError as e:
+ raise RuntimeError(
+ "Could not connect to calibration catalog proxy. This proxy allows "
+ "unauthenticated access inside the XFEL/DESY network. To look up "
+ "calibration constants from outside, you will need to create an Oauth "
+ "client ID & secret in the CalCat web interface. You will still not "
+ "be able to load constants without the constant store folder."
+ ) from e
+
+
def inject_ccv(const_src, ccv_root, report_to=None, client=None):
"""Inject new CCV into CalCat.
@@ -410,7 +461,6 @@ def inject_ccv(const_src, ccv_root, report_to=None, client=None):
RuntimeError: If CalCat POST request fails.
"""
if client is None:
- from cal_tools.calcat_interface2 import get_client
client = get_client()
pdu_name, calibration, _ = ccv_root.lstrip('/').split('/')
diff --git a/src/cal_tools/restful_config.py b/src/cal_tools/restful_config.py
index 671dec619..1e2017dec 100644
--- a/src/cal_tools/restful_config.py
+++ b/src/cal_tools/restful_config.py
@@ -8,7 +8,7 @@ config_dir = Path(__file__).parent.resolve()
# Default fles.
settings_files = [
- config_dir / "restful_config.yaml",
+ config_dir / "restful_test_config.yaml",
config_dir / "restful_config.secrets.yaml",
Path("~/.config/pycalibration/cal_tools/restful_config.yaml").expanduser(),
]
@@ -44,10 +44,10 @@ def calibration_client():
scope='')
-def extra_calibration_client():
+def extra_calibration_client(inject=False):
"""Obtain an initialized CalCatAPIClient object."""
- from cal_tools import calcat_interface2
+ from cal_tools import calcat_interface2, constants
calcat_config = restful_config.get('calcat')
user_id = user_secret = None
@@ -57,13 +57,23 @@ def extra_calibration_client():
base_api_url = calcat_config['base-api-url'].rstrip('/')
assert base_api_url.endswith('/api')
base_url = base_api_url[:-4]
-
- calcat_interface2.setup_client(
- base_url,
- client_id=user_id,
- client_secret=user_secret,
- user_email=calcat_config['user-email'],
- )
- if calcat_config['caldb-root']:
- calcat_interface2.set_default_caldb_root(Path(calcat_config['caldb-root']))
- return calcat_interface2.get_client()
+ if inject:
+ constants.setup_client(
+ base_url,
+ client_id=user_id,
+ client_secret=user_secret,
+ user_email=calcat_config['user-email'],
+ )
+ if calcat_config['caldb-root']:
+ calcat_interface2.set_default_caldb_root(Path(calcat_config['caldb-root']))
+ return constants.get_client()
+ else:
+ calcat_interface2.setup_client(
+ base_url,
+ client_id=user_id,
+ client_secret=user_secret,
+ user_email=calcat_config['user-email'],
+ )
+ if calcat_config['caldb-root']:
+ calcat_interface2.set_default_caldb_root(Path(calcat_config['caldb-root']))
+ return calcat_interface2.get_client()
--
GitLab