Don't pass default snapshot_at=None to calibration_client

46864296 · Thomas Kluyver · ee7bddf8 · 46864296
Commit 46864296 authored 2 years ago by Thomas Kluyver
--- a/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
+++ b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
@@ -160,7 +160,7 @@
    "        resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(\n",
    "            client, karabo_id, list(calibrations.keys()),\n",
    "            {'parameters_conditions_attributes': condition},\n",
-    "            karabo_da=k_da, event_at=creation_time.isoformat(), snapshot_at=None)\n",
+    "            karabo_da=k_da, event_at=creation_time.isoformat())\n",
    "\n",
    "        if not resp[\"success\"]:\n",
    "            print(f\"ERROR: Constants {list(calibrations.values())} \"\n",

 %% Cell type:markdown id: tags:
 # LPD Retrieving Constants Pre-correction #
 Author: European XFEL Detector Group, Version: 1.0
 The following notebook provides a constants metadata in a YAML file to use while correcting LPD images.
 %% Cell type:code id: tags:
 ``` python
 # Input parameters
 in_folder = "/gpfs/exfel/exp/FXE/202201/p003073/raw/"  # the folder to read data from, required
 out_folder = "/gpfs/exfel/data/scratch/ahmedk/test/remove/LPD_test"  # the folder to output to, required
 metadata_folder = ''  # Directory containing calibration_metadata.yml when run by xfel-calibrate.
 modules = [-1]  # Modules indices to correct, use [-1] for all, only used when karabo_da is empty
 karabo_da = ['']  # Data aggregators names to correct, use [''] for all
 run = 10  # run to process, required
 # Source parameters
 karabo_id = 'FXE_DET_LPD1M-1'  # Karabo domain for detector.
 # CalCat parameters
 creation_time = ""  # The timestamp to use with Calibration DB. Required Format: "YYYY-MM-DD hh:mm:ss" e.g. 2019-07-04 11:02:41
 # Operating conditions
 mem_cells = 512  # Memory cells, LPD constants are always taken with 512 cells.
 bias_voltage = 250.0  # Detector bias voltage.
 capacitor = '5pF'  # Capacitor setting: 5pF or 50pF
 photon_energy = 9.2  # Photon energy in keV.
 category = 0  # Whom to blame.
 ```
 %% Cell type:code id: tags:
 ``` python
 from pathlib import Path
 from time import perf_counter
 from calibration_client import CalibrationClient
 from calibration_client.modules import CalibrationConstantVersion
 from cal_tools.tools import (
    CalibrationMetadata,
    calcat_creation_time,
    save_constant_metadata,
 )
 from cal_tools.restful_config import restful_config
 ```
 %% Cell type:code id: tags:
 ``` python
 out_folder = Path(out_folder)
 out_folder.mkdir(exist_ok=True)
 metadata = CalibrationMetadata(metadata_folder or out_folder)
 # Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml
 retrieved_constants = metadata.setdefault("retrieved-constants", {})
 creation_time = calcat_creation_time(in_folder, run, creation_time)
 print(f'Using {creation_time.isoformat()} as creation time')
 # Pick all modules/aggregators or those selected.
 if not karabo_da or karabo_da == ['']:
    if not modules or modules == [-1]:
        modules = list(range(16))
    karabo_da = [f'LPD{i:02d}' for i in modules]
 ```
 %% Cell type:code id: tags:
 ``` python
 # Connect to CalCat.
 calcat_config = restful_config['calcat']
 client = CalibrationClient(
    base_api_url=calcat_config['base-api-url'],
    use_oauth2=calcat_config['use-oauth2'],
    client_id=calcat_config['user-id'],
    client_secret=calcat_config['user-secret'],
    user_email=calcat_config['user-email'],
    token_url=calcat_config['token-url'],
    refresh_url=calcat_config['refresh-url'],
    auth_url=calcat_config['auth-url'],
    scope='')
 ```
 %% Cell type:code id: tags:
 ``` python
 dark_calibrations = {
    1: 'Offset',
    14: 'BadPixelsDark',
 }
 dark_condition = [
    dict(parameter_id=1, value=bias_voltage),  # Sensor bias voltage
    dict(parameter_id=7, value=mem_cells),  # Memory cells
    dict(parameter_id=15, value=capacitor),  # Feedback capacitor
    dict(parameter_id=13, value=256),  # Pixels X
    dict(parameter_id=14, value=256),  # Pixels Y
 ]
 illuminated_calibrations = {
    20: 'BadPixelsFF',
    42: 'GainAmpMap',
    43: 'FFMap',
    44: 'RelativeGain',
 }
 illuminated_condition = dark_condition.copy()
 illuminated_condition += [
    dict(parameter_id=3, value=photon_energy),  # Source energy
    dict(parameter_id=25, value=category)  # category
 ]
 ```
 %% Cell type:code id: tags:
 ``` python
 const_data = {}
 print('Querying calibration database', end='', flush=True)
 start = perf_counter()
 for k_da in karabo_da:
    pdu = None
    if k_da in retrieved_constants:
        print(f"Constant for {k_da} already in {metadata.filename}, won't query again.")  # noqa
        continue
    retrieved_constants[k_da] = dict()
    const_mdata = retrieved_constants[k_da]["constants"] = dict()
    for calibrations, condition in [
        (dark_calibrations, dark_condition),
        (illuminated_calibrations, illuminated_condition)
    ]:
        resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(
            client, karabo_id, list(calibrations.keys()),
            {'parameters_conditions_attributes': condition},
-            karabo_da=k_da, event_at=creation_time.isoformat(), snapshot_at=None)
+            karabo_da=k_da, event_at=creation_time.isoformat())
        if not resp["success"]:
            print(f"ERROR: Constants {list(calibrations.values())} "
            f"were not retrieved, {resp['app_info']}")
            for cname in calibrations.values():
                const_mdata[cname] = dict()
                const_mdata[cname]["file-path"] = None
                const_mdata[cname]["dataset-name"] = None
                const_mdata[cname]["creation-time"] = None
            continue
        for ccv in resp["data"]:
            cc = ccv['calibration_constant']
            cname = calibrations[cc['calibration_id']]
            const_mdata[cname] = dict()
            const_mdata[cname]["file-path"] = str(Path(ccv['path_to_file']) / ccv['file_name'])
            const_mdata[cname]["dataset-name"] = ccv['data_set_name']
            const_mdata[cname]["creation-time"] = ccv['begin_at']
            pdu = ccv['physical_detector_unit']['physical_name']
        print('.', end='', flush=True)
    retrieved_constants[k_da]["physical-detector-unit"] = pdu
 metadata.save()
 total_time = perf_counter() - start
 print(f'{total_time:.1f}s')
 print(f"Stored retrieved constants in {metadata.filename}")
 ```