Skip to content
Snippets Groups Projects
Commit 57df52f1 authored by Thomas Kluyver's avatar Thomas Kluyver
Browse files

Error out if no offset constant found

parent 5148b9c7
No related branches found
No related tags found
1 merge request!851[LPD] Automatically decide whether to inject & use memory cell order
Hide whitespace changes
Inline Side-by-side
notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
+ 2
− 0
View file @ 57df52f1
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# LPD Retrieving Constants Pre-correction # # LPD Retrieving Constants Pre-correction #
Author: European XFEL Detector Group, Version: 1.0 Author: European XFEL Detector Group, Version: 1.0
The following notebook provides a constants metadata in a YAML file to use while correcting LPD images. The following notebook provides a constants metadata in a YAML file to use while correcting LPD images.
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
# Input parameters # Input parameters
in_folder = "/gpfs/exfel/exp/FXE/202201/p003073/raw/" # the folder to read data from, required in_folder = "/gpfs/exfel/exp/FXE/202201/p003073/raw/" # the folder to read data from, required
out_folder = "/gpfs/exfel/data/scratch/ahmedk/test/remove/LPD_test" # the folder to output to, required out_folder = "/gpfs/exfel/data/scratch/ahmedk/test/remove/LPD_test" # the folder to output to, required
metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate. metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate.
modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty
karabo_da = [''] # Data aggregators names to correct, use [''] for all karabo_da = [''] # Data aggregators names to correct, use [''] for all
run = 10 # run to process, required run = 10 # run to process, required
# Source parameters # Source parameters
karabo_id = 'FXE_DET_LPD1M-1' # Karabo domain for detector. karabo_id = 'FXE_DET_LPD1M-1' # Karabo domain for detector.
input_source = '{karabo_id}/DET/{module_index}CH0:xtdf' # Input fast data source. input_source = '{karabo_id}/DET/{module_index}CH0:xtdf' # Input fast data source.
# CalCat parameters # CalCat parameters
creation_time = "" # The timestamp to use with Calibration DB. Required Format: "YYYY-MM-DD hh:mm:ss" e.g. 2019-07-04 11:02:41 creation_time = "" # The timestamp to use with Calibration DB. Required Format: "YYYY-MM-DD hh:mm:ss" e.g. 2019-07-04 11:02:41
# Operating conditions # Operating conditions
mem_cells = 512 # Memory cells, LPD constants are always taken with 512 cells. mem_cells = 512 # Memory cells, LPD constants are always taken with 512 cells.
bias_voltage = 250.0 # Detector bias voltage. bias_voltage = 250.0 # Detector bias voltage.
capacitor = '5pF' # Capacitor setting: 5pF or 50pF capacitor = '5pF' # Capacitor setting: 5pF or 50pF
photon_energy = 9.2 # Photon energy in keV. photon_energy = 9.2 # Photon energy in keV.
category = 0 # Whom to blame. category = 0 # Whom to blame.
use_cell_order = False # Whether to use memory cell order as a detector condition (not stored for older constants) use_cell_order = False # Whether to use memory cell order as a detector condition (not stored for older constants)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
from pathlib import Path from pathlib import Path
from time import perf_counter from time import perf_counter
import numpy as np import numpy as np
from calibration_client import CalibrationClient from calibration_client import CalibrationClient
from calibration_client.modules import CalibrationConstantVersion from calibration_client.modules import CalibrationConstantVersion
import extra_data as xd import extra_data as xd
from cal_tools.lpdlib import get_mem_cell_order from cal_tools.lpdlib import get_mem_cell_order
from cal_tools.tools import ( from cal_tools.tools import (
CalibrationMetadata, CalibrationMetadata,
calcat_creation_time, calcat_creation_time,
save_constant_metadata, save_constant_metadata,
) )
from cal_tools.restful_config import restful_config from cal_tools.restful_config import restful_config
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
out_folder = Path(out_folder) out_folder = Path(out_folder)
out_folder.mkdir(exist_ok=True) out_folder.mkdir(exist_ok=True)
metadata = CalibrationMetadata(metadata_folder or out_folder) metadata = CalibrationMetadata(metadata_folder or out_folder)
# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml # Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml
retrieved_constants = metadata.setdefault("retrieved-constants", {}) retrieved_constants = metadata.setdefault("retrieved-constants", {})
creation_time = calcat_creation_time(in_folder, run, creation_time) creation_time = calcat_creation_time(in_folder, run, creation_time)
print(f'Using {creation_time.isoformat()} as creation time') print(f'Using {creation_time.isoformat()} as creation time')
# Pick all modules/aggregators or those selected. # Pick all modules/aggregators or those selected.
if not karabo_da or karabo_da == ['']: if not karabo_da or karabo_da == ['']:
if not modules or modules == [-1]: if not modules or modules == [-1]:
modules = list(range(16)) modules = list(range(16))
karabo_da = [f'LPD{i:02d}' for i in modules] karabo_da = [f'LPD{i:02d}' for i in modules]
# List of detector sources. # List of detector sources.
det_inp_sources = [input_source.format(karabo_id=karabo_id, module_index=int(da[-2:])) for da in karabo_da] det_inp_sources = [input_source.format(karabo_id=karabo_id, module_index=int(da[-2:])) for da in karabo_da]
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
# Connect to CalCat. # Connect to CalCat.
calcat_config = restful_config['calcat'] calcat_config = restful_config['calcat']
client = CalibrationClient( client = CalibrationClient(
base_api_url=calcat_config['base-api-url'], base_api_url=calcat_config['base-api-url'],
use_oauth2=calcat_config['use-oauth2'], use_oauth2=calcat_config['use-oauth2'],
client_id=calcat_config['user-id'], client_id=calcat_config['user-id'],
client_secret=calcat_config['user-secret'], client_secret=calcat_config['user-secret'],
user_email=calcat_config['user-email'], user_email=calcat_config['user-email'],
token_url=calcat_config['token-url'], token_url=calcat_config['token-url'],
refresh_url=calcat_config['refresh-url'], refresh_url=calcat_config['refresh-url'],
auth_url=calcat_config['auth-url'], auth_url=calcat_config['auth-url'],
scope='') scope='')
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
dark_calibrations = { dark_calibrations = {
1: 'Offset', 1: 'Offset',
14: 'BadPixelsDark', 14: 'BadPixelsDark',
} }
base_condition = [ base_condition = [
dict(parameter_id=1, value=bias_voltage), # Sensor bias voltage dict(parameter_id=1, value=bias_voltage), # Sensor bias voltage
dict(parameter_id=7, value=mem_cells), # Memory cells dict(parameter_id=7, value=mem_cells), # Memory cells
dict(parameter_id=15, value=capacitor), # Feedback capacitor dict(parameter_id=15, value=capacitor), # Feedback capacitor
dict(parameter_id=13, value=256), # Pixels X dict(parameter_id=13, value=256), # Pixels X
dict(parameter_id=14, value=256), # Pixels Y dict(parameter_id=14, value=256), # Pixels Y
] ]
if use_cell_order: if use_cell_order:
# Read the order of memory cells used # Read the order of memory cells used
raw_data = xd.RunDirectory(Path(in_folder, f'r{run:04d}')) raw_data = xd.RunDirectory(Path(in_folder, f'r{run:04d}'))
cell_ids_pattern_s = get_mem_cell_order(raw_data, det_inp_sources) cell_ids_pattern_s = get_mem_cell_order(raw_data, det_inp_sources)
print("Memory cells order:", cell_ids_pattern_s) print("Memory cells order:", cell_ids_pattern_s)
dark_condition = base_condition + [ dark_condition = base_condition + [
dict(parameter_id=30, value=cell_ids_pattern_s), # Memory cell order dict(parameter_id=30, value=cell_ids_pattern_s), # Memory cell order
] ]
else: else:
dark_condition = base_condition.copy() dark_condition = base_condition.copy()
illuminated_calibrations = { illuminated_calibrations = {
20: 'BadPixelsFF', 20: 'BadPixelsFF',
42: 'GainAmpMap', 42: 'GainAmpMap',
43: 'FFMap', 43: 'FFMap',
44: 'RelativeGain', 44: 'RelativeGain',
} }
illuminated_condition = base_condition + [ illuminated_condition = base_condition + [
dict(parameter_id=3, value=photon_energy), # Source energy dict(parameter_id=3, value=photon_energy), # Source energy
dict(parameter_id=25, value=category) # category dict(parameter_id=25, value=category) # category
] ]
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
const_data = {} const_data = {}
print('Querying calibration database', end='', flush=True) print('Querying calibration database', end='', flush=True)
start = perf_counter() start = perf_counter()
for k_da in karabo_da: for k_da in karabo_da:
pdu = None pdu = None
retrieved_constants[k_da] = dict() retrieved_constants[k_da] = dict()
const_mdata = retrieved_constants[k_da]["constants"] = dict() const_mdata = retrieved_constants[k_da]["constants"] = dict()
for calibrations, condition in [ for calibrations, condition in [
(dark_calibrations, dark_condition), (dark_calibrations, dark_condition),
(illuminated_calibrations, illuminated_condition) (illuminated_calibrations, illuminated_condition)
]: ]:
resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions( resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(
client, karabo_id, list(calibrations.keys()), client, karabo_id, list(calibrations.keys()),
{'parameters_conditions_attributes': condition}, {'parameters_conditions_attributes': condition},
karabo_da=k_da, event_at=creation_time.isoformat()) karabo_da=k_da, event_at=creation_time.isoformat())
if not resp["success"]: if not resp["success"]:
print(f"ERROR: Constants {list(calibrations.values())} " print(f"ERROR: Constants {list(calibrations.values())} "
f"were not retrieved, {resp['app_info']}") f"were not retrieved, {resp['app_info']}")
for cname in calibrations.values(): for cname in calibrations.values():
if cname == 'Offset':
raise Exception("Could not find offset constant, will not correct data")
const_mdata[cname] = dict() const_mdata[cname] = dict()
const_mdata[cname]["file-path"] = None const_mdata[cname]["file-path"] = None
const_mdata[cname]["dataset-name"] = None const_mdata[cname]["dataset-name"] = None
const_mdata[cname]["creation-time"] = None const_mdata[cname]["creation-time"] = None
continue continue
for ccv in resp["data"]: for ccv in resp["data"]:
cc = ccv['calibration_constant'] cc = ccv['calibration_constant']
cname = calibrations[cc['calibration_id']] cname = calibrations[cc['calibration_id']]
const_mdata[cname] = dict() const_mdata[cname] = dict()
const_mdata[cname]["file-path"] = str(Path(ccv['path_to_file']) / ccv['file_name']) const_mdata[cname]["file-path"] = str(Path(ccv['path_to_file']) / ccv['file_name'])
const_mdata[cname]["dataset-name"] = ccv['data_set_name'] const_mdata[cname]["dataset-name"] = ccv['data_set_name']
const_mdata[cname]["creation-time"] = ccv['begin_at'] const_mdata[cname]["creation-time"] = ccv['begin_at']
pdu = ccv['physical_detector_unit']['physical_name'] pdu = ccv['physical_detector_unit']['physical_name']
print('.', end='', flush=True) print('.', end='', flush=True)
retrieved_constants[k_da]["physical-detector-unit"] = pdu retrieved_constants[k_da]["physical-detector-unit"] = pdu
metadata.save() metadata.save()
total_time = perf_counter() - start total_time = perf_counter() - start
print(f'{total_time:.1f}s') print(f'{total_time:.1f}s')
print(f"Stored retrieved constants in {metadata.filename}") print(f"Stored retrieved constants in {metadata.filename}")
``` ```
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts