Skip to content
Snippets Groups Projects
Commit 070d574f authored by Karim Ahmed's avatar Karim Ahmed
Browse files

remove LPD changes

parent 58845f6f
No related branches found
No related tags found
1 merge request!738New calcat interface
Hide whitespace changes
Inline Side-by-side
notebooks/LPD/LPD_Correct_Fast.ipynb
+ 87
− 22
View file @ 070d574f
...@@ -26,7 +26,7 @@ ...@@ -26,7 +26,7 @@
"metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate.\n", "metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate.\n",
"sequences = [-1] # Sequences to correct, use [-1] for all\n", "sequences = [-1] # Sequences to correct, use [-1] for all\n",
"modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty\n", "modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty\n",
"karabo_da = ['all'] # Data aggregators names to correct, use [''] for all\n", "karabo_da = [''] # Data aggregators names to correct, use [''] for all\n",
"run = 10 # run to process, required\n", "run = 10 # run to process, required\n",
"\n", "\n",
"# Source parameters\n", "# Source parameters\n",
...@@ -104,7 +104,10 @@ ...@@ -104,7 +104,10 @@
"\n", "\n",
"from extra_data.components import LPD1M\n", "from extra_data.components import LPD1M\n",
"\n", "\n",
<<<<<<< HEAD
"from cal_tools.calcat_interface import CalCatError, LPD_CalibrationData\n", "from cal_tools.calcat_interface import CalCatError, LPD_CalibrationData\n",
=======
>>>>>>> remove LPD changes
"from cal_tools.lpdalgs import correct_lpd_frames\n", "from cal_tools.lpdalgs import correct_lpd_frames\n",
"from cal_tools.tools import CalibrationMetadata, calcat_creation_time\n", "from cal_tools.tools import CalibrationMetadata, calcat_creation_time\n",
"from cal_tools.files import DataFile\n", "from cal_tools.files import DataFile\n",
...@@ -135,20 +138,16 @@ ...@@ -135,20 +138,16 @@
"cal_db_root = Path(cal_db_root)\n", "cal_db_root = Path(cal_db_root)\n",
"\n", "\n",
"metadata = CalibrationMetadata(metadata_folder or out_folder)\n", "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
"# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
"retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
"\n", "\n",
"creation_time = calcat_creation_time(in_folder, run, creation_time)\n", "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
"print(f'Using {creation_time.isoformat()} as creation time')\n", "print(f'Using {creation_time.isoformat()} as creation time')\n",
"\n", "\n",
"# Pick all modules/aggregators or those selected.\n", "# Pick all modules/aggregators or those selected.\n",
"if karabo_da == ['all']:\n", "if not karabo_da or karabo_da == ['']:\n",
" if modules == [-1]:\n", " if not modules or modules == [-1]:\n",
" modules = list(range(16))\n", " modules = list(range(16))\n",
"\n", "\n",
" karabo_da = [f'LPD{i:02d}' for i in modules]\n", " karabo_da = [f'LPD{i:02d}' for i in modules]\n",
"else:\n",
" modules = [int(x[-2:]) for x in karabo_da]\n",
" \n", " \n",
"# Pick all sequences or those selected.\n", "# Pick all sequences or those selected.\n",
"if not sequences or sequences == [-1]:\n", "if not sequences or sequences == [-1]:\n",
...@@ -198,17 +197,6 @@ ...@@ -198,17 +197,6 @@
"# Obtain and prepare calibration constants" "# Obtain and prepare calibration constants"
] ]
}, },
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
"# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
"const_yaml = metadata.setdefault(\"retrieved-constants\", {})"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
...@@ -232,11 +220,23 @@ ...@@ -232,11 +220,23 @@
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
"metadata": { "metadata": {},
"tags": [] "outputs": [],
}, "source": [
<<<<<<< HEAD
=======
"metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
"# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
"const_yaml = metadata.setdefault(\"retrieved-constants\", {})"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
>>>>>>> remove LPD changes
"const_data = {}\n", "const_data = {}\n",
"const_load_mp = psh.ProcessContext(num_workers=24)\n", "const_load_mp = psh.ProcessContext(num_workers=24)\n",
"\n", "\n",
...@@ -248,6 +248,7 @@ ...@@ -248,6 +248,7 @@
"\n", "\n",
" dtype = np.uint32 if calibration_name.startswith('BadPixels') else np.float32\n", " dtype = np.uint32 if calibration_name.startswith('BadPixels') else np.float32\n",
"\n", "\n",
<<<<<<< HEAD
<<<<<<< HEAD <<<<<<< HEAD
" const_data[(da, calibration_name)] = dict(\n", " const_data[(da, calibration_name)] = dict(\n",
" path=Path(ccv['path']),\n", " path=Path(ccv['path']),\n",
...@@ -255,6 +256,40 @@ ...@@ -255,6 +256,40 @@
" data=const_load_mp.alloc(shape=(256, 256, mem_cells, 3), dtype=dtype)\n", " data=const_load_mp.alloc(shape=(256, 256, mem_cells, 3), dtype=dtype)\n",
" )\n", " )\n",
"else: # Retrieve constants from CALCAT.\n", "else: # Retrieve constants from CALCAT.\n",
=======
" const_data[(da, calibration_name)] = dict(\n",
" path=Path(ccv['file-path']),\n",
" dataset=ccv['dataset-name'],\n",
" data=const_load_mp.alloc(shape=(256, 256, mem_cells, 3), dtype=dtype)\n",
" )\n",
"else: # Retrieve constants from CALCAT.\n",
" dark_calibrations = {\n",
" 1: 'Offset', # np.float32\n",
" 14: 'BadPixelsDark' # should be np.uint32, but is np.float64\n",
" }\n",
"\n",
" dark_condition = [\n",
" dict(parameter_id=1, value=bias_voltage), # Sensor bias voltage\n",
" dict(parameter_id=7, value=mem_cells), # Memory cells\n",
" dict(parameter_id=15, value=capacitor), # Feedback capacitor\n",
" dict(parameter_id=13, value=256), # Pixels X\n",
" dict(parameter_id=14, value=256), # Pixels Y\n",
" ]\n",
"\n",
" illuminated_calibrations = {\n",
" 20: 'BadPixelsFF', # np.uint32\n",
" 42: 'GainAmpMap', # np.float32\n",
" 43: 'FFMap', # np.float32\n",
" 44: 'RelativeGain' # np.float32\n",
" }\n",
"\n",
" illuminated_condition = dark_condition.copy()\n",
" illuminated_condition += [\n",
" dict(parameter_id=3, value=photon_energy), # Source energy\n",
" dict(parameter_id=25, value=category) # category\n",
" ]\n",
"\n",
>>>>>>> remove LPD changes
" print('Querying calibration database', end='', flush=True)\n", " print('Querying calibration database', end='', flush=True)\n",
" start = perf_counter()\n", " start = perf_counter()\n",
" for calibrations, condition in [\n", " for calibrations, condition in [\n",
...@@ -264,8 +299,12 @@ ...@@ -264,8 +299,12 @@
" resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(\n", " resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(\n",
" client, karabo_id, list(calibrations.keys()),\n", " client, karabo_id, list(calibrations.keys()),\n",
" {'parameters_conditions_attributes': condition},\n", " {'parameters_conditions_attributes': condition},\n",
<<<<<<< HEAD
" karabo_da='', event_at=creation_time.isoformat()\n", " karabo_da='', event_at=creation_time.isoformat()\n",
" )\n", " )\n",
=======
" karabo_da='', event_at=creation_time.isoformat(), snapshot_at=None)\n",
>>>>>>> remove LPD changes
"\n", "\n",
" if not resp['success']:\n", " if not resp['success']:\n",
" raise RuntimeError(resp)\n", " raise RuntimeError(resp)\n",
...@@ -284,6 +323,7 @@ ...@@ -284,6 +323,7 @@
" )\n", " )\n",
" print('.', end='', flush=True)\n", " print('.', end='', flush=True)\n",
" \n", " \n",
<<<<<<< HEAD
======= =======
"else:\n", "else:\n",
" lpd_cal = LPD_CalibrationData(\n", " lpd_cal = LPD_CalibrationData(\n",
...@@ -319,6 +359,30 @@ ...@@ -319,6 +359,30 @@
"print(f'{total_time:.1f}s')" "print(f'{total_time:.1f}s')"
] ]
}, },
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"def load_constant_dataset(wid, index, const_descr):\n",
" ccv_entry = const_data[const_descr]\n",
" \n",
" with h5py.File(cal_db_root / ccv_entry['path'], 'r') as fp:\n",
" fp[ccv_entry['dataset'] + '/data'].read_direct(ccv_entry['data'])\n",
" \n",
" print('.', end='', flush=True)\n",
"\n",
"print('Loading calibration data', end='', flush=True)\n",
"start = perf_counter()\n",
"const_load_mp.map(load_constant_dataset, list(const_data.keys()))\n",
=======
>>>>>>> remove LPD changes
"total_time = perf_counter() - start\n",
"\n",
"print(f'{total_time:.1f}s')"
]
},
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": null,
...@@ -588,7 +652,8 @@ ...@@ -588,7 +652,8 @@
"ExecuteTime": { "ExecuteTime": {
"end_time": "2018-11-13T18:24:57.547563Z", "end_time": "2018-11-13T18:24:57.547563Z",
"start_time": "2018-11-13T18:24:56.995005Z" "start_time": "2018-11-13T18:24:56.995005Z"
} },
"scrolled": false
}, },
"outputs": [], "outputs": [],
"source": [ "source": [
......
notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
+ 25
− 38
View file @ 070d574f
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# LPD Retrieving Constants Pre-correction # # LPD Retrieving Constants Pre-correction #
Author: European XFEL Detector Group, Version: 1.0 Author: European XFEL Detector Group, Version: 1.0
The following notebook provides a constants metadata in a YAML file to use while correcting LPD images. The following notebook provides a constants metadata in a YAML file to use while correcting LPD images.
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
# Input parameters # Input parameters
in_folder = "/gpfs/exfel/exp/FXE/202201/p003073/raw/" # the folder to read data from, required in_folder = "/gpfs/exfel/exp/FXE/202201/p003073/raw/" # the folder to read data from, required
out_folder = "/gpfs/exfel/data/scratch/ahmedk/test/remove/LPD_test" # the folder to output to, required out_folder = "/gpfs/exfel/data/scratch/ahmedk/test/remove/LPD_test" # the folder to output to, required
metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate. metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate.
modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty
karabo_da = ['all'] # Data aggregators names to correct, use [''] for all karabo_da = [''] # Data aggregators names to correct, use [''] for all
run = 10 # run to process, required run = 10 # run to process, required
# Source parameters # Source parameters
karabo_id = 'FXE_DET_LPD1M-1' # Karabo domain for detector. karabo_id = 'FXE_DET_LPD1M-1' # Karabo domain for detector.
# CalCat parameters # CalCat parameters
creation_time = "" # The timestamp to use with Calibration DB. Required Format: "YYYY-MM-DD hh:mm:ss" e.g. 2019-07-04 11:02:41 creation_time = "" # The timestamp to use with Calibration DB. Required Format: "YYYY-MM-DD hh:mm:ss" e.g. 2019-07-04 11:02:41
# Operating conditions # Operating conditions
mem_cells = 512 # Memory cells, LPD constants are always taken with 512 cells. mem_cells = 512 # Memory cells, LPD constants are always taken with 512 cells.
bias_voltage = 250.0 # Detector bias voltage. bias_voltage = 250.0 # Detector bias voltage.
capacitor = '5pF' # Capacitor setting: 5pF or 50pF capacitor = '5pF' # Capacitor setting: 5pF or 50pF
photon_energy = 9.2 # Photon energy in keV. photon_energy = 9.2 # Photon energy in keV.
category = 0 # Whom to blame. category = 0 # Whom to blame.
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
from logging import warning
from pathlib import Path from pathlib import Path
from time import perf_counter
from calibration_client import CalibrationClient
from calibration_client.modules import CalibrationConstantVersion
from cal_tools.calcat_interface import LPD_CalibrationData
from cal_tools.tools import ( from cal_tools.tools import (
CalibrationMetadata, CalibrationMetadata,
calcat_creation_time, calcat_creation_time,
save_constant_metadata,
) )
from calibration_client import CalibrationClient
from cal_tools.restful_config import restful_config from cal_tools.restful_config import restful_config
from cal_tools.step_timing import StepTimer
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
out_folder = Path(out_folder) out_folder = Path(out_folder)
out_folder.mkdir(exist_ok=True) out_folder.mkdir(exist_ok=True)
metadata = CalibrationMetadata(metadata_folder or out_folder) metadata = CalibrationMetadata(metadata_folder or out_folder)
# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml # Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml
retrieved_constants = metadata.setdefault("retrieved-constants", {}) retrieved_constants = metadata.setdefault("retrieved-constants", {})
creation_time = calcat_creation_time(in_folder, run, creation_time) creation_time = calcat_creation_time(in_folder, run, creation_time)
print(f'Using {creation_time.isoformat()} as creation time') print(f'Using {creation_time.isoformat()} as creation time')
# Pick all modules/aggregators or those selected. # Pick all modules/aggregators or those selected.
if karabo_da == ['all']: if not karabo_da or karabo_da == ['']:
if modules == [-1]: if not modules or modules == [-1]:
modules = list(range(16)) modules = list(range(16))
karabo_da = [f'LPD{i:02d}' for i in modules]
else:
modules = [int(x[-2:]) for x in karabo_da]
```
%% Cell type:code id: tags: karabo_da = [f'LPD{i:02d}' for i in modules]
``` python
step_timer = StepTimer()
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
# Connect to CalCat. # Connect to CalCat.
calcat_config = restful_config['calcat'] calcat_config = restful_config['calcat']
client = CalibrationClient( client = CalibrationClient(
base_api_url=calcat_config['base-api-url'], base_api_url=calcat_config['base-api-url'],
use_oauth2=calcat_config['use-oauth2'], use_oauth2=calcat_config['use-oauth2'],
client_id=calcat_config['user-id'], client_id=calcat_config['user-id'],
client_secret=calcat_config['user-secret'], client_secret=calcat_config['user-secret'],
user_email=calcat_config['user-email'], user_email=calcat_config['user-email'],
token_url=calcat_config['token-url'], token_url=calcat_config['token-url'],
refresh_url=calcat_config['refresh-url'], refresh_url=calcat_config['refresh-url'],
auth_url=calcat_config['auth-url'], auth_url=calcat_config['auth-url'],
scope='') scope='')
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
step_timer.start() dark_calibrations = {
lpd_cal = LPD_CalibrationData( 1: 'Offset',
detector_name=karabo_id, 14: 'BadPixelsDark',
modules=karabo_da, }
sensor_bias_voltage=bias_voltage,
memory_cells=mem_cells, dark_condition = [
feedback_capacitor=capacitor, dict(parameter_id=1, value=bias_voltage), # Sensor bias voltage
source_energy=photon_energy, dict(parameter_id=7, value=mem_cells), # Memory cells
category=category, dict(parameter_id=15, value=capacitor), # Feedback capacitor
event_at=creation_time, dict(parameter_id=13, value=256), # Pixels X
snapshot_at=creation_time, dict(parameter_id=14, value=256), # Pixels Y
client=client,
)
constant_names = [
"Offset", "BadPixelsDark",
"BadPixelsFF", "GainAmpMap",
"FFMap", "RelativeGain",
] ]
print('Querying calibration database', end='', flush=True) print('Querying calibration database', end='', flush=True)
start = perf_counter() start = perf_counter()
for k_da in karabo_da: for k_da in karabo_da:
pdu = None pdu = None
if k_da in retrieved_constants: if k_da in retrieved_constants:
print(f"Constant for {k_da} already in {metadata.filename}, won't query again.") # noqa print(f"Constant for {k_da} already in {metadata.filename}, won't query again.") # noqa
continue continue
retrieved_constants[k_da] = dict() retrieved_constants[k_da] = dict()
const_mdata = retrieved_constants[k_da]["constants"] = dict() const_mdata = retrieved_constants[k_da]["constants"] = dict()
for calibrations, condition in [ for calibrations, condition in [
(dark_calibrations, dark_condition), (dark_calibrations, dark_condition),
(illuminated_calibrations, illuminated_condition) (illuminated_calibrations, illuminated_condition)
]: ]:
resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions( resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(
client, karabo_id, list(calibrations.keys()), client, karabo_id, list(calibrations.keys()),
{'parameters_conditions_attributes': condition}, {'parameters_conditions_attributes': condition},
karabo_da=k_da, event_at=creation_time.isoformat()) karabo_da=k_da, event_at=creation_time.isoformat())
lpd_cal_metadata = lpd_cal.metadata() lpd_cal_metadata = lpd_cal.metadata()
# Validate the constants availability and raise/warn correspondingly. # Validate the constants availability and raise/warn correspondingly.
for mod, ccv_dict in lpd_cal_metadata.items(): for mod, ccv_dict in lpd_cal_metadata.items():
missing_dark_constants = set( missing_dark_constants = set(
c for c in ["Offset", "BadPixelsDark"] if c not in ccv_dict.keys()) c for c in ["Offset", "BadPixelsDark"] if c not in ccv_dict.keys())
missing_gain_constants = set( missing_gain_constants = set(
c for c in ["BadPixelsFF", "GainAmpMap", "FFMap", "RelativeGain"] if c not in ccv_dict.keys()) # noqa c for c in ["BadPixelsFF", "GainAmpMap", "FFMap", "RelativeGain"] if c not in ccv_dict.keys()) # noqa
if missing_dark_constants: if missing_dark_constants:
raise KeyError( raise KeyError(
f"Dark constants {missing_dark_constants} are not available for correction. Module: {mod}") # noqa f"Dark constants {missing_dark_constants} are not available for correction. Module: {mod}") # noqa
if missing_gain_constants: if missing_gain_constants:
warning( warning(
f"Gain constants {missing_gain_constants} were not retrieved. Module: {mod}") f"Gain constants {missing_gain_constants} were not retrieved. Module: {mod}")
for mod, ccv_dict in lpd_cal_metadata.items(): for mod, ccv_dict in lpd_cal_metadata.items():
mdata_dict = {"constants": dict()} mdata_dict = {"constants": dict()}
for cname, ccv_metadata in ccv_dict.items(): for cname, ccv_metadata in ccv_dict.items():
mdata_dict["constants"][cname] = { mdata_dict["constants"][cname] = {
"path": str(lpd_cal.caldb_root / ccv_metadata["path"]), "path": str(lpd_cal.caldb_root / ccv_metadata["path"]),
"dataset": ccv_metadata["dataset"], "dataset": ccv_metadata["dataset"],
"creation-time": ccv_metadata["begin_validity_at"], "creation-time": ccv_metadata["begin_validity_at"],
} }
mdata_dict["physical-name"] = ccv_metadata["physical_name"] mdata_dict["physical-name"] = ccv_metadata["physical_name"]
retrieved_constants[mod] = mdata_dict retrieved_constants[mod] = mdata_dict
metadata.save() metadata.save()
step_timer.done_step(f"Stored retrieved constants in {metadata.filename}.") total_time = perf_counter() - start
print(f'{total_time:.1f}s')
print(f"Stored retrieved constants in {metadata.filename}")
``` ```
......
notebooks/ePix100/Correction_ePix100_NBC.ipynb
+ 2
− 2
View file @ 070d574f
...@@ -881,9 +881,9 @@ ...@@ -881,9 +881,9 @@
], ],
"metadata": { "metadata": {
"kernelspec": { "kernelspec": {
"display_name": ".cal2_venv", "display_name": "cal_venv",
"language": "python", "language": "python",
"name": "python3" "name": "cal_venv"
}, },
"language_info": { "language_info": {
"codemirror_mode": { "codemirror_mode": {
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts