Skip to content
Snippets Groups Projects
Commit 32dde25f authored by Karim Ahmed's avatar Karim Ahmed
Browse files

update lpd notebook

parent d14acc38
Branches fix/manual_edit_conditions
No related tags found
1 merge request!738New calcat interface
Hide whitespace changes
Inline Side-by-side
notebooks/LPD/LPD_Correct_Fast.ipynb
+ 8
− 3
View file @ 32dde25f
...@@ -605,9 +605,9 @@ ...@@ -605,9 +605,9 @@
], ],
"metadata": { "metadata": {
"kernelspec": { "kernelspec": {
"display_name": "pycal", "display_name": ".cal2_venv",
"language": "python", "language": "python",
"name": "pycal" "name": "python3"
}, },
"language_info": { "language_info": {
"codemirror_mode": { "codemirror_mode": {
...@@ -619,7 +619,12 @@ ...@@ -619,7 +619,12 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.8.11" "version": "3.8.11 (default, Jul 2 2021, 14:23:46) \n[GCC 4.8.5 20150623 (Red Hat 4.8.5-44)]"
},
"vscode": {
"interpreter": {
"hash": "8817b9fbc45d4ec68a62bc54fa52c1c5ac00d5619f7fe87d01def1798ea4889e"
}
} }
}, },
"nbformat": 4, "nbformat": 4,
......
notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
+ 43
− 24
View file @ 32dde25f
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# LPD Retrieving Constants Pre-correction # # LPD Retrieving Constants Pre-correction #
Author: European XFEL Detector Group, Version: 1.0 Author: European XFEL Detector Group, Version: 1.0
The following notebook provides a constants metadata in a YAML file to use while correcting LPD images. The following notebook provides a constants metadata in a YAML file to use while correcting LPD images.
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
# Input parameters # Input parameters
in_folder = "/gpfs/exfel/exp/FXE/202201/p003073/raw/" # the folder to read data from, required in_folder = "/gpfs/exfel/exp/FXE/202201/p003073/raw/" # the folder to read data from, required
out_folder = "/gpfs/exfel/data/scratch/ahmedk/test/remove/LPD_test" # the folder to output to, required out_folder = "/gpfs/exfel/data/scratch/ahmedk/test/remove/LPD_test" # the folder to output to, required
metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate. metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate.
modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty
karabo_da = ['all'] # Data aggregators names to correct, use [''] for all karabo_da = ['all'] # Data aggregators names to correct, use [''] for all
run = 10 # run to process, required run = 10 # run to process, required
# Source parameters # Source parameters
karabo_id = 'FXE_DET_LPD1M-1' # Karabo domain for detector. karabo_id = 'FXE_DET_LPD1M-1' # Karabo domain for detector.
# CalCat parameters # CalCat parameters
creation_time = "" # The timestamp to use with Calibration DB. Required Format: "YYYY-MM-DD hh:mm:ss" e.g. 2019-07-04 11:02:41 creation_time = "" # The timestamp to use with Calibration DB. Required Format: "YYYY-MM-DD hh:mm:ss" e.g. 2019-07-04 11:02:41
# Operating conditions # Operating conditions
mem_cells = 512 # Memory cells, LPD constants are always taken with 512 cells. mem_cells = 512 # Memory cells, LPD constants are always taken with 512 cells.
bias_voltage = 250.0 # Detector bias voltage. bias_voltage = 250.0 # Detector bias voltage.
capacitor = '5pF' # Capacitor setting: 5pF or 50pF capacitor = '5pF' # Capacitor setting: 5pF or 50pF
photon_energy = 9.2 # Photon energy in keV. photon_energy = 9.2 # Photon energy in keV.
category = 0 # Whom to blame. category = 0 # Whom to blame.
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
from logging import warning
from pathlib import Path from pathlib import Path
from cal_tools.calcat_interface import CalCatError, LPD_CalibrationData from cal_tools.calcat_interface import LPD_CalibrationData
from cal_tools.tools import ( from cal_tools.tools import (
CalibrationMetadata, CalibrationMetadata,
calcat_creation_time, calcat_creation_time,
) )
from cal_tools.restful_config import restful_config from cal_tools.restful_config import restful_config
from cal_tools.step_timing import StepTimer from cal_tools.step_timing import StepTimer
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
out_folder = Path(out_folder) out_folder = Path(out_folder)
out_folder.mkdir(exist_ok=True) out_folder.mkdir(exist_ok=True)
metadata = CalibrationMetadata(metadata_folder or out_folder) metadata = CalibrationMetadata(metadata_folder or out_folder)
# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml # Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml
retrieved_constants = metadata.setdefault("retrieved-constants", {}) retrieved_constants = metadata.setdefault("retrieved-constants", {})
creation_time = calcat_creation_time(in_folder, run, creation_time) creation_time = calcat_creation_time(in_folder, run, creation_time)
print(f'Using {creation_time.isoformat()} as creation time') print(f'Using {creation_time.isoformat()} as creation time')
# Pick all modules/aggregators or those selected. # Pick all modules/aggregators or those selected.
if karabo_da == ['all']: if karabo_da == ['all']:
if modules == [-1]: if modules == [-1]:
modules = list(range(16)) modules = list(range(16))
karabo_da = [f'LPD{i:02d}' for i in modules] karabo_da = [f'LPD{i:02d}' for i in modules]
else: else:
modules = [int(x[-2:]) for x in karabo_da] modules = [int(x[-2:]) for x in karabo_da]
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
step_timer = StepTimer() step_timer = StepTimer()
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
const_data = {} step_timer.start()
lpd_cal = LPD_CalibrationData(
detector_name=karabo_id,
modules=karabo_da,
sensor_bias_voltage=bias_voltage,
memory_cells=mem_cells,
feedback_capacitor=capacitor,
source_energy=photon_energy,
category=category,
event_at=creation_time,
snapshot_at=creation_time,
)
constant_names = [
"Offset", "BadPixelsDark",
"BadPixelsFF", "GainAmpMap",
"FFMap", "RelativeGain",
]
print('Querying calibration database', end='', flush=True) print('Querying calibration database', end='', flush=True)
start = perf_counter() start = perf_counter()
for k_da in karabo_da: for k_da in karabo_da:
pdu = None pdu = None
if k_da in retrieved_constants: if k_da in retrieved_constants:
print(f"Constant for {k_da} already in {metadata.filename}, won't query again.") # noqa print(f"Constant for {k_da} already in {metadata.filename}, won't query again.") # noqa
continue continue
retrieved_constants[k_da] = dict() retrieved_constants[k_da] = dict()
const_mdata = retrieved_constants[k_da]["constants"] = dict() const_mdata = retrieved_constants[k_da]["constants"] = dict()
for calibrations, condition in [ for calibrations, condition in [
(dark_calibrations, dark_condition), (dark_calibrations, dark_condition),
(illuminated_calibrations, illuminated_condition) (illuminated_calibrations, illuminated_condition)
]: ]:
resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions( resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(
client, karabo_id, list(calibrations.keys()), client, karabo_id, list(calibrations.keys()),
{'parameters_conditions_attributes': condition}, {'parameters_conditions_attributes': condition},
karabo_da=k_da, event_at=creation_time.isoformat()) karabo_da=k_da, event_at=creation_time.isoformat())
if not resp["success"]: lpd_cal_metadata = lpd_cal.metadata()
print(f"ERROR: Constants {list(calibrations.values())} "
f"were not retrieved, {resp['app_info']}")
for cname in calibrations.values():
const_mdata[cname] = dict()
const_mdata[cname]["file-path"] = None
const_mdata[cname]["dataset-name"] = None
const_mdata[cname]["creation-time"] = None
continue
for ccv in resp["data"]:
cc = ccv['calibration_constant']
cname = calibrations[cc['calibration_id']]
const_mdata[cname] = dict()
const_mdata[cname]["file-path"] = str(Path(ccv['path_to_file']) / ccv['file_name'])
const_mdata[cname]["dataset-name"] = ccv['data_set_name']
const_mdata[cname]["creation-time"] = ccv['begin_at']
pdu = ccv['physical_detector_unit']['physical_name']
print('.', end='', flush=True) # Validate the constants availability and raise/warn correspondingly.
retrieved_constants[k_da]["physical-detector-unit"] = pdu for mod, ccv_dict in lpd_cal_metadata.items():
missing_dark_constants = set(
c for c in ["Offset", "BadPixelsDark"] if c not in ccv_dict.keys())
missing_gain_constants = set(
c for c in ["BadPixelsFF", "GainAmpMap", "FFMap", "RelativeGain"] if c not in ccv_dict.keys()) # noqa
if missing_dark_constants:
raise KeyError(
f"Dark constants {missing_dark_constants} are not available for correction. Module: {mod}") # noqa
if missing_gain_constants:
warning(
f"Gain constants {missing_gain_constants} were not retrieved. Module: {mod}")
for mod, ccv_dict in lpd_cal_metadata.items():
mdata_dict = {"constants": dict()}
for cname, ccv_metadata in ccv_dict.items():
mdata_dict["constants"][cname] = {
"path": str(lpd_cal.caldb_root / ccv_metadata["path"]),
"dataset": ccv_metadata["dataset"],
"creation-time": ccv_metadata["begin_validity_at"],
}
mdata_dict["physical-name"] = ccv_metadata["physical_name"]
retrieved_constants[mod] = mdata_dict
metadata.save() metadata.save()
total_time = perf_counter() - start step_timer.done_step(f"Stored retrieved constants in {metadata.filename}.")
print(f'{total_time:.1f}s')
print(f"Stored retrieved constants in {metadata.filename}")
``` ```
......
notebooks/ePix100/Correction_ePix100_NBC.ipynb
+ 8
− 3
View file @ 32dde25f
...@@ -846,9 +846,9 @@ ...@@ -846,9 +846,9 @@
], ],
"metadata": { "metadata": {
"kernelspec": { "kernelspec": {
"display_name": "cal_venv", "display_name": ".cal2_venv",
"language": "python", "language": "python",
"name": "cal_venv" "name": "python3"
}, },
"language_info": { "language_info": {
"codemirror_mode": { "codemirror_mode": {
...@@ -860,7 +860,7 @@ ...@@ -860,7 +860,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.8.11" "version": "3.8.11 (default, Jul 2 2021, 14:23:46) \n[GCC 4.8.5 20150623 (Red Hat 4.8.5-44)]"
}, },
"latex_envs": { "latex_envs": {
"LaTeX_envs_menu_present": true, "LaTeX_envs_menu_present": true,
...@@ -878,6 +878,11 @@ ...@@ -878,6 +878,11 @@
"latex_user_defs": false, "latex_user_defs": false,
"report_style_numbering": false, "report_style_numbering": false,
"user_envs_cfg": false "user_envs_cfg": false
},
"vscode": {
"interpreter": {
"hash": "8817b9fbc45d4ec68a62bc54fa52c1c5ac00d5619f7fe87d01def1798ea4889e"
}
} }
}, },
"nbformat": 4, "nbformat": 4,
......
notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
+ 3
− 3
View file @ 32dde25f
...@@ -193,9 +193,9 @@ ...@@ -193,9 +193,9 @@
], ],
"metadata": { "metadata": {
"kernelspec": { "kernelspec": {
"display_name": "Python 3.8.11 ('.cal4_venv')", "display_name": "cal2_venv",
"language": "python", "language": "python",
"name": "python3" "name": "cal2_venv"
}, },
"language_info": { "language_info": {
"codemirror_mode": { "codemirror_mode": {
...@@ -207,7 +207,7 @@ ...@@ -207,7 +207,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.8.11" "version": "3.8.11 (default, Jul 2 2021, 14:23:46) \n[GCC 4.8.5 20150623 (Red Hat 4.8.5-44)]"
}, },
"orig_nbformat": 4, "orig_nbformat": 4,
"vscode": { "vscode": {
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts