diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index dae62fab736d82aa5fa07a4cfc25d934ad21f9c2..aeafce49cdd1c00adea7ea7433fc4ef4ff5360c4 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -18,7 +18,7 @@
"outputs": [],
"source": [
"in_folder = \"/gpfs/exfel/exp/MID/202201/p002834/raw\" # the folder to read data from, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_\" # the folder to output to, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/esobolev/pycal_litfrm/p002834/r0225\" # the folder to output to, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
@@ -36,18 +36,19 @@
"\n",
"slopes_ff_from_files = \"\" # Path to locally stored SlopesFF and BadPixelsFF constants, loaded in precorrection notebook\n",
"\n",
- "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
+ "use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
"cal_db_interface = \"tcp://max-exfl016:8015#8045\" # the database interface to use\n",
"cal_db_timeout = 30000 # in milliseconds\n",
"creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
"\n",
- "mem_cells = -1 # Number of memory cells used, set to 0 to automatically infer\n",
- "bias_voltage = -1 # bias voltage, set to 0 to use stored value in slow data.\n",
- "acq_rate = -1. # the detector acquisition rate, use 0 to try to auto-determine\n",
+ "mem_cells = 0 # Number of memory cells used, set to 0 to automatically infer\n",
+ "bias_voltage = 0 # bias voltage, set to 0 to use stored value in slow data.\n",
+ "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
"gain_setting = -1 # the gain setting, use -1 to use value stored in slow data.\n",
"gain_mode = -1 # gain mode (0: adaptive, 1-3 fixed high/med/low, -1: read from CONTROL data)\n",
+ "overwrite = True # set to True if existing data should be overwritten\n",
"max_pulses = [0, 352, 1] # range list [st, end, step] of memory cell indices to be processed within a train. 3 allowed maximum list input elements.\n",
- "mem_cells_db = -1 # set to a value different than 0 to use this value for DB queries\n",
+ "mem_cells_db = 0 # set to a value different than 0 to use this value for DB queries\n",
"integration_time = -1 # integration time, negative values for auto-detection.\n",
"\n",
"# Correction parameters\n",
@@ -122,6 +123,7 @@
"outputs": [],
"source": [
"import itertools\n",
+ "import os\n",
"import math\n",
"import multiprocessing\n",
"import re\n",
@@ -139,7 +141,7 @@
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"import yaml\n",
- "from extra_data import RunDirectory, stack_detector_data\n",
+ "from extra_data import H5File, RunDirectory, stack_detector_data, by_id\n",
"from extra_geom import AGIPD_1MGeometry, AGIPD_500K2GGeometry\n",
"from matplotlib import cm as colormap\n",
"from matplotlib.colors import LogNorm\n",
@@ -162,14 +164,11 @@
")\n",
"from cal_tools.ana_tools import get_range\n",
"from cal_tools.enums import AgipdGainMode, BadPixels\n",
- "from cal_tools.restful_config import restful_config\n",
"from cal_tools.step_timing import StepTimer\n",
- "from cal_tools.tools import (\n",
- " CalibrationMetadata,\n",
- " calcat_creation_time,\n",
- " map_modules_from_folder,\n",
- " module_index_to_qm,\n",
- ")"
+ "\n",
+ "sns.set()\n",
+ "sns.set_context(\"paper\", font_scale=1.4)\n",
+ "sns.set_style(\"ticks\")"
]
},
{
@@ -283,7 +282,7 @@
"else:\n",
" modules = [int(x[-2:]) for x in karabo_da]\n",
"\n",
- "print(\"Process modules:\", ', '.join(module_index_to_qm(x) for x in modules))\n",
+ "print(\"Process modules:\", ', '.join(cal_tools.tools.module_index_to_qm(x) for x in modules))\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"print(f\"Instrument {instrument}\")\n",
"print(f\"Detector instance {dinstance}\")"
@@ -326,7 +325,7 @@
"outputs": [],
"source": [
"# set everything up filewise\n",
- "mapped_files, _, total_sequences, _, _ = map_modules_from_folder(\n",
+ "mapped_files, _, total_sequences, _, _ = cal_tools.tools.map_modules_from_folder(\n",
" str(in_folder), run, path_template, karabo_da, sequences\n",
")\n",
"file_list = []\n",
@@ -376,21 +375,22 @@
"metadata": {},
"outputs": [],
"source": [
- "# Run's creation time:\n",
- "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
- "offset = parser.parse(creation_date_offset)\n",
- "delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
- "creation_time += delta\n",
- "print(f\"Creation time: {creation_time}\")\n",
- "\n",
- "if acq_rate == -1.:\n",
+ "# Evaluate creation time\n",
+ "creation_time = None\n",
+ "if use_dir_creation_date:\n",
+ " creation_time = cal_tools.tools.get_dir_creation_date(str(in_folder), run)\n",
+ " offset = parser.parse(creation_date_offset)\n",
+ " delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
+ " creation_time += delta\n",
+ "\n",
+ "if acq_rate == 0.:\n",
" acq_rate = agipd_cond.get_acq_rate()\n",
- "if mem_cells == -1:\n",
+ "if mem_cells == 0.:\n",
" mem_cells = agipd_cond.get_num_cells()\n",
"# TODO: look for alternative for passing creation_time\n",
"if gain_setting == -1:\n",
" gain_setting = agipd_cond.get_gain_setting(creation_time)\n",
- "if bias_voltage == -1:\n",
+ "if bias_voltage == 0.:\n",
" bias_voltage = agipd_cond.get_bias_voltage(karabo_id_control)\n",
"if integration_time == -1:\n",
" integration_time = agipd_cond.get_integration_time()\n",
@@ -407,9 +407,9 @@
"outputs": [],
"source": [
"if mem_cells is None:\n",
- " raise ValueError(f\"No raw images found for {instrument_src_mod}\")\n",
+ " raise ValueError(f\"No raw images found in {filename}\")\n",
"\n",
- "mem_cells_db = mem_cells if mem_cells_db == -1 else mem_cells_db\n",
+ "mem_cells_db = mem_cells if mem_cells_db == 0 else mem_cells_db\n",
"\n",
"print(f\"Maximum memory cells to calibrate: {mem_cells}\")"
]
@@ -551,37 +551,6 @@
"module_index_to_karabo_da = {mod: da for (mod, da) in zip(modules, karabo_da)}"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# NOTE: this notebook will not overwrite calibration metadata file\n",
- "const_yaml = metadata.get(\"retrieved-constants\", {})"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Connect to CalCat.\n",
- "calcat_config = restful_config['calcat']\n",
- "client = CalibrationClient(\n",
- " base_api_url=calcat_config['base-api-url'],\n",
- " use_oauth2=calcat_config['use-oauth2'],\n",
- " client_id=calcat_config['user-id'],\n",
- " client_secret=calcat_config['user-secret'],\n",
- " user_email=calcat_config['user-email'],\n",
- " token_url=calcat_config['token-url'],\n",
- " refresh_url=calcat_config['refresh-url'],\n",
- " auth_url=calcat_config['auth-url'],\n",
- " scope='')"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -601,6 +570,7 @@
"\n",
" Metadata for constants is taken from yml file or retrieved from the DB\n",
" \"\"\"\n",
+ " err = \"\"\n",
" k_da = module_index_to_karabo_da[mod]\n",
" # check if there is a yaml file in out_folder that has the device constants.\n",
" if k_da in const_yaml:\n",
@@ -638,50 +608,6 @@
"print(f\"Constants were loaded in {perf_counter()-ts:.01f}s\")"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# if the yml file contains \"retrieved-constants\", that means a leading\n",
- "# notebook got processed and the reporting would be generated from it.\n",
- "fst_print = True\n",
- "timestamps = {}\n",
- "\n",
- "for i, (modno, when, k_da) in enumerate(const_out):\n",
- " qm = module_index_to_qm(modno)\n",
- "\n",
- " if k_da not in const_yaml:\n",
- " if fst_print:\n",
- " print(\"Constants are retrieved with creation time: \")\n",
- " fst_print = False\n",
- "\n",
- " module_timestamps = {}\n",
- "\n",
- " print(f\"{qm}:\")\n",
- " for key, item in when.items():\n",
- " if hasattr(item, 'strftime'):\n",
- " item = item.strftime('%y-%m-%d %H:%M')\n",
- " when[key] = item\n",
- " print('{:.<12s}'.format(key), item)\n",
- "\n",
- " # Store few time stamps if exists\n",
- " # Add NA to keep array structure\n",
- " for key in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
- " if when and key in when and when[key]:\n",
- " module_timestamps[key] = when[key]\n",
- " else:\n",
- " module_timestamps[key] = \"NA\"\n",
- " timestamps[qm] = module_timestamps\n",
- "\n",
- "seq = sequences[0] if sequences else 0\n",
- "\n",
- "if timestamps:\n",
- " with open(f\"{out_folder}/retrieved_constants_s{seq}.yml\",\"w\") as fd:\n",
- " yaml.safe_dump({\"time-summary\": {f\"S{seq}\": timestamps}}, fd)"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -830,8 +756,11 @@
"fst_print = True\n",
"timestamps = {}\n",
"\n",
- "for i, (modno, when, k_da) in enumerate(const_out):\n",
- " qm = module_index_to_qm(modno)\n",
+ "for i, (error, modno, when, k_da) in enumerate(const_out):\n",
+ " qm = cal_tools.tools.module_index_to_qm(modno)\n",
+ " # expose errors while applying correction\n",
+ " if error:\n",
+ " print(\"Error: {}\".format(error) )\n",
"\n",
" if k_da not in const_yaml:\n",
" if fst_print:\n",
@@ -840,12 +769,14 @@
"\n",
" module_timestamps = {}\n",
"\n",
- " print(f\"{qm}:\")\n",
- " for key, item in when.items():\n",
- " if hasattr(item, 'strftime'):\n",
- " item = item.strftime('%y-%m-%d %H:%M')\n",
- " when[key] = item\n",
- " print('{:.<12s}'.format(key), item)\n",
+ " # If correction is crashed\n",
+ " if not error:\n",
+ " print(f\"{qm}:\")\n",
+ " for key, item in when.items():\n",
+ " if hasattr(item, 'strftime'):\n",
+ " item = item.strftime('%y-%m-%d %H:%M')\n",
+ " when[key] = item\n",
+ " print('{:.<12s}'.format(key), item)\n",
"\n",
" # Store few time stamps if exists\n",
" # Add NA to keep array structure\n",
@@ -853,7 +784,10 @@
" if when and key in when and when[key]:\n",
" module_timestamps[key] = when[key]\n",
" else:\n",
- " module_timestamps[key] = \"NA\"\n",
+ " if error is not None:\n",
+ " module_timestamps[key] = \"Err\"\n",
+ " else:\n",
+ " module_timestamps[key] = \"NA\"\n",
" timestamps[qm] = module_timestamps\n",
"\n",
"seq = sequences[0] if sequences else 0\n",
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 62f64d8c5f14840681c6b0fff79d4ff2950ac819..4f155d4a8c82d35192fe855734d580ad065ff00b 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -20,25 +20,28 @@
"in_folder = \"/gpfs/exfel/exp/SPB/202030/p900119/raw\" # the folder to read data from, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_\" # the folder to output to, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
+ "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
"run = 80 # runs to process, required\n",
"\n",
"karabo_id = \"SPB_DET_AGIPD1M-1\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
+ "path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # the template to use to access data\n",
"ctrl_source_template = '{}/MDL/FPGA_COMP_TEST' # path to control information\n",
"instrument_source_template = '{}/DET/{}:xtdf' # path in the HDF5 file to images\n",
"receiver_template = \"{}CH0\" # inset for receiver devices\n",
"karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for control device\n",
"\n",
- "# Parameters for calibration database.\n",
+ "use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
"cal_db_interface = \"tcp://max-exfl016:8015#8045\" # the database interface to use\n",
"creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
- "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
"\n",
"slopes_ff_from_files = \"\" # Path to locally stored SlopesFF and BadPixelsFF constants\n",
- "mem_cells = -1 # number of memory cells used, set to 0 to automatically infer\n",
- "bias_voltage = -1 # bias voltage, set to 0 to use stored value in slow data.\n",
- "acq_rate = -1. # the detector acquisition rate, use 0 to try to auto-determine\n",
+ "calfile = \"\" # path to calibration file. Leave empty if all data should come from DB\n",
+ "nodb = False # if set only file-based constants will be used\n",
+ "mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
+ "bias_voltage = 0 # bias voltage, set to 0 to use stored value in slow data.\n",
+ "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
"gain_setting = -1 # the gain setting, use -1 to use value stored in slow data.\n",
"gain_mode = -1 # gain mode (0: adaptive, 1-3 fixed high/med/low, -1: read from CONTROL data)\n",
"integration_time = -1 # integration time, negative values for auto-detection.\n",
@@ -50,6 +53,7 @@
"blc_noise = False # if set, baseline correction via noise peak location is attempted\n",
"blc_stripes = False # if set, baseline corrected via stripes\n",
"blc_hmatch = False # if set, base line correction via histogram matching is attempted\n",
+ "match_asics = False # if set, inner ASIC borders are matched to the same signal level\n",
"adjust_mg_baseline = False # adjust medium gain baseline to match highest high gain value"
]
},
@@ -86,7 +90,10 @@
"from pathlib import Path\n",
"from typing import Tuple\n",
"\n",
+ "import matplotlib\n",
+ "import matplotlib.pyplot as plt\n",
"import multiprocessing\n",
+ "import numpy as np\n",
"from datetime import timedelta\n",
"from dateutil import parser\n",
"from extra_data import RunDirectory\n",
@@ -115,9 +122,7 @@
"# slopes_ff_from_files left as str for now\n",
"in_folder = Path(in_folder)\n",
"out_folder = Path(out_folder)\n",
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
- "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})"
+ "metadata = tools.CalibrationMetadata(metadata_folder or out_folder)"
]
},
{
@@ -126,17 +131,21 @@
"metadata": {},
"outputs": [],
"source": [
- "# Run's creation time:\n",
- "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
- "offset = parser.parse(creation_date_offset)\n",
- "delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
- "creation_time += delta\n",
- "print(f\"Creation time: {creation_time}\")\n",
+ "creation_time = None\n",
+ "if use_dir_creation_date:\n",
+ " creation_time = tools.get_dir_creation_date(str(in_folder), run)\n",
+ " offset = parser.parse(creation_date_offset)\n",
+ " delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
+ " creation_time += delta\n",
+ " print(f\"Using {creation_time} as creation time\")\n",
+ "\n",
+ "if sequences[0] == -1:\n",
+ " sequences = None\n",
" \n",
"print(f\"Outputting to {out_folder}\")\n",
"out_folder.mkdir(parents=True, exist_ok=True)\n",
"\n",
- "melt_snow = False if corr_bools[\"only_offset\"] else SnowResolution.NONE"
+ "melt_snow = False if corr_bools[\"only_offset\"] else agipdlib.SnowResolution.NONE"
]
},
{
@@ -191,7 +200,7 @@
"metadata": {},
"outputs": [],
"source": [
- "agipd_cond = AgipdCtrl(\n",
+ "agipd_cond = agipdlib.AgipdCtrl(\n",
" run_dc=run_dc,\n",
" image_src=None, # Not neededed, as we wont read mem_cells or acq_rate.\n",
" ctrl_src=ctrl_src,\n",
@@ -199,7 +208,7 @@
"\n",
"if gain_setting == -1:\n",
" gain_setting = agipd_cond.get_gain_setting(creation_time)\n",
- "if bias_voltage == -1:\n",
+ "if bias_voltage == 0.:\n",
" bias_voltage = agipd_cond.get_bias_voltage(karabo_id_control)\n",
"if integration_time == -1:\n",
" integration_time = agipd_cond.get_integration_time()\n",
@@ -364,36 +373,39 @@
"metadata": {},
"outputs": [],
"source": [
- "def find_cells_acq_rate_module(idx):\n",
- " \"\"\"Find memory cells and acquisition rate for a module.\"\"\"\n",
- " agipd_cond.image_src = instrument_src.format(idx)\n",
- "\n",
- " if agipd_cond.image_src not in instr_dc.all_sources:\n",
- " warning(f\"No raw images found for {module_index_to_qm(idx)}({k_da}).\")\n",
- " return None, None\n",
- " \n",
- " _mem_cells = agipd_cond.get_num_cells() if mem_cells == -1 else mem_cells\n",
- " _acq_rate = agipd_cond.get_acq_rate() if acq_rate == -1 else acq_rate\n",
- "\n",
- " return _mem_cells, _acq_rate\n",
- "\n",
- "\n",
- "all_mem_cells = set()\n",
- "all_acq_rates = set()\n",
+ "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
+ "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "pc_bools = [corr_bools.get(\"rel_gain\"),\n",
+ " corr_bools.get(\"adjust_mg_baseline\"),\n",
+ " corr_bools.get('blc_noise'),\n",
+ " corr_bools.get('blc_hmatch'),\n",
+ " corr_bools.get('blc_stripes'),\n",
+ " melt_snow]\n",
+ "inp = []\n",
+ "only_dark = False\n",
+ "nodb_with_dark = False\n",
+ "if not nodb:\n",
+ " only_dark = (calfile != \"\")\n",
+ "if calfile != \"\" and not corr_bools[\"only_offset\"]:\n",
+ " nodb_with_dark = nodb\n",
"\n",
- "with multiprocessing.Pool(processes=nmods) as pool:\n",
- " results = pool.map(find_cells_acq_rate_module, modules)\n",
+ "da_to_qm = dict()\n",
+ "for module_index, k_da in zip(modules, karabo_da):\n",
+ " da_to_qm[k_da] = tools.module_index_to_qm(module_index)\n",
+ " if k_da in retrieved_constants:\n",
+ " print(\n",
+ " f\"Constant for {k_da} already in calibration_metadata.yml, won't query again.\")\n",
+ " continue\n",
"\n",
- "for (_mem_cells, _acq_rate) in results:\n",
- " all_mem_cells.add(_mem_cells)\n",
- " all_acq_rates.add(_acq_rate)\n",
- "# Validate that mem_cells and acq_rate are the same for all modules.\n",
- "if len(all_mem_cells) != 1 or len(all_acq_rates) != 1:\n",
- " warning(\n",
- " \"Number of memory cells or acquisition rate are not identical for all modules.\\n\"\n",
- " f\"different mem_cells values: {all_mem_cells}.\\ndifferent acq_rate values: {all_acq_rates}.\")\n",
- "mem_cells = all_mem_cells.pop()\n",
- "acq_rate = all_acq_rates.pop()"
+ " inp.append((k_da, module_index))"
]
},
{
@@ -466,25 +478,6 @@
"\n",
"metadata.save()"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "print(\"\\nRetrieved constants for modules:\",\n",
- " ', '.join([module_index_to_qm(x) for x in modules]))\n",
- "print(f\"Operating conditions are:\")\n",
- "print(f\"• Bias voltage: {bias_voltage}\")\n",
- "print(f\"• Memory cells: {mem_cells}\")\n",
- "print(f\"• Acquisition rate: {acq_rate}\")\n",
- "print(f\"• Gain mode: {gain_mode.name}\")\n",
- "print(f\"• Gain setting: {gain_setting}\")\n",
- "print(f\"• Integration time: {integration_time}\")\n",
- "print(f\"• Photon Energy: 9.2\")\n",
- "step_timer.done_step(f\"Constant metadata is saved under \\\"retrieved-constants\\\" in {metadata.filename}\")"
- ]
}
],
"metadata": {
diff --git a/src/cal_tools/agipdlib.py b/src/cal_tools/agipdlib.py
index 13781f7cd78cfb9197bfbf0f5352d18aea735dc0..fe7e1079d9e4d278a6c1156411bead1397a9b0c5 100644
--- a/src/cal_tools/agipdlib.py
+++ b/src/cal_tools/agipdlib.py
@@ -2,19 +2,20 @@ import os
import posixpath
import zlib
from datetime import datetime
-from logging import warning
from multiprocessing import Manager
from multiprocessing.pool import ThreadPool
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Optional, Tuple
import h5py
import numpy as np
import sharedmem
from dateutil import parser
from extra_data import DataCollection, H5File, by_id, components
+from iCalibrationDB import Conditions, Constants
from cal_tools import agipdalgs as calgs
from cal_tools.agipdutils import (
+ assemble_constant_dict,
baseline_correct_via_noise,
baseline_correct_via_stripe,
correct_baseline_via_hist,
@@ -26,6 +27,7 @@ from cal_tools.agipdutils import (
)
from cal_tools.enums import AgipdGainMode, BadPixels, SnowResolution
from cal_tools.h5_copy_except import h5_copy_except_paths
+from cal_tools.tools import get_from_db
class AgipdCtrl:
@@ -1337,12 +1339,12 @@ class AgipdCorrections:
cons_data = dict()
when = dict()
variant = dict()
- db_module = const_yaml[karabo_da]["physical-name"]
+ db_module = const_yaml[karabo_da]["physical-detector-unit"]
for cname, mdata in const_yaml[karabo_da]["constants"].items():
base_key = f"{db_module}/{cname}/0"
when[cname] = mdata["creation-time"]
if when[cname]:
- with h5py.File(mdata["path"], "r") as cf:
+ with h5py.File(mdata["file-path"], "r") as cf:
cons_data[cname] = np.copy(cf[f"{base_key}/data"])
# Set variant to 0 if the attribute is missing
# as for old constants.
@@ -1352,43 +1354,54 @@ class AgipdCorrections:
variant[cname] = 0
else:
# Create empty constant using the list elements
- cons_data[cname] = getattr(np, mdata["path"][0])(mdata["path"][1]) # noqa
+ cons_data[cname] = getattr(np, mdata["file-path"][0])(mdata["file-path"][1]) # noqa
self.init_constants(cons_data, when, module_idx, variant)
return when
- def initialize_from_db(
- self, karabo_id: str,
- karabo_da: str,
- module_idx: int,
- creation_time: datetime,
- memory_cells: int,
- bias_voltage: int,
- gain_setting: float,
- gain_mode: int,
- acquisition_rate: float,
- integration_time: int,
- photon_energy: float = 9.2,
- client=None,
- ):
+ def initialize_from_db(self, karabo_id: str, karabo_da: str,
+ cal_db_interface: str,
+ creation_time: datetime,
+ memory_cells: float, bias_voltage: int,
+ photon_energy: float, gain_setting: float,
+ acquisition_rate: float, integration_time: int,
+ module_idx: int, only_dark: bool = False):
""" Initialize calibration constants from the calibration database
- :param karabo_id: karabo detector identifier.
- :param karabo_da: karabo module data aggregator name.
- :param module_idx: module index to save retrieved CCV in sharedmem.
- :param creation_time: time for desired calibration constant version.
- :param memory_cells: number of memory cells used for CCV conditions.
- :param bias_voltage: bias voltage used for CCV conditions.
- :param gain_setting: gain setting used for CCV conditions.
- :param gain_mode: gain mode used for CCV conditions.
- :param acquisition_rate: acquistion rate used for CCV conditions.
- :param integration_time: integration time used for CCV conditions.
- :param photon_energy: photon energy value used for CCV conditions.
- :param client: calibration_client object for CalibrationData objects.
-
- Loading the following constants based on the
- correction configuration:
+ :param karabo_id: karabo identifier
+ :param karabo_da: karabo data aggregator
+ :param cal_db_interface: database interaface port
+ :param creation_time: time for desired calibration constant version
+ :param memory_cells: number of memory cells used for CCV conditions
+ :param bias_voltage: bias voltage used for CCV conditions
+ :param photon_energy: photon energy used for CCV conditions
+ :param gain_setting: gain setting used for CCV conditions
+ :param acquisition_rate: acquistion rate used for CCV conditions
+ :param integration_time: integration time used for CCV conditions
+ :param module_idx: module index to save retrieved CCV in sharedmem
+ :param only_dark: load only dark image derived constants. This
+ implies that a `calfile` is used to load the remaining
+ constants. Useful to reduce DB traffic and interactions
+ for non-frequently changing constants, i.e. such which are
+ not usually updated during a beamtime.
+
+ The `cal_db_interface` parameter in the `dbparms` tuple may be in
+ one of the following notations:
+ * tcp://host:port to directly identify the host and port to
+ connect to
+ * tcp://host:port_low#port_high to specify a port range from
+ which a random port will be picked. E.g. specifying
+
+ tcp://max-exfl016:8015#8025
+
+ will randomly pick an address in the range max-exfl016:8015 and
+ max-exfl016:8025.
+
+ The latter notation allows for load-balancing.
+
+ This routine loads the following constants as given in
+ `iCalibrationDB`:
Dark Image Derived
------------------
@@ -1402,77 +1415,50 @@ class AgipdCorrections:
-----------------------
* Constants.AGIPD.SlopesPC
- * Constants.AGIPD.BadPixelsPC
Flat-Field Derived
* Constants.AGIPD.SlopesFF
- * Constants.AGIPD.BadPixelsFF
+
"""
- from cal_tools.calcat_interface import AGIPD_CalibrationData
+
+ const_dict = assemble_constant_dict(
+ self.corr_bools,
+ self.pc_bools,
+ memory_cells,
+ bias_voltage,
+ gain_setting,
+ acquisition_rate,
+ photon_energy,
+ beam_energy=None,
+ only_dark=only_dark,
+ integration_time=integration_time
+ )
when = {}
cons_data = {}
variant = {}
- agipd_cal = AGIPD_CalibrationData(
- detector_name=karabo_id,
- modules=[karabo_da],
- sensor_bias_voltage=bias_voltage,
- memory_cells=memory_cells,
- acquisition_rate=acquisition_rate,
- integration_time=integration_time,
- source_energy=photon_energy,
- gain_mode=gain_mode,
- gain_setting=gain_setting,
- event_at=creation_time,
- client=client,
- )
-
- dark_constants = [
- "Offset",
- "Noise",
- "BadPixelsDark"
- ] if self.gain_mode else [
- "ThresholdsDark",
- "Offset",
- "Noise",
- "BadPixelsDark"
- ]
+ for cname, cval in const_dict.items():
+ condition = getattr(
+ Conditions, cval[2][0]).AGIPD(**cval[2][1])
+ cdata, md = get_from_db(
+ karabo_id=karabo_id,
+ karabo_da=karabo_da,
+ constant=getattr(Constants.AGIPD, cname)(),
+ condition=condition,
+ empty_constant=getattr(np, cval[0])(cval[1]),
+ cal_db_interface=cal_db_interface,
+ creation_time=creation_time,
+ verbosity=0,
+ )
+ cons_data[cname] = cdata
+ variant[cname] = md.calibration_constant_version.variant
- gain_constants = []
- if any(self.pc_bools):
- gain_constants += ["SlopesPC", "BadPixelsPC"]
- if self.corr_bools.get("xray_corr"):
- gain_constants += ["SlopesFF", "BadPixelsFF"]
-
- constants = dark_constants
- constants += gain_constants
- metadata = agipd_cal.metadata(constants)
- # Validate the constants availability and raise/warn correspondingly.
- for mod, ccv_dict in metadata.items():
- missing_dark_constants = set(
- c for c in dark_constants if c not in ccv_dict.keys())
- missing_gain_constants = set(
- c for c in gain_constants if c not in ccv_dict.keys())
- if missing_dark_constants:
- raise KeyError(
- f"Dark constants {missing_dark_constants} are not available"
- f" for correction. Module: {mod}")
- if missing_gain_constants:
- warning(
- f"Gain constants {missing_gain_constants} were"
- f" not retrieved. Module: {mod}")
-
- for cname in constants:
- if metadata[karabo_da].get(cname):
- when[cname] = metadata[karabo_da][cname]["begin_validity_at"]
- dataset = metadata[karabo_da][cname]["dataset"]
- with h5py.File(agipd_cal.caldb_root / metadata[karabo_da][cname]["path"], "r") as cf: # noqa
- cons_data[cname] = np.copy(cf[f"{dataset}/data"])
- variant[cname] = cf[dataset].attrs["variant"] if cf[dataset].attrs.keys() else 0 # noqa
- else:
- when[cname] = None
+ when[cname] = None
+ # Read the CCV begin at if constant was retrieved successfully.
+ if md and md.comm_db_success:
+ when[cname] = md.calibration_constant_version.begin_at
self.init_constants(cons_data, when, module_idx, variant)
diff --git a/tests/test_cal_tools.py b/tests/test_cal_tools.py
index 55601886b3c3d9c719213f94fd45fe44b7f05b7e..ad493e46c1bf1cd15336508c9dbc98f296ae80af 100644
--- a/tests/test_cal_tools.py
+++ b/tests/test_cal_tools.py
@@ -9,7 +9,6 @@ from extra_data import open_run
from iCalibrationDB import Conditions, ConstantMetaData, Constants
from cal_tools.agipdlib import AgipdCorrections, CellRange
-from cal_tools.calcat_interface import CalCatError
from cal_tools.plotting import show_processed_modules
from cal_tools.tools import (
creation_date_file_metadata,
@@ -26,7 +25,6 @@ from cal_tools.tools import (
ACQ_RATE = 1.1
BIAS_VOLTAGE = 300
GAIN_SETTING = 0
-GAIN_MODE = 0
INTEGRATION_TIME = 12
MEM_CELLS = 352
PHOTON_ENERGY = 9.2
@@ -404,37 +402,6 @@ def test_get_pdu_from_db(_agipd_const_cond):
"CAL_PHYSICAL_DETECTOR_UNIT-2_TEST"]
-# TODO add a marker for accessing zmq end_point
-@pytest.mark.requires_gpfs
-@pytest.mark.requires_caldb
-def test_raise_initialize_from_db():
- creation_time = datetime.strptime(
- "2020-01-07 13:26:48.00", "%Y-%m-%d %H:%M:%S.%f")
-
- agipd_corr = AgipdCorrections(
- max_cells=MEM_CELLS,
- cell_sel=CellRange([0, 500, 1], MEM_CELLS))
-
- agipd_corr.allocate_constants(
- modules=[0],
- constant_shape=(3, MEM_CELLS, 512, 128))
-
- # CalCatError: calibration_constant_version not found !
- with pytest.raises(CalCatError):
- agipd_corr.initialize_from_db(
- karabo_id="TEST_DET_CAL_CI-1",
- karabo_da="TEST_DET_CAL_DA1",
- creation_time=creation_time,
- memory_cells=MEM_CELLS,
- bias_voltage=BIAS_VOLTAGE,
- photon_energy=PHOTON_ENERGY,
- gain_setting=GAIN_SETTING,
- gain_mode=GAIN_MODE,
- acquisition_rate=ACQ_RATE,
- integration_time=INTEGRATION_TIME,
- module_idx=0,
- )
-
# TODO add a marker for accessing zmq end_point
@pytest.mark.requires_gpfs
@pytest.mark.requires_caldb
@@ -449,25 +416,45 @@ def test_initialize_from_db():
agipd_corr.allocate_constants(
modules=[0],
constant_shape=(3, MEM_CELLS, 512, 128))
- agipd_corr
+
dark_const_time_dict = agipd_corr.initialize_from_db(
- karabo_id=AGIPD_KARABO_ID,
- karabo_da="AGIPD00",
+ karabo_id="TEST_DET_CAL_CI-1",
+ karabo_da="TEST_DET_CAL_DA1",
+ cal_db_interface=CAL_DB_INTERFACE,
creation_time=creation_time,
memory_cells=MEM_CELLS,
bias_voltage=BIAS_VOLTAGE,
photon_energy=PHOTON_ENERGY,
gain_setting=GAIN_SETTING,
- gain_mode=GAIN_MODE,
- integration_time=INTEGRATION_TIME,
acquisition_rate=ACQ_RATE,
+ integration_time=INTEGRATION_TIME,
module_idx=0,
+ only_dark=False,
+ )
+
+ assert dark_const_time_dict == {
+ "Offset": None,
+ "Noise": None,
+ "ThresholdsDark": None,
+ "BadPixelsDark": None,
+ }
+
+ dark_const_time_dict = agipd_corr.initialize_from_db(
+ karabo_id=AGIPD_KARABO_ID,
+ karabo_da="AGIPD00",
+ cal_db_interface=CAL_DB_INTERFACE,
+ creation_time=creation_time,
+ memory_cells=MEM_CELLS, bias_voltage=BIAS_VOLTAGE,
+ photon_energy=PHOTON_ENERGY, gain_setting=GAIN_SETTING,
+ integration_time=INTEGRATION_TIME,
+ acquisition_rate=ACQ_RATE, module_idx=0,
+ only_dark=False,
)
# A retrieved constant has a value of datetime creation_time
- assert isinstance(dark_const_time_dict["Offset"], str)
- assert set(dark_const_time_dict.keys()) == {
- "Offset", "Noise", "ThresholdsDark", "BadPixelsDark"}
+ assert isinstance(dark_const_time_dict["Offset"], datetime)
+ assert list(dark_const_time_dict.keys()) == [
+ "Offset", "Noise", "ThresholdsDark", "BadPixelsDark"]
def test_module_index_to_qm():