diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 03f09957c094b7c97b90c31efb4934b707cbeb26..925d9b077ab2dc8309b28ab909089e8ffbbb151f 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -41,6 +41,7 @@
"cal_db_interface = \"tcp://max-exfl-cal001:8015#8045\" # the database interface to use\n",
"cal_db_timeout = 30000 # in milliseconds\n",
"creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
+ "cal_db_root = '/gpfs/exfel/d/cal/caldb_store' # The calibration database root path to access constant files. For example accessing constants from the test database.\n",
"\n",
"mem_cells = -1 # Number of memory cells used, set to 0 to automatically infer\n",
"bias_voltage = -1 # bias voltage, set to 0 to use stored value in slow data.\n",
@@ -123,21 +124,20 @@
"outputs": [],
"source": [
"import itertools\n",
- "import os\n",
"import math\n",
"import multiprocessing\n",
- "import re\n",
+ "import os\n",
"import warnings\n",
"from datetime import timedelta\n",
"from logging import warning\n",
"from pathlib import Path\n",
- "from time import perf_counter\n",
"\n",
"import tabulate\n",
"from dateutil import parser\n",
"from IPython.display import Latex, Markdown, display\n",
"\n",
"warnings.filterwarnings('ignore')\n",
+ "import h5py\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"import yaml\n",
@@ -155,6 +155,7 @@
"sns.set_context(\"paper\", font_scale=1.4)\n",
"sns.set_style(\"ticks\")\n",
"\n",
+ "import cal_tools.restful_config as rest_cfg\n",
"from cal_tools import agipdalgs as calgs\n",
"from cal_tools.agipdlib import (\n",
" AgipdCorrections,\n",
@@ -163,13 +164,17 @@
" LitFrameSelection,\n",
")\n",
"from cal_tools.ana_tools import get_range\n",
+ "from cal_tools.calcat_interface import (\n",
+ " AGIPD_CalibrationData,\n",
+ " CalCatError,\n",
+ ")\n",
"from cal_tools.enums import AgipdGainMode, BadPixels\n",
"from cal_tools.step_timing import StepTimer\n",
"from cal_tools.tools import (\n",
- " CalibrationMetadata,\n",
" calcat_creation_time,\n",
" map_modules_from_folder,\n",
" module_index_to_qm,\n",
+ " write_constants_fragment,\n",
")"
]
},
@@ -181,7 +186,9 @@
"source": [
"in_folder = Path(in_folder)\n",
"out_folder = Path(out_folder)\n",
- "run_folder = in_folder / f'r{run:04d}'"
+ "run_folder = in_folder / f'r{run:04d}'\n",
+ "\n",
+ "step_timer = StepTimer()"
]
},
{
@@ -368,7 +375,6 @@
"\n",
"instrument_src_mod = [\n",
" s for s in list(dc.all_sources) if f\"{first_mod_channel}CH\" in s][0]\n",
- "mod_channel = int(re.findall(rf\".*{first_mod_channel}CH([0-9]+):.*\", instrument_src_mod)[0])\n",
"\n",
"agipd_cond = AgipdCtrl(\n",
" run_dc=dc,\n",
@@ -509,13 +515,6 @@
" print(f'Photon energy for rounding: {photon_energy:.3f} keV')"
]
},
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Data processing ##"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -550,13 +549,67 @@
" agipd_corr.recast_image_fields['data'] = np.dtype(recast_image_data)"
]
},
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Retrieving constants"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
- "module_index_to_karabo_da = {mod: da for (mod, da) in zip(modules, karabo_da)}"
+ "step_timer.start()\n",
+ "# Instantiate agipd_cal with the read operating conditions.\n",
+ "agipd_cal = AGIPD_CalibrationData(\n",
+ " detector_name=karabo_id,\n",
+ " modules=karabo_da,\n",
+ " sensor_bias_voltage=bias_voltage,\n",
+ " memory_cells=mem_cells,\n",
+ " acquisition_rate=acq_rate,\n",
+ " integration_time=integration_time,\n",
+ " source_energy=9.2,\n",
+ " gain_mode=gain_mode,\n",
+ " gain_setting=gain_setting,\n",
+ " event_at=creation_time,\n",
+ " client=rest_cfg.calibration_client(),\n",
+ " caldb_root=Path(cal_db_root),\n",
+ ")\n",
+ "\n",
+ "# Prepare lists of expected calibrations\n",
+ "dark_constants = [\"Offset\", \"Noise\", \"BadPixelsDark\"]\n",
+ "if not gain_mode: # Adaptive gain\n",
+ " dark_constants.append(\"ThresholdsDark\")\n",
+ "gain_constants = []\n",
+ "if any(agipd_corr.pc_bools):\n",
+ " gain_constants += [\"SlopesPC\", \"BadPixelsPC\"]\n",
+ "if agipd_corr.corr_bools.get('xray_corr'):\n",
+ " gain_constants += agipd_cal.illuminated_calibrations\n",
+ "\n",
+ "# First retrieve dark constants\n",
+ "agipd_metadata = agipd_cal.metadata(dark_constants)\n",
+ "if gain_constants:\n",
+ " # Then retrieve gain constants without\n",
+ " # using the `gain_mode` condition.\n",
+ " agipd_cal.gain_mode = None\n",
+ " try:\n",
+ " illum_metadata = agipd_cal.metadata(gain_constants)\n",
+ " for key, value in illum_metadata.items():\n",
+ " agipd_metadata.setdefault(key, {}).update(value)\n",
+ " except CalCatError as e: # TODO: replace when API errors are improved.\n",
+ " warning(f\"CalCatError: {e}\")\n",
+ "step_timer.done_step(\"Constants were retrieved in\")\n",
+ "\n",
+ "print(\"Preparing constants (\"\n",
+ " f\"FF: {agipd_corr.corr_bools.get('xray_corr', False)}, \"\n",
+ " f\"PC: {any(agipd_corr.pc_bools)}, \"\n",
+ " f\"BLC: {any(agipd_corr.blc_bools)})\")\n",
+ "# Display retrieved calibration constants timestamps\n",
+ "agipd_cal.display_markdown_retrieved_constants(metadata=agipd_metadata)"
]
},
{
@@ -565,54 +618,117 @@
"metadata": {},
"outputs": [],
"source": [
- "# Retrieve calibration constants to RAM\n",
+ "# Validate constants availability and exclude modules with no offsets.\n",
+ "for da, calibrations in agipd_metadata.items():\n",
+ " mod = modules[karabo_da.index(da)]\n",
+ " # Constants to error out for when missing.\n",
+ " error_missing_constants = {\"Offset\"}\n",
+ " if not gain_mode:\n",
+ " error_missing_constants |= {\"ThresholdsDark\"}\n",
+ "\n",
+ " error_missing_constants -= set(calibrations)\n",
+ " if error_missing_constants:\n",
+ " warning(f\"Offset constant is not available to correct {da}.\")\n",
+ " # Remove module from files to process.\n",
+ " del mapped_files[module_index_to_qm(mod)]\n",
+ " karabo_da.drop(da)\n",
+ " modules.drop(mod)\n",
+ "\n",
+ " warn_missing_constants = set(dark_constants + gain_constants)\n",
+ " warn_missing_constants -= error_missing_constants\n",
+ " warn_missing_constants -= set(calibrations)\n",
+ " if warn_missing_constants:\n",
+ " warning(f\"Constants {warn_missing_constants} were not retrieved for {da}.\")\n",
+ "\n",
+ "if not mapped_files: # Offsets are missing for all modules.\n",
+ " raise Exception(\"Could not find offset constants for any modules, will not correct data.\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Record constant details in YAML metadata\n",
+ "write_constants_fragment(\n",
+ " out_folder=(metadata_folder or out_folder),\n",
+ " det_metadata=agipd_metadata,\n",
+ " caldb_root=agipd_cal.caldb_root)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Load calibration constants to RAM\n",
"agipd_corr.allocate_constants(modules, (3, mem_cells_db, 512, 128))\n",
"\n",
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# NOTE: this notebook will not overwrite calibration metadata file\n",
- "const_yaml = metadata.get(\"retrieved-constants\", {})\n",
- "\n",
- "def retrieve_constants(mod):\n",
+ "def load_constants(da, module):\n",
" \"\"\"\n",
- " Retrieve calibration constants and load them to shared memory\n",
+ " Initialize constants data from previously retrieved metadata.\n",
"\n",
- " Metadata for constants is taken from yml file or retrieved from the DB\n",
+ " Args:\n",
+ " da (str): Data Aggregator (Karabo DA)\n",
+ " module (int): Module index\n",
+ "\n",
+ " Returns:\n",
+ " (int, dict, str): Module index, {constant name: creation time}, Karabo DA\n",
" \"\"\"\n",
- " k_da = module_index_to_karabo_da[mod]\n",
- " # check if there is a yaml file in out_folder that has the device constants.\n",
- " if k_da in const_yaml:\n",
- " when = agipd_corr.initialize_from_yaml(k_da, const_yaml, mod)\n",
- " print(f\"Found constants for {k_da} in calibration_metadata.yml\")\n",
- " else:\n",
- " try:\n",
- " # TODO: replace with proper retrieval (as done in pre-correction)\n",
- " when = agipd_corr.initialize_from_db(\n",
- " karabo_id=karabo_id,\n",
- " karabo_da=k_da,\n",
- " cal_db_interface=cal_db_interface,\n",
- " creation_time=creation_time,\n",
- " memory_cells=mem_cells_db,\n",
- " bias_voltage=bias_voltage,\n",
- " photon_energy=9.2,\n",
- " gain_setting=gain_setting,\n",
- " acquisition_rate=acq_rate,\n",
- " integration_time=integration_time,\n",
- " module_idx=mod,\n",
- " only_dark=False,\n",
- " )\n",
- " print(f\"Queried CalCat for {k_da}\")\n",
- " except Exception as e:\n",
- " warning(f\"Module: {k_da}, {e}\")\n",
- " when = None\n",
- " return mod, when, k_da\n",
+ " const_data = dict()\n",
+ " variant = dict()\n",
+ " for cname, mdata in agipd_metadata[da].items():\n",
+ " dataset = mdata[\"dataset\"]\n",
+ " with h5py.File(agipd_cal.caldb_root / mdata[\"path\"], \"r\") as cf: # noqa\n",
+ " const_data[cname] = np.copy(cf[f\"{dataset}/data\"])\n",
+ " variant[cname] = cf[dataset].attrs[\"variant\"] if cf[dataset].attrs.keys() else 0 # noqa\n",
+ " agipd_corr.init_constants(const_data, module, variant)\n",
"\n",
"\n",
- "print(f'Preparing constants (FF: {agipd_corr.corr_bools.get(\"xray_corr\", False)}, PC: {any(agipd_corr.pc_bools)}, '\n",
- " f'BLC: {any(agipd_corr.blc_bools)})')\n",
- "ts = perf_counter()\n",
+ "step_timer.start()\n",
"with multiprocessing.Pool(processes=len(modules)) as pool:\n",
- " const_out = pool.map(retrieve_constants, modules)\n",
- "print(f\"Constants were loaded in {perf_counter()-ts:.01f}s\")"
+ " pool.starmap(load_constants, zip(karabo_da, modules))\n",
+ "step_timer.done_step(f'Constants were loaded in ')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Store timestamps for Offset, SlopesPC, and SlopesFF\n",
+ "# in YAML file for time-summary table.\n",
+ "timestamps = {}\n",
+ "\n",
+ "for mod, mod_mdata in agipd_metadata.items():\n",
+ " modno = int(mod[-2:])\n",
+ "\n",
+ " module_timestamps = {}\n",
+ "\n",
+ " # Store few time stamps if exists\n",
+ " # Add NA to keep array structure\n",
+ " for key in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
+ " if key in mod_mdata:\n",
+ " module_timestamps[key] = mod_mdata[key][\"begin_validity_at\"]\n",
+ " else:\n",
+ " module_timestamps[key] = \"NA\"\n",
+ "\n",
+ " timestamps[module_index_to_qm(modno)] = module_timestamps\n",
+ "\n",
+ "seq = sequences[0] if sequences else 0\n",
+ "\n",
+ "with open(f\"{out_folder}/retrieved_constants_s{seq}.yml\",\"w\") as fd:\n",
+ " yaml.safe_dump({\"time-summary\": {f\"S{seq}\": timestamps}}, fd)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Data processing ##"
]
},
{
@@ -660,15 +776,6 @@
" yield i_proc, i * n_img // n_chunks, (i+1) * n_img // n_chunks"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "step_timer = StepTimer()"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -752,50 +859,6 @@
"step_timer.print_summary()"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# if the yml file contains \"retrieved-constants\", that means a leading\n",
- "# notebook got processed and the reporting would be generated from it.\n",
- "fst_print = True\n",
- "timestamps = {}\n",
- "\n",
- "for i, (modno, when, k_da) in enumerate(const_out):\n",
- " qm = module_index_to_qm(modno)\n",
- "\n",
- " if k_da not in const_yaml:\n",
- " if fst_print:\n",
- " print(\"Constants are retrieved with creation time: \")\n",
- " fst_print = False\n",
- "\n",
- " module_timestamps = {}\n",
- "\n",
- " print(f\"{qm}:\")\n",
- " for key, item in when.items():\n",
- " if hasattr(item, 'strftime'):\n",
- " item = item.strftime('%y-%m-%d %H:%M')\n",
- " when[key] = item\n",
- " print('{:.<12s}'.format(key), item)\n",
- "\n",
- " # Store few time stamps if exists\n",
- " # Add NA to keep array structure\n",
- " for key in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
- " if when and key in when and when[key]:\n",
- " module_timestamps[key] = when[key]\n",
- " else:\n",
- " module_timestamps[key] = \"NA\"\n",
- " timestamps[qm] = module_timestamps\n",
- "\n",
- "seq = sequences[0] if sequences else 0\n",
- "\n",
- "if timestamps:\n",
- " with open(f\"{out_folder}/retrieved_constants_s{seq}.yml\",\"w\") as fd:\n",
- " yaml.safe_dump({\"time-summary\": {f\"S{seq}\": timestamps}}, fd)"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -989,9 +1052,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"pulse_range = [np.min(pulseId[pulseId>=0]), np.max(pulseId[pulseId>=0])]\n",
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
index 5de75ff6e72aa42ae279044455f624b17745d50e..36e21e9220cc0fcda5f1f4f4a235fcdef1208034 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
@@ -27,22 +27,13 @@
"metadata": {},
"outputs": [],
"source": [
- "import re\n",
- "import warnings\n",
"from pathlib import Path\n",
"\n",
- "import dateutil.parser\n",
- "import numpy as np\n",
"import yaml\n",
"\n",
- "warnings.filterwarnings('ignore')\n",
- "\n",
- "import matplotlib.pyplot as plt\n",
- "\n",
- "%matplotlib inline\n",
"import tabulate\n",
"from cal_tools.tools import CalibrationMetadata\n",
- "from IPython.display import Latex, Markdown, display"
+ "from IPython.display import Latex, display"
]
},
{
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
deleted file mode 100644
index d0b20cadfbc25215b6f79641d1b89daff25ee125..0000000000000000000000000000000000000000
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ /dev/null
@@ -1,468 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# AGIPD Retrieving Constants Pre-correction #\n",
- "\n",
- "Author: European XFEL Detector Group, Version: 1.0\n",
- "\n",
- "Retrieving Required Constants for Offline Calibration of the AGIPD Detector"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "in_folder = \"/gpfs/exfel/exp/SPB/202030/p900119/raw\" # the folder to read data from, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_\" # the folder to output to, required\n",
- "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
- "modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
- "run = 80 # runs to process, required\n",
- "\n",
- "karabo_id = \"SPB_DET_AGIPD1M-1\" # karabo karabo_id\n",
- "karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
- "ctrl_source_template = '{}/MDL/FPGA_COMP_TEST' # path to control information\n",
- "instrument_source_template = '{}/DET/{}:xtdf' # path in the HDF5 file to images\n",
- "receiver_template = \"{}CH0\" # inset for receiver devices\n",
- "karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for control device\n",
- "\n",
- "# Parameters for calibration database.\n",
- "cal_db_interface = \"tcp://max-exfl-cal001:8015#8045\" # the database interface to use\n",
- "creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
- "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
- "\n",
- "slopes_ff_from_files = \"\" # Path to locally stored SlopesFF and BadPixelsFF constants\n",
- "mem_cells = -1 # number of memory cells used, set to 0 to automatically infer\n",
- "bias_voltage = -1 # bias voltage, set to 0 to use stored value in slow data.\n",
- "acq_rate = -1. # the detector acquisition rate, use 0 to try to auto-determine\n",
- "gain_setting = -1 # the gain setting, use -1 to use value stored in slow data.\n",
- "gain_mode = -1 # gain mode (0: adaptive, 1-3 fixed high/med/low, -1: read from CONTROL data)\n",
- "integration_time = -1 # integration time, negative values for auto-detection.\n",
- "\n",
- "# Correction Booleans\n",
- "only_offset = False # Apply only Offset correction. if False, Offset is applied by Default. if True, Offset is only applied.\n",
- "rel_gain = False # do relative gain correction based on PC data\n",
- "xray_gain = True # do relative gain correction based on xray data\n",
- "blc_noise = False # if set, baseline correction via noise peak location is attempted\n",
- "blc_stripes = False # if set, baseline corrected via stripes\n",
- "blc_hmatch = False # if set, base line correction via histogram matching is attempted\n",
- "adjust_mg_baseline = False # adjust medium gain baseline to match highest high gain value"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Fill dictionaries comprising bools and arguments for correction and data analysis\n",
- "# Here the hierarichy and dependencies for correction booleans are defined \n",
- "corr_bools = {}\n",
- "\n",
- "# offset is at the bottom of AGIPD correction pyramid.\n",
- "corr_bools[\"only_offset\"] = only_offset\n",
- "\n",
- "# Dont apply any corrections if only_offset is requested \n",
- "if not only_offset:\n",
- " corr_bools[\"adjust_mg_baseline\"] = adjust_mg_baseline\n",
- " corr_bools[\"rel_gain\"] = rel_gain\n",
- " corr_bools[\"xray_corr\"] = xray_gain\n",
- " corr_bools[\"blc_noise\"] = blc_noise\n",
- " corr_bools[\"blc_hmatch\"] = blc_hmatch"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import numpy as np\n",
- "from logging import warning\n",
- "from pathlib import Path\n",
- "from typing import Tuple\n",
- "\n",
- "import multiprocessing\n",
- "from datetime import timedelta\n",
- "from dateutil import parser\n",
- "from extra_data import RunDirectory\n",
- "\n",
- "from cal_tools.agipdlib import (\n",
- " AgipdCtrl,\n",
- " SnowResolution,\n",
- " assemble_constant_dict,\n",
- ")\n",
- "from cal_tools.enums import AgipdGainMode\n",
- "from cal_tools.tools import (\n",
- " calcat_creation_time,\n",
- " get_from_db,\n",
- " module_index_to_qm,\n",
- " CalibrationMetadata,\n",
- ")\n",
- "from iCalibrationDB import Conditions, Constants, Detectors"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# slopes_ff_from_files left as str for now\n",
- "in_folder = Path(in_folder)\n",
- "out_folder = Path(out_folder)\n",
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
- "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Run's creation time:\n",
- "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
- "offset = parser.parse(creation_date_offset)\n",
- "delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
- "creation_time += delta\n",
- "print(f\"Creation time: {creation_time}\")\n",
- " \n",
- "print(f\"Outputting to {out_folder}\")\n",
- "out_folder.mkdir(parents=True, exist_ok=True)\n",
- "\n",
- "melt_snow = False if corr_bools[\"only_offset\"] else SnowResolution.NONE"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "ctrl_src = ctrl_source_template.format(karabo_id_control)\n",
- "\n",
- "print(f\"Detector in use is {karabo_id}\")\n",
- "\n",
- "# Extracting Instrument string\n",
- "instrument = karabo_id.split(\"_\")[0]\n",
- "# Evaluate detector instance for mapping\n",
- "if instrument == \"SPB\":\n",
- " nmods = 16\n",
- "elif instrument == \"MID\":\n",
- " nmods = 16\n",
- "elif instrument == \"HED\":\n",
- " nmods = 8\n",
- "\n",
- "print(f\"Instrument {instrument}\")\n",
- "\n",
- "if karabo_da[0] == '-1':\n",
- " if modules[0] == -1:\n",
- " modules = list(range(nmods))\n",
- " karabo_da = [\"AGIPD{:02d}\".format(i) for i in modules]\n",
- "else:\n",
- " modules = [int(x[-2:]) for x in karabo_da]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "run_dc = RunDirectory(in_folder / f\"r{run:04d}\")\n",
- "instrument_src = instrument_source_template.format(karabo_id, receiver_template)\n",
- "\n",
- "instr_dc = run_dc.select(instrument_src.format(\"*\"))\n",
- "\n",
- "for m in modules:\n",
- " # Remove empty sources from `instr_dc`\n",
- " if instr_dc[instrument_src.format(m), 'image.data'].shape[0] == 0:\n",
- " instr_dc = instr_dc.deselect(instrument_src.format(m))\n",
- "\n",
- "if not instr_dc.all_sources:\n",
- " raise ValueError(f\"No images found for {in_folder / f'r{run:04d}'}\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "agipd_cond = AgipdCtrl(\n",
- " run_dc=run_dc,\n",
- " image_src=None, # Not neededed, as we wont read mem_cells or acq_rate.\n",
- " ctrl_src=ctrl_src,\n",
- ")\n",
- "\n",
- "if gain_setting == -1:\n",
- " gain_setting = agipd_cond.get_gain_setting(creation_time)\n",
- "if bias_voltage == -1:\n",
- " bias_voltage = agipd_cond.get_bias_voltage(karabo_id_control)\n",
- "if integration_time == -1:\n",
- " integration_time = agipd_cond.get_integration_time()\n",
- "if gain_mode == -1:\n",
- " gain_mode = agipd_cond.get_gain_mode()\n",
- "else:\n",
- " gain_mode = AgipdGainMode(gain_mode)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Retrieve Constants ##"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "pc_bools = [ # flags that points to the need for retrieving SlopesPC and BadPixelsPC constants.\n",
- " corr_bools.get(\"rel_gain\"),\n",
- " corr_bools.get(\"adjust_mg_baseline\"),\n",
- " corr_bools.get('blc_noise'),\n",
- " corr_bools.get('blc_hmatch'),\n",
- " corr_bools.get('blc_stripes'),\n",
- " melt_snow,\n",
- "]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "def retrieve_constants(\n",
- " k_da: str, idx: int\n",
- ") -> Tuple[str, str, float, float, str, dict]:\n",
- " \"\"\"\n",
- " Retrieve constants for a module.\n",
- "\n",
- " :return:\n",
- " k_da: karabo data aggregator.\n",
- " acq_rate: acquisition rate parameter.\n",
- " mem_cells: number of memory cells.\n",
- " mdata_dict: (DICT) dictionary with the metadata for the retrieved constants.\n",
- " \"\"\"\n",
- " # check if this module has images to process.\n",
- " if instrument_src.format(idx) not in instr_dc.all_sources:\n",
- " print(\"ERROR: No raw images found for \"\n",
- " f\"{module_index_to_qm(idx)}({k_da}).\")\n",
- "\n",
- " return None, k_da, None, None\n",
- "\n",
- " agipd_cond.image_src = instrument_src.format(idx)\n",
- "\n",
- " if mem_cells == -1:\n",
- " # Read value from fast data.\n",
- " local_mem_cells = agipd_cond.get_num_cells()\n",
- " else:\n",
- " # or use overriding notebook parameter.\n",
- " local_mem_cells = mem_cells\n",
- "\n",
- " if acq_rate == -1.:\n",
- " local_acq_rate = agipd_cond.get_acq_rate()\n",
- " else:\n",
- " local_acq_rate = acq_rate\n",
- "\n",
- " const_dict = assemble_constant_dict(\n",
- " corr_bools,\n",
- " pc_bools,\n",
- " local_mem_cells,\n",
- " bias_voltage,\n",
- " gain_setting,\n",
- " local_acq_rate,\n",
- " photon_energy=9.2,\n",
- " gain_mode=gain_mode,\n",
- " beam_energy=None,\n",
- " only_dark=False,\n",
- " integration_time=integration_time\n",
- " )\n",
- "\n",
- " # Retrieve multiple constants through an input dictionary\n",
- " # to return a dict of useful metadata.\n",
- " mdata_dict = dict()\n",
- " mdata_dict[\"constants\"] = dict()\n",
- " mdata_dict[\"physical-detector-unit\"] = None # initialization\n",
- "\n",
- " for const_name, (const_init_fun, const_shape, (cond_type, cond_param)) in const_dict.items(): # noqa\n",
- " if gain_mode and const_name in (\"ThresholdsDark\",):\n",
- " continue\n",
- " \n",
- " # saving metadata in a dict\n",
- " const_mdata = dict()\n",
- " mdata_dict[\"constants\"][const_name] = const_mdata\n",
- "\n",
- " if slopes_ff_from_files and const_name in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
- " const_mdata[\"file-path\"] = (\n",
- " f\"{slopes_ff_from_files}/slopesff_bpmask_module_{module_index_to_qm(idx)}.h5\") # noqa\n",
- " const_mdata[\"creation-time\"] = \"00:00:00\"\n",
- " continue\n",
- " \n",
- " if gain_mode and const_name in (\n",
- " \"BadPixelsPC\", \"SlopesPC\", \"BadPixelsFF\", \"SlopesFF\"\n",
- " ):\n",
- " param_copy = cond_param.copy()\n",
- " del param_copy[\"gain_mode\"]\n",
- " condition = getattr(Conditions, cond_type).AGIPD(**param_copy)\n",
- " else:\n",
- " condition = getattr(Conditions, cond_type).AGIPD(**cond_param)\n",
- "\n",
- " _, mdata = get_from_db(\n",
- " karabo_id,\n",
- " k_da,\n",
- " getattr(Constants.AGIPD, const_name)(),\n",
- " condition,\n",
- " getattr(np, const_init_fun)(const_shape),\n",
- " cal_db_interface,\n",
- " creation_time,\n",
- " meta_only=True,\n",
- " verbosity=0,\n",
- " )\n",
- " mdata_const = mdata.calibration_constant_version\n",
- " # check if constant was sucessfully retrieved.\n",
- " if mdata.comm_db_success:\n",
- " const_mdata[\"file-path\"] = (\n",
- " f\"{mdata_const.hdf5path}\" f\"{mdata_const.filename}\"\n",
- " )\n",
- " const_mdata[\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
- " mdata_dict[\"physical-detector-unit\"] = mdata_const.device_name\n",
- " else:\n",
- " const_mdata[\"file-path\"] = const_dict[const_name][:2]\n",
- " const_mdata[\"creation-time\"] = None\n",
- "\n",
- " return mdata_dict, k_da, local_acq_rate, local_mem_cells"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "inp = []\n",
- "da_to_qm = dict()\n",
- "for module_index, k_da in zip(modules, karabo_da):\n",
- " da_to_qm[k_da] = module_index_to_qm(module_index)\n",
- " if k_da in retrieved_constants:\n",
- " print(\n",
- " f\"Constant for {k_da} already in calibration_metadata.yml, won't query again.\")\n",
- " continue\n",
- "\n",
- " inp.append((k_da, module_index))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "with multiprocessing.Pool(processes=nmods) as pool:\n",
- " results = pool.starmap(retrieve_constants, inp)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "acq_rate_mods = []\n",
- "mem_cells_mods = []\n",
- "for md_dict, k_da, acq_rate, mem_cells in results:\n",
- " if acq_rate is None and mem_cells is None:\n",
- " continue\n",
- " md_dict, k_da, acq_rate, mem_cells\n",
- " retrieved_constants[k_da] = md_dict\n",
- " mem_cells_mods.append(mem_cells)\n",
- " acq_rate_mods.append(acq_rate)\n",
- "\n",
- "# Validate that mem_cells and acq_rate are the same for all modules.\n",
- "# TODO: Should a warning be enough?\n",
- "if len(set(mem_cells_mods)) != 1 or len(set(acq_rate_mods)) != 1:\n",
- " print(\n",
- " \"WARNING: Number of memory cells or \"\n",
- " \"acquisition rate are not identical for all modules.\\n\"\n",
- " f\"mem_cells: {mem_cells_mods}.\\nacq_rate: {acq_rate_mods}.\")\n",
- "\n",
- "# check if it is requested not to retrieve any constants from the database\n",
- "print(\"\\nRetrieved constants for modules:\",\n",
- " ', '.join([module_index_to_qm(x) for x in modules]))\n",
- "print(f\"Operating conditions are:\")\n",
- "print(f\"• Bias voltage: {bias_voltage}\")\n",
- "print(f\"• Memory cells: {mem_cells}\")\n",
- "print(f\"• Acquisition rate: {acq_rate}\")\n",
- "print(f\"• Gain mode: {gain_mode.name}\")\n",
- "print(f\"• Gain setting: {gain_setting}\")\n",
- "print(f\"• Integration time: {integration_time}\")\n",
- "print(f\"• Photon Energy: 9.2\")\n",
- "print(\"Constant metadata is saved under \\\"retrieved-constants\\\" in calibration_metadata.yml\\n\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "print(\"Using constants with creation times:\")\n",
- "timestamps = {}\n",
- "\n",
- "for k_da, module_name in da_to_qm.items():\n",
- " if k_da not in retrieved_constants.keys():\n",
- " continue\n",
- " module_timestamps = timestamps[module_name] = {}\n",
- " module_constants = retrieved_constants[k_da]\n",
- "\n",
- " print(f\"{module_name}:\")\n",
- " for cname, mdata in module_constants[\"constants\"].items():\n",
- " if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
- " mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
- " print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
- "\n",
- " for cname in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
- " if cname in module_constants[\"constants\"]:\n",
- " module_timestamps[cname] = module_constants[\"constants\"][cname][\"creation-time\"]\n",
- " else:\n",
- " module_timestamps[cname] = \"NA\"\n",
- "\n",
- "time_summary = retrieved_constants.setdefault(\"time-summary\", {})\n",
- "time_summary[\"SAll\"] = timestamps\n",
- "\n",
- "metadata.save()"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.8.12"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}
diff --git a/src/cal_tools/agipdlib.py b/src/cal_tools/agipdlib.py
index 038ca342b5e60ecd076f3697be30d4842a1b5db1..60e245ab4527bee12e8e890512fb508b4330565d 100644
--- a/src/cal_tools/agipdlib.py
+++ b/src/cal_tools/agipdlib.py
@@ -4,18 +4,16 @@ import zlib
from datetime import datetime
from multiprocessing import Manager
from multiprocessing.pool import ThreadPool
-from typing import Any, Dict, List, Optional, Tuple
+from typing import List, Optional
import h5py
import numpy as np
import sharedmem
from dateutil import parser
from extra_data import DataCollection, H5File, by_id, components
-from iCalibrationDB import Conditions, Constants
from cal_tools import agipdalgs as calgs
from cal_tools.agipdutils import (
- assemble_constant_dict,
baseline_correct_via_noise,
baseline_correct_via_stripe,
correct_baseline_via_hist,
@@ -27,7 +25,6 @@ from cal_tools.agipdutils import (
)
from cal_tools.enums import AgipdGainMode, BadPixels, SnowResolution
from cal_tools.h5_copy_except import h5_copy_except_paths
-from cal_tools.tools import get_from_db
class AgipdCtrl:
@@ -1127,7 +1124,7 @@ class AgipdCorrections:
fletcher32=True)
def init_constants(
- self, cons_data: dict, when: dict, module_idx: int, variant: dict):
+ self, cons_data: dict, module_idx: int, variant: dict):
"""
For CI derived gain, a mean multiplication factor of 4.48 compared
to medium gain is used, as no reliable CI data for all memory cells
@@ -1166,8 +1163,6 @@ class AgipdCorrections:
rel_low gain = _rel_medium gain * 4.48
:param cons_data: A dictionary for each retrieved constant value.
- :param when: A dictionary for the creation time
- of each retrieved constant.
:param module_idx: A module_idx index
:param variant: A dictionary for the variant of each retrieved CCV.
:return:
@@ -1177,11 +1172,17 @@ class AgipdCorrections:
# assuming this method runs in parallel.
calgs_opts = dict(num_threads=os.cpu_count() // len(self.offset))
- calgs.transpose_constant(self.offset[module_idx], cons_data['Offset'], **calgs_opts)
- calgs.transpose_constant(self.noise[module_idx], cons_data['Noise'], **calgs_opts)
+ calgs.transpose_constant(
+ self.offset[module_idx], cons_data["Offset"], **calgs_opts)
+
+ # In case noise wasn't retrieved no need for transposing.
+ if "Noise" in cons_data:
+ calgs.transpose_constant(
+ self.noise[module_idx], cons_data["Noise"], **calgs_opts)
+
if self.gain_mode is AgipdGainMode.ADAPTIVE_GAIN:
calgs.transpose_constant(self.thresholds[module_idx],
- cons_data['ThresholdsDark'][..., :3],
+ cons_data["ThresholdsDark"][..., :3],
**calgs_opts)
if self.corr_bools.get("low_medium_gap"):
@@ -1192,12 +1193,12 @@ class AgipdCorrections:
bpixels = cons_data["BadPixelsDark"].astype(np.uint32)
if self.corr_bools.get("xray_corr"):
- if when["BadPixelsFF"]:
+ if "BadPixelsFF" in cons_data:
bpixels |= cons_data["BadPixelsFF"].astype(np.uint32)[...,
:bpixels.shape[2], # noqa
None]
- if when["SlopesFF"]: # Checking if constant was retrieved
+ if "SlopesFF" in cons_data: # Checking if constant was retrieved
slopesFF = cons_data["SlopesFF"]
# This could be used for backward compatibility
@@ -1237,7 +1238,7 @@ class AgipdCorrections:
# add additional bad pixel information
if any(self.pc_bools):
- if when["BadPixelsPC"]:
+ if "BadPixelsPC" in cons_data:
bppc = np.moveaxis(cons_data["BadPixelsPC"].astype(np.uint32),
0, 2)
bpixels |= bppc[..., :bpixels.shape[2], None]
@@ -1245,7 +1246,7 @@ class AgipdCorrections:
# calculate relative gain from the constants
rel_gain = np.ones((128, 512, self.max_cells, 3), np.float32)
- if when["SlopesPC"]:
+ if "SlopesPC" in cons_data:
slopesPC = cons_data["SlopesPC"].astype(np.float32, copy=False)
# This will handle some historical data in a different format
@@ -1339,149 +1340,6 @@ class AgipdCorrections:
return
- def initialize_from_yaml(
- self, karabo_da: str, const_yaml: Dict[str, Any], module_idx: int
- ) -> Dict[str, Any]:
- """Initialize calibration constants from a yaml file
-
- :param karabo_da: a karabo data aggregator
- :param const_yaml: from the "retrieved-constants" part of a yaml file
- from pre-notebook, which consists of metadata of either the constant
- file path or the empty constant shape, and the creation-time of the
- retrieved constants
- :param module_idx: Index of module
- :return when: dict of retrieved constants with their creation-time
- """
-
- # string of the device name.
- cons_data = dict()
- when = dict()
- variant = dict()
- db_module = const_yaml[karabo_da]["physical-detector-unit"]
- for cname, mdata in const_yaml[karabo_da]["constants"].items():
- base_key = f"{db_module}/{cname}/0"
- when[cname] = mdata["creation-time"]
- if when[cname]:
- with h5py.File(mdata["file-path"], "r") as cf:
- cons_data[cname] = np.copy(cf[f"{base_key}/data"])
- # Set variant to 0 if the attribute is missing
- # as for old constants.
- if "variant" in cf[base_key].attrs.keys():
- variant[cname] = cf[base_key].attrs["variant"]
- else:
- variant[cname] = 0
- else:
- # Create empty constant using the list elements
- cons_data[cname] = getattr(np, mdata["file-path"][0])(mdata["file-path"][1]) # noqa
-
- self.init_constants(cons_data, when, module_idx, variant)
-
- return when
-
- def initialize_from_db(self, karabo_id: str, karabo_da: str,
- cal_db_interface: str,
- creation_time: datetime,
- memory_cells: float, bias_voltage: int,
- photon_energy: float, gain_setting: float,
- acquisition_rate: float, integration_time: int,
- module_idx: int, only_dark: bool = False):
- """ Initialize calibration constants from the calibration database
-
- :param karabo_id: karabo identifier
- :param karabo_da: karabo data aggregator
- :param cal_db_interface: database interaface port
- :param creation_time: time for desired calibration constant version
- :param memory_cells: number of memory cells used for CCV conditions
- :param bias_voltage: bias voltage used for CCV conditions
- :param photon_energy: photon energy used for CCV conditions
- :param gain_setting: gain setting used for CCV conditions
- :param acquisition_rate: acquistion rate used for CCV conditions
- :param integration_time: integration time used for CCV conditions
- :param module_idx: module index to save retrieved CCV in sharedmem
- :param only_dark: load only dark image derived constants. This
- implies that a `calfile` is used to load the remaining
- constants. Useful to reduce DB traffic and interactions
- for non-frequently changing constants, i.e. such which are
- not usually updated during a beamtime.
-
- The `cal_db_interface` parameter in the `dbparms` tuple may be in
- one of the following notations:
- * tcp://host:port to directly identify the host and port to
- connect to
- * tcp://host:port_low#port_high to specify a port range from
- which a random port will be picked. E.g. specifying
-
- tcp://max-exfl-cal001:8015#8025
-
- will randomly pick an address in the range max-exfl-cal001:8015 and
- max-exfl-cal001:8025.
-
- The latter notation allows for load-balancing.
-
- This routine loads the following constants as given in
- `iCalibrationDB`:
-
- Dark Image Derived
- ------------------
-
- * Constants.AGIPD.Offset
- * Constants.AGIPD.Noise
- * Constants.AGIPD.BadPixelsDark
- * Constants.AGIPD.ThresholdsDark
-
- Pulse Capacitor Derived
- -----------------------
-
- * Constants.AGIPD.SlopesPC
-
- Flat-Field Derived
-
- * Constants.AGIPD.SlopesFF
-
- """
-
- const_dict = assemble_constant_dict(
- self.corr_bools,
- self.pc_bools,
- memory_cells,
- bias_voltage,
- gain_setting,
- acquisition_rate,
- photon_energy,
- beam_energy=None,
- only_dark=only_dark,
- integration_time=integration_time
- )
-
- when = {}
- cons_data = {}
- variant = {}
-
- for cname, cval in const_dict.items():
- condition = getattr(
- Conditions, cval[2][0]).AGIPD(**cval[2][1])
- cdata, md = get_from_db(
- karabo_id=karabo_id,
- karabo_da=karabo_da,
- constant=getattr(Constants.AGIPD, cname)(),
- condition=condition,
- empty_constant=getattr(np, cval[0])(cval[1]),
- cal_db_interface=cal_db_interface,
- creation_time=creation_time,
- verbosity=0,
- )
- cons_data[cname] = cdata
- variant[cname] = md.calibration_constant_version.variant
-
- when[cname] = None
- # Read the CCV begin at if constant was retrieved successfully.
- if md and md.comm_db_success:
- when[cname] = md.calibration_constant_version.begin_at
-
- self.init_constants(cons_data, when, module_idx, variant)
-
- return when
-
def allocate_constants(self, modules, constant_shape):
"""
Allocate memory for correction constants
diff --git a/src/cal_tools/agipdutils.py b/src/cal_tools/agipdutils.py
index 6d859edbe2ab7dc7c573e858d41d196c21532664..a7ee52de1c110921dbc652325c6441afed3cc008 100644
--- a/src/cal_tools/agipdutils.py
+++ b/src/cal_tools/agipdutils.py
@@ -6,80 +6,7 @@ from scipy.signal import cwt, find_peaks_cwt, ricker
from sklearn.mixture import GaussianMixture
from sklearn.preprocessing import StandardScaler
-from cal_tools.enums import AgipdGainMode, BadPixels, SnowResolution
-
-
-def assemble_constant_dict(
- corr_bools,
- pc_bools,
- memory_cells,
- bias_voltage,
- gain_setting,
- acquisition_rate,
- photon_energy,
- beam_energy=None,
- only_dark=False,
- gain_mode=AgipdGainMode.ADAPTIVE_GAIN,
- integration_time=None
-):
- """
- Assemble a dictionary with the iCalibrationDB constant names and
- the operating conditions for retrieving the required constants
- for correction.
-
- :param corr_bools: (Dict) A dict of booleans for applying
- specific corrections
- :param pc_bools: (List) A list of booleans to enable SlopesPC retrieval
- :param memory_cells: (Int) Number of memory cells
- :param bias_voltage: (Int) Bias Voltage
- :param gain_setting: (Float) Gain setting
- :param acquisition_rate: (Float) Acquisition rate
- :param photon_energy: (Float) Photon energy
- :param integration_time: (Float) Integration time
- :param beam_energy: (Float) Beam Energy
- :param only_dark: (Bool) Indicating a retrieval for dark constants only from db
- :param gain_mode: Operation mode of the detector (default to adaptive gain)
- :return: const_dict: (Dict) An assembled dictionary that can be used
- to retrieve the required constants
- """
-
- darkcond = [
- "Dark",
- {
- "memory_cells": memory_cells,
- "bias_voltage": bias_voltage,
- "acquisition_rate": acquisition_rate,
- "gain_setting": gain_setting,
- "gain_mode": gain_mode,
- "integration_time": integration_time,
- "pixels_x": 512,
- "pixels_y": 128,
- },
- ]
- const_dict = {
- "Offset": ["zeros", (128, 512, memory_cells, 3), darkcond],
- "Noise": ["zeros", (128, 512, memory_cells, 3), darkcond],
- "ThresholdsDark": ["ones", (128, 512, memory_cells, 5), darkcond],
- "BadPixelsDark": ["zeros", (128, 512, memory_cells, 3), darkcond],
- }
-
- if not (corr_bools.get("only_offset") or only_dark):
- if any(pc_bools):
- const_dict["BadPixelsPC"] = ["zeros", (memory_cells, 128, 512), darkcond]
- const_dict["SlopesPC"] = ["ones", (128, 512, memory_cells, 10), darkcond]
-
- if corr_bools.get("xray_corr"):
- # Add illuminated conditions
- illumcond = [
- "Illuminated",
- {"beam_energy": beam_energy, "photon_energy": photon_energy},
- ]
- illumcond[1].update(darkcond[1])
-
- const_dict["BadPixelsFF"] = ["zeros", (128, 512, memory_cells), illumcond]
- const_dict["SlopesFF"] = ["ones", (128, 512, memory_cells, 2), illumcond]
-
- return const_dict
+from cal_tools.enums import BadPixels, SnowResolution
# contiguous_regions() by Joe Kington on Stackoverflow
diff --git a/src/cal_tools/calcat_interface.py b/src/cal_tools/calcat_interface.py
index 03f3d1437d28a423ee447f370550d5c0df70e9b1..db67afc900f82e1949dd2ef9ce37d76502a4eda4 100644
--- a/src/cal_tools/calcat_interface.py
+++ b/src/cal_tools/calcat_interface.py
@@ -176,7 +176,7 @@ class CalCatApi(metaclass=ClientWrapper):
for pdu in resp_pdus["data"]
}
else:
- raise ValueError(f"{module_naming} is unknown!")
+ raise ValueError(f"{module_naming} is unknown!. Expected da, modno, or qm")
@lru_cache()
@@ -728,7 +728,7 @@ class CalibrationData:
table = [["Modules"] + cal_group]
# Loop over calibrations and modules to form the next rows.
- for mod in list(self.mod_to_pdu):
+ for mod in metadata:
mod_consts = []
for cname in cal_group:
@@ -1037,10 +1037,12 @@ class AGIPD_CalibrationData(SplitConditionCalibrationData):
cond = super()._build_condition(parameters)
# Fix-up some database quirks.
- if int(cond.get("Gain mode", -1)) == 0:
- del cond["Gain mode"]
+ if cond.get("Gain mode", None):
+ cond["Gain mode"] = 1
+ else:
+ cond.pop("Gain mode", None)
- if int(cond.get("Integration time", -1)) == 12:
+ if cond.get("Integration time", None) == 12:
del cond["Integration time"]
return cond
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index 9e2ce6c836b391f055c6f643d72586f13d607afc..fef99624c513b4f7087c7d2a1767717b97d9f1cc 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -28,7 +28,6 @@ notebooks = {
"cluster cores": 16},
},
"CORRECT": {
- "pre_notebooks": ["notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb"],
"notebook": "notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb",
"dep_notebooks": [
"notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb"],
diff --git a/tests/test_cal_tools.py b/tests/test_cal_tools.py
index 2eb84eac7cb6d62ee6f61c714e4c73179be1115f..3eb962b0ec1a12c29efd03584b77d5340cd277cf 100644
--- a/tests/test_cal_tools.py
+++ b/tests/test_cal_tools.py
@@ -9,7 +9,6 @@ import zmq
from extra_data import open_run
from iCalibrationDB import Conditions, ConstantMetaData, Constants
-from cal_tools.agipdlib import AgipdCorrections, CellRange
from cal_tools.plotting import show_processed_modules
from cal_tools.tools import (
creation_date_file_metadata,
@@ -29,9 +28,7 @@ from cal_tools.tools import (
ACQ_RATE = 1.1
BIAS_VOLTAGE = 300
GAIN_SETTING = 0
-INTEGRATION_TIME = 12
MEM_CELLS = 352
-PHOTON_ENERGY = 9.2
AGIPD_KARABO_ID = "SPB_DET_AGIPD1M-1"
WRONG_AGIPD_MODULE = "AGIPD_**"
@@ -406,61 +403,6 @@ def test_get_pdu_from_db(_agipd_const_cond):
"CAL_PHYSICAL_DETECTOR_UNIT-2_TEST"]
-# TODO add a marker for accessing zmq end_point
-@pytest.mark.requires_gpfs
-@pytest.mark.requires_caldb
-def test_initialize_from_db():
- creation_time = datetime.strptime(
- "2020-01-07 13:26:48.00", "%Y-%m-%d %H:%M:%S.%f")
-
- agipd_corr = AgipdCorrections(
- max_cells=MEM_CELLS,
- cell_sel=CellRange([0, 500, 1], MEM_CELLS))
-
- agipd_corr.allocate_constants(
- modules=[0],
- constant_shape=(3, MEM_CELLS, 512, 128))
-
- dark_const_time_dict = agipd_corr.initialize_from_db(
- karabo_id="TEST_DET_CAL_CI-1",
- karabo_da="TEST_DET_CAL_DA1",
- cal_db_interface=CAL_DB_INTERFACE,
- creation_time=creation_time,
- memory_cells=MEM_CELLS,
- bias_voltage=BIAS_VOLTAGE,
- photon_energy=PHOTON_ENERGY,
- gain_setting=GAIN_SETTING,
- acquisition_rate=ACQ_RATE,
- integration_time=INTEGRATION_TIME,
- module_idx=0,
- only_dark=False,
- )
-
- assert dark_const_time_dict == {
- "Offset": None,
- "Noise": None,
- "ThresholdsDark": None,
- "BadPixelsDark": None,
- }
-
- dark_const_time_dict = agipd_corr.initialize_from_db(
- karabo_id=AGIPD_KARABO_ID,
- karabo_da="AGIPD00",
- cal_db_interface=CAL_DB_INTERFACE,
- creation_time=creation_time,
- memory_cells=MEM_CELLS, bias_voltage=BIAS_VOLTAGE,
- photon_energy=PHOTON_ENERGY, gain_setting=GAIN_SETTING,
- integration_time=INTEGRATION_TIME,
- acquisition_rate=ACQ_RATE, module_idx=0,
- only_dark=False,
- )
-
- # A retrieved constant has a value of datetime creation_time
- assert isinstance(dark_const_time_dict["Offset"], datetime)
- assert list(dark_const_time_dict.keys()) == [
- "Offset", "Noise", "ThresholdsDark", "BadPixelsDark"]
-
-
def test_module_index_to_qm():
assert module_index_to_qm(0) == 'Q1M1'