From 960a74d240644b3c5667a08b926b144715662843 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Tue, 11 Jul 2023 14:17:44 +0200
Subject: [PATCH] Remove precorrection notebook
- Remove precorrection notebook
- Remove function retrieving constant from YAML
- Use fragment file
- WIP: start moving constant retrieval function to the notebook
---
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 105 ++--
...AGIPD_Correct_and_Verify_Summary_NBC.ipynb | 11 +-
...IPD_Retrieve_Constants_Precorrection.ipynb | 456 ------------------
src/cal_tools/agipdlib.py | 43 +-
src/xfel_calibrate/notebooks.py | 1 -
5 files changed, 73 insertions(+), 543 deletions(-)
delete mode 100644 notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 943454e71..cfe04e181 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -41,6 +41,7 @@
"cal_db_interface = \"tcp://max-exfl-cal001:8015#8045\" # the database interface to use\n",
"cal_db_timeout = 30000 # in milliseconds\n",
"creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
+ "cal_db_root = '/gpfs/exfel/d/cal/caldb_store' # The calibration database root path to access constant files. For example accessing constants from the test database.\n",
"\n",
"mem_cells = -1 # Number of memory cells used, set to 0 to automatically infer\n",
"bias_voltage = -1 # bias voltage, set to 0 to use stored value in slow data.\n",
@@ -166,10 +167,10 @@
"from cal_tools.enums import AgipdGainMode, BadPixels\n",
"from cal_tools.step_timing import StepTimer\n",
"from cal_tools.tools import (\n",
- " CalibrationMetadata,\n",
" calcat_creation_time,\n",
" map_modules_from_folder,\n",
" module_index_to_qm,\n",
+ " write_constants_fragment,\n",
")"
]
},
@@ -538,43 +539,79 @@
"# Retrieve calibration constants to RAM\n",
"agipd_corr.allocate_constants(modules, (3, mem_cells_db, 512, 128))\n",
"\n",
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# NOTE: this notebook will not overwrite calibration metadata file\n",
- "const_yaml = metadata.get(\"retrieved-constants\", {})\n",
"\n",
- "def retrieve_constants(mod):\n",
- " \"\"\"\n",
- " Retrieve calibration constants and load them to shared memory\n",
+ "from cal_tools.calcat_interface import (\n",
+ " AGIPD_CalibrationData,\n",
+ " CalCatError,\n",
+ ")\n",
+ "\n",
+ "when = {}\n",
+ "cons_data = {}\n",
+ "variant = {}\n",
+ "\n",
+ "agipd_cal = AGIPD_CalibrationData(\n",
+ " detector_name=karabo_id,\n",
+ " modules=[karabo_da],\n",
+ " sensor_bias_voltage=bias_voltage,\n",
+ " memory_cells=mem_cells,\n",
+ " acquisition_rate=acq_rate,\n",
+ " integration_time=integration_time,\n",
+ " source_energy=photon_energy,\n",
+ " gain_mode=gain_mode,\n",
+ " gain_setting=gain_setting,\n",
+ " event_at=creation_time,\n",
+ " client=rest_cfg.calibration_client(),\n",
+ " caldb_root=Path(cal_db_root),\n",
+ ")\n",
"\n",
- " Metadata for constants is taken from yml file or retrieved from the DB\n",
- " \"\"\"\n",
- " k_da = module_index_to_karabo_da[mod]\n",
- " # check if there is a yaml file in out_folder that has the device constants.\n",
- "\n",
- " if k_da in const_yaml:\n",
- " when = agipd_corr.initialize_from_yaml(k_da, const_yaml, mod)\n",
- " print(f\"Found constants for {k_da} in calibration_metadata.yml\")\n",
- " else: # Contact the database for module's constants.\n",
- " when = agipd_corr.initialize_from_db(\n",
- " karabo_id=karabo_id,\n",
- " karabo_da=k_da,\n",
- " creation_time=creation_time,\n",
- " memory_cells=mem_cells_db,\n",
- " bias_voltage=bias_voltage,\n",
- " photon_energy=9.2,\n",
- " gain_setting=gain_setting,\n",
- " acquisition_rate=acq_rate,\n",
- " integration_time=integration_time,\n",
- " gain_mode=gain_mode,\n",
- " module_idx=mod,\n",
- " client=rest_cfg.calibration_client(),\n",
- " )\n",
- " print(f\"Queried CalCat for {k_da}\")\n",
"\n",
- " # `when` is expected to be None if dark constants are not available\n",
- " # and correction should be skipped for this module.\n",
- " return mod, when, k_da\n",
+ "dark_constants = [\"Offset\",\"Noise\", \"ThresholdsDark\", \"BadPixelsDark\"]\n",
+ "pc_constants = [\"SlopesPC\", \"BadPixelsPC\"]\n",
+ "ff_constants = agipd_cal.illuminated_calibrations\n",
"\n",
+ "agipd_metadata = agipd_cal.metadata()\n",
+ "\n",
+ "agipd_metadata = agipd_cal.metadata(dark_constants)\n",
+ "for gain_constants in [pc_constants, ff_constants]:\n",
+ " try:\n",
+ " illum_metadata = agipd_cal.metadata(gain_constants)\n",
+ " for key, value in illum_metadata.items():\n",
+ " agipd_metadata.setdefault(key, {}).update(value)\n",
+ " except CalCatError as e: # TODO: replace when API errors are improved.\n",
+ " warning(f\"CalCatError: {e}\")\n",
+ "\n",
+ "agipd_md = dict()\n",
+ "\n",
+ "for cname in self.all_constants:\n",
+ " if cname in self.gain_constants:\n",
+ " agipd_cal.gain_mode = None\n",
+ " else:\n",
+ " agipd_cal.gain_mode = gain_mode\n",
+ " try:\n",
+ " agipd_md = agipd_cal.metadata([cname])[karabo_da]\n",
+ " except CalCatError as e:\n",
+ " pass # a validation of missing constants is done later.\n",
+ " if agipd_md.get(cname):\n",
+ " when[cname] = agipd_md[cname][\"begin_validity_at\"]\n",
+ " dataset = agipd_md[cname][\"dataset\"]\n",
+ " with h5py.File(agipd_cal.caldb_root / agipd_md[cname][\"path\"], \"r\") as cf: # noqa\n",
+ " cons_data[cname] = np.copy(cf[f\"{dataset}/data\"])\n",
+ " variant[cname] = cf[dataset].attrs[\"variant\"] if cf[dataset].attrs.keys() else 0 # noqa\n",
+ "\n",
+ "# skip initializing the constants if a dark constant is missing.\n",
+ "if self.validate_constants_availability(cons_data, karabo_da):\n",
+ " self.init_constants(cons_data, when, module_idx, variant)\n",
+ " return when\n",
+ "else:\n",
+ " # Avoid initializing the constants and return None\n",
+ " # to skip correcting this module.\n",
+ " return\n",
+ "\n",
+ "# Record constant details in YAML metadata\n",
+ "write_constants_fragment(\n",
+ " out_folder=(metadata_folder or out_folder),\n",
+ " det_metadata=jf_metadata,\n",
+ " caldb_root=jf_cal.caldb_root)\n",
"\n",
"print(f'Preparing constants (FF: {agipd_corr.corr_bools.get(\"xray_corr\", False)}, PC: {any(agipd_corr.pc_bools)}, '\n",
" f'BLC: {any(agipd_corr.blc_bools)})')\n",
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
index 5de75ff6e..36e21e922 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb
@@ -27,22 +27,13 @@
"metadata": {},
"outputs": [],
"source": [
- "import re\n",
- "import warnings\n",
"from pathlib import Path\n",
"\n",
- "import dateutil.parser\n",
- "import numpy as np\n",
"import yaml\n",
"\n",
- "warnings.filterwarnings('ignore')\n",
- "\n",
- "import matplotlib.pyplot as plt\n",
- "\n",
- "%matplotlib inline\n",
"import tabulate\n",
"from cal_tools.tools import CalibrationMetadata\n",
- "from IPython.display import Latex, Markdown, display"
+ "from IPython.display import Latex, display"
]
},
{
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
deleted file mode 100644
index 46b2dad4a..000000000
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ /dev/null
@@ -1,456 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# AGIPD Retrieving Constants Pre-correction #\n",
- "\n",
- "Author: European XFEL Detector Group, Version: 1.0\n",
- "\n",
- "Retrieving Required Constants for Offline Calibration of the AGIPD Detector"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "in_folder = \"/gpfs/exfel/exp/SPB/202030/p900119/raw\" # the folder to read data from, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_\" # the folder to output to, required\n",
- "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
- "modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
- "run = 80 # runs to process, required\n",
- "\n",
- "karabo_id = \"SPB_DET_AGIPD1M-1\" # karabo karabo_id\n",
- "karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
- "ctrl_source_template = '{}/MDL/FPGA_COMP_TEST' # path to control information\n",
- "instrument_source_template = '{}/DET/{}:xtdf' # path in the HDF5 file to images\n",
- "receiver_template = \"{}CH0\" # inset for receiver devices\n",
- "karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for control device\n",
- "\n",
- "# Parameters for calibration database.\n",
- "cal_db_interface = \"tcp://max-exfl-cal001:8015#8045\" # the database interface to use\n",
- "creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
- "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
- "\n",
- "slopes_ff_from_files = \"\" # Path to locally stored SlopesFF and BadPixelsFF constants\n",
- "mem_cells = -1 # number of memory cells used, set to 0 to automatically infer\n",
- "bias_voltage = -1 # bias voltage, set to 0 to use stored value in slow data.\n",
- "acq_rate = -1. # the detector acquisition rate, use 0 to try to auto-determine\n",
- "gain_setting = -1 # the gain setting, use -1 to use value stored in slow data.\n",
- "gain_mode = -1 # gain mode (0: adaptive, 1-3 fixed high/med/low, -1: read from CONTROL data)\n",
- "integration_time = -1 # integration time, negative values for auto-detection.\n",
- "\n",
- "# Correction Booleans\n",
- "only_offset = False # Apply only Offset correction. if False, Offset is applied by Default. if True, Offset is only applied.\n",
- "rel_gain = False # do relative gain correction based on PC data\n",
- "xray_gain = True # do relative gain correction based on xray data\n",
- "blc_noise = False # if set, baseline correction via noise peak location is attempted\n",
- "blc_stripes = False # if set, baseline corrected via stripes\n",
- "blc_hmatch = False # if set, base line correction via histogram matching is attempted\n",
- "adjust_mg_baseline = False # adjust medium gain baseline to match highest high gain value"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Fill dictionaries comprising bools and arguments for correction and data analysis\n",
- "# Here the hierarichy and dependencies for correction booleans are defined \n",
- "corr_bools = {}\n",
- "\n",
- "# offset is at the bottom of AGIPD correction pyramid.\n",
- "corr_bools[\"only_offset\"] = only_offset\n",
- "\n",
- "# Dont apply any corrections if only_offset is requested \n",
- "if not only_offset:\n",
- " corr_bools[\"adjust_mg_baseline\"] = adjust_mg_baseline\n",
- " corr_bools[\"rel_gain\"] = rel_gain\n",
- " corr_bools[\"xray_corr\"] = xray_gain\n",
- " corr_bools[\"blc_noise\"] = blc_noise\n",
- " corr_bools[\"blc_hmatch\"] = blc_hmatch"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from logging import warning\n",
- "from pathlib import Path\n",
- "\n",
- "import multiprocessing\n",
- "from datetime import timedelta\n",
- "from dateutil import parser\n",
- "from extra_data import RunDirectory\n",
- "\n",
- "import cal_tools.restful_config as rest_cfg\n",
- "from cal_tools.agipdlib import AgipdCtrl, SnowResolution\n",
- "from cal_tools.calcat_interface import AGIPD_CalibrationData, CalCatError\n",
- "from cal_tools.step_timing import StepTimer\n",
- "from cal_tools.tools import CalibrationMetadata, calcat_creation_time, module_index_to_qm\n",
- "from cal_tools.enums import AgipdGainMode"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# slopes_ff_from_files left as str for now\n",
- "in_folder = Path(in_folder)\n",
- "out_folder = Path(out_folder)\n",
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
- "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Run's creation time:\n",
- "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
- "offset = parser.parse(creation_date_offset)\n",
- "delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
- "creation_time += delta\n",
- "print(f\"Creation time: {creation_time}\")\n",
- " \n",
- "print(f\"Outputting to {out_folder}\")\n",
- "out_folder.mkdir(parents=True, exist_ok=True)\n",
- "\n",
- "melt_snow = False if corr_bools[\"only_offset\"] else SnowResolution.NONE"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "ctrl_src = ctrl_source_template.format(karabo_id_control)\n",
- "\n",
- "print(f\"Detector in use is {karabo_id}\")\n",
- "\n",
- "# Extracting Instrument string\n",
- "instrument = karabo_id.split(\"_\")[0]\n",
- "# Evaluate detector instance for mapping\n",
- "if instrument == \"SPB\":\n",
- " nmods = 16\n",
- "elif instrument == \"MID\":\n",
- " nmods = 16\n",
- "elif instrument == \"HED\":\n",
- " nmods = 8\n",
- "\n",
- "print(f\"Instrument {instrument}\")\n",
- "\n",
- "if karabo_da[0] == '-1':\n",
- " if modules[0] == -1:\n",
- " modules = list(range(nmods))\n",
- " karabo_da = [\"AGIPD{:02d}\".format(i) for i in modules]\n",
- "else:\n",
- " modules = [int(x[-2:]) for x in karabo_da]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "run_dc = RunDirectory(in_folder / f\"r{run:04d}\")\n",
- "instrument_src = instrument_source_template.format(karabo_id, receiver_template)\n",
- "\n",
- "instr_dc = run_dc.select(instrument_src.format(\"*\"))\n",
- "\n",
- "for m in modules:\n",
- " # Remove empty sources from `instr_dc`\n",
- " if instr_dc[instrument_src.format(m), 'image.data'].shape[0] == 0:\n",
- " instr_dc = instr_dc.deselect(instrument_src.format(m))\n",
- "\n",
- "if not instr_dc.all_sources:\n",
- " raise ValueError(f\"No images found for {in_folder / f'r{run:04d}'}\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "agipd_cond = AgipdCtrl(\n",
- " run_dc=run_dc,\n",
- " image_src=None, # Not neededed, as we wont read mem_cells or acq_rate.\n",
- " ctrl_src=ctrl_src,\n",
- ")\n",
- "\n",
- "if gain_setting == -1:\n",
- " gain_setting = agipd_cond.get_gain_setting(creation_time)\n",
- "if bias_voltage == -1:\n",
- " bias_voltage = agipd_cond.get_bias_voltage(karabo_id_control)\n",
- "if integration_time == -1:\n",
- " integration_time = agipd_cond.get_integration_time()\n",
- "if gain_mode == -1:\n",
- " gain_mode = agipd_cond.get_gain_mode()\n",
- "else:\n",
- " gain_mode = AgipdGainMode(gain_mode)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Retrieve Constants ##"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "pc_bools = [ # flags that points to the need for retrieving SlopesPC and BadPixelsPC constants.\n",
- " corr_bools.get(\"rel_gain\"),\n",
- " corr_bools.get(\"adjust_mg_baseline\"),\n",
- " corr_bools.get('blc_noise'),\n",
- " corr_bools.get('blc_hmatch'),\n",
- " corr_bools.get('blc_stripes'),\n",
- " melt_snow,\n",
- "]\n",
- "\n",
- "da_to_qm = dict()\n",
- "for module_index, k_da in zip(modules, karabo_da):\n",
- " da_to_qm[k_da] = module_index_to_qm(module_index)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "def find_cells_acq_rate_module(idx):\n",
- " \"\"\"Find memory cells and acquisition rate for a module.\"\"\"\n",
- " agipd_cond.image_src = instrument_src.format(idx)\n",
- "\n",
- " if agipd_cond.image_src not in instr_dc.all_sources:\n",
- " warning(f\"No raw images found for {module_index_to_qm(idx)}({k_da}).\")\n",
- " return None, None\n",
- " \n",
- " _mem_cells = agipd_cond.get_num_cells() if mem_cells == -1 else mem_cells\n",
- " _acq_rate = agipd_cond.get_acq_rate() if acq_rate == -1 else acq_rate\n",
- "\n",
- " return _mem_cells, _acq_rate\n",
- "\n",
- "\n",
- "all_mem_cells = set()\n",
- "all_acq_rates = set()\n",
- "\n",
- "with multiprocessing.Pool(processes=nmods) as pool:\n",
- " results = pool.map(find_cells_acq_rate_module, modules)\n",
- "\n",
- "for (_mem_cells, _acq_rate) in results:\n",
- " all_mem_cells.add(_mem_cells)\n",
- " all_acq_rates.add(_acq_rate)\n",
- "# Validate that mem_cells and acq_rate are the same for all modules.\n",
- "if len(all_mem_cells) != 1 or len(all_acq_rates) != 1:\n",
- " warning(\n",
- " \"Number of memory cells or acquisition rate are not identical for all modules.\\n\"\n",
- " f\"different mem_cells values: {all_mem_cells}.\\ndifferent acq_rate values: {all_acq_rates}.\")\n",
- "mem_cells = all_mem_cells.pop()\n",
- "acq_rate = all_acq_rates.pop()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "dark_constants = {\"Offset\", \"Noise\", \"BadPixelsDark\"}\n",
- "if not gain_mode:\n",
- " dark_constants.add(\"ThresholdsDark\")\n",
- "\n",
- "gain_constants = set()\n",
- "if pc_bools:\n",
- " gain_constants |= {\"SlopesPC\", \"BadPixelsPC\"}\n",
- "if xray_gain:\n",
- " gain_constants |= {\"SlopesFF\", \"BadPixelsFF\"}\n",
- "\n",
- "all_constants = dark_constants.copy()\n",
- "all_constants |= gain_constants"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "step_timer = StepTimer()\n",
- "step_timer.start()\n",
- "agipd_cal = AGIPD_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " modules=karabo_da,\n",
- " sensor_bias_voltage=bias_voltage,\n",
- " memory_cells=mem_cells,\n",
- " acquisition_rate=acq_rate,\n",
- " integration_time=integration_time,\n",
- " source_energy=9.2,\n",
- " gain_mode=gain_mode,\n",
- " gain_setting=gain_setting,\n",
- " event_at=creation_time,\n",
- " client=rest_cfg.calibration_client(),\n",
- ")\n",
- "\n",
- "# The main metadata dict for all modules and retrieved CCVs.\n",
- "agipd_metadata = {mod:{} for mod in agipd_cal.modules}\n",
- "\n",
- "# This is done to alter the condition for gain constants and avoid using gain_mode.\n",
- "for cname in all_constants:\n",
- " if cname in gain_constants:\n",
- " agipd_cal.gain_mode = None\n",
- " else:\n",
- " agipd_cal.gain_mode = gain_mode\n",
- " try:\n",
- " a = agipd_cal.metadata([cname])\n",
- " for mod, ccv_entry in a.items():\n",
- " agipd_metadata[mod].update(ccv_entry)\n",
- " except CalCatError as e:\n",
- " pass # Validating missing constants is done later."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "missing_dark_modules = set()\n",
- "# Validate the constants availability and raise/warn correspondingly. \n",
- "for mod, ccv_dict in agipd_metadata.items():\n",
- "\n",
- " missing_dark_constants = dark_constants - set(ccv_dict)\n",
- " if missing_dark_constants:\n",
- " warning(\n",
- " f\"Dark constants {missing_dark_constants} are not available to correct {mod}\") # noqa\n",
- " missing_dark_modules.add(mod)\n",
- "\n",
- " missing_gain_constants = gain_constants - set(ccv_dict)\n",
- " if missing_gain_constants:\n",
- " warning(\n",
- " f\"Gain constants {missing_gain_constants} were not retrieved for {mod}\")\n",
- "\n",
- "if missing_dark_modules == set(karabo_da):\n",
- " raise ValueError(\n",
- " \"Dark constants are not available for all modules.\"\n",
- " \"No correction can be performed.\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "for mod, ccv_dict in agipd_metadata.items():\n",
- " mdata_dict = {\"constants\": dict()}\n",
- " for cname, ccv_metadata in ccv_dict.items():\n",
- " if slopes_ff_from_files and cname in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
- " mdata_dict[\"constants\"][cname] = {\n",
- " \"path\": f\"{slopes_ff_from_files}/slopesff_bpmask_module_{module_index_to_qm(int(mod[-2:]))}.h5\", # noqa\n",
- " \"dataset\": \"\",\n",
- " \"creation-time\": \"00:00:00\",\n",
- " }\n",
- " else:\n",
- " mdata_dict[\"constants\"][cname] = {\n",
- " \"path\": str(agipd_cal.caldb_root / ccv_metadata[\"path\"]),\n",
- " \"dataset\": ccv_metadata[\"dataset\"],\n",
- " \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
- " \"ccv_id\": ccv_metadata[\"ccv_id\"],\n",
- " }\n",
- " mdata_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n",
- " retrieved_constants[mod] = mdata_dict"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "print(\"Using constants with creation times:\")\n",
- "timestamps = {}\n",
- "\n",
- "for k_da, module_name in da_to_qm.items():\n",
- " if k_da not in retrieved_constants.keys():\n",
- " continue\n",
- " module_timestamps = timestamps[module_name] = {}\n",
- " module_constants = retrieved_constants[k_da]\n",
- "\n",
- " print(f\"{module_name}:\")\n",
- " for cname, mdata in module_constants[\"constants\"].items():\n",
- " print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
- "\n",
- " for cname in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
- " if cname in module_constants[\"constants\"]:\n",
- " module_timestamps[cname] = module_constants[\"constants\"][cname][\"creation-time\"]\n",
- " else:\n",
- " module_timestamps[cname] = \"NA\"\n",
- "\n",
- "time_summary = retrieved_constants.setdefault(\"time-summary\", {})\n",
- "time_summary[\"SAll\"] = timestamps\n",
- "\n",
- "metadata.save()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "print(\"\\nRetrieved constants for modules:\",\n",
- " ', '.join([module_index_to_qm(x) for x in modules]))\n",
- "print(f\"Operating conditions are:\")\n",
- "print(f\"• Bias voltage: {bias_voltage}\")\n",
- "print(f\"• Memory cells: {mem_cells}\")\n",
- "print(f\"• Acquisition rate: {acq_rate}\")\n",
- "print(f\"• Gain mode: {gain_mode.name}\")\n",
- "print(f\"• Gain setting: {gain_setting}\")\n",
- "print(f\"• Integration time: {integration_time}\")\n",
- "print(f\"• Photon Energy: 9.2\")\n",
- "step_timer.done_step(f\"Constant metadata is saved under \\\"retrieved-constants\\\" in {metadata.filename}\")"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.8.12"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}
diff --git a/src/cal_tools/agipdlib.py b/src/cal_tools/agipdlib.py
index 7ab0087de..530f86c52 100644
--- a/src/cal_tools/agipdlib.py
+++ b/src/cal_tools/agipdlib.py
@@ -1373,48 +1373,6 @@ class AgipdCorrections:
else:
return True
- def initialize_from_yaml(
- self, karabo_da: str, const_yaml: Dict[str, Any], module_idx: int,
- ) -> Dict[str, Any]:
- """Initialize calibration constants from a yaml file
-
- :param karabo_da: a karabo data aggregator
- :param const_yaml: from the "retrieved-constants" part of a yaml file
- from pre-notebook, which consists of metadata of either the constant
- file path or the empty constant shape, and the creation-time of the
- retrieved constants
- :param module_idx: Index of module
- :return when: dict of retrieved constants with their creation-time
- """
-
- cons_data = dict()
- when = dict()
- variant = dict()
-
- # string of the device name.
- db_module = const_yaml[karabo_da]["physical-name"]
- for cname, mdata in const_yaml[karabo_da]["constants"].items():
- base_key = f"{db_module}/{cname}/0"
- when[cname] = mdata["creation-time"]
- if when[cname]:
- with h5py.File(mdata["path"], "r") as cf:
- cons_data[cname] = np.copy(cf[f"{base_key}/data"])
- # Set variant to 0 if the attribute is missing
- # as for old constants.
- if "variant" in cf[base_key].attrs.keys():
- variant[cname] = cf[base_key].attrs["variant"]
- else:
- variant[cname] = 0
-
- # skip initializing the constants if a dark constant is missing.
- if self.validate_constants_availability(cons_data, karabo_da):
- self.init_constants(cons_data, when, module_idx, variant)
- return when
- else:
- # Avoid initializing the constants and return None
- # to skip correcting this module.
- return
-
def initialize_from_db(
self, karabo_id: str, karabo_da: str,
creation_time: datetime, memory_cells: float,
@@ -1457,6 +1415,7 @@ class AgipdCorrections:
* Constants.AGIPD.BadPixelsPC
Flat-Field Derived
+ ------------------
* Constants.AGIPD.SlopesFF
* Constants.AGIPD.BadPixelsFF
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index 9e2ce6c83..fef99624c 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -28,7 +28,6 @@ notebooks = {
"cluster cores": 16},
},
"CORRECT": {
- "pre_notebooks": ["notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb"],
"notebook": "notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb",
"dep_notebooks": [
"notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb"],
--
GitLab