diff --git a/notebooks/LPD/LPD_Correct_Fast.ipynb b/notebooks/LPD/LPD_Correct_Fast.ipynb
index 4fd91a297b48fe4f592193f0f5912462ae85d602..d9ff08114dce628a847dedac56635c24aea9ade5 100644
--- a/notebooks/LPD/LPD_Correct_Fast.ipynb
+++ b/notebooks/LPD/LPD_Correct_Fast.ipynb
@@ -108,7 +108,11 @@
"from cal_tools.calcat_interface import CalCatError, LPD_CalibrationData\n",
"from cal_tools.lpdalgs import correct_lpd_frames\n",
"from cal_tools.lpdlib import get_mem_cell_pattern, make_cell_order_condition\n",
- "from cal_tools.tools import CalibrationMetadata, calcat_creation_time\n",
+ "from cal_tools.tools import (\n",
+ " CalibrationMetadata,\n",
+ " calcat_creation_time,\n",
+ " write_constants_fragment,\n",
+ ")\n",
"from cal_tools.files import DataFile"
]
},
@@ -230,17 +234,6 @@
"# Obtain and prepare calibration constants"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
- "const_yaml = metadata.setdefault(\"retrieved-constants\", {})"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -249,55 +242,41 @@
},
"outputs": [],
"source": [
- "const_data = dict() # {\"ModuleName\": {\"ConstantName\": ndarray}}\n",
"start = perf_counter()\n",
- "if const_yaml:\n",
- " const_load_mp = psh.ProcessContext(num_workers=24)\n",
- "\n",
- " for mod, constants in const_yaml.items():\n",
- " const_data[mod] = {} # An empty dictionary stays for a module with no constants.\n",
- " for cname, cmdata in constants[\"constants\"].items():\n",
- " const_data[mod][cname] = const_load_mp.alloc(\n",
- " shape=(256, 256, mem_cells, 3), # All LPD constants have the same shape.\n",
- " dtype=np.uint32 if cname.startswith('BadPixels') else np.float32\n",
- " )\n",
- " def load_constant_dataset(wid, index, mod):\n",
- " for cname, mdata in const_yaml[mod][\"constants\"].items():\n",
- " with h5py.File(mdata[\"path\"], \"r\") as cf:\n",
- " cf[f\"{mdata['dataset']}/data\"].read_direct(const_data[mod][cname])\n",
- "\n",
- " const_load_mp.map(load_constant_dataset, karabo_da)\n",
- "else:\n",
- " cell_ids_pattern_s = None\n",
- " if use_cell_order != 'never':\n",
- " # Read the order of memory cells used\n",
- " raw_data = xd.DataCollection.from_paths([e[1] for e in data_to_process])\n",
- " cell_ids_pattern_s = make_cell_order_condition(\n",
- " use_cell_order, get_mem_cell_pattern(raw_data, det_inp_sources)\n",
- " )\n",
- " print(\"Memory cells order:\", cell_ids_pattern_s)\n",
- " lpd_cal = LPD_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " modules=karabo_da,\n",
- " sensor_bias_voltage=bias_voltage,\n",
- " memory_cells=mem_cells,\n",
- " feedback_capacitor=capacitor,\n",
- " source_energy=photon_energy,\n",
- " memory_cell_order=cell_ids_pattern_s,\n",
- " category=category,\n",
- " event_at=creation_time,\n",
- " client=rest_cfg.calibration_client(),\n",
- " caldb_root=Path(cal_db_root),\n",
+ "\n",
+ "cell_ids_pattern_s = None\n",
+ "if use_cell_order != 'never':\n",
+ " # Read the order of memory cells used\n",
+ " raw_data = xd.DataCollection.from_paths([e[1] for e in data_to_process])\n",
+ " cell_ids_pattern_s = make_cell_order_condition(\n",
+ " use_cell_order, get_mem_cell_pattern(raw_data, det_inp_sources)\n",
" )\n",
- " const_data = lpd_cal.ndarray_map([\"Offset\", \"BadPixelsDark\"])\n",
- " try:\n",
- " illum_const_data = lpd_cal.ndarray_map(lpd_cal.illuminated_calibrations)\n",
- " for key, value in illum_const_data.items():\n",
- " const_data.setdefault(key, {}).update(value)\n",
- " except CalCatError as e: # TODO: replace when API errors are improved.\n",
- " warning(f\"CalCatError: {e}\")\n",
+ "print(\"Memory cells order:\", cell_ids_pattern_s)\n",
+ "\n",
+ "lpd_cal = LPD_CalibrationData(\n",
+ " detector_name=karabo_id,\n",
+ " modules=karabo_da,\n",
+ " sensor_bias_voltage=bias_voltage,\n",
+ " memory_cells=mem_cells,\n",
+ " feedback_capacitor=capacitor,\n",
+ " source_energy=photon_energy,\n",
+ " memory_cell_order=cell_ids_pattern_s,\n",
+ " category=category,\n",
+ " event_at=creation_time,\n",
+ " client=rest_cfg.calibration_client(),\n",
+ " caldb_root=Path(cal_db_root),\n",
+ ")\n",
+ "\n",
+ "lpd_metadata = lpd_cal.metadata([\"Offset\", \"BadPixelsDark\"])\n",
+ "try:\n",
+ " illum_metadata = lpd_cal.metadata(lpd_cal.illuminated_calibrations)\n",
+ " for key, value in illum_metadata.items():\n",
+ " lpd_metadata.setdefault(key, {}).update(value)\n",
+ "except CalCatError as e: # TODO: replace when API errors are improved.\n",
+ " warning(f\"CalCatError: {e}\")\n",
+ "\n",
"total_time = perf_counter() - start\n",
- "print(f'Loading constants {total_time:.1f}s')"
+ "print(f'Looking up constants {total_time:.1f}s')"
]
},
{
@@ -307,7 +286,7 @@
"outputs": [],
"source": [
"# Validate the constants availability and raise/warn accordingly.\n",
- "for mod, calibrations in const_data.items():\n",
+ "for mod, calibrations in lpd_metadata.items():\n",
" missing_offset = {\"Offset\"} - set(calibrations)\n",
" warn_missing_constants = {\n",
" \"BadPixelsDark\", \"BadPixelsFF\", \"GainAmpMap\",\n",
@@ -324,6 +303,23 @@
"data_to_process = [(mod, in_f, out_f) for mod, in_f, out_f in data_to_process if mod in karabo_da]"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# write constants metadata to fragment YAML\n",
+ "write_constants_fragment(\n",
+ " out_folder=(metadata_folder or out_folder),\n",
+ " det_metadata=lpd_metadata,\n",
+ " caldb_root=lpd_cal.caldb_root,\n",
+ ")\n",
+ "\n",
+ "# Load constants data for all constants\n",
+ "const_data = lpd_cal.ndarray_map(metadata=lpd_metadata)"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
deleted file mode 100644
index 27b78bc715629114466f4dd7e6e845c494cc3381..0000000000000000000000000000000000000000
--- a/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
+++ /dev/null
@@ -1,205 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# LPD Retrieving Constants Pre-correction #\n",
- "\n",
- "Author: European XFEL Detector Group, Version: 1.0\n",
- "\n",
- "The following notebook provides a constants metadata in a YAML file to use while correcting LPD images."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Input parameters\n",
- "in_folder = \"/gpfs/exfel/exp/FXE/202201/p003073/raw/\" # the folder to read data from, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/LPD_test\" # the folder to output to, required\n",
- "metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate.\n",
- "modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty\n",
- "karabo_da = [''] # Data aggregators names to correct, use [''] for all\n",
- "run = 10 # run to process, required\n",
- "\n",
- "# Source parameters\n",
- "karabo_id = 'FXE_DET_LPD1M-1' # Karabo domain for detector.\n",
- "input_source = '{karabo_id}/DET/{module_index}CH0:xtdf' # Input fast data source.\n",
- "\n",
- "# CalCat parameters\n",
- "creation_time = \"\" # The timestamp to use with Calibration DB. Required Format: \"YYYY-MM-DD hh:mm:ss\" e.g. 2019-07-04 11:02:41\n",
- "\n",
- "# Operating conditions\n",
- "mem_cells = 512 # Memory cells, LPD constants are always taken with 512 cells.\n",
- "bias_voltage = 250.0 # Detector bias voltage.\n",
- "capacitor = '5pF' # Capacitor setting: 5pF or 50pF\n",
- "photon_energy = 9.2 # Photon energy in keV.\n",
- "category = 0 # Whom to blame.\n",
- "use_cell_order = 'auto' # Whether to use memory cell order as a detector condition (not stored for older constants)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "from logging import warning\n",
- "from pathlib import Path\n",
- "\n",
- "import numpy as np\n",
- "\n",
- "from calibration_client import CalibrationClient\n",
- "from calibration_client.modules import CalibrationConstantVersion\n",
- "import extra_data as xd\n",
- "\n",
- "from cal_tools.lpdlib import get_mem_cell_pattern, make_cell_order_condition\n",
- "import cal_tools.restful_config as rest_cfg\n",
- "from cal_tools.calcat_interface import CalCatError, LPD_CalibrationData\n",
- "from cal_tools.tools import (\n",
- " CalibrationMetadata,\n",
- " calcat_creation_time,\n",
- ")\n",
- "from cal_tools.step_timing import StepTimer"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "out_folder = Path(out_folder)\n",
- "out_folder.mkdir(exist_ok=True)\n",
- "\n",
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
- "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
- "\n",
- "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
- "print(f'Using {creation_time.isoformat()} as creation time')\n",
- "\n",
- "# Pick all modules/aggregators or those selected.\n",
- "if not karabo_da or karabo_da == ['']:\n",
- " if not modules or modules == [-1]:\n",
- " modules = list(range(16))\n",
- "\n",
- " karabo_da = [f'LPD{i:02d}' for i in modules]\n",
- "else:\n",
- " modules = [int(x[-2:]) for x in karabo_da]\n",
- "\n",
- "# List of detector sources.\n",
- "det_inp_sources = [input_source.format(karabo_id=karabo_id, module_index=int(da[-2:])) for da in karabo_da]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "step_timer = StepTimer()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "step_timer.start()\n",
- "\n",
- "cell_ids_pattern_s = None\n",
- "if use_cell_order != 'never':\n",
- " # Read the order of memory cells used\n",
- " raw_data = xd.RunDirectory(Path(in_folder, f'r{run:04d}'))\n",
- " cell_ids_pattern_s = make_cell_order_condition(\n",
- " use_cell_order, get_mem_cell_pattern(raw_data, det_inp_sources)\n",
- " )\n",
- " print(\"Memory cells order:\", cell_ids_pattern_s)\n",
- "\n",
- "lpd_cal = LPD_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " modules=karabo_da,\n",
- " sensor_bias_voltage=bias_voltage,\n",
- " memory_cells=mem_cells,\n",
- " feedback_capacitor=capacitor,\n",
- " source_energy=photon_energy,\n",
- " memory_cell_order=cell_ids_pattern_s,\n",
- " category=category,\n",
- " event_at=creation_time,\n",
- " client=rest_cfg.calibration_client(),\n",
- ")\n",
- "\n",
- "# Initialize const_data in case no constants were retrieved.\n",
- "lpd_cal_metadata = {mod: {\"no-constants-retrieved\": None} for mod in karabo_da}\n",
- "\n",
- "constant_data = lpd_cal.metadata([\"Offset\", \"BadPixelsDark\"])\n",
- "try:\n",
- " illum_const_data = lpd_cal.metadata(lpd_cal.illuminated_calibrations)\n",
- " for key, value in illum_const_data.items():\n",
- " const_data.setdefault(key, {}).update(value)\n",
- "except CalCatError as e: # TODO: replace when API errors are improved.\n",
- " warning(f\"CalCatError: {e}\")\n",
- "\n",
- "# Validate the constants availability and raise/warn correspondingly. \n",
- "for mod, ccv_dict in lpd_cal_metadata.items():\n",
- " missing_offset = {\"Offset\"} - set(ccv_dict)\n",
- " warn_missing_constants = {\n",
- " \"BadPixelsDark\", \"BadPixelsFF\", \"GainAmpMap\", \"FFMap\", \"RelativeGain\"} - set(ccv_dict)\n",
- " if missing_offset:\n",
- " warning(f\"Offset constant is not available to correct {mod}\")\n",
- " karabo_da.remove(mod)\n",
- " if warn_missing_constants:\n",
- " warning(f\"Constants {warn_missing_constants} were not retrieved for {mod}.\")\n",
- "if not karabo_da: # Offsets are missing for all modules.\n",
- " raise Exception(\"Could not find offset constants for all modules, will not correct data.\")\n",
- "\n",
- "for mod, ccv_dict in lpd_cal_metadata.items():\n",
- " mdata_dict = {\"constants\": dict()}\n",
- " for cname, ccv_metadata in ccv_dict.items():\n",
- " mdata_dict[\"constants\"][cname] = {\n",
- " \"path\": str(lpd_cal.caldb_root / ccv_metadata[\"path\"]),\n",
- " \"dataset\": ccv_metadata[\"dataset\"],\n",
- " \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
- " \"ccv_id\": ccv_metadata[\"ccv_id\"],\n",
- " }\n",
- " mdata_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n",
- " retrieved_constants[mod] = mdata_dict\n",
- "metadata.save()\n",
- "\n",
- "step_timer.done_step(f\"Stored retrieved constants in {metadata.filename}.\")"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3.8.11 ('.cal4_venv')",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.8.11"
- },
- "orig_nbformat": 4,
- "vscode": {
- "interpreter": {
- "hash": "ccde353e8822f411c1c49844e1cbe3edf63293a69efd975d1b44f5e852832668"
- }
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index a35bdaacf97d371abdf06ed5fbbe3d8f4f21a3da..b505e7530d4daf3fe438be191fc0e1743dfca0b9 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -76,8 +76,6 @@ notebooks = {
"cluster cores": 8},
},
"CORRECT": {
- "pre_notebooks": [
- "notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb"],
"notebook": "notebooks/LPD/LPD_Correct_Fast.ipynb",
"concurrency": {"parameter": "sequences",
"default concurrency": [-1],