diff --git a/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..de8bae278277b1817d2dca9f1920670528d0857b
--- /dev/null
+++ b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
@@ -0,0 +1,262 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# LPD Retrieving Constants Pre-correction #\n",
+ "\n",
+ "Author: European XFEL Detector Group, Version: 1.0\n",
+ "\n",
+ "The following notebook provides a constants metadata in a YAML file to use while correcting LPD images."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Input parameters\n",
+ "in_folder = \"/gpfs/exfel/exp/FXE/202201/p003073/raw/\" # the folder to read data from, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/LPD_test\" # the folder to output to, required\n",
+ "metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate.\n",
+ "sequences = [-1] # Sequences to correct, use [-1] for all\n",
+ "modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty\n",
+ "karabo_da = [''] # Data aggregators names to correct, use [''] for all\n",
+ "run = 10 # runs to process, required\n",
+ "\n",
+ "# Source parameters\n",
+ "karabo_id = 'FXE_DET_LPD1M-1' # Karabo domain for detector.\n",
+ "input_source = '{karabo_id}/DET/{module_index}CH0:xtdf' # Input fast data source.\n",
+ "output_source = '' # Output fast data source, empty to use same as input.\n",
+ "\n",
+ "# CalCat parameters\n",
+ "use_dir_creation_date = True # Use the creation date of the directory for database time derivation.\n",
+ "cal_db_root = '/gpfs/exfel/d/cal/caldb_store'\n",
+ "\n",
+ "# Operating conditions\n",
+ "mem_cells = 512 # Memory cells, LPD constants are always taken with 512 cells.\n",
+ "bias_voltage = 250.0 # Detector bias voltage.\n",
+ "capacitor = '5pF' # Capacitor setting: 5pF or 50pF\n",
+ "photon_energy = 9.2 # Photon energy in keV.\n",
+ "category = 0 # Whom to blame.\n",
+ "\n",
+ "# Correction parameters\n",
+ "offset_corr = True # Offset correction.\n",
+ "rel_gain = True # Gain correction based on RelativeGain constant.\n",
+ "ff_map = True # Gain correction based on FFMap constant.\n",
+ "gain_amp_map = True # Gain correction based on GainAmpMap constant.\n",
+ "\n",
+ "# Output options\n",
+ "overwrite = True # set to True if existing data should be overwritten\n",
+ "chunks_data = 1 # HDF chunk size for pixel data in number of frames.\n",
+ "chunks_ids = 32 # HDF chunk size for cellId and pulseId datasets.\n",
+ "\n",
+ "# Parallelization options\n",
+ "max_nodes = 8 # Maximum number of SLURM jobs to split correction work into\n",
+ "num_workers = 8 # Worker processes per node, 8 is safe on 768G nodes but won't work on 512G.\n",
+ "num_threads_per_worker = 32 # Number of threads per worker."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from pathlib import Path\n",
+ "from time import perf_counter\n",
+ "import gc\n",
+ "import re\n",
+ "\n",
+ "from calibration_client import CalibrationClient\n",
+ "from calibration_client.modules import CalibrationConstantVersion\n",
+ "import extra_data as xd\n",
+ "import extra_geom as xg\n",
+ "import pasha as psh\n",
+ "\n",
+ "from extra_data.components import LPD1M\n",
+ "\n",
+ "from cal_tools.tools import (\n",
+ " CalibrationMetadata,\n",
+ " get_dir_creation_date,\n",
+ " save_constant_metadata,\n",
+ " write_compressed_frames,\n",
+ ")\n",
+ "from cal_tools.files import DataFile\n",
+ "from cal_tools.restful_config import restful_config"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "file_re = re.compile(r'^RAW-R(\\d{4})-(\\w+\\d+)-S(\\d{5})$') # This should probably move to cal_tools\n",
+ "\n",
+ "run_folder = Path(in_folder) / f'r{run:04d}'\n",
+ "out_folder = Path(out_folder)\n",
+ "out_folder.mkdir(exist_ok=True)\n",
+ "\n",
+ "output_source = output_source or input_source\n",
+ "\n",
+ "cal_db_root = Path(cal_db_root)\n",
+ "\n",
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
+ "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
+ "\n",
+ "if use_dir_creation_date:\n",
+ " creation_time = get_dir_creation_date(in_folder, run) \n",
+ "else:\n",
+ " from datetime import datetime\n",
+ " creation_time = datetime.now()\n",
+ " \n",
+ "print(f'Using {creation_time.isoformat()} as creation time')\n",
+ "\n",
+ "# Pick all modules/aggregators or those selected.\n",
+ "if not karabo_da or karabo_da == ['']:\n",
+ " if not modules or modules == [-1]:\n",
+ " modules = list(range(16))\n",
+ "\n",
+ " karabo_da = [f'LPD{i:02d}' for i in modules]\n",
+ " \n",
+ "# Pick all sequences or those selected.\n",
+ "if not sequences or sequences == [-1]:\n",
+ " do_sequence = lambda seq: True\n",
+ "else:\n",
+ " do_sequence = [int(x) for x in sequences].__contains__ \n",
+ " \n",
+ "# List of detector sources.\n",
+ "det_inp_sources = [input_source.format(karabo_id=karabo_id, module_index=int(da[-2:])) for da in karabo_da]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Connect to CalCat.\n",
+ "calcat_config = restful_config['calcat']\n",
+ "client = CalibrationClient(\n",
+ " base_api_url=calcat_config['base-api-url'],\n",
+ " use_oauth2=calcat_config['use-oauth2'],\n",
+ " client_id=calcat_config['user-id'],\n",
+ " client_secret=calcat_config['user-secret'],\n",
+ " user_email=calcat_config['user-email'],\n",
+ " token_url=calcat_config['token-url'],\n",
+ " refresh_url=calcat_config['refresh-url'],\n",
+ " auth_url=calcat_config['auth-url'],\n",
+ " scope='')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dark_calibrations = {\n",
+ " 11: 'Offset',\n",
+ " 14: 'BadPixelsDark',\n",
+ "}\n",
+ "\n",
+ "dark_condition = [\n",
+ " dict(parameter_id=1, value=bias_voltage), # Sensor bias voltage\n",
+ " dict(parameter_id=7, value=mem_cells), # Memory cells\n",
+ " dict(parameter_id=15, value=capacitor), # Feedback capacitor\n",
+ " dict(parameter_id=13, value=256), # Pixels X\n",
+ " dict(parameter_id=14, value=256), # Pixels Y\n",
+ "]\n",
+ "\n",
+ "illuminated_calibrations = {\n",
+ " 20: 'BadPixelsFF',\n",
+ " 432: 'GainAmpMap',\n",
+ " 413: 'FFMap',\n",
+ " 424: 'RelativeGain',\n",
+ "}\n",
+ "\n",
+ "illuminated_condition = dark_condition.copy()\n",
+ "illuminated_condition += [\n",
+ " dict(parameter_id=3, value=photon_energy), # Source energy\n",
+ " dict(parameter_id=25, value=category) # category\n",
+ "]\n",
+ "\n",
+ "const_data = {}\n",
+ "const_load_mp = psh.ProcessContext(num_workers=24)\n",
+ "\n",
+ "print('Querying calibration database', end='', flush=True)\n",
+ "start = perf_counter()\n",
+ "for k_da in karabo_da:\n",
+ " pdu = None\n",
+ " if k_da in retrieved_constants:\n",
+ " print(f\"Constant for {k_da} already in {metadata.filename}, won't query again.\") # noqa\n",
+ " continue\n",
+ " retrieved_constants[k_da] = dict()\n",
+ " const_mdata = retrieved_constants[k_da][\"constants\"] = dict()\n",
+ " for calibrations, condition in [\n",
+ " (dark_calibrations, dark_condition),\n",
+ " (illuminated_calibrations, illuminated_condition)\n",
+ " ]:\n",
+ " resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(\n",
+ " client, karabo_id, list(calibrations.keys()),\n",
+ " {'parameters_conditions_attributes': condition},\n",
+ " karabo_da=k_da, event_at=creation_time.isoformat(), snapshot_at=None)\n",
+ "\n",
+ " if not resp[\"success\"]:\n",
+ " print(f\"ERROR: Constants {list(calibrations.values())} \"\n",
+ " f\"were not retrieved, {resp['app_info']}\")\n",
+ " const_mdata[\"file-path\"] = None\n",
+ " const_mdata[\"dataset-name\"] = None\n",
+ " const_mdata[\"creation-time\"] = None \n",
+ " continue\n",
+ "\n",
+ " for ccv in resp[\"data\"]:\n",
+ " cc = ccv['calibration_constant']\n",
+ " calibration_name = calibrations[cc['calibration_id']]\n",
+ " const_mdata[\"file-path\"] = str(Path(ccv['path_to_file']) / ccv['file_name'])\n",
+ " const_mdata[\"dataset-name\"] = ccv['data_set_name']\n",
+ " const_mdata[\"creation-time\"] = ccv['begin_at']\n",
+ " pdu = ccv['physical_detector_unit']['physical_name']\n",
+ "\n",
+ " print('.', end='', flush=True)\n",
+ " retrieved_constants[k_da][\"physical-detector-unit\"] = pdu\n",
+ "metadata.save()\n",
+ "\n",
+ "total_time = perf_counter() - start\n",
+ "print(f'{total_time:.1f}s')\n",
+ "print(f\"Stored retrieved constants in {metadata.filename}\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.8.11 ('.cal4_venv')",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.11"
+ },
+ "orig_nbformat": 4,
+ "vscode": {
+ "interpreter": {
+ "hash": "ccde353e8822f411c1c49844e1cbe3edf63293a69efd975d1b44f5e852832668"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}