diff --git a/notebooks/DSSC/DSSC_Correct_and_Verify.ipynb b/notebooks/DSSC/DSSC_Correct_and_Verify.ipynb
index b29f2e2d93e5ac08ed6b83afff63b2272f3ea237..c020a491e6c33f0fddfdb19d7003aee3cee21788 100644
--- a/notebooks/DSSC/DSSC_Correct_and_Verify.ipynb
+++ b/notebooks/DSSC/DSSC_Correct_and_Verify.ipynb
@@ -24,7 +24,8 @@
"source": [
"cluster_profile = \"noDB\" # The ipcluster profile to use\n",
"in_folder = \"/gpfs/exfel/exp/SQS/202131/p900210/raw\" # path to input data, required\n",
- "out_folder = \"/gpfs/exfel/data/scratch/samartse/data/DSSC\" # path to output to, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/dssc\" # path to output to, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
"sequences = [-1] # sequence files to evaluate.\n",
"modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
"run = 20 #runs to process, required\n",
@@ -100,6 +101,7 @@
" map_modules_from_folder,\n",
" parse_runs,\n",
" run_prop_seq_from_path,\n",
+ " CalibrationMetadata,\n",
")\n",
"from dateutil import parser\n",
"from iCalibrationDB import Conditions, ConstantMetaData, Constants, Detectors, Versions"
@@ -115,7 +117,9 @@
"if use_dir_creation_date:\n",
" creation_time = get_dir_creation_date(in_folder, run)\n",
" print(f\"Using {creation_time} as creation time\")\n",
- "\n",
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
+ "const_yaml = metadata.setdefault(\"retrieved-constants\", {})\n",
"if sequences[0] == -1:\n",
" sequences = None\n",
" \n",
@@ -162,7 +166,7 @@
},
"outputs": [],
"source": [
- "# set everything up filewise\n",
+ "# set everything up file wise\n",
"mmf = map_modules_from_folder(in_folder, run, path_template, karabo_da, sequences)\n",
"mapped_files, mod_ids, total_sequences, sequences_qm, file_size = mmf\n",
"MAX_PAR = min(MAX_PAR, total_sequences)"
@@ -229,7 +233,7 @@
"\n",
"def correct_module(total_sequences, sequences_qm, karabo_id, dinstance, mask_noisy_asic, \n",
" mask_cold_asic, noisy_pix_threshold, chunksize, mem_cells, bias_voltage,\n",
- " cal_db_timeout, creation_time, cal_db_interface, h5path, h5path_idx, inp):\n",
+ " cal_db_timeout, creation_time, cal_db_interface, h5path, h5path_idx, const_yaml, inp):\n",
" \n",
" import binascii\n",
" import copy\n",
@@ -240,7 +244,7 @@
" import numpy as np\n",
" from cal_tools.dssclib import get_dssc_ctrl_data, get_pulseid_checksum\n",
" from cal_tools.enums import BadPixels\n",
- " from cal_tools.tools import get_constant_from_db_and_time\n",
+ " from cal_tools.tools import get_constant_from_db_and_time, read_const_yaml\n",
" from iCalibrationDB import (\n",
" Conditions,\n",
" ConstantMetaData,\n",
@@ -276,32 +280,32 @@
" dists = np.array([(o-maxcell) for o in options])\n",
" dists[dists<0] = 10000 # assure to always go higher\n",
" return options[np.argmin(dists)]\n",
- " \n",
+ "\n",
" if mem_cells == 0:\n",
" mem_cells = get_num_cells(filename, h5path)\n",
" \n",
" pulseid_checksum = get_pulseid_checksum(filename, h5path, h5path_idx)\n",
" \n",
" print(f\"Memcells: {mem_cells}\")\n",
- " \n",
- " condition = Conditions.Dark.DSSC(bias_voltage=bias_voltage, memory_cells=mem_cells,\\\n",
- " pulseid_checksum=pulseid_checksum,\\\n",
- " acquisition_rate=conditions['acquisition_rate'],\\\n",
- " target_gain=conditions['target_gain'],\\\n",
- " encoded_gain=conditions['encoded_gain'])\n",
- " \n",
- " detinst = getattr(Detectors, dinstance)\n",
- " device = getattr(detinst, qm)\n",
- " with h5py.File(filename, \"r\") as infile:\n",
- " y = infile[f\"{h5path}/data\"].shape[2]\n",
- " x = infile[f\"{h5path}/data\"].shape[3]\n",
- " offset, when = get_constant_from_db_and_time(karabo_id, karabo_da,\n",
- " Constants.DSSC.Offset(),\n",
- " condition,\n",
- " None,\n",
- " cal_db_interface,\n",
- " creation_time=creation_time,\n",
- " timeout=cal_db_timeout)\n",
+ " if const_yaml:\n",
+ " const_dict, when_dict = read_const_yaml(const_yaml[karabo_da][\"constants\"])\n",
+ " offset = const_dict[\"Offset\"]\n",
+ " when = when_dict[\"Offset\"]\n",
+ " else:\n",
+ " condition = Conditions.Dark.DSSC(bias_voltage=bias_voltage, memory_cells=mem_cells,\\\n",
+ " pulseid_checksum=pulseid_checksum,\\\n",
+ " acquisition_rate=conditions['acquisition_rate'],\\\n",
+ " target_gain=conditions['target_gain'],\\\n",
+ " encoded_gain=conditions['encoded_gain'])\n",
+ " \n",
+ "\n",
+ " offset, when = get_constant_from_db_and_time(karabo_id, karabo_da,\n",
+ " Constants.DSSC.Offset(),\n",
+ " condition,\n",
+ " None,\n",
+ " cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " timeout=cal_db_timeout)\n",
" if offset is not None:\n",
" offset = np.moveaxis(np.moveaxis(offset[...], 2, 0), 2, 1)\n",
" else:\n",
@@ -456,7 +460,7 @@
" karabo_id, dinstance, mask_noisy_asic, mask_cold_asic,\n",
" noisy_pix_threshold, chunk_size_idim, mem_cells,\n",
" bias_voltage, cal_db_timeout, creation_time, cal_db_interface,\n",
- " h5path, h5path_idx)\n",
+ " h5path, h5path_idx, const_yaml)\n",
"\n",
" r = view.map_sync(p, inp)\n",
" #r = list(map(p, inp))\n",
diff --git a/notebooks/DSSC/DSSC_retrieve_constants_precorrection.ipynb b/notebooks/DSSC/DSSC_retrieve_constants_precorrection.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..7f7392d3fe8742573563c1b3c9d651f394ad27c3
--- /dev/null
+++ b/notebooks/DSSC/DSSC_retrieve_constants_precorrection.ipynb
@@ -0,0 +1,235 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# DSSC Retrieving Constants Pre-correction #\n",
+ "\n",
+ "Author: European XFEL Detector Group, Version: 1.0\n",
+ "\n",
+ "The following notebook provides a constants metadata in a YAML file to use while correcting DSSC images."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = \"/gpfs/exfel/exp/SCS/202122/p002937/raw\" # path to input data, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/dssc_scs\" # path to output to, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
+ "sequences = [-1] # sequence files to evaluate.\n",
+ "modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
+ "run = 1723 # runs to process, required\n",
+ "\n",
+ "# Parameters for accessing the raw data.\n",
+ "karabo_id = \"SCS_DET_DSSC1M-1\" # karabo karabo_id\n",
+ "karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
+ "receiver_id = \"{}CH0\" # inset for receiver devices\n",
+ "path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # the template to use to access data\n",
+ "h5path = 'INSTRUMENT/{}/DET/{}:xtdf/image' # path in the HDF5 file to images\n",
+ "h5path_idx = '/INDEX/{}/DET/{}:xtdf/image' # path in the HDF5 file to images\n",
+ "slow_data_pattern = 'RAW-R{}-DA{}-S00000.h5'\n",
+ "slow_data_aggregators = [1, 2, 3, 4] # quadrant/aggregator\n",
+ "slow_data_path = \"SCS_CDIDET_DSSC/FPGA/PPT_Q\"\n",
+ "\n",
+ "# Parameters affecting writing corrected data.\n",
+ "overwrite = True # set to True if existing data should be overwritten\n",
+ "\n",
+ "# Parameters for the calibration database.\n",
+ "use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
+ "cal_db_interface = \"tcp://max-exfl016:8020#8025\" # the database interface to use\n",
+ "cal_db_timeout = 300000 # in milliseconds\n",
+ "\n",
+ "# Conditions for retrieving calibration constants.\n",
+ "mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
+ "max_pulses = 800 # maximum number of pulses per train\n",
+ "bias_voltage = 100 # detector bias voltage"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from pathlib import Path\n",
+ "\n",
+ "from cal_tools import dssclib\n",
+ "from cal_tools.tools import (\n",
+ " get_from_db,\n",
+ " get_dir_creation_date,\n",
+ " map_modules_from_folder,\n",
+ " module_index_to_qm,\n",
+ " save_constant_metadata,\n",
+ " CalibrationMetadata,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = Path(in_folder)\n",
+ "out_folder = Path(out_folder)\n",
+ "\n",
+ "run_folder = in_folder / f\"r{run:04d}\"\n",
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
+ "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
+ "\n",
+ "instrument_src_template = h5path.format(karabo_id, receiver_id)\n",
+ "index_src_template = h5path_idx.format(karabo_id, receiver_id)\n",
+ "\n",
+ "if karabo_da[0] == \"-1\":\n",
+ " if modules[0] == -1:\n",
+ " modules = list(range(16))\n",
+ " karabo_da = [\"DSSC{:02d}\".format(i) for i in modules]\n",
+ "else:\n",
+ " modules = [int(x[-2:]) for x in karabo_da]\n",
+ "print(\n",
+ " \"Retrieve constants for modules: \",\n",
+ " \", \".join([module_index_to_qm(x) for x in modules]),\n",
+ ")\n",
+ "\n",
+ "if out_folder.exists() and not overwrite:\n",
+ " raise AttributeError(\"Output path exists! Exiting\")\n",
+ "else:\n",
+ " out_folder.mkdir(parents=True, exist_ok=True)\n",
+ "\n",
+ "creation_time = None\n",
+ "if use_dir_creation_date:\n",
+ " creation_time = get_dir_creation_date(in_folder, run)\n",
+ " print(f\"Using {creation_time} as creation time\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mapped_files, _, total_sequences, _, _ = map_modules_from_folder(\n",
+ " in_folder,\n",
+ " run,\n",
+ " path_template,\n",
+ " karabo_da,\n",
+ " None if sequences[0] == -1 else sequences,\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "tGain, encodedGain, operatingFreq = dssclib.get_dssc_ctrl_data(\n",
+ " in_folder / f\"r{run:04d}\",\n",
+ " slow_data_pattern,\n",
+ " slow_data_aggregators,\n",
+ " run,\n",
+ " slow_data_path,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for i, k_da in zip(modules, karabo_da):\n",
+ " if k_da in retrieved_constants:\n",
+ " print(f\"Constant for {k_da} already in {metadata.filename}, won't query again.\") # noqa\n",
+ " continue\n",
+ " qm = module_index_to_qm(i)\n",
+ " instrument_src = instrument_src_template.format(i)\n",
+ " index_src = index_src_template.format(i)\n",
+ "\n",
+ " if qm in mapped_files:\n",
+ " if not mapped_files[qm].empty():\n",
+ " filename = str(mapped_files[qm].get())\n",
+ " print(filename)\n",
+ " else:\n",
+ " print(f\"{qm} file is missing\")\n",
+ " continue\n",
+ " else:\n",
+ " print(f\"Skipping {qm}\")\n",
+ " continue\n",
+ "\n",
+ " conditions = {}\n",
+ " conditions[\"acquisition_rate\"] = operatingFreq[qm]\n",
+ " conditions[\"target_gain\"] = tGain[qm]\n",
+ " conditions[\"encoded_gain\"] = encodedGain[qm]\n",
+ "\n",
+ " if mem_cells == 0:\n",
+ " mem_cells = dssclib.get_num_cells(filename, instrument_src)\n",
+ "\n",
+ " pulseid_checksum = dssclib.get_pulseid_checksum(\n",
+ " filename,\n",
+ " instrument_src,\n",
+ " index_src,\n",
+ " )\n",
+ " condition = Conditions.Dark.DSSC(\n",
+ " bias_voltage=bias_voltage,\n",
+ " memory_cells=mem_cells,\n",
+ " pulseid_checksum=pulseid_checksum,\n",
+ " acquisition_rate=conditions[\"acquisition_rate\"],\n",
+ " target_gain=conditions[\"target_gain\"],\n",
+ " encoded_gain=conditions[\"encoded_gain\"],\n",
+ " )\n",
+ " mdata_dict = dict()\n",
+ " mdata_dict[\"constants\"] = dict()\n",
+ " offset, mdata = get_from_db(\n",
+ " karabo_id=karabo_id,\n",
+ " karabo_da=k_da,\n",
+ " constant=Constants.DSSC.Offset(),\n",
+ " condition=condition,\n",
+ " empty_constant=None,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " timeout=cal_db_timeout,\n",
+ " )\n",
+ " save_constant_metadata(mdata_dict[\"constants\"], mdata, \"Offset\")\n",
+ " mdata_dict[\"physical-detector-unit\"] = mdata.calibration_constant_version.device_name\n",
+ " retrieved_constants[k_da] = mdata_dict\n",
+ "metadata.save()\n",
+ "\n",
+ "print(f\"Stored retrieved constants in {metadata.filename}\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.8.11 ('.cal4_venv': venv)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.11"
+ },
+ "orig_nbformat": 4,
+ "vscode": {
+ "interpreter": {
+ "hash": "25ceec0b6126c0ccf883616a02d86b8eaec8ca3fe33700925044adbe0a704e39"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/src/cal_tools/dssclib.py b/src/cal_tools/dssclib.py
index 59ca874d2c896224418347577f2a991789fb00b4..7550d42463bd166142e7905e03f9c3c44d4e2be5 100644
--- a/src/cal_tools/dssclib.py
+++ b/src/cal_tools/dssclib.py
@@ -9,6 +9,17 @@ import h5py
import numpy as np
+def get_num_cells(fname, h5path):
+
+ with h5py.File(fname, "r") as f:
+ cells = f[f"{h5path}/cellId"][()]
+ maxcell = np.max(cells)
+ options = [100, 200, 400, 500, 600, 700, 800]
+ dists = np.array([(o-maxcell) for o in options])
+ dists[dists<0] = 10000 # assure to always go higher
+ return options[np.argmin(dists)]
+
+
def get_pulseid_checksum(fname, h5path, h5path_idx):
"""generates hash value from pulse pattern (veto defined)."""
with h5py.File(fname, "r") as infile:
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index 1e4fcdee37dbf729a502336614a7be22d121f004..ee4cfefac2c5c97b2d974cc0a0c03e3ef489ebf3 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -247,6 +247,7 @@ notebooks = {
"cluster cores": 8},
},
"CORRECT": {
+ "pre_notebooks": ["notebooks/DSSC/DSSC_retrieve_constants_precorrection.ipynb"],
"notebook": "notebooks/DSSC/DSSC_Correct_and_Verify.ipynb",
"concurrency": {"parameter": "sequences",
"use function": "balance_sequences",