diff --git a/notebooks/DynamicFF/Characterize_DynamicFF_NBC.ipynb b/notebooks/DynamicFF/Characterize_DynamicFF_NBC.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..9339eae9c535260072335592a462bbcff1d162e6
--- /dev/null
+++ b/notebooks/DynamicFF/Characterize_DynamicFF_NBC.ipynb
@@ -0,0 +1,351 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Characterization of dark and flat field for Dynamic Flat Field correction\n",
+ "\n",
+ "Author: Egor Sobolev\n",
+ "\n",
+ "Computation of dark offsets and flat-field principal components"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = \"/gpfs/exfel/exp/SPB/202430/p900425/raw\" # input folder, required\n",
+ "out_folder = '/gpfs/exfel/data/scratch/esobolev/test/shimadzu' # output folder, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
+ "run_high = 1 # run number in which dark data was recorded, required\n",
+ "run_low = 2 # run number in which flat-field data was recorded, required\n",
+ "operation_mode = \"PCA_DynamicFF\" # Detector operation mode, optional (defaults to \"PCA_DynamicFF\")\n",
+ "\n",
+ "# Data files parameters.\n",
+ "karabo_da = ['-1'] # data aggregators\n",
+ "karabo_id = \"SPB_MIC_HPVX2\" # karabo prefix of Shimadzu HPV-X2 devices\n",
+ "\n",
+ "# Database access parameters.\n",
+ "cal_db_interface = \"tcp://max-exfl-cal001:8021\" # Unused, calibration DB interface to use\n",
+ "cal_db_timeout = 30000 # Unused, calibration DB timeout\n",
+ "db_output = False # if True, the notebook sends dark constants to the calibration database\n",
+ "local_output = True # if True, the notebook saves dark constants locally\n",
+ "\n",
+ "# Calibration constants parameters\n",
+ "n_components = 50 # Number of principal components of flat-field to compute (default: 50)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import datetime\n",
+ "import os\n",
+ "import warnings\n",
+ "from logging import warning\n",
+ "from shutil import copyfile\n",
+ "from tempfile import NamedTemporaryFile\n",
+ "\n",
+ "warnings.filterwarnings('ignore')\n",
+ "\n",
+ "import time\n",
+ "import numpy as np\n",
+ "import matplotlib.pyplot as plt\n",
+ "from IPython.display import display, Markdown\n",
+ "\n",
+ "from extra_data import RunDirectory\n",
+ "\n",
+ "%matplotlib inline\n",
+ "from cal_tools.step_timing import StepTimer\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " run_prop_seq_from_path,\n",
+ " save_dict_to_hdf5\n",
+ ")\n",
+ "from cal_tools.restful_config import calibration_client, extra_calibration_client\n",
+ "from cal_tools.shimadzu import ShimadzuHPVX2\n",
+ "from cal_tools.constants import write_ccv, inject_ccv\n",
+ "\n",
+ "import dynflatfield as dffc\n",
+ "from dynflatfield.draw import plot_images, plot_camera_image"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "extra_calibration_client() # Configure CalibrationData.\n",
+ "\n",
+ "cc = calibration_client()\n",
+ "pdus = cc.get_all_phy_det_units_from_detector(\n",
+ " {\"detector_identifier\": karabo_id}) # TODO: Use creation_time for snapshot_at\n",
+ "\n",
+ "if not pdus[\"success\"]:\n",
+ " raise ValueError(\"Failed to retrieve PDUs\")\n",
+ "\n",
+ "detector_info = pdus['data'][0]['detector']\n",
+ "detector = ShimadzuHPVX2(detector_info[\"source_name_pattern\"])\n",
+ "\n",
+ "print(f\"Instrument {detector.instrument}\")\n",
+ "print(f\"Detector in use is {karabo_id}\")\n",
+ "\n",
+ "modules = {}\n",
+ "for pdu_no, pdu in enumerate(pdus[\"data\"]):\n",
+ " db_module = pdu[\"physical_name\"]\n",
+ " module = pdu[\"module_number\"]\n",
+ " da = pdu[\"karabo_da\"]\n",
+ " if karabo_da[0] != \"-1\" and da not in karabo_da:\n",
+ " continue\n",
+ "\n",
+ " instrument_source_name = detector.instrument_source(module)\n",
+ " print('-', da, db_module, module, instrument_source_name)\n",
+ "\n",
+ " modules[da] = dict(\n",
+ " db_module=db_module,\n",
+ " module=module,\n",
+ " raw_source_name=instrument_source_name,\n",
+ " pdu_no=pdu_no,\n",
+ " )\n",
+ "\n",
+ "constants = {}\n",
+ "\n",
+ "step_timer = StepTimer()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Offset map"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dark_run = run_high\n",
+ "dark_creation_time = get_dir_creation_date(in_folder, dark_run)\n",
+ "print(f\"Using {dark_creation_time} as creation time of Offset constant.\")\n",
+ "\n",
+ "for da, meta in modules.items():\n",
+ " source_name = detector.instrument_source(meta[\"module\"])\n",
+ " image_key = detector.image_key\n",
+ "\n",
+ " display(Markdown(f\"## {source_name}\"))\n",
+ "\n",
+ " # read\n",
+ " step_timer.start()\n",
+ " file_da, _, _ = da.partition('/')\n",
+ " dark_dc = RunDirectory(f\"{in_folder}/r{dark_run:04d}\",\n",
+ " include=f\"RAW-R{dark_run:04d}-{file_da}-S*.h5\")\n",
+ "\n",
+ " if source_name not in dark_dc.all_sources:\n",
+ " raise ValueError(f\"Could not find source {source_name} for module {da} in dark data\")\n",
+ "\n",
+ " dark_dc = dark_dc.select([(source_name, image_key)])\n",
+ " conditions = detector.conditions(dark_dc, meta[\"module\"])\n",
+ "\n",
+ " key_data = dark_dc[source_name, image_key]\n",
+ " images_dark = key_data.ndarray()\n",
+ " ntrain, npulse, ny, nx = images_dark.shape\n",
+ "\n",
+ " print(f\"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})\")\n",
+ " print(f\"Image size: {ny} x {nx} px\")\n",
+ " step_timer.done_step(\"Read dark images\")\n",
+ "\n",
+ " # process\n",
+ " step_timer.start()\n",
+ " dark = dffc.process_dark(images_dark) # Amounts to a per-pixel mean right now.\n",
+ "\n",
+ " # put results in the dict\n",
+ " module_constants = constants.setdefault(meta[\"db_module\"], {})\n",
+ " module_constants[\"Offset\"] = dict(\n",
+ " conditions=conditions, data=dark, pdu_no=meta[\"pdu_no\"],\n",
+ " creation_time=dark_creation_time, dims=['ss', 'fs']\n",
+ " )\n",
+ " step_timer.done_step(\"Process dark images\")\n",
+ " display()\n",
+ "\n",
+ " # draw plots\n",
+ " step_timer.start()\n",
+ " plot_camera_image(dark)\n",
+ " plt.show()\n",
+ " step_timer.done_step(\"Draw offsets\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Flat-field PCA decomposition"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "flat_run = run_low\n",
+ "flat_creation_time = get_dir_creation_date(in_folder, flat_run)\n",
+ "print(f\"Using {flat_creation_time} as creation time of DynamicFF constant.\")\n",
+ "\n",
+ "for da, meta in modules.items():\n",
+ " source_name = detector.instrument_source(meta[\"module\"])\n",
+ " image_key = detector.image_key\n",
+ "\n",
+ " display(Markdown(f\"## {source_name}\"))\n",
+ "\n",
+ " # read\n",
+ " step_timer.start()\n",
+ " file_da, _, _ = da.partition('/')\n",
+ " flat_dc = RunDirectory(f\"{in_folder}/r{flat_run:04d}\",\n",
+ " include=f\"RAW-R{flat_run:04d}-{file_da}-S*.h5\")\n",
+ "\n",
+ " if source_name not in flat_dc.all_sources:\n",
+ " raise ValueError(f\"Could not find source {source_name} for module {da} in flatfield data\")\n",
+ "\n",
+ " flat_dc = flat_dc.select([(source_name, image_key)])\n",
+ " conditions = detector.conditions(flat_dc, meta[\"module\"])\n",
+ "\n",
+ " dark = constants[meta[\"db_module\"]][\"Offset\"][\"data\"]\n",
+ " dark_conditions = constants[meta[\"db_module\"]][\"Offset\"][\"conditions\"]\n",
+ "\n",
+ " if conditions != dark_conditions:\n",
+ " raise ValueError(f\"The conditions for flat-field run {conditions}) do not match \"\n",
+ " f\"the dark run conditions ({dark_conditions}). Skip flat-field characterization.\")\n",
+ "\n",
+ " key_data = flat_dc[source_name][image_key]\n",
+ " images_flat = key_data.ndarray()\n",
+ " ntrain, npulse, ny, nx = images_flat.shape\n",
+ "\n",
+ " print(f\"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})\")\n",
+ " print(f\"Image size: {ny} x {nx} px\")\n",
+ " step_timer.done_step(\"Read flat-field images\")\n",
+ "\n",
+ " # process\n",
+ " step_timer.start()\n",
+ " flat, components, explained_variance_ratio = dffc.process_flat(\n",
+ " images_flat, dark, n_components)\n",
+ " flat_data = np.concatenate([flat[None, ...], components])\n",
+ "\n",
+ " # put results in the dict\n",
+ " conditions = detector.conditions(flat_dc, meta[\"module\"])\n",
+ " module_constants = constants.setdefault(meta[\"db_module\"], {})\n",
+ " module_constants[\"DynamicFF\"] = dict(\n",
+ " conditions=conditions, data=flat_data, pdu_no=meta[\"pdu_no\"],\n",
+ " creation_time=flat_creation_time, dims=['component', 'ss', 'fs']\n",
+ " )\n",
+ " step_timer.done_step(\"Process flat-field images\")\n",
+ "\n",
+ " # draw plots\n",
+ " step_timer.start()\n",
+ " display(Markdown(\"### Average flat-field\"))\n",
+ " plot_camera_image(flat)\n",
+ " plt.show()\n",
+ "\n",
+ " display(Markdown(\"### Explained variance ratio\"))\n",
+ " fig, ax = plt.subplots(1, 1, figsize=(10,4), tight_layout=True)\n",
+ " ax.semilogy(explained_variance_ratio, 'o')\n",
+ " ax.set_xticks(np.arange(len(explained_variance_ratio)))\n",
+ " ax.set_xlabel(\"Component no.\")\n",
+ " ax.set_ylabel(\"Variance fraction\")\n",
+ " plt.show()\n",
+ "\n",
+ " display(Markdown(\"### The first principal components (up to 20)\"))\n",
+ " plot_images(components[:20], figsize=(13, 8))\n",
+ " plt.show()\n",
+ "\n",
+ " step_timer.done_step(\"Draw flat-field\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Calibration constants"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "step_timer.start()\n",
+ "\n",
+ "_, proposal, _ = run_prop_seq_from_path(in_folder)\n",
+ "\n",
+ "# Output Folder Creation:\n",
+ "if local_output:\n",
+ " os.makedirs(out_folder, exist_ok=True)\n",
+ "\n",
+ "for db_module, module_constants in constants.items():\n",
+ " for constant_name, constant in module_constants.items():\n",
+ " conditions = constant[\"conditions\"]\n",
+ " pdu = pdus[\"data\"][constant[\"pdu_no\"]]\n",
+ "\n",
+ " with NamedTemporaryFile() as tempf:\n",
+ " ccv_root = write_ccv(\n",
+ " tempf.name,\n",
+ " pdu['physical_name'], pdu['uuid'], pdu['detector_type']['name'],\n",
+ " constant_name, conditions, constant['creation_time'],\n",
+ " proposal, [dark_run, flat_run],\n",
+ " constant[\"data\"], constant['dims'])\n",
+ " \n",
+ " if db_output:\n",
+ " inject_ccv(tempf.name, ccv_root, metadata_folder)\n",
+ " \n",
+ " if local_output:\n",
+ " ofile = f\"{out_folder}/const_{constant_name}_{db_module}.h5\"\n",
+ " \n",
+ " if os.path.isfile(ofile):\n",
+ " print(f'File {ofile} already exists and will be overwritten')\n",
+ " \n",
+ " copyfile(tempf.name, ofile)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"Total processing time {step_timer.timespan():.01f} s\")\n",
+ "step_timer.print_summary()"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.11"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/notebooks/DynamicFF/Correct_DynamicFF_NBC.ipynb b/notebooks/DynamicFF/Correct_DynamicFF_NBC.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..3e6a1875c027ef1796d7f5687588becf8539fdf1
--- /dev/null
+++ b/notebooks/DynamicFF/Correct_DynamicFF_NBC.ipynb
@@ -0,0 +1,343 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Dynamic Flat-field Offline Correction\n",
+ "\n",
+ "Author: Egor Sobolev\n",
+ "\n",
+ "Offline dynamic flat-field correction"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = \"/gpfs/exfel/exp/SPB/202430/p900425/raw\" # input folder, required\n",
+ "out_folder =\"/gpfs/exfel/exp/SPB/202430/p900425/scratch/proc/r0003\" # output folder, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
+ "run = 3 # which run to read data from, required\n",
+ "\n",
+ "# Data files parameters.\n",
+ "karabo_da = ['-1'] # data aggregators\n",
+ "karabo_id = \"SPB_MIC_HPVX2\" # karabo prefix of Shimadzu HPV-X2 devices\n",
+ "\n",
+ "# Database access parameters.\n",
+ "cal_db_interface = \"tcp://max-exfl-cal001:8021\" # Unused, calibration DB interface to use\n",
+ "cal_db_timeout = 30000 # Unused, calibration DB timeout\n",
+ "\n",
+ "# Correction parameters\n",
+ "n_components = 20 # number of principal components of flat-field to use in correction\n",
+ "downsample_factors = [1, 1] # list of downsample factors for each image dimention (y, x)\n",
+ "\n",
+ "num_proc = 32 # number of processes running correction in parallel"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import h5py\n",
+ "import warnings\n",
+ "from logging import warning\n",
+ "\n",
+ "warnings.filterwarnings('ignore')\n",
+ "\n",
+ "import numpy as np\n",
+ "import matplotlib.pyplot as plt\n",
+ "from IPython.display import display, Markdown\n",
+ "from datetime import datetime\n",
+ "\n",
+ "from extra_data import RunDirectory, by_id\n",
+ "\n",
+ "%matplotlib inline\n",
+ "from cal_tools.step_timing import StepTimer\n",
+ "from cal_tools.files import sequence_trains, DataFile\n",
+ "from cal_tools.tools import get_dir_creation_date\n",
+ "\n",
+ "from cal_tools.restful_config import calibration_client, extra_calibration_client\n",
+ "from cal_tools.calcat_interface2 import CalibrationData\n",
+ "from cal_tools.shimadzu import ShimadzuHPVX2\n",
+ "\n",
+ "from dynflatfield import (\n",
+ " DynamicFlatFieldCorrectionCython as DynamicFlatFieldCorrection,\n",
+ " FlatFieldCorrectionFileProcessor\n",
+ ")\n",
+ "from dynflatfield.draw import plot_images, plot_camera_image"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "creation_time = get_dir_creation_date(in_folder, run)\n",
+ "print(f\"Creation time is {creation_time}\")\n",
+ "\n",
+ "extra_calibration_client() # Configure CalibrationData API.\n",
+ "\n",
+ "cc = calibration_client()\n",
+ "pdus = cc.get_all_phy_det_units_from_detector(\n",
+ " {\"detector_identifier\": karabo_id}) # TODO: Use creation_time for snapshot_at\n",
+ "\n",
+ "if not pdus[\"success\"]:\n",
+ " raise ValueError(\"Failed to retrieve PDUs\")\n",
+ "\n",
+ "detector_info = pdus['data'][0]['detector']\n",
+ "detector = ShimadzuHPVX2(detector_info[\"source_name_pattern\"])\n",
+ "index_group = detector.image_index_group\n",
+ "image_key = detector.image_key\n",
+ "\n",
+ "print(f\"Instrument {detector.instrument}\")\n",
+ "print(f\"Detector in use is {karabo_id}\")\n",
+ "\n",
+ "modules = {}\n",
+ "for pdu in pdus[\"data\"]:\n",
+ " db_module = pdu[\"physical_name\"]\n",
+ " module = pdu[\"module_number\"]\n",
+ " da = pdu[\"karabo_da\"]\n",
+ " if karabo_da[0] != \"-1\" and da not in karabo_da:\n",
+ " continue\n",
+ "\n",
+ " instrument_source_name = detector.instrument_source(module)\n",
+ " corrected_source_name = detector.corrected_source(module)\n",
+ " print('-', da, db_module, module, instrument_source_name)\n",
+ " \n",
+ " modules[da] = dict(\n",
+ " db_module=db_module,\n",
+ " module=module,\n",
+ " raw_source_name=instrument_source_name,\n",
+ " corrected_source_name=corrected_source_name,\n",
+ " )\n",
+ "\n",
+ "step_timer = StepTimer()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Calibration constants"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "step_timer.start()\n",
+ "\n",
+ "dc = RunDirectory(f\"{in_folder}/r{run:04d}\")\n",
+ "conditions = detector.conditions(dc)\n",
+ "\n",
+ "caldata = CalibrationData.from_condition(\n",
+ " conditions, 'SPB_MIC_HPVX2', event_at=creation_time)\n",
+ "\n",
+ "aggregators = {}\n",
+ "corrections = {}\n",
+ "for da in modules:\n",
+ " try:\n",
+ " dark = caldata[\"Offset\", da].ndarray()\n",
+ " flat = caldata[\"DynamicFF\", da].ndarray()\n",
+ " \n",
+ " components = flat[1:][:n_components]\n",
+ " flat = flat[0]\n",
+ "\n",
+ " dffc = DynamicFlatFieldCorrection.from_constants(\n",
+ " dark, flat, components, downsample_factors)\n",
+ "\n",
+ " corrections[da] = dffc\n",
+ " \n",
+ " file_da, _, _ = da.partition('/')\n",
+ " aggregators.setdefault(file_da, []).append(da)\n",
+ " except (KeyError, FileNotFoundError):\n",
+ " warning(f\"Constants are not found for module {da}. \"\n",
+ " \"The module will not calibrated\")\n",
+ "\n",
+ "step_timer.done_step(\"Load calibration constants\") "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Correction"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Output Folder Creation:\n",
+ "os.makedirs(out_folder, exist_ok=True)\n",
+ "\n",
+ "report = []\n",
+ "for file_da, file_modules in aggregators.items():\n",
+ " dc = RunDirectory(f\"{in_folder}/r{run:04d}\", f\"RAW-R{run:04d}-{file_da}-S*.h5\")\n",
+ "\n",
+ " # build train IDs\n",
+ " train_ids = set()\n",
+ " process_modules = []\n",
+ " for da in file_modules:\n",
+ " instrument_source = modules[da][\"raw_source_name\"]\n",
+ " if instrument_source in dc.all_sources:\n",
+ " keydata = dc[instrument_source][image_key].drop_empty_trains()\n",
+ " train_ids.update(keydata.train_ids)\n",
+ " process_modules.append(da)\n",
+ " else:\n",
+ " print(f\"Source {instrument_source} for module {da} is missed\")\n",
+ " \n",
+ " train_ids = np.array(sorted(train_ids))\n",
+ " ts = dc.select_trains(by_id[train_ids]).train_timestamps().astype(np.uint64)\n",
+ "\n",
+ " # correct and write sequence files\n",
+ " for seq_id, train_mask in sequence_trains(train_ids, 200):\n",
+ " step_timer.start()\n",
+ " print('* sequence', seq_id)\n",
+ " seq_train_ids = train_ids[train_mask]\n",
+ " seq_timestamps = ts[train_mask]\n",
+ " dc_seq = dc.select_trains(by_id[seq_train_ids])\n",
+ " ntrains = len(seq_train_ids)\n",
+ "\n",
+ " # create output file\n",
+ " channels = [f\"{modules[da]['corrected_source_name']}/{index_group}\"\n",
+ " for da in process_modules]\n",
+ "\n",
+ " f = DataFile.from_details(out_folder, file_da, run, seq_id)\n",
+ " f.create_metadata(like=dc, instrument_channels=channels)\n",
+ " f.create_index(seq_train_ids, timestamps=seq_timestamps)\n",
+ "\n",
+ " # create file structure\n",
+ " seq_report = {}\n",
+ " file_datasets = {}\n",
+ " for da in process_modules:\n",
+ " instrument_source = modules[da][\"raw_source_name\"]\n",
+ " keydata = dc_seq[instrument_source][image_key].drop_empty_trains()\n",
+ " count = keydata.data_counts(labelled=False)\n",
+ " i = np.flatnonzero(count)\n",
+ " raw_images = keydata.select_trains(np.s_[i]).ndarray()\n",
+ "\n",
+ " # not pulse resolved\n",
+ " shape = keydata.shape\n",
+ " count = np.in1d(seq_train_ids, keydata.train_ids).astype(int)\n",
+ "\n",
+ " corrected_source = modules[da][\"corrected_source_name\"]\n",
+ " src = f.create_instrument_source(corrected_source)\n",
+ " src.create_index(index_group=count)\n",
+ "\n",
+ " # create key for images\n",
+ " ds_data = src.create_key(image_key, shape=shape, dtype=np.float32)\n",
+ " module_datasets = {image_key: ds_data}\n",
+ "\n",
+ " # create keys for image parameters\n",
+ " for key in detector.copy_keys:\n",
+ " keydata = dc_seq[instrument_source][key].drop_empty_trains()\n",
+ " module_datasets[key] = (keydata, src.create_key(\n",
+ " key, shape=keydata.shape, dtype=keydata.dtype))\n",
+ "\n",
+ " file_datasets[da] = module_datasets\n",
+ "\n",
+ " step_timer.done_step(\"Create output file\")\n",
+ "\n",
+ " # correct and write data to file\n",
+ " for da in process_modules:\n",
+ " step_timer.start()\n",
+ " dc_seq = dc.select_trains(by_id[seq_train_ids])\n",
+ "\n",
+ " dffc = corrections[da]\n",
+ " instrument_source = modules[da][\"raw_source_name\"]\n",
+ " proc = FlatFieldCorrectionFileProcessor(dffc, num_proc, instrument_source, image_key)\n",
+ "\n",
+ " proc.start_workers()\n",
+ " proc.run(dc_seq)\n",
+ " proc.join_workers()\n",
+ "\n",
+ " # not pulse resolved\n",
+ " corrected_images = np.stack(proc.rdr.results, 0)\n",
+ " file_datasets[da][image_key][:] = corrected_images\n",
+ "\n",
+ " # copy image parameters\n",
+ " for key in detector.copy_keys:\n",
+ " keydata, ds = file_datasets[da][key]\n",
+ " ds[:] = keydata.ndarray()\n",
+ "\n",
+ " seq_report[da] = (raw_images[0, 0], corrected_images[:20, 0])\n",
+ " step_timer.done_step(\"Correct flat-field\")\n",
+ "\n",
+ " f.close()\n",
+ " report.append(seq_report)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [],
+ "source": [
+ "step_timer.start()\n",
+ "if report:\n",
+ " for da, (raw_image, corrected_images) in report[0].items():\n",
+ " source = modules[da][\"raw_source_name\"]\n",
+ " display(Markdown(f\"## {source}\"))\n",
+ "\n",
+ " display(Markdown(\"### The first raw image\"))\n",
+ " plot_camera_image(raw_images[0, 0])\n",
+ " plt.show()\n",
+ "\n",
+ " display(Markdown(\"### The first corrected image\"))\n",
+ " plot_camera_image(corrected_images[0])\n",
+ " plt.show()\n",
+ "\n",
+ " display(Markdown(\"### The first corrected images in the trains (up to 20)\"))\n",
+ " plot_images(corrected_images, figsize=(13, 8))\n",
+ " plt.show()\n",
+ "\n",
+ "step_timer.done_step(\"Draw images\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"Total processing time {step_timer.timespan():.01f} s\")\n",
+ "step_timer.print_summary()"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.11"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
diff --git a/setup.py b/setup.py
index 9177c82c32ff8cceab356c6a180d548940c27b5f..84a7bfdca72eb808eab9349b1fa2809ad30997b8 100644
--- a/setup.py
+++ b/setup.py
@@ -62,6 +62,7 @@ install_requires = [
"dill==0.3.0",
"docutils==0.17.1",
"dynaconf==3.1.4",
+ "dynflatfield==1.0.0",
"env_cache==0.1",
"extra_data==1.15.1",
"extra_geom==1.10.0",
diff --git a/src/cal_tools/calcat_interface2.py b/src/cal_tools/calcat_interface2.py
index 9048db129d60e69991e63edd14f659dc902643f7..bc598d5726741c11a0a9a01983b640687702e7dc 100644
--- a/src/cal_tools/calcat_interface2.py
+++ b/src/cal_tools/calcat_interface2.py
@@ -837,3 +837,13 @@ class DSSCConditions(ConditionsBase):
"Offset": _params,
"Noise": _params,
}
+
+
+@dataclass
+class ShimadzuHPVX2Conditions(ConditionsBase):
+ burst_frame_count: float
+
+ calibration_types = {
+ 'Offset': ['Burst Frame Count'],
+ 'DynamicFF': ['Burst Frame Count'],
+ }
diff --git a/src/cal_tools/constants.py b/src/cal_tools/constants.py
new file mode 100644
index 0000000000000000000000000000000000000000..cd4abb00f819ef75f2da7b3315adaf4f863bd3c6
--- /dev/null
+++ b/src/cal_tools/constants.py
@@ -0,0 +1,199 @@
+
+from datetime import datetime, timezone
+from struct import pack, unpack
+from pathlib import Path
+from shutil import copyfile
+from hashlib import md5
+import binascii
+import time
+
+import numpy as np
+import h5py
+
+from calibration_client import CalibrationClient
+from cal_tools.calcat_interface2 import _get_default_caldb_root, get_client
+from cal_tools.tools import run_prop_seq_from_path
+from cal_tools.restful_config import calibration_client
+
+
+def write_ccv(
+ const_path,
+ pdu_name, pdu_uuid, detector_type,
+ calibration, conditions, created_at, proposal, runs,
+ data, dims, key='0'
+):
+ """Write CCV data file.
+
+ Args:
+ const_path (os.PathLike): Path to CCV file to write
+ pdu_name (str): Physical detector unit name
+ pdu_uuid (int): Physical detector unit UUID
+ detector_type (str): Detector type name
+ calibration (str): Calibration name
+ conditions (ConditionsBase): Detector operating conditions
+ created_at (datetime): Validity start for calibration
+ proposal (int): Raw data proposal the calibration data is
+ generated from
+ runs (Iterable of int): Raw data runs the calibration data is
+ generated from
+ data (ndarray): Calibration constant data
+ dims (Iterable of str):
+ key (str, optional):
+
+ Returns:
+ (str) CCV HDF group name.
+
+ """
+
+ if data.ndim != len(dims):
+ raise ValueError('data.ndims != len(dims)')
+
+ with h5py.File(const_path, 'a') as const_file:
+ const_file.attrs['version'] = 0
+
+ pdu_group = const_file.require_group(pdu_name)
+ pdu_group.attrs['uuid'] = pdu_uuid
+ pdu_group.attrs['detector_type'] = detector_type
+
+ calibration_group = pdu_group.require_group(calibration)
+
+ if key is None:
+ key = str(len(calibration_group))
+
+ ccv_group = calibration_group.create_group(key)
+ ccv_group.attrs['begin_at'] = created_at.isoformat()
+ ccv_group.attrs['proposal'] = proposal
+ ccv_group.attrs['runs'] = np.array(runs, dtype=np.int32)
+ ccv_group_name = ccv_group.name
+
+ opcond_group = ccv_group.create_group('operating_condition')
+ opcond_dict = conditions.make_dict(
+ conditions.calibration_types[calibration])
+ for db_name, value in opcond_dict.items():
+ key = db_name.lower().replace(' ', '_')
+ dset = opcond_group.create_dataset(key, data=value,
+ dtype=np.float64)
+ dset.attrs['lower_deviation'] = 0.0
+ dset.attrs['upper_deviation'] = 0.0
+ dset.attrs['database_name'] = db_name
+
+ dset = ccv_group.create_dataset('data', data=data)
+ dset.attrs['dims'] = dims
+
+ return ccv_group_name
+
+
+def inject_ccv(const_src, ccv_root, report_to=None):
+ """Inject new CCV into CalCat.
+
+ Args:
+ const_path (str or Path): Path to CCV data file.
+ ccv_root (str): CCV HDF group name.
+ report_to (str): Metadata location.
+
+ Returns:
+ None
+
+ Raises:
+ RuntimeError: If CalCat POST request fails.
+ """
+
+ pdu_name, calibration, key = ccv_root.lstrip('/').split('/')
+
+ with h5py.File(const_src, 'r') as const_file:
+ pdu_group = const_file[pdu_name]
+ pdu_uuid = pdu_group.attrs['uuid']
+ detector_type = pdu_group.attrs['detector_type']
+
+ ccv_group = const_file[ccv_root]
+ proposal, runs = ccv_group.attrs['proposal'], ccv_group.attrs['runs']
+ begin_at_str = ccv_group.attrs['begin_at']
+
+ condition_group = ccv_group['operating_condition']
+
+ cond_params = []
+
+ # It's really not ideal we're mixing conditionS and condition now.
+ for parameter in condition_group:
+ param_dset = condition_group[parameter]
+ cond_params.append({
+ 'parameter_name': param_dset.attrs['database_name'],
+ 'value': float(param_dset[()]),
+ 'lower_deviation_value': param_dset.attrs['lower_deviation'],
+ 'upper_deviation_value': param_dset.attrs['upper_deviation'],
+ 'flg_available': True
+ })
+
+ const_rel_path = f'xfel/cal/{detector_type.lower()}/{pdu_name.lower()}'
+ const_filename = f'cal.{time.time()}.h5'
+
+ if proposal and len(runs) > 0:
+ raw_data_location = 'proposal:{} runs: {}'.format(
+ proposal, ' '.join([str(x) for x in runs]))
+ else:
+ pass # Fallback for non-run based constants
+
+ # Generate condition name.
+ unique_name = detector_type[:detector_type.index('-Type')] + ' Def'
+ cond_hash = md5(pdu_name.encode())
+ cond_hash.update(int(pdu_uuid).to_bytes(
+ length=8, byteorder='little', signed=False))
+
+ for param_dict in cond_params:
+ cond_hash.update(str(param_dict['parameter_name']).encode())
+ cond_hash.update(str(param_dict['value']).encode())
+
+ unique_name += binascii.b2a_base64(cond_hash.digest()).decode()
+ unique_name = unique_name[:60]
+
+ # Add PDU "UUID" to parameters.
+ cond_params.append({
+ 'parameter_name': 'Detector UUID',
+ 'value': unpack('d', pack('q', pdu_uuid))[0],
+ 'lower_deviation_value': 0.0,
+ 'upper_deviation_value': 0.0,
+ 'flg_available': True
+ })
+
+ inject_h = {
+ 'detector_condition': {
+ 'name': unique_name,
+ 'parameters': cond_params
+ },
+ 'calibration_constant': {
+ 'calibration_name': calibration,
+ 'detector_type_name': detector_type,
+ 'flg_auto_approve': True
+ },
+ 'calibration_constant_version': {
+ 'raw_data_location': raw_data_location,
+ 'file_name': const_filename,
+ 'path_to_file': const_rel_path,
+ 'data_set_name': f'{pdu_name}/{calibration}/0',
+ 'start_idx': '0',
+ 'end_idx': '0',
+ 'begin_validity_at': begin_at_str,
+ 'end_validity_at': '',
+ 'begin_at': begin_at_str,
+ 'pdu_physical_name': pdu_name,
+ 'flg_good_quality': True
+ }
+ }
+
+ if report_to:
+ report_path = Path(report_to).absolute().with_suffix('.pdf')
+ inject_h['report'] = {
+ 'name': report_path.stem,
+ 'file_path': str(report_path)
+ }
+
+ const_dest = _get_default_caldb_root() / const_rel_path / const_filename
+ const_dest.parent.mkdir(parents=True, exist_ok=True)
+ copyfile(const_src, const_dest)
+
+ resp = CalibrationClient.inject_new_calibration_constant_version(
+ calibration_client(), inject_h)
+
+ if not resp['success']:
+ const_dest.unlink() # Delete already copied CCV file.
+ raise RuntimeError(resp)
diff --git a/src/cal_tools/restful_config.py b/src/cal_tools/restful_config.py
index 618140c0b372cfd344ca1341697a8ec50eba63b3..5a6d2e9e5245e6d4355ec40ae20c6ef557590c14 100644
--- a/src/cal_tools/restful_config.py
+++ b/src/cal_tools/restful_config.py
@@ -1,21 +1,33 @@
+
from pathlib import Path
+from os import getenv
from dynaconf import Dynaconf
config_dir = Path(__file__).parent.resolve()
+# Default fles.
+settings_files = [
+ config_dir / "restful_config.yaml",
+ config_dir / "restful_config.secrets.yaml",
+ Path("~/.config/pycalibration/cal_tools/restful_config.yaml").expanduser(),
+]
+
+if getenv('CAL_RESTFUL_TEST_CONFIG', None) == '1':
+ settings_files.append(Path(
+ "~/.config/pycalibration/cal_tools/restful_test_config.yaml").expanduser())
+
+
restful_config = Dynaconf(
envvar_prefix="CAL_CAL_TOOLS",
- settings_files=[
- config_dir / "restful_config.yaml",
- config_dir / "restful_config.secrets.yaml",
- Path("~/.config/pycalibration/cal_tools/restful_config.yaml").expanduser(),
- ],
+ settings_files=settings_files,
merge_enabled=True,
)
def calibration_client():
+ """Obtain an initialized CalibrationClient object."""
+
from calibration_client import CalibrationClient
# Create client for CalCat.
@@ -30,3 +42,33 @@ def calibration_client():
refresh_url=calcat_config['refresh-url'],
auth_url=calcat_config['auth-url'],
scope='')
+
+
+def extra_calibration_client():
+ """Obtain an initialized CalCatAPIClient object."""
+
+ from cal_tools import calcat_interface2
+
+ calcat_config = restful_config.get('calcat')
+ if calcat_config['use-oauth2']:
+ from oauth2_xfel_client import Oauth2ClientBackend
+ oauth_client = Oauth2ClientBackend(
+ client_id=calcat_config['user-id'],
+ client_secret=calcat_config['user-secret'],
+ token_url=calcat_config['token-url'],
+ scope='',
+ )
+ else:
+ oauth_client = None
+
+ if calcat_config['caldb-root']:
+ calcat_interface2._default_caldb_root = Path(calcat_config['caldb-root'])
+
+ client = calcat_interface2.CalCatAPIClient(
+ base_api_url=calcat_config['base-api-url'],
+ oauth_client=oauth_client,
+ user_email=calcat_config['user-email'],
+ )
+
+ calcat_interface2.global_client = client
+ return client
diff --git a/src/cal_tools/restful_config.yaml b/src/cal_tools/restful_config.yaml
index 899fb07d0886b1f4dd4951edb26b93280d3b9bfc..0cecfd4a3844edcf8b850d733580cc2dc9fab4e3 100644
--- a/src/cal_tools/restful_config.yaml
+++ b/src/cal_tools/restful_config.yaml
@@ -19,3 +19,4 @@ calcat:
user-email: calibration@example.com
user-id: '@note add this to secrets file'
user-secret: '@note add this to secrets file'
+ caldb-root: ''
diff --git a/src/cal_tools/shimadzu.py b/src/cal_tools/shimadzu.py
new file mode 100644
index 0000000000000000000000000000000000000000..d34d8e85f9d63c39a7c84f93903f24f6f72cf187
--- /dev/null
+++ b/src/cal_tools/shimadzu.py
@@ -0,0 +1,55 @@
+from dataclasses import dataclass
+
+from cal_tools.calcat_interface2 import ShimadzuHPVX2Conditions
+
+
+class ShimadzuHPVX2:
+ image_key = "data.image.pixels"
+ copy_keys = [
+ "data.image.binning",
+ "data.image.dimTypes",
+ "data.image.dims",
+ "data.image.flipX",
+ "data.image.flipY",
+ "data.image.roiOffsets",
+ "data.image.rotation",
+ ]
+
+ def __init__(self, source_name_pattern: str, channel=None, image_key=None):
+ self.source_name_pattern = source_name_pattern
+ if channel is not None:
+ self.channel = channel
+ if image_key is not None:
+ self.image_key = image_key
+ self.image_index_group = self.image_key.partition('.')[0]
+ self.instrument = source_name_pattern.split('_')[0]
+
+ def conditions(self, dc: "DataCollection", module=None): # noqa: F821
+ if module is None:
+ source_pattern = self.source_name_pattern.format(modno='*')
+ det_dc = dc.select(source_pattern)
+ if not det_dc.instrument_sources:
+ raise ValueError("No detector sources are found")
+
+ source_name = list(det_dc.instrument_sources)[0]
+ else:
+ source_name = self.instrument_source(module)
+ keydata = dc[source_name, self.image_key]
+ num_frames = keydata.shape[-3]
+ return ShimadzuHPVX2Conditions(burst_frame_count=float(num_frames))
+
+ def instrument_source(self, module: int):
+ return self.source_name_pattern.format(modno=module)
+
+ def corrected_source(self, module: int):
+ source_name = self.source_name_pattern.format(modno=module)
+
+ # Replace type with CORR.
+ parts = source_name.split('/')
+ parts[1] = "CORR"
+ source_name = '/'.join(parts)
+
+ # Replace channel with output.
+ source_name = source_name[:source_name.index(':')] + ':output'
+
+ return source_name
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index e1f308faab5228ef145cfa19b57f891d43630e4d..cf23aa919da4dce0a2fba1af87170fc75ce31eeb 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -304,6 +304,26 @@ notebooks = {
},
},
},
+ "DYNAMICFF": {
+ "DARK": {
+ "notebook": "notebooks/DynamicFF/Characterize_DynamicFF_NBC.ipynb",
+ "concurrency": {
+ "parameter": None,
+ "use function": None,
+ "default concurrency": None,
+ "cluster cores": 1,
+ },
+ },
+ "CORRECT": {
+ "notebook": "notebooks/DynamicFF/Correct_DynamicFF_NBC.ipynb",
+ "concurrency": {
+ "parameter": None,
+ "use function": None,
+ "default concurrency": None,
+ "cluster cores": 1,
+ },
+ },
+ },
"TEST": {
"TEST-CLI": {
"notebook": "notebooks/test/test-cli.ipynb",