diff --git a/notebooks/ShimadzuHPVX2/Characterize_Darks_ShimadzuHPVX2_NBC.ipynb b/notebooks/ShimadzuHPVX2/Characterize_Darks_ShimadzuHPVX2_NBC.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..0a31e9e7afad8bf828ff54f6ef8c3742ca09aa37
--- /dev/null
+++ b/notebooks/ShimadzuHPVX2/Characterize_Darks_ShimadzuHPVX2_NBC.ipynb
@@ -0,0 +1,354 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Shimadzu HPVX2 Characterization of dark and flat field\n",
+ "\n",
+ "Author: Egor Sobolev\n",
+ "\n",
+ "Computation of dark offsets and flat-field principal components"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = \"/gpfs/exfel/exp/SPB/202121/p002919/raw/\" # input folder, required\n",
+ "out_folder = '/gpfs/exfel/data/scratch/esobolev/test/shimadzu' # output folder, required\n",
+ "metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
+ "dark_run = 59 # which run to read data from, required\n",
+ "flat_run = 40 # which run to read\n",
+ "\n",
+ "# Data files parameters.\n",
+ "karabo_da = ['HPVX01'] # data aggregators\n",
+ "karabo_id = \"SPB_EHD_HPVX2_2\" # karabo prefix of Shimadzu HPV-X2 devices\n",
+ "\n",
+ "#receiver_id = \"PNCCD_FMT-0\" # inset for receiver devices\n",
+ "#path_template = 'RAW-R{:04d}-{}-S{{:05d}}.h5' # the template to use to access data\n",
+ "instrument_source_template = '{}/CAM/CAMERA:daqOutput' # data source path in h5file. Template filled with karabo_id and receiver_id\n",
+ "image_key = \"data.image.pixels\" # image data key in Karabo or exdf notation\n",
+ "\n",
+ "# Database access parameters.\n",
+ "use_dir_creation_date = True # use dir creation date as data production reference date\n",
+ "cal_db_interface = \"tcp://max-exfl-cal001:8021\" # calibration DB interface to use\n",
+ "cal_db_timeout = 300000 # timeout on caldb requests\n",
+ "db_output = False # if True, the notebook sends dark constants to the calibration database\n",
+ "local_output = True # if True, the notebook saves dark constants locally\n",
+ "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.00 e.g. 2019-07-04 11:02:41.00\n",
+ "\n",
+ "n_components = 50"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import datetime\n",
+ "import os\n",
+ "import warnings\n",
+ "\n",
+ "warnings.filterwarnings('ignore')\n",
+ "\n",
+ "import time\n",
+ "import numpy as np\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "from extra_data import RunDirectory\n",
+ "\n",
+ "%matplotlib inline\n",
+ "from cal_tools.step_timing import StepTimer\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " get_pdu_from_db,\n",
+ " get_random_db_interface,\n",
+ " get_report,\n",
+ " save_const_to_h5,\n",
+ " save_dict_to_hdf5,\n",
+ " send_to_db,\n",
+ " run_prop_seq_from_path,\n",
+ ")\n",
+ "\n",
+ "import dffc\n",
+ "from dffc.draw import plot_images, plot_camera_image"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "creation_time=None\n",
+ "if use_dir_creation_date:\n",
+ " creation_time = get_dir_creation_date(in_folder, max(dark_run, flat_run))\n",
+ "\n",
+ "print(f\"Using {creation_time} as creation time of constant.\")\n",
+ "\n",
+ "run, prop, seq = run_prop_seq_from_path(in_folder)\n",
+ "file_loc = f'proposal: {prop}, runs: {dark_run} {flat_run}'\n",
+ "\n",
+ "# Read report path and create file location tuple to add with the injection\n",
+ "file_loc = f\"proposal:{prop} runs:{dark_run} {flat_run}\"\n",
+ "\n",
+ "report = get_report(metadata_folder)\n",
+ "cal_db_interface = get_random_db_interface(cal_db_interface)\n",
+ "print(f'Calibration database interface: {cal_db_interface}')\n",
+ "\n",
+ "instrument = karabo_id.split(\"_\")[0]\n",
+ "source = instrument_source_template.format(karabo_id)\n",
+ "\n",
+ "print(f\"Detector in use is {karabo_id}\")\n",
+ "print(f\"Instrument {instrument}\")\n",
+ "\n",
+ "step_timer = StepTimer()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Offset map"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "step_timer.start()\n",
+ "\n",
+ "dark_dc = RunDirectory(f\"{in_folder}/r{dark_run:04d}\")\n",
+ "dark_dc = dark_dc.select([(source, image_key)])\n",
+ "key_data = dark_dc[source][image_key]\n",
+ "\n",
+ "images_dark = key_data.ndarray()\n",
+ "ntrain, npulse, ny, nx = images_dark.shape\n",
+ "\n",
+ "print(f\"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})\")\n",
+ "print(f\"Image size: {ny} x {nx} px\")\n",
+ "step_timer.done_step(\"Read dark images\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "step_timer.start()\n",
+ "dark = dffc.process_dark(images_dark)\n",
+ "step_timer.done_step(\"Process dark images\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "step_timer.start()\n",
+ "plot_camera_image(dark)\n",
+ "plt.show()\n",
+ "step_timer.done_step(\"Draw offset map\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Flat-field PCA decomposition"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "step_timer.start()\n",
+ "\n",
+ "flat_dc = RunDirectory(f\"{in_folder}/r{flat_run:04d}\")\n",
+ "flat_dc = flat_dc.select([(source, image_key)])\n",
+ "key_data = flat_dc[source][image_key]\n",
+ "\n",
+ "images_flat = key_data.ndarray()\n",
+ "ntrain, npulse, ny, nx = images_flat.shape\n",
+ "\n",
+ "print(f\"N image: {ntrain * npulse} (ntrain: {ntrain}, npulse: {npulse})\")\n",
+ "print(f\"Image size: {ny} x {nx} px\")\n",
+ "step_timer.done_step(\"Read flat-field images\")\n",
+ "\n",
+ "tm0 = time.monotonic()\n",
+ "tm_cm = time.monotonic() - tm0\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "step_timer.start()\n",
+ "flat, components, explained_variance_ratio = dffc.process_flat(\n",
+ " images_flat, dark, n_components)\n",
+ "step_timer.done_step(\"Process flat-field images\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Average flat-field"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "step_timer.start()\n",
+ "plot_camera_image(flat)\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Explained variance ratio"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fig, ax = plt.subplots(1, 1, figsize=(10,4), tight_layout=True)\n",
+ "ax.semilogy(explained_variance_ratio, 'o')\n",
+ "ax.set_xticks(np.arange(len(explained_variance_ratio)))\n",
+ "ax.set_xlabel(\"Component no.\")\n",
+ "ax.set_ylabel(\"Variance fraction\")\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# The first principal components (up to 20)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "plot_images(components[:20], figsize=(13, 8))\n",
+ "plt.show()\n",
+ "step_timer.done_step(\"Draw flat-field map and components\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Calibration constants"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "step_timer.start()\n",
+ "\n",
+ "# Output Folder Creation:\n",
+ "os.makedirs(out_folder, exist_ok=True)\n",
+ "\n",
+ "db_module = \"SHIMADZU_HPVX2_M001\"\n",
+ "\n",
+ "constant_name = \"Offset\"\n",
+ "\n",
+ "conditions = {\n",
+ " 'Memory cells': {'value': 128},\n",
+ " 'Pixels X': {'value': flat.shape[1]},\n",
+ " 'Pixels Y': {'value': flat.shape[0]},\n",
+ " 'FF components': {'value': components.shape[0]}\n",
+ "}\n",
+ "\n",
+ "data_to_store = {\n",
+ " 'condition': conditions,\n",
+ " 'db_module': db_module,\n",
+ " 'karabo_id': karabo_id,\n",
+ " 'constant': constant_name,\n",
+ " 'data': dark,\n",
+ " 'creation_time': creation_time.replace(microsecond=0),\n",
+ " 'file_loc': file_loc,\n",
+ " 'report': report,\n",
+ "}\n",
+ "\n",
+ "ofile = f\"{out_folder}/const_{constant_name}_{db_module}.h5\"\n",
+ "if os.path.isfile(ofile):\n",
+ " print(f'File {ofile} already exists and will be overwritten')\n",
+ "save_dict_to_hdf5(data_to_store, ofile)\n",
+ "\n",
+ "\n",
+ "constant_name = \"ComponentsFF\"\n",
+ "\n",
+ "data_to_store.update({\n",
+ " 'constant': constant_name,\n",
+ " 'data': np.concatenate([flat[None, ...], components]),\n",
+ "})\n",
+ "\n",
+ "ofile = f\"{out_folder}/const_{constant_name}_{db_module}.h5\"\n",
+ "if os.path.isfile(ofile):\n",
+ " print(f'File {ofile} already exists and will be overwritten')\n",
+ "save_dict_to_hdf5(data_to_store, ofile)\n",
+ "\n",
+ "step_timer.done_step(\"Storing calibration constants\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"Total processing time {step_timer.timespan():.01f} s\")\n",
+ "step_timer.print_summary()"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.11"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}