From b7feeb4e7ab5fa47cbf836d3a7225c20e860d3eb Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Thu, 21 Apr 2022 18:08:10 +0200
Subject: [PATCH] Draft: Gotthard2 dark notebook
---
.../Characterize_Darks_Gotthard2_NBC.ipynb | 408 ++++++++++++++++++
1 file changed, 408 insertions(+)
create mode 100644 notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
diff --git a/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb b/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
new file mode 100644
index 000000000..1a3a82901
--- /dev/null
+++ b/notebooks/Gotthard2/Characterize_Darks_Gotthard2_NBC.ipynb
@@ -0,0 +1,408 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "49b6577f-96a5-4dd2-bdd9-da661b2c4619",
+ "metadata": {},
+ "source": [
+ "# Gotthard2 Dark Image Characterization\n",
+ "\n",
+ "Author: European XFEL Detector Group, Version: 1.0"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "818e24e8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "in_folder = \"/gpfs/exfel/exp/DETLAB/202230/p900276/raw\" # the folder to read data from, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/gotthard2/darks\" # the folder to output to, required\n",
+ "run_high = 10 # run number for G0 dark run, required\n",
+ "run_med = 10 # run number for G1 dark run, required\n",
+ "run_low = 10 # run number for G2 dark run, required\n",
+ "sequences = [-1] # sequences to correct, set to [-1] for all, range allowed\n",
+ "sequences_per_node = 1 # number of sequence files per node if notebook executed through xfel-calibrate, set to 0 to not run SLURM parallel\n",
+ "\n",
+ "# Parameters used to access raw data.\n",
+ "karabo_id = \"DET_LAB_G2\" # karabo prefix of Jungfrau devices\n",
+ "karabo_da = [\"DA01\"] # data aggregators\n",
+ "receiver_template = \"GOT{:02d}\" # receiver template used to read INSTRUMENT keys.\n",
+ "control_template = \"CTRL{:02d}\" # control template used to read CONTROL keys.\n",
+ "instrument_source_template = '{}/DET/{}:daqOutput' # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput'\n",
+ "ctrl_source_template = '{}/DET/{}' # template for control source name (filled with karabo_id_control)\n",
+ "karabo_id_control = \"\" # if control is on a different ID, set to empty string if it is the same a karabo-id\n",
+ "\n",
+ "# Parameters for the calibration database.\n",
+ "use_dir_creation_date = True\n",
+ "cal_db_interface = \"tcp://max-exfl016:8020\" # calibration DB interface to use\n",
+ "cal_db_timeout = 300000 # timeout on caldb requests\n",
+ "db_output = False # Output constants to the calibration database\n",
+ "local_output = True # Output constants locally\n",
+ "\n",
+ "# Conditions used for injected calibration constants.\n",
+ "\n",
+ "# Parameters used during selecting raw data trains.\n",
+ "min_trains = 1 # Minimum number of trains that should be available to process dark constants. Default 1.\n",
+ "max_trains = 1000 # Maximum number of trains to use for processing dark constants. Set to 0 to use all available trains.\n",
+ "\n",
+ "# Don't delete! myMDC sends this by default.\n",
+ "operation_mode = '' # Detector operation mode, optional"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8085f9aa",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import h5py\n",
+ "import numpy as np\n",
+ "import matplotlib.pyplot as plt\n",
+ "import multiprocessing\n",
+ "import pasha as psh\n",
+ "from extra_data import RunDirectory\n",
+ "from pathlib import Path\n",
+ "\n",
+ "from cal_tools.enums import BadPixels\n",
+ "from cal_tools.gotthard2algs import convert_to_10bit\n",
+ "from cal_tools.step_timing import StepTimer\n",
+ "from cal_tools.tools import (\n",
+ " get_dir_creation_date,\n",
+ " get_from_db,\n",
+ " get_pdu_from_db,\n",
+ " get_report,\n",
+ " save_const_to_h5,\n",
+ " send_to_db,\n",
+ ")\n",
+ "from iCalibrationDB import Conditions, Constants"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "18fe4379",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "run_nums = [run_high, run_med, run_low]\n",
+ "in_folder = Path(in_folder)\n",
+ "out_folder = Path(out_folder)\n",
+ "out_folder.mkdir(exist_ok=True)\n",
+ "\n",
+ "print(f\"Process modules: {karabo_da}\")\n",
+ "\n",
+ "run_dc = RunDirectory(in_folder / f\"r{run_high:04d}\")\n",
+ "file_loc = f\"proposal:{run_dc.run_metadata()['proposalNumber']} runs:{run_high} {run_med} {run_low}\" # noqa\n",
+ "\n",
+ "instrument_src = instrument_source_template.format(\n",
+ " karabo_id, receiver_template)\n",
+ "ctrl_src = ctrl_source_template.format(\n",
+ " karabo_id_control, control_template)\n",
+ "\n",
+ "# Read report path to associate it later with injected constants.\n",
+ "report = get_report(out_folder)\n",
+ "\n",
+ "if use_dir_creation_date:\n",
+ " creation_time = get_dir_creation_date(in_folder, run_high)\n",
+ " print(f\"Using {creation_time.isoformat()} as creation time\")\n",
+ "\n",
+ "if not karabo_id_control:\n",
+ " karabo_id_control = karabo_id"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "108be688",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "step_timer = StepTimer()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fb80b98e",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# load constants temporarely using defined local paths.\n",
+ "\n",
+ "constants_file = \"/gpfs/exfel/data/user/mramilli/gotthard2/constants/GH2-0124/calibration_constants_GH2-0124.h5\"\n",
+ "\n",
+ "with h5py.File(constants_file, 'r') as cfile:\n",
+ " lut = cfile[\"LUT\"][()]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ff9149fc",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Read parameter conditions\n",
+ "\n",
+ "step_timer.start()\n",
+ "run_dcs_dict = dict()\n",
+ "\n",
+ "ctrl_src = ctrl_source_template.format(karabo_id_control, control_template)\n",
+ "\n",
+ "for gain, run in enumerate(run_nums):\n",
+ " run_dc = RunDirectory(in_folder / f\"r{run:04d}/\")\n",
+ " run_dcs_dict[run] = [gain, run_dc]\n",
+ "\n",
+ " # TODO: Read every condition from slow data.\n",
+ "\n",
+ " # TODO: Validate that the conditions are the same and as expected for all runs."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ac9c5dc3-bc66-4e7e-b6a1-360259be535c",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def specifiy_trains_to_process(\n",
+ " img_key_data: \"extra_data.KeyData\", # noqa\n",
+ " max_trains: int = 0,\n",
+ " min_trains: int = 0,\n",
+ "):\n",
+ " \"\"\"Specify total number of trains to process.\n",
+ " Based on given min_trains and max_trains, if given.\n",
+ "\n",
+ " Print number of trains to process and number of empty trains.\n",
+ " Raise ValueError if specified trains are less than min_trains.\n",
+ " \"\"\"\n",
+ " # Specifies total number of trains to proccess.\n",
+ " n_trains = img_key_data.shape[0]\n",
+ " all_trains = len(img_key_data.train_ids)\n",
+ " print(f\"{mod} has {all_trains - n_trains} \"\n",
+ " f\"trains with empty frames out of {all_trains} trains\")\n",
+ "\n",
+ " if n_trains < min_trains:\n",
+ " raise ValueError(\n",
+ " f\"Less than {min_trains} trains are available in RAW data.\"\n",
+ " \" Not enough data to process darks.\")\n",
+ "\n",
+ " if max_trains > 0:\n",
+ " n_trains = min(n_trains, max_trains)\n",
+ "\n",
+ " print(f\"Processing {n_trains} trains.\")\n",
+ "\n",
+ " return n_trains"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e2eb2fc0-df9c-4887-9691-f81474f8c131",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def convert_train(wid, index, tid, d):\n",
+ " \"\"\"Convert a Gotthard2 train from 12bit to 10bit.\"\"\"\n",
+ " d_10bit = np.zeros_like(d[instr_mod_src][\"data.adc\"], dtype=np.float32)\n",
+ " convert_to_10bit(d[instr_mod_src][\"data.adc\"], lut, d_10bit)\n",
+ " data_10bit[index, ...] = d_10bit\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4e8ffeae",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Calculate noise and offset per pixel and global average, std and median\n",
+ "\n",
+ "noise_map = dict()\n",
+ "offset_map = dict()\n",
+ "bad_pixels_map = dict()\n",
+ "\n",
+ "context = psh.context.ProcessContext(num_workers=multiprocessing.cpu_count())\n",
+ "\n",
+ "\n",
+ "for mod in karabo_da:\n",
+ " # Path to pixels ADC values\n",
+ " instr_mod_src = instrument_src.format(int(mod[-2:]))\n",
+ " pixels_src = (instr_mod_src, \"data.adc\")\n",
+ " data_path = \"INSTRUMENT/\"+instr_mod_src+\"/data\"\n",
+ "\n",
+ " # TODO: Validate the final shape to store constants.\n",
+ " cshape = (3, 2, 1280)\n",
+ " offset_map[mod] = np.zeros(cshape, dtype=np.float32)\n",
+ " noise_map[mod] = np.zeros_like(offset_map[mod])\n",
+ " bad_pixels_map[mod] = np.zeros_like(offset_map[mod], dtype=np.uint32)\n",
+ "\n",
+ " for run_num, [gain, run_dc] in run_dcs_dict.items():\n",
+ " step_timer.start()\n",
+ " n_trains = specifiy_trains_to_process(run_dc[pixels_src])\n",
+ "\n",
+ " # Select requested number of trains to process.\n",
+ " dc = run_dc.select(*pixels_src, require_all=True).select_trains(np.s_[:n_trains]) # noqa\n",
+ " dshape = dc[instr_mod_src, \"data.adc\"].shape\n",
+ " step_timer.done_step(\"preparing raw data\")\n",
+ "\n",
+ " step_timer.start()\n",
+ " # Convert 12bit data to 10bit\n",
+ " data_10bit = context.alloc(shape=dshape, dtype=np.float32)\n",
+ " context.map(convert_train, dc)\n",
+ " step_timer.done_step(\"convert to 10bit\")\n",
+ "\n",
+ " step_timer.start()\n",
+ " even_data = data_10bit[:, ::2, :]\n",
+ " odd_data = data_10bit[:, 1::2, :]\n",
+ " \n",
+ " def offset_noise_cell(wid, index, d):\n",
+ " offset[index] = np.mean(d, axis=(0, 1))\n",
+ " noise[index] = np.std(d, axis=(0, 1))\n",
+ "\n",
+ " offset = context.alloc(shape=cshape[-2:], dtype=np.float32)\n",
+ " noise = context.alloc(like=offset)\n",
+ " context.map(offset_noise_cell, (even_data, odd_data))\n",
+ " offset_map[mod][gain, ...] = offset.copy()\n",
+ " noise_map[mod][gain, ...] = noise.copy()\n",
+ "\n",
+ " step_timer.done_step(\"Processing darks\")\n",
+ " "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "6e10ed93-66de-4fb1-bf97-f8d25af22edb",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# # set the operating condition\n",
+ "# # TODO: add the final conditions for constants.\n",
+ "# condition = Conditions.Dark.Gotthard2(\n",
+ "# bias_voltage=bias_voltage,\n",
+ "# )\n",
+ "\n",
+ "# db_modules = get_pdu_from_db(\n",
+ "# karabo_id=karabo_id,\n",
+ "# karabo_da=karabo_da,\n",
+ "# constant=Constants.Gotthard2.Offset(),\n",
+ "# condition=condition,\n",
+ "# cal_db_interface=cal_db_interface,\n",
+ "# snapshot_at=creation_time)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "fde8e1cf-bc74-462f-b6e5-cfee8279090d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from XFELDetAna.plotting.heatmap import heatmapPlot\n",
+ "unit = '[ADCu]'\n",
+ "\n",
+ "for mod in karabo_da:\n",
+ " for _, [gain, _] in run_dcs_dict.items():\n",
+ " heatmapPlot(\n",
+ " offset_map[mod][gain],\n",
+ " y_label=\"Row\",\n",
+ " x_label=\"Column\",\n",
+ " lut_label=unit,\n",
+ " title=f\"Even / Odd Offset map G{gain} - Module {mod}\", # TODO: add PDU name({pdu})',\n",
+ " )\n",
+ " plt.show()\n",
+ " heatmapPlot(\n",
+ " noise_map[mod][gain],\n",
+ " y_label=\"Row\",\n",
+ " x_label=\"Column\",\n",
+ " lut_label=unit,\n",
+ " title=f\"Even / Odd noise map G{gain} - Module {mod}\", # TODO: add PDU name({pdu})',\n",
+ " )\n",
+ " plt.show()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1c4eddf7-7d6e-49f4-8cbb-12d2bc496a8f",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "step_timer.start()\n",
+ "for mod, db_mod in zip(karabo_da, db_modules):\n",
+ " constants = {\n",
+ " 'Offset': offset_map[mod],\n",
+ " 'Noise': noise_map[mod],\n",
+ " }\n",
+ "\n",
+ " md = None\n",
+ "\n",
+ " for key, const_data in constants.items():\n",
+ "\n",
+ " const = getattr(Constants.Gotthard2, key)()\n",
+ " const.data = const_data\n",
+ "\n",
+ " if db_output:\n",
+ " md = send_to_db(\n",
+ " db_module=db_mod,\n",
+ " karabo_id=karabo_id,\n",
+ " constant=const,\n",
+ " condition=condition,\n",
+ " file_loc=file_loc,\n",
+ " report_path=report,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " timeout=cal_db_timeout,\n",
+ " )\n",
+ " if local_output:\n",
+ " md = save_const_to_h5(\n",
+ " db_module=db_mod,\n",
+ " karabo_id=karabo_id,\n",
+ " constant=const,\n",
+ " condition=condition,\n",
+ " data=const.data,\n",
+ " file_loc=file_loc,\n",
+ " report=report,\n",
+ " creation_time=creation_time,\n",
+ " out_folder=out_folder,\n",
+ " )\n",
+ " print(f\"Calibration constant {key} is stored locally at {out_folder}.\\n\")\n",
+ "\n",
+ "print(\"Constants parameter conditions are:\\n\")\n",
+ "# TODO: add the final conditions for constants.\n",
+ "print(\n",
+ " f\"• Bias voltage: {bias_voltage}\\n\"\n",
+ " f\"• Creation time: {md.calibration_constant_version.begin_at if md is not None else creation_time}\\n\") # noqa\n",
+ "step_timer.done_step(\"Injecting constants.\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "interpreter": {
+ "hash": "767d51c1340bd893661ea55ea3124f6de3c7a262a8b4abca0554b478b1e2ff90"
+ },
+ "kernelspec": {
+ "display_name": "cal4_venv",
+ "language": "python",
+ "name": "cal4_venv"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.11"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
--
GitLab