From 48f8d8499d90fcc1315dd27b18dd6095cf0b744d Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Thu, 11 Jan 2024 17:21:58 +0100
Subject: [PATCH] merge notebooks and update notebooks.py
---
.../gainCal_JF_Fit_Spectra_Histos.ipynb | 314 ------------------
src/xfel_calibrate/notebooks.py | 23 +-
2 files changed, 4 insertions(+), 333 deletions(-)
delete mode 100644 notebooks/Jungfrau/gainCal_JF_Fit_Spectra_Histos.ipynb
diff --git a/notebooks/Jungfrau/gainCal_JF_Fit_Spectra_Histos.ipynb b/notebooks/Jungfrau/gainCal_JF_Fit_Spectra_Histos.ipynb
deleted file mode 100644
index 3cd8cfd71..000000000
--- a/notebooks/Jungfrau/gainCal_JF_Fit_Spectra_Histos.ipynb
+++ /dev/null
@@ -1,314 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# G0 map from single photon spectra\n",
- "\n",
- "Author: European XFEL Detector Group, Version: 1.0\n",
- "\n",
- "Converts single photon flat fields into a G0 map"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "in_folder = \"/gpfs/exfel/exp/HED/202231/p900297/raw\"\n",
- "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/FFDATA/HED/p900297/gain_fit\"\n",
- "runs = [26] # can be a list of runs\n",
- "dir_date_iso = \"\" ## string to save the date of the creation of the run\n",
- "\n",
- "# to be used when injecting constants (third separate nb)\n",
- "sensor_size = [512, 1024] # size of the array in the 'row' and 'col' dimensions\n",
- "chunk_size = 10 # n of trains per chunk\n",
- "block_size = [128, 64] # dimension of the chunks in 'row' and 'col'\n",
- "mod_n = 1 # module number\n",
- "karabo_id = \"HED_IA1_JF500K1\" # karabo prefix of Jungfrau devices\n",
- "da_name = f\"JNGFR{mod_n:02d}\"\n",
- "da_control = \"JNGFRCTRL00\"\n",
- "ctrl_source_template = \"{}/DET/CONTROL\" # template for control source name (filled with karabo_id_control)\n",
- "karabo_id_control = \"\" # if control is on a different ID, set to empty string if it is the same a karabo-id\n",
- "\n",
- "bias_voltage = 180.0 # bias voltage - should be derived from CONTROL file\n",
- "integration_time = 10.0 # integration time - should be derived from CONTROL file.\n",
- "memory_cells = 1 # number of memory cells - should be derived from CONTROL file\n",
- "sc_start = 15 # storage cell start value - should be derived from CONTROL file\n",
- "gain_mode = 0 # number of memory cells - Set to -1 to derive from CONTROL file.\n",
- "\n",
- "_fit_func = \"CHARGE_SHARING\" ## which function will be used to fit the histogram\n",
- "_h_range = (200.0, 350.0) # range of the histogram in x-axis units\n",
- "rebin = 1\n",
- "# parameters for the peak finder\n",
- "n_sigma = 20.0 # n of sigma abov pedestal threshold\n",
- "ratio = 0.99 # ratio of the next peak amplitude in the peak_finder"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import multiprocessing\n",
- "import time\n",
- "from functools import partial\n",
- "from logging import warning\n",
- "from pathlib import Path\n",
- "\n",
- "import numpy as np\n",
- "from extra_data import RunDirectory\n",
- "from h5py import File as h5file\n",
- "\n",
- "from cal_tools.jungfrau import jungfrau_ff\n",
- "from cal_tools.jungfraulib import JungfrauCtrl"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "out_folder = Path(out_folder)\n",
- "out_folder.mkdir(parents=True, exist_ok=True)\n",
- "in_folder = Path(in_folder)\n",
- "\n",
- "if karabo_id_control == \"\":\n",
- " karabo_id_control = karabo_id"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Opening log file"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "h_spectra = None\n",
- "edges = None\n",
- "noise_map = None\n",
- "\n",
- "ctrl_src = ctrl_source_template.format(karabo_id_control)\n",
- "run_dc = RunDirectory(in_folder / f\"r{runs[0]:04d}\")\n",
- "\n",
- "ctrl_data = JungfrauCtrl(run_dc, ctrl_src)\n",
- "\n",
- "if memory_cells < 0:\n",
- " memory_cells, sc_start = ctrl_data.get_memory_cells()\n",
- "\n",
- " mem_cells_name = \"single cell\" if memory_cells == 1 else \"burst\"\n",
- " print(f\"Run is in {mem_cells_name} mode.\\nStorage cell start: {sc_start:02d}\")\n",
- "else:\n",
- " mem_cells_name = \"single cell\" if memory_cells == 1 else \"burst\"\n",
- " print(\n",
- " f\"Run is in manually set to {mem_cells_name} mode. With {memory_cells} memory cells\"\n",
- " )\n",
- "\n",
- "mode = \"Single\"\n",
- "if memory_cells > 1:\n",
- " mode = \"Burst\"\n",
- "\n",
- "file_h_name = (\n",
- " f\"R{runs[0]:04d}_Gain_{mode}_Spectra_{da_name}_Histo.h5\" # histogram file name\n",
- ")\n",
- "\n",
- "begin_stuff = time.localtime()\n",
- "i_cut = file_h_name.find(\"_Histo\")\n",
- "fout_temp = file_h_name[:i_cut]\n",
- "fout_temp += f\"_{_fit_func}_Fit\"\n",
- "\n",
- "print(f\"block_size: {block_size}\")\n",
- "\n",
- "if integration_time < 0:\n",
- " integration_time = ctrl_data.get_integration_time()\n",
- "if bias_voltage < 0:\n",
- " bias_voltage = ctrl_data.get_bias_voltage()\n",
- "# if gain_setting < 0:\n",
- "# gain_setting = ctrl_data.get_gain_setting()\n",
- "if gain_mode < 0:\n",
- " gain_mode = ctrl_data.get_gain_mode()\n",
- "\n",
- "print(f\"Integration time is {integration_time} us\")\n",
- "# print(f\"Gain setting is {gain_setting} (run settings: {ctrl_data.run_settings})\")\n",
- "print(f\"Gain mode is {gain_mode} ({ctrl_data.run_mode})\")\n",
- "print(f\"Bias voltage is {bias_voltage} V\")\n",
- "print(f\"Number of memory cells are {memory_cells}\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Opening Histo File"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# transposition here is just to make saved histo compatible with my other notebooks\n",
- "with h5file(out_folder / file_h_name, \"r\") as f:\n",
- " print(f\"opening histo in file {file_h_name}\")\n",
- " if \"histos\" in f.keys():\n",
- " h_spectra = np.transpose(np.array(f[\"histos\"]))\n",
- " print(f\"histogram found: {h_spectra.shape}\")\n",
- " else:\n",
- " raise AttributeError(\"No histo in file!\")\n",
- "\n",
- " edges = np.array(f[\"edges\"])\n",
- "\n",
- " x = (edges[1:] + edges[:-1]) / 2.0\n",
- " if \"noise_map\" in f.keys():\n",
- " noise_map = np.array(f[\"noise_map\"])\n",
- " print(f\"noise map found: {noise_map.shape}\")\n",
- " else:\n",
- " warning(\"noise map not found!\")\n",
- "\n",
- " print(\"Reading control data from histogram files.\")\n",
- " if \"integration_time\" in f.attrs.keys():\n",
- " integration_time = np.float32(f.attrs[\"integration_time\"])\n",
- " else:\n",
- " print(f\"integration_time is not found! using default value{integration_time}\")\n",
- " if \"bias_voltage\" in f.attrs.keys():\n",
- " bias_voltage = np.float32(f.attrs[\"bias_voltage\"])\n",
- " else:\n",
- " warning(f\"bias_voltage not found! using default value: {bias_voltage}\")\n",
- "\n",
- " if \"creation_time\" in f.attrs.keys():\n",
- " dir_date_iso = str(f.attrs[\"creation_time\"])\n",
- " if len(dir_date_iso) == 0:\n",
- " warning(\"Dir date is empty string\")\n",
- " else:\n",
- " print(\"dir_date not found!\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Fitting histograms"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "chunks = jungfrau_ff.chunk_Multi([h_spectra], block_size)\n",
- "pool = multiprocessing.Pool()\n",
- "\n",
- "st = time.perf_counter()\n",
- "\n",
- "partial_fit = partial(\n",
- " jungfrau_ff.fit_histogram,\n",
- " x,\n",
- " _fit_func,\n",
- " n_sigma,\n",
- " rebin,\n",
- " ratio,\n",
- " noise_map,\n",
- ")\n",
- "\n",
- "print(\"starting spectra fit\")\n",
- "r_maps = pool.map(partial_fit, chunks)\n",
- "print(\"r_maps calculation\", time.perf_counter() - st)\n",
- "\n",
- "g0_map = np.zeros((memory_cells, sensor_size[0], sensor_size[1]), dtype=np.float32)\n",
- "sigma_map = np.zeros((memory_cells, sensor_size[0], sensor_size[1]), dtype=np.float32)\n",
- "chi2ndf_map = np.zeros((memory_cells, sensor_size[0], sensor_size[1]), dtype=np.float32)\n",
- "alpha_map = np.zeros((memory_cells, sensor_size[0], sensor_size[1]), dtype=np.float32)\n",
- "\n",
- "for i, r in enumerate(r_maps):\n",
- " g0_chk, sigma_chk, chi2ndf_chk, alpha_chk = r\n",
- "\n",
- " n_blocks_col = int(g0_map.shape[-1] / block_size[1])\n",
- " irow = int(np.floor(i / n_blocks_col)) * block_size[0]\n",
- " icol = i % n_blocks_col * block_size[1]\n",
- "\n",
- " g0_map[..., irow : irow + block_size[0], icol : icol + block_size[1]] = g0_chk\n",
- " sigma_map[..., irow : irow + block_size[0], icol : icol + block_size[1]] = sigma_chk\n",
- " chi2ndf_map[\n",
- " ..., irow : irow + block_size[0], icol : icol + block_size[1]\n",
- " ] = chi2ndf_chk\n",
- " alpha_map[..., irow : irow + block_size[0], icol : icol + block_size[1]] = alpha_chk\n",
- "\n",
- "print(\"loading r_maps calculation results\", time.perf_counter() - st)\n",
- "\n",
- "pool.close()\n",
- "\n",
- "print(\"... done\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Final steps\n",
- "### Saving fit results"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "fout_path = f\"{out_folder}/{fout_temp}.h5\"\n",
- "\n",
- "with h5file(fout_path, \"w\") as fout:\n",
- " print(\"saving noise map ...\")\n",
- "\n",
- " ##trasposition is to make it compatible with my other nb\n",
- " dset_noi = fout.create_dataset(\"noise_map\", data=np.transpose(noise_map))\n",
- " print(\"saving fit results ...\")\n",
- "\n",
- " dset_chi2 = fout.create_dataset(\"chi2map\", data=np.transpose(chi2ndf_map))\n",
- " dset_gmap_fit = fout.create_dataset(\"gainMap_fit\", data=np.transpose(g0_map))\n",
- " dset_std = fout.create_dataset(\"sigmamap\", data=np.transpose(sigma_map))\n",
- " dset_alpha = fout.create_dataset(\"alphamap\", data=np.transpose(alpha_map))\n",
- " fout.attrs[\"memory_cells\"] = memory_cells # TODO: Why memory cells are not saved here. What about the other conditions??\n",
- " fout.attrs[\"integration_time\"] = integration_time\n",
- " fout.attrs[\"bias_voltage\"] = bias_voltage\n",
- " fout.attrs[\"dir_date_iso\"] = dir_date_iso\n",
- " fout.attrs[\"karabo_id\"] = karabo_id\n",
- " fout.attrs[\"da_name\"] = da_name\n",
- "\n",
- "print(\"closing\")"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": ".cal2_venv",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.8.11"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 4
-}
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index d68669469..3e2c56237 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -206,30 +206,15 @@ notebooks = {
"FF_HISTS": {
"notebook":
"notebooks/Jungfrau/gainCal_JF_Create_Spectra_Histos.ipynb",
- "dep_notebooks": [
- "notebooks/Jungfrau/gainCal_JF_Fit_Spectra_Histos.ipynb"],
- "concurrency": {"parameter": None,
- "default concurrency": None,
+ "concurrency": {"parameter": "karabo_da",
+ "default concurrency": list(range(8)),
"cluster cores": 4},
},
"FF": {
"notebook":
"notebooks/Jungfrau/create_gain_map.ipynb",
- "dep_notebooks": [
- "notebooks/Jungfrau/gainCal_JF_Fit_sendDB_NBC.ipynb"],
- "concurrency": {"parameter": None,
- "default concurrency": None,
- "cluster cores": 4},
- },
- "FF_ALL": {
- "notebook":
- "notebooks/Jungfrau/gainCal_JF_Create_Spectra_Histos.ipynb",
- "dep_notebooks": [
- "notebooks/Jungfrau/gainCal_JF_Fit_Spectra_Histos.ipynb",
- "notebooks/Jungfrau/create_gain_map.ipynb",
- "notebooks/Jungfrau/gainCal_JF_Fit_sendDB_NBC.ipynb",],
- "concurrency": {"parameter": None,
- "default concurrency": None,
+ "concurrency": {"parameter": "karabo_da",
+ "default concurrency": list(range(8)),
"cluster cores": 4},
},
},
--
GitLab