From ab6138dc3e4a7201530db69d58ab70e71070ad08 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Tue, 2 Jan 2024 16:19:00 +0100
Subject: [PATCH] FF_ALL for tests
---
.../gainCal_JF_Create_Spectra_Histos.ipynb | 247 ++++++++++++------
src/xfel_calibrate/notebooks.py | 15 +-
2 files changed, 175 insertions(+), 87 deletions(-)
diff --git a/notebooks/Jungfrau/gainCal_JF_Create_Spectra_Histos.ipynb b/notebooks/Jungfrau/gainCal_JF_Create_Spectra_Histos.ipynb
index 4dbe7b8aa..aa47b0026 100644
--- a/notebooks/Jungfrau/gainCal_JF_Create_Spectra_Histos.ipynb
+++ b/notebooks/Jungfrau/gainCal_JF_Create_Spectra_Histos.ipynb
@@ -84,6 +84,7 @@
"import os\n",
"from h5py import File as h5file\n",
"from functools import partial\n",
+ "from logging import warning\n",
"\n",
"import pasha as psh\n",
"import cal_tools.restful_config as rest_cfg\n",
@@ -128,18 +129,12 @@
"source": [
"begin_stuff = time.localtime()\n",
"\n",
- "\n",
"first_run_folder = in_folder / f'r{g0_runs[0]:04d}'\n",
"ctrl_data = JungfrauCtrl(RunDirectory(first_run_folder), control_src)\n",
"\n",
- "if memory_cells < 0 and sc_start < 0:\n",
- " memory_cells, sc_start = ctrl_data.get_memory_cells()\n",
- "\n",
- "_mode = 'Burst'\n",
- "if memory_cells == 1:\n",
- " _mode = 'Single'\n",
- "\n",
- "fout_temp = f'R{g0_runs[0]:04d}_Gain_{_mode}_Spectra_{da_name}'\n",
+ "# Run's creation time:\n",
+ "creation_time = calcat_creation_time(in_folder, g0_runs[0], creation_time)\n",
+ "print(f\"Creation time: {creation_time}\")\n",
"\n",
"print(\"Chunked trains: \", chunked_trains)"
]
@@ -170,53 +165,125 @@
"metadata": {},
"outputs": [],
"source": [
- "# def read_conditions_and_retrieve_constants(\n",
- "# run_dc,\n",
- "# run_number,\n",
- "# integration_time,\n",
- "# bias_voltage,\n",
- "# gain_mode,\n",
- "# creation_time,\n",
- "# ):\n",
- "# \"\"\"Retrieve offset and noise calibration constants\n",
- "# for one jungfrau module.\n",
- "# Args:\n",
- "# run_dc(extra_data.DataCollection):\n",
- "# run_number(int):\n",
- "# integration_time(float)\n",
- "# bias_voltage(float):\n",
- "# gain_mode(int):\n",
- "# creation_time():\n",
- "# Returns:\n",
- "# ndarray, ndarray: Offset and Noise calibration constants.\n",
- "# \"\"\"\n",
- "run_folder = in_folder / f'r{g0_runs[0]:04d}' # first run for now\n",
- "run_number = g0_runs[0]\n",
- "## Retrieve DB constants\n",
- "ctrl_data = JungfrauCtrl(RunDirectory(run_folder), control_src)\n",
- "if integration_time < 0:\n",
- " integration_time = ctrl_data.get_integration_time()\n",
- "if bias_voltage < 0:\n",
- " bias_voltage = ctrl_data.get_bias_voltage()\n",
- "# gain_setting = ctrl_data.get_gain_setting() TODO: Where is the gain_setting?\n",
- "if gain_mode < 0:\n",
- " gain_mode = ctrl_data.get_gain_mode()\n",
- "\n",
- "# TODO: clarify this way in getting the control data.\n",
- "# We are using RUN. We can of course update and start using `as_single_value` method.\n",
+ "def read_detector_conditions(\n",
+ " run_dc,\n",
+ " run_number,\n",
+ " bias_voltage,\n",
+ " memory_cells,\n",
+ " integration_time,\n",
+ " gain_mode,\n",
+ " gain_setting,\n",
+ "):\n",
+ " \"\"\"Get parameter conditions for a run.\n",
+ "\n",
+ " Args:\n",
+ " run_dc(extra_data.DataCollection):\n",
+ " run_number(int):\n",
+ " integration_time(float)\n",
+ " bias_voltage(float):\n",
+ " gain_mode(int):\n",
+ " gain_setting(int):\n",
+ "\n",
+ " Returns:\n",
+ " float, int, float, int, int, int\n",
+ " \"\"\"\n",
+ "\n",
+ " ## Retrieve DB constants\n",
+ " ctrl_data = JungfrauCtrl(run_dc, control_src)\n",
+ "\n",
+ " if memory_cells < 0:\n",
+ " memory_cells, sc_start = ctrl_data.get_memory_cells()\n",
+ "\n",
+ " if integration_time < 0:\n",
+ " integration_time = ctrl_data.get_integration_time()\n",
+ "\n",
+ " if bias_voltage < 0:\n",
+ " bias_voltage = ctrl_data.get_bias_voltage()\n",
+ "\n",
+ " # gain_setting = ctrl_data.get_gain_setting() TODO: Where is the gain_setting?\n",
+ " if gain_mode < 0:\n",
+ " gain_mode = ctrl_data.get_gain_mode()\n",
+ "\n",
+ "\n",
+ " return (\n",
+ " bias_voltage,\n",
+ " memory_cells,\n",
+ " sc_start,\n",
+ " integration_time,\n",
+ " gain_mode,\n",
+ " gain_setting,\n",
+ " run_number,\n",
+ " )\n",
+ "\n",
+ "condition_lists = {\n",
+ " \"integration_time\": [],\n",
+ " \"memory_cells\": [],\n",
+ " \"sc_start\": [],\n",
+ " \"bias_voltage\": [],\n",
+ " \"gain_setting\": [],\n",
+ " \"gain_mode\": [],\n",
+ " \"runs\": [],\n",
+ "}\n",
+ "\n",
+ "for run in g0_runs:\n",
+ " voltage, cells, sc_start, integration, mode, setting, run_number = read_detector_conditions( # noqa\n",
+ " RunDirectory(in_folder / f\"r{run:04d}\"),\n",
+ " run,\n",
+ " bias_voltage=bias_voltage,\n",
+ " memory_cells=memory_cells,\n",
+ " integration_time=integration_time,\n",
+ " gain_mode=gain_mode,\n",
+ " gain_setting=gain_setting)\n",
+ "\n",
+ " condition_lists[\"bias_voltage\"].append(voltage)\n",
+ " condition_lists[\"memory_cells\"].append(cells)\n",
+ " condition_lists[\"integration_time\"].append(integration)\n",
+ " condition_lists[\"gain_mode\"].append(mode)\n",
+ " condition_lists[\"gain_setting\"].append(setting)\n",
+ " condition_lists[\"sc_start\"].append(sc_start)\n",
+ "\n",
+ "# Validate run conditions\n",
+ "runs_results = condition_lists[\"runs\"]\n",
+ "for c, lst in condition_lists.items():\n",
+ " if c == \"runs\":\n",
+ " continue\n",
+ " if not all(val == lst[0] for val in lst):\n",
+ " warning(\n",
+ " f\"{c} is not the same for all runs: {lst} \"\n",
+ " f\"for runs {runs_results}, respectively\"\n",
+ " )\n",
+ "\n",
+ "memory_cells = condition_lists[\"memory_cells\"][0]\n",
+ "sc_start = condition_lists[\"sc_start\"][0]\n",
"print(f\"Memory cells: {memory_cells:d}\")\n",
- "print(f\"Exposure Time: {integration_time:1.2f} us\")\n",
+ "bias_voltage = condition_lists[\"bias_voltage\"][0]\n",
"print(f\"Bias Voltage: {bias_voltage:3.1f} V\")\n",
+ "integration_time = condition_lists[\"integration_time\"][0]\n",
+ "print(f\"Exposure Time: {integration_time:1.2f} us\")\n",
+ "gain_mode = condition_lists[\"gain_mode\"][0]\n",
+ "print(f\"Gain mode: {gain_mode}\")\n",
+ "gain_setting = condition_lists[\"gain_setting\"][0]\n",
+ "print(f\"Gain setting: {gain_setting}\")\n",
"\n",
- "# Run's creation time:\n",
- "creation_time = calcat_creation_time(in_folder, run_number, creation_time)\n",
- "print(f\"Creation time: {creation_time}\")\n",
+ "_mode = 'Burst'\n",
+ "if memory_cells == 1:\n",
+ " _mode = 'Single'\n",
"\n",
+ "fout_temp = f'R{g0_runs[0]:04d}_Gain_{_mode}_Spectra_{da_name}'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Retrieve calibration constants\n",
"jf_cal = JUNGFRAU_CalibrationData(\n",
" detector_name=karabo_id,\n",
" sensor_bias_voltage=bias_voltage,\n",
" event_at=creation_time,\n",
- " modules=karabo_da[0],\n",
+ " modules=karabo_da,\n",
" memory_cells=memory_cells,\n",
" integration_time=integration_time,\n",
" # No gain setting 1? I use 0 for now as I don't see gain_setting used anywhere for the 3 notebooks.\n",
@@ -224,40 +291,49 @@
" gain_mode=gain_mode,\n",
" client=rest_cfg.calibration_client(),\n",
")\n",
- "jf_metadata = jf_cal.metadata(calibrations=[\"Offset10Hz\", \"Noise10Hz\"])\n",
+ "\n",
+ "jf_metadata = jf_cal.metadata(\n",
+ " calibrations=[\"Offset10Hz\", \"Noise10Hz\"])\n",
+ "\n",
"# TODO: display CCV timestamp\n",
"const_data = jf_cal.ndarray_map(metadata=jf_metadata)\n",
"\n",
"constants_data = const_data[karabo_da[0]]\n",
"\n",
- "if \"Offset10Hz\" in constants_data:\n",
- " offset_map = np.array(\n",
- " np.transpose(np.swapaxes(constants_data[\"Offset10Hz\"], 0, 1)),\n",
- " dtype=np.float32\n",
- " )\n",
- " print(f'Retrieved Offset Map {offset_map.shape}')\n",
- "else:\n",
- " # Create offset empty constant.\n",
- " offset_map = np.zeros((3, memory_cells, 512, 1024), dtype=np.float32)\n",
- " print(\"Offset map not found\")\n",
- "\n",
- "if \"Noise10Hz\" in constants_data:\n",
- " noise_map = np.array(\n",
- " np.transpose(np.swapaxes(constants_data[\"Noise10Hz\"], 0, 1)),\n",
- " dtype=np.float32\n",
- " )\n",
- " print(f'Retrieved Noise Map {noise_map.shape}')\n",
- "else:\n",
- " noise_map = None\n",
- " print(\"Noise map not found\")\n",
- "\n",
- "if memory_cells > 1:\n",
- " # move from x, y, cell, gain to cell, x, y, gain\n",
- " offset_map = np.moveaxis(offset_map, [0, 1], [1, 2])\n",
- "else:\n",
- " offset_map = np.squeeze(offset_map)\n",
- "\n",
- " # return offset, noise"
+ "for mod in karabo_da:\n",
+ " constants_data = const_data[mod]\n",
+ "\n",
+ " if \"Offset10Hz\" in constants_data:\n",
+ " constants_data[\"Offset10Hz\"] = np.array(\n",
+ " np.transpose(np.swapaxes(constants_data[\"Offset10Hz\"], 0, 1)),\n",
+ " dtype=np.float32\n",
+ " )\n",
+ " print(\n",
+ " f'Retrieved Offset Map {constants_data[\"Noise10Hz\"].shape}')\n",
+ " else:\n",
+ " # Create offset empty constant.\n",
+ " constants_data[\"Offset10Hz\"] = np.zeros(\n",
+ " (3, memory_cells, 512, 1024),\n",
+ " dtype=np.float32,\n",
+ " )\n",
+ " warning(f\"Offset map is not found for {mod}\")\n",
+ "\n",
+ " if \"Noise10Hz\" in constants_data:\n",
+ " constants_data[\"Noise10Hz\"] = np.array(\n",
+ " np.transpose(np.swapaxes(constants_data[\"Noise10Hz\"], 0, 1)),\n",
+ " dtype=np.float32\n",
+ " )\n",
+ " print(\n",
+ " f'Retrieved Noise Map {constants_data[\"Noise10Hz\"].shape} for {mod}')\n",
+ " else:\n",
+ " constants_data[\"Noise10Hz\"] = None\n",
+ " print(f\"Noise map is not found for {mod}\")\n",
+ "\n",
+ " if memory_cells > 1:\n",
+ " # move from x, y, cell, gain to cell, x, y, gain\n",
+ " offset_map = np.moveaxis(offset_map, [0, 1], [1, 2])\n",
+ " else:\n",
+ " offset_map = np.squeeze(offset_map)"
]
},
{
@@ -347,14 +423,13 @@
"\n",
" run_dc = run_dc.select(selection)\n",
"\n",
- " # offset_map, noise_map = read_conditions_and_retrieve_constants(\n",
- " # run_dc=run_dc,\n",
- " # run_number=g0_run,\n",
- " # integration_time=integration_time,\n",
- " # bias_voltage=bias_voltage,\n",
- " # gain_mode=gain_mode,\n",
- " # creation_time=creation_time,\n",
- " # )\n",
+ " offset_map, noise_map = retrieve_calibration_constants(\n",
+ " run_number=g0_run,\n",
+ " integration_time=integration_time,\n",
+ " bias_voltage=bias_voltage,\n",
+ " gain_mode=gain_mode,\n",
+ " creation_time=creation_time,\n",
+ " )\n",
"\n",
" ## Offset subtraction & Histogram filling\n",
" ### performs offset subtraction and fills the histogram\n",
@@ -404,6 +479,7 @@
"\n",
" h_spectra[..., irow:irow+block_size[0], icol:icol+block_size[1]] += h_chk\n",
" step_timer.done_step(\"Create histogram\")\n",
+ "\n",
"x = (edges[1:] + edges[:-1])/2."
]
},
@@ -432,6 +508,7 @@
"source": [
"fout_h_path = f'{out_folder}/{fout_temp}_Histo.h5'\n",
"hists = np.transpose(h_spectra) # this is to make it compatible with my other notebooks, can be removed\n",
+ "\n",
"with h5file(fout_h_path, 'w') as fout_h: \n",
" print(f\"Saving histograms at {fout_h_path}.\")\n",
" dset_h = fout_h.create_dataset(\"histos\", data=hists)\n",
diff --git a/src/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
index 51650bc14..d68669469 100644
--- a/src/xfel_calibrate/notebooks.py
+++ b/src/xfel_calibrate/notebooks.py
@@ -206,8 +206,8 @@ notebooks = {
"FF_HISTS": {
"notebook":
"notebooks/Jungfrau/gainCal_JF_Create_Spectra_Histos.ipynb",
-# "dep_notebooks": [
-# "notebooks/Jungfrau/gainCal_JF_Fit_Spectra_Histos.ipynb"],
+ "dep_notebooks": [
+ "notebooks/Jungfrau/gainCal_JF_Fit_Spectra_Histos.ipynb"],
"concurrency": {"parameter": None,
"default concurrency": None,
"cluster cores": 4},
@@ -221,6 +221,17 @@ notebooks = {
"default concurrency": None,
"cluster cores": 4},
},
+ "FF_ALL": {
+ "notebook":
+ "notebooks/Jungfrau/gainCal_JF_Create_Spectra_Histos.ipynb",
+ "dep_notebooks": [
+ "notebooks/Jungfrau/gainCal_JF_Fit_Spectra_Histos.ipynb",
+ "notebooks/Jungfrau/create_gain_map.ipynb",
+ "notebooks/Jungfrau/gainCal_JF_Fit_sendDB_NBC.ipynb",],
+ "concurrency": {"parameter": None,
+ "default concurrency": None,
+ "cluster cores": 4},
+ },
},
"GOTTHARD2": {
"CORRECT": {
--
GitLab