diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 11d4438584b2db40bea2970558a1066269cc6e2c..25df1619293f0d6461b6061bde489d877a50671b 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -1,3 +1,6 @@
+from pathlib import Path
+from typing import Optional, Tuple
+
import h5py
import numpy as np
import sharedmem
@@ -22,21 +25,65 @@ def get_num_cells(fname, loc, module):
return options[np.argmin(dists)]
-def get_acq_rate(fname, loc, module):
- with h5py.File(fname, "r") as f:
- try:
- pulses = \
- np.squeeze(f[f"INSTRUMENT/{loc}/DET/{module}CH0:xtdf/image/pulseId"][:2]) # noqa
- diff = pulses[1] - pulses[0]
- except:
- diff = 0
- options = {8: 0.5, 4: 1.1, 2: 2.2, 1: 4.5}
- return options.get(diff, None)
+def get_acq_rate(fast_paths: Tuple[str, str, int],
+ slow_paths: Optional[Tuple[str, str]] = ('', '')
+ ) -> Optional[float]:
+ """Get the acquisition rate from said detector module.
+
+ If the data is available from the middlelayer FPGA_COMP device, then it is
+ retrieved from there. If not, the rate is calculated from two different
+ pulses time.
+
+ The first entry is deliberatly not used, as the detector just began
+ operating, and it might have skipped a train.
+ :param slow_paths: in which file and h5 path to look for slow data.
+ The first string is the filename with complete path,
+ the second string is the key `karabo_id_control`
-def get_gain_setting(fname, h5path_ctrl):
+ :param fast_paths: in which module file and h5 path to look for pulses.
+ The first string is the filename with complete path,
+ the second string is the module device name `karabo_id`,
+ the third parameter is the module number, used to
+ navigate through the h5 file structure.
+
+ :return acq_rate: the acquisition rate.
+ If not found in either files, return None.
+ """
+ # Attempt to look for acquisition rate in slow data
+ slow_data_file, karabo_id_control = slow_paths
+ slow_data_file = Path(slow_data_file)
+ if slow_data_file.is_file():
+ slow_data_path = f'CONTROL/{karabo_id_control}/MDL/FPGA_COMP/bunchStructure/repetitionRate/value' # noqa
+ with h5py.File(slow_data_file, "r") as fin:
+ if slow_data_path in fin:
+ # The acquisition rate value is stored in a 1D array of type
+ # float. Use the 3rd value, arbitrarily chosen. It's okay to
+ # loose precision here because the usage is about defining the
+ # rate for meta-data.
+ return round(fin[slow_data_path][3], 1)
+
+ # Compute acquisition rate from fast data
+ fast_data_file, karabo_id, module = fast_paths
+ fast_data_file = Path(fast_data_file)
+ if fast_data_file.is_file():
+ fast_data_path = f'INSTRUMENT/{karabo_id}/DET/{module}CH0:xtdf/image/pulseId' # noqa
+ with h5py.File(fast_data_file) as fin:
+ if fast_data_path in fin:
+ # pulses is of shape (NNNN, 1), of type uint8.
+ # Squeeze out the data, and subtract the 3rd entry from the 2nd
+ # to get a rate.
+ pulses = np.squeeze(fin[fast_data_path][1:3])
+ diff = pulses[1] - pulses[0]
+ options = {8: 0.5, 4: 1.1, 2: 2.2, 1: 4.5}
+ return options.get(diff, None)
+
+
+def get_gain_setting(fname: str, h5path_ctrl: str) -> int:
"""
- Return gain setting base on setupr and patternTypeIndex
+ If the data is available from the middlelayer FPGA_COMP device, then it is
+ retrieved from there.
+ If not, the setting is calculated off `setupr` and `patternTypeIndex`
gain-setting 1: setupr@dark=8, setupr@slopespc=40
gain-setting 0: setupr@dark=0, setupr@slopespc=32
@@ -50,11 +97,18 @@ def get_gain_setting(fname, h5path_ctrl):
:param h5path_ctrl: path to control information inside the file
:return: gain setting
"""
- with h5py.File(fname, "r") as f:
- train_id = f["INDEX/trainId"][()]
+ gain_path = f'{h5path_ctrl}/gain/value'
+ with h5py.File(fname, "r") as fin:
+ if gain_path in fin:
+ return fin[gain_path][0]
+
+ # Get the index at which the train is not zero.
+ train_id = fin["INDEX/trainId"][()]
idx = np.nonzero(train_id)[0][0]
- setupr = f[f'{h5path_ctrl}/setupr/value'][idx]
- pattern_type_idx = f[f'{h5path_ctrl}/patternTypeIndex/value'][idx]
+
+ setupr = fin[f'{h5path_ctrl}/setupr/value'][idx]
+ pattern_type_idx = fin[f'{h5path_ctrl}/patternTypeIndex/value'][idx]
+
if (setupr == 0 and pattern_type_idx < 4) or (
setupr == 32 and pattern_type_idx == 4):
return 0
@@ -65,6 +119,30 @@ def get_gain_setting(fname, h5path_ctrl):
raise ValueError('Could not derive gain setting from setupr and patternTypeIndex') # noqa
+def get_bias_voltage(fname: str, karabo_id_control: str,
+ module: Optional[int] = 0) -> int:
+ """Read the voltage information from the FPGA device of module 0.
+
+ Different modules may operate at different voltages. In practice, they all
+ operate at the same voltage. As such, it is okay to read a single module's
+ value.
+
+ This value is read from slow data.
+
+ If the file cannot be accessed, an OSError will be raised.
+ If the hdf5 path cannot be accessed, None will be returned.
+
+ :param fname: path to slow data file with control information
+ :param karabo_id: The detector Karabo id, for creating the hdf5 path
+ :param module: defaults to module 0
+ :return: voltage, a uint16
+ """
+ voltage_path = f'/CONTROL/{karabo_id_control}/FPGA/M_{module}/highVoltage/actual/value' # noqa
+ with h5py.File(fname, "r") as fin:
+ if voltage_path in fin:
+ return fin[voltage_path][0]
+
+
class AgipdCorrections:
def __init__(self, max_cells, max_pulses,
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index b8f695160e5ee6ff621158eb9a49d10e9e237e38..0f46f959ec4433b264c5591f08ed5c1bdeb572a4 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -7,6 +7,7 @@ from os.path import isfile, splitext
from queue import Queue
import re
from time import sleep
+from typing import Optional
from urllib.parse import urljoin
import dateutil.parser
@@ -227,7 +228,9 @@ def get_run_info(proposal, run):
return resp.json()
-def get_dir_creation_date(directory, run, tsdir=False, verbosity=0):
+def get_dir_creation_date(directory: str, run: int,
+ tsdir: Optional[bool] = False,
+ verbosity: Optional[int] = 0):
"""
Return run starting time from the MDC.
If not succeeded, return modification time of oldest file.h5
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index d7803821cf31e4baf5ecc115a3851ea8ff2b361d..e906d9921aa33d3f733bc70e47e11b0cb61f442e 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -313,7 +313,7 @@
"\n",
"# Evaluate aquisition rate\n",
"if acq_rate == 0:\n",
- " acq_rate = get_acq_rate(filename, karabo_id, channel)\n",
+ " acq_rate = get_acq_rate((filename, karabo_id, channel))\n",
"else:\n",
" acq_rate = None\n",
"\n",
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 8f3df25d24065fb556f678d4e27dc4618fa06532..8a2ded9840b0484a95478c069917efc61489167d 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -271,7 +271,7 @@
" break\n",
"\n",
" if acq_rate == 0.:\n",
- " acq_rate = get_acq_rate(f, karabo_id, idx)\n",
+ " acq_rate = get_acq_rate((f, karabo_id, idx))\n",
" else:\n",
" acq_rate = None\n",
"\n",
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 2b39c0bc11e203dc0ed78bb8f4b8afcd2b37614b..5dc3a6c48990ee9e1ef68ad05dfc3f0bd9014668 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -50,7 +50,7 @@
"db_output = False # output constants to database\n",
"\n",
"mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
- "bias_voltage = 300 # detector bias voltage\n",
+ "bias_voltage = 0 # detector bias voltage\n",
"gain_setting = 0.1 # the gain setting, use 0.1 to try to auto-determine\n",
"acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
"interlaced = False # assume interlaced data format, for data prior to Dec. 2017\n",
@@ -91,6 +91,7 @@
"warnings.filterwarnings('ignore')\n",
"from collections import OrderedDict\n",
"import os\n",
+ "from typing import Tuple, List\n",
"import h5py\n",
"import numpy as np\n",
"import matplotlib\n",
@@ -111,7 +112,7 @@
"from cal_tools.plotting import (create_constant_overview,\n",
" plot_badpix_3d, show_processed_modules,\n",
" show_overview)\n",
- "from cal_tools.agipdlib import get_gain_setting\n",
+ "from cal_tools.agipdlib import get_bias_voltage, get_gain_setting\n",
"\n",
"# make sure a cluster is running with ipcluster start --n=32, give it a while to start\n",
"from ipyparallel import Client\n",
@@ -155,6 +156,9 @@
" dinstance = \"AGIPD500K\"\n",
" nmods = 8\n",
"\n",
+ "control_names = [f'{in_folder}/r{r:04d}/RAW-R{r:04d}-{karabo_da_control}-S00000.h5'\n",
+ " for r in (run_high, run_med, run_low)] \n",
+ "\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"print(f\"Instrument {instrument}\")\n",
"print(f\"Detector instance {dinstance}\")\n",
@@ -218,6 +222,12 @@
"h5path = h5path.format(karabo_id, receiver_id)\n",
"h5path_idx = h5path_idx.format(karabo_id, receiver_id)\n",
"\n",
+ "if bias_voltage == 0: \n",
+ " # Read the bias voltage from files, if recorded.\n",
+ " # If not available, make use of the historical voltage the detector is running at\n",
+ " bias_voltage = get_bias_voltage(control_names[0], karabo_id_control)\n",
+ " bias_voltage = bias_voltage if bias_voltage is not None else 300\n",
+ "\n",
"print(\"Parameters are:\")\n",
"print(f\"Proposal: {prop}\")\n",
"print(f\"Memory cells: {mem_cells}/{max_cells}\")\n",
@@ -273,55 +283,64 @@
"end_time": "2019-02-20T10:50:55.839958Z",
"start_time": "2019-02-20T10:50:55.468134Z"
},
- "scrolled": false
+ "scrolled": true
},
"outputs": [],
"source": [
"import copy\n",
"from functools import partial\n",
- "def characterize_module(il_mode, cells, bp_thresh, rawversion, loc, acq_rate,\n",
- " h5path, h5path_idx, inp):\n",
+ "def characterize_module(il_mode: bool,\n",
+ " cells: int,\n",
+ " bp_thresh: Tuple[List[int], float, List[int], float], \n",
+ " rawversion: int,\n",
+ " loc: str, \n",
+ " acq_rate: float,\n",
+ " h5path: str,\n",
+ " h5path_idx: str,\n",
+ " control_names: List[str],\n",
+ " karabo_id_control: str,\n",
+ " inp: Tuple[str, int, int]) -> Tuple[np.array, np.array, np.array, np.array, int, np.array, int, float]:\n",
" import numpy as np\n",
" import copy\n",
" import h5py\n",
" from cal_tools.enums import BadPixels\n",
" from cal_tools.agipdlib import get_num_cells, get_acq_rate\n",
"\n",
- " filename, channel, gg = inp\n",
+ " fast_data_filename, channel, gg = inp\n",
" \n",
" if cells == 0:\n",
- " cells = get_num_cells(filename, loc, channel)\n",
+ " cells = get_num_cells(fast_data_filename, loc, channel)\n",
"\n",
" print(f\"Using {cells} memory cells\")\n",
" \n",
" if acq_rate == 0.:\n",
- " acq_rate = get_acq_rate(filename, loc, channel)\n",
+ " slow_paths = control_names[gg], karabo_id_control\n",
+ " fast_paths = fast_data_filename, loc, channel\n",
+ " acq_rate = get_acq_rate(fast_paths, slow_paths)\n",
"\n",
" thresholds_offset, thresholds_offset_sigma, thresholds_noise, thresholds_noise_sigma = bp_thresh \n",
" thresholds_offset_hard = thresholds_offset[gg]\n",
" thresholds_noise_hard = thresholds_noise[gg]\n",
- " infile = h5py.File(filename, \"r\", driver=\"core\")\n",
" \n",
" h5path = h5path.format(channel)\n",
" h5path_idx = h5path_idx.format(channel)\n",
" \n",
- " if rawversion == 2:\n",
- " count = np.squeeze(infile[f\"{h5path_idx}/count\"])\n",
- " first = np.squeeze(infile[f\"{h5path_idx}/first\"])\n",
- " last_index = int(first[count != 0][-1]+count[count != 0][-1])\n",
- " first_index = int(first[count != 0][0])\n",
- " else:\n",
- " status = np.squeeze(infile[f\"{h5path_idx}/status\"])\n",
- " if np.count_nonzero(status != 0) == 0:\n",
- " return\n",
- " last = np.squeeze(infile[f\"{h5path_idx}/last\"])\n",
- " first = np.squeeze(infile[f\"{h5path_idx}/first\"])\n",
- " last_index = int(last[status != 0][-1]) + 1\n",
- " first_index = int(first[status != 0][0])\n",
- " im = np.array(infile[f\"{h5path}/data\"][first_index:last_index,...]) \n",
- " cellIds = np.squeeze(infile[f\"{h5path}/cellId\"][first_index:last_index,...]) \n",
- " \n",
- " infile.close()\n",
+ " with h5py.File(fast_data_filename, \"r\", driver=\"core\") as infile:\n",
+ " if rawversion == 2:\n",
+ " count = np.squeeze(infile[f\"{h5path_idx}/count\"])\n",
+ " first = np.squeeze(infile[f\"{h5path_idx}/first\"])\n",
+ " last_index = int(first[count != 0][-1]+count[count != 0][-1])\n",
+ " first_index = int(first[count != 0][0])\n",
+ " else:\n",
+ " status = np.squeeze(infile[f\"{h5path_idx}/status\"])\n",
+ " if np.count_nonzero(status != 0) == 0:\n",
+ " return\n",
+ " last = np.squeeze(infile[f\"{h5path_idx}/last\"])\n",
+ " first = np.squeeze(infile[f\"{h5path_idx}/first\"])\n",
+ " last_index = int(last[status != 0][-1]) + 1\n",
+ " first_index = int(first[status != 0][0])\n",
+ " im = np.array(infile[f\"{h5path}/data\"][first_index:last_index,...]) \n",
+ " cellIds = np.squeeze(infile[f\"{h5path}/cellId\"][first_index:last_index,...]) \n",
"\n",
" if il_mode:\n",
" ga = im[1::2, 0, ...]\n",
@@ -412,7 +431,8 @@
"p = partial(characterize_module, IL_MODE, max_cells,\n",
" (thresholds_offset_hard, thresholds_offset_sigma,\n",
" thresholds_noise_hard, thresholds_noise_sigma),\n",
- " rawversion, karabo_id, acq_rate, h5path, h5path_idx)\n",
+ " rawversion, karabo_id, acq_rate, h5path, h5path_idx,\n",
+ " control_names, karabo_id_control)\n",
"\n",
"# Don't remove. Used for Debugging.\n",
"#results = list(map(p, inp))\n",
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
index 9f4a3351b56b1857bd6f79465aa94b5249a03bd5..6c7a36b8f44b6dfabe902d4f38a5ac7504249481 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb
@@ -189,7 +189,7 @@
"channel = 0\n",
"fname = fbase.format(runs[0], runs[0].upper(), channel, sequences[0])\n",
"if acqrate == 0.:\n",
- " acqrate = get_acq_rate(fname, loc, channel)\n",
+ " acqrate = get_acq_rate((fname, loc, channel))\n",
" \n",
"\n",
"if mem_cells == 0:\n",
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
index bd919cef03f613c3466e04f9060a13c4b4674b31..59c499d2a2dc8bd005b57780d42a8dad1911aadc 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
@@ -160,7 +160,7 @@
" print('Reading ',fname)\n",
" \n",
" if acq_rate == 0.:\n",
- " acq_rate = get_acq_rate(fname, loc, channel)\n",
+ " acq_rate = get_acq_rate((fname, loc, channel))\n",
" print(\"Acquisition rate set from file: {} MHz\".format(acq_rate))\n",
"\n",
" if mem_cells == 0:\n",
diff --git a/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb b/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb
index 551f5f98a568931bcc44e659bcf9d141b406e3a0..f727a1a94440240014378ad353bb276210798570 100644
--- a/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb
+++ b/notebooks/AGIPD/playground/AGIPD_SingleM_test_Dark.ipynb
@@ -260,7 +260,7 @@
" cells = get_num_cells(filename, loc, channel)\n",
" \n",
" if acq_rate == 0.:\n",
- " acq_rate = get_acq_rate(filename, loc, channel)\n",
+ " acq_rate = get_acq_rate((filename, loc, channel))\n",
" \n",
" thresholds_offset, thresholds_offset_sigma, thresholds_noise, thresholds_noise_sigma = bp_thresh \n",
" thresholds_offset_hard = thresholds_offset[gg]\n",
diff --git a/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb b/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
index d0f9c7d44522dd8baee23f26877bdedf4b2540cb..5747928d6c6e309d653ed911767b1d142dc3f773 100644
--- a/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
+++ b/notebooks/generic/overallmodules_Darks_Summary_NBC.ipynb
@@ -61,7 +61,7 @@
" elif \"MID\" in karabo_id:\n",
" dinstance = \"AGIPD1M2\"\n",
" nmods = 16\n",
- " elif \"HED\" in karabo_id or \"DETLAB\" in karabo_id :\n",
+ " elif \"HED\" in karabo_id or \"DETLAB\" in karabo_id:\n",
" dinstance = \"AGIPD500K\"\n",
" nmods = 8\n",
" display(Markdown(\"\"\"\n",
diff --git a/requirements.txt b/requirements.txt
index 76a713f32e287b96e697f3b4a5022792ac01ed60..451df43f035744e740dbff94ea08ca2ad438c9bd 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
-git+file:///gpfs/exfel/sw/calsoft/git/cal_db_interactive@1.5.3
+git+file:///gpfs/exfel/sw/calsoft/git/cal_db_interactive@1.6.1
git+file:///gpfs/exfel/sw/calsoft/git/nbparameterise@0.3
git+file:///gpfs/exfel/sw/calsoft/git/pyDetLib@2.5.3-2.7.0#subdirectory=lib
astcheck == 0.2.5