diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 68216a55d59c2460a69a40a2eb6bdaafbb15efa9..19822fd6eb510fdf37f619f92f497c4aebfe74a0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -47,6 +47,9 @@ pytest:
- export LANG=C # Hopefully detect anything relying on locale
- python3 -m pip install ".[test]" --index-url https://git.xfel.eu/api/v4/groups/501/-/packages/pypi/simple
- python3 -m pytest --color yes --verbose --cov=cal_tools --cov=xfel_calibrate --cov-report html:htmlcov --cov-report term
+ - config_dir=$(mktemp -d)
+ - git clone "https://${CONFIG_REPO_DEPLOY_TOKEN}@git.xfel.eu/detectors/calibration_configurations.git" "$config_dir"
+ - python3 -m xfel_calibrate.validate_nbs_config --config-dir "$config_dir"
coverage: '/TOTAL.*? (\d+(?:\.\d+)?\%)$/'
artifacts:
expose_as: 'Coverage report'
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 371720faf4c2559e822358dd4e1d9dd9f008b317..55a2a2f78a61b91913bceaa039b1b37c2e804c9b 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -43,7 +43,7 @@
"sort_runs = True # Sort the selected dark runs. This flag is added for old data (e.g. 900174 r0011).\n",
"\n",
"mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
- "bias_voltage = 0 # bias voltage, set to 0 to use stored value in slow data.\n",
+ "bias_voltage = 0. # bias voltage, set to 0 to use stored value in slow data.\n",
"gain_setting = -1 # the gain setting, use -1 to use value stored in slow data.\n",
"gain_mode = -1 # gain mode, use -1 to use value stored in slow data.\n",
"integration_time = -1 # integration time, negative values for auto-detection.\n",
diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
index 2e3b47d3f2f09c971585de7ceb416a3575a2c584..f4c8d91ca93ff4dad003bed49f5d72ae6723d573 100644
--- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
@@ -50,12 +50,12 @@
"replace_wrong_gain_value = 0 # Force gain value into the chosen gain [0, 1, or 2] for pixels specified in `wrong_gain_pixels`. This has no effect if wrong_gain_pixels = [-1]\n",
"\n",
"# Parameters for retrieving calibration constants\n",
- "integration_time = -1 # integration time in us. set to -1 to overwrite by value in file.\n",
+ "integration_time = -1. # integration time in us. set to -1 to overwrite by value in file.\n",
"exposure_timeout = -1 # Exposure timeout. set to -1 to overwrite by value in file.\n",
"gain_setting = -1 # 0 for dynamic gain, 1 for dynamic HG0. set to -1 to overwrite by value in file.\n",
"gain_mode = -1 # 0 for runs with dynamic gain setting, 1 for fixed gain. Set to -1 to overwrite by value in file.\n",
"mem_cells = -1 # Set mem_cells to -1 to automatically use the value stored in RAW data.\n",
- "bias_voltage = -1 # Bias Voltage. Set to -1 to overwrite by value in file.\n",
+ "bias_voltage = -1. # Bias Voltage. Set to -1 to overwrite by value in file.\n",
"\n",
"# Parameters for plotting\n",
"skip_plots = False # exit after writing corrected files\n",
diff --git a/notebooks/LPDMini/LPD_Mini_Correct.ipynb b/notebooks/LPDMini/LPD_Mini_Correct.ipynb
index 0898a4fdc23eb08282020092a14dd37ef56cfb1f..7e2b27d78dcc524cfb781816783b8a25e9ba33a1 100644
--- a/notebooks/LPDMini/LPD_Mini_Correct.ipynb
+++ b/notebooks/LPDMini/LPD_Mini_Correct.ipynb
@@ -41,8 +41,8 @@
"cal_db_root = '/gpfs/exfel/d/cal/caldb_store'\n",
"\n",
"# Operating conditions\n",
- "bias_voltage_0 = -1 # bias voltage for minis 1, 3, 5, 7; Setting -1 will read the value from files\n",
- "bias_voltage_1 = -1 # bias voltage for minis 2, 4, 6, 8; Setting -1 will read the value from files\n",
+ "bias_voltage_0 = -1. # bias voltage for minis 1, 3, 5, 7; Setting -1 will read the value from files\n",
+ "bias_voltage_1 = -1. # bias voltage for minis 2, 4, 6, 8; Setting -1 will read the value from files\n",
"capacitor = '5pF' # Capacitor setting: 5pF or 50pF\n",
"photon_energy = 9.3 # Photon energy in keV.\n",
"use_cell_order = 'auto' # Whether to use memory cell order as a detector condition (auto = used only when memory cells wrap around)\n",
diff --git a/src/xfel_calibrate/validate_nbs_config.py b/src/xfel_calibrate/validate_nbs_config.py
new file mode 100644
index 0000000000000000000000000000000000000000..ff949658ff0b16e9c9a732a4484fd1e5f471cc6c
--- /dev/null
+++ b/src/xfel_calibrate/validate_nbs_config.py
@@ -0,0 +1,188 @@
+import argparse
+import sys
+from pathlib import Path
+
+import nbformat
+import nbparameterise
+import yaml
+
+from .notebooks import notebooks
+
+PKG_DIR = Path(__file__).parent
+ACTIONS = ("CORRECT", "DARK") # Only these use this config system
+
+
+def params_from_nb(rel_path):
+ abs_path = PKG_DIR / rel_path
+ nb = nbformat.read(abs_path, as_version=4)
+ return nbparameterise.code.extract_parameter_dict(nb)
+
+
+def load_nb_params():
+ res = {}
+ n = 0
+ for detector, det_nbs in notebooks.items():
+ res[detector] = det_params = {}
+ for action in ACTIONS:
+ if action not in det_nbs:
+ continue
+
+ det_params[action] = params_from_nb(det_nbs[action]["notebook"])
+ n += 1
+
+ for dep_nb in det_nbs[action].get("dep_notebooks", []):
+ dep_params = params_from_nb(dep_nb)
+ extras = set(dep_params) - set(det_params[action])
+ if extras:
+ print(f"Extra parameters in dep notebook for {detector} {action}")
+ print(" ", dep_nb)
+ print(" ", sorted(extras))
+
+ print(f"Loaded parameters for {n} correct/dark notebooks\n")
+ return res
+
+n_probs = 0
+
+def config_problem(problem, detail, file, key):
+ global n_probs
+ n_probs += 1
+ print(f"{problem}\n {detail}\n file: {file}\n key: {'/'.join(key)}\n")
+
+
+def bad_list_element_types(value, param: nbparameterise.Parameter):
+ # param.type == list
+ nb_types = {type(e) for e in param.value}
+ if nb_types == {float, int}:
+ nb_types = {float}
+ assert len(nb_types) == 1, f"{param.value}"
+ nb_type = nb_types.pop()
+
+ if not isinstance(value, list):
+ value = [value] # In the xfel-calibrate CLI, a list of 1 item is not distinct
+
+ if "RANGE ALLOWED" in (param.comment or "").upper():
+ new_value = []
+ # Try to parse ranges
+ for e in value:
+ if not isinstance(e, str):
+ new_value.append(e)
+ continue
+ try:
+ for rcomp in e.split(","):
+ if "-" in rcomp:
+ start, end = rcomp.split("-")
+ new_value += list(range(int(start), int(end)))
+ else:
+ new_value += [int(rcomp)]
+ except ValueError:
+ return "Invalid range specified", repr(e)
+ value = new_value
+
+ if isinstance(value, list):
+ if any(not like_type(e, nb_type) for e in value):
+ bad_types = sorted({type(e) for e in value})
+ return ("List element type differs from default in notebook",
+ f"notebook: {nb_type}, config: {bad_types}")
+ return None, None
+
+
+def like_type(value, param_type):
+ if isinstance(value, param_type):
+ return True
+ if param_type is float and isinstance(value, int):
+ return True
+ return False
+
+
+def check_yaml_file(path: Path, config_dir: Path, nb_params, detectors):
+ rel_path = path.relative_to(config_dir)
+ with path.open('r') as f:
+ cfg = yaml.safe_load(f)
+
+ kid_to_det = {}
+ for kid, d in cfg['data-mapping'].items():
+ det = d['detector-type'].upper()
+ if det in nb_params:
+ kid_to_det[kid] = det
+ else:
+ config_problem(
+ "Unknown detector type in data mapping", det,
+ rel_path, ('data-mapping', kid, 'detector-type')
+ )
+
+ for action in ACTIONS:
+ action = action.lower()
+ for instrument in cfg[action]:
+ inst_cfg = cfg[action][instrument]
+ for kid, det_cfg in inst_cfg.items():
+ try:
+ det = kid_to_det[kid]
+ except KeyError:
+ config_problem(
+ "Detector Karabo ID not in data mapping", kid,
+ rel_path, (action, instrument)
+ )
+ continue
+
+ if detectors and (det not in detectors):
+ continue
+
+ try:
+ det_params = nb_params[det][action.upper()]
+ except KeyError:
+ config_problem(
+ f"Action not available for detector {det}", action,
+ rel_path, (action, instrument, kid)
+ )
+ continue
+
+ for k, v in (det_cfg or {}).items():
+ pname = k.replace('-', '_')
+ try:
+ param = det_params[pname]
+ except KeyError:
+ config_problem(
+ "Parameter name not found in notebook", pname,
+ rel_path, (action, instrument, kid)
+ )
+ continue
+
+ if param.type is list:
+ pblm, detail = bad_list_element_types(v, param)
+ if pblm:
+ config_problem(
+ pblm, detail, rel_path, (action, instrument, kid, k)
+ )
+ elif not like_type(v, param.type):
+ config_problem(
+ "Type differs from default in notebook",
+ f"notebook: {param.value!r}, config: {v!r}",
+ rel_path, (action, instrument, kid, k)
+ )
+
+
+
+def main(argv=None):
+ ap = argparse.ArgumentParser()
+ ap.add_argument('--config-dir', type=Path, default="~/calibration_config/")
+ ap.add_argument('--detector', choices=notebooks.keys(), action='append')
+ args = ap.parse_args(argv)
+
+ config_dir = args.config_dir.expanduser()
+ if not config_dir.is_dir():
+ sys.exit(f"No such directory: {config_dir}")
+
+ nb_params = load_nb_params()
+
+ files = [config_dir / 'default.yaml'] + list(config_dir.glob('*/*.yaml'))
+ print(f"Checking {len(files)} files in {config_dir}...\n")
+
+ for file in files:
+ check_yaml_file(file, config_dir, nb_params, detectors=args.detector)
+
+ print(f"{n_probs} config issues found")
+ sys.exit(1 if n_probs else 0)
+
+
+if __name__ == "__main__":
+ main()