Merge branch 'test/check_retrieved_constants' into 'master'

[Tests] Validate metadata for retrieved constants before validating the corrected files

See merge request !942

.gitlab-ci.yml

@@ -64,7 +64,7 @@ automated_test:
   <<: *before_script
   script:
     - export LANG=C  # Hopefully detect anything relying on locale
-    - python3 -m pip install ".[automated_test]"
+    - python3 -m pip install ".[test]"
     - echo "Running automated test. This can take sometime to finish depending on the test data."
     - echo "Given variables are REFERENCE=$REFERENCE, OUTPUT=$OUTPUT, DETECTORS=$DETECTORS, CALIBRATION=$CALIBRATION"
     - python3 -m pytest ./tests/test_reference_runs --color yes --verbose --release-test --reference-folder /gpfs/exfel/d/cal_tst/reference_folder --out-folder /gpfs/exfel/data/scratch/xcaltst/test/$OUTPUT  --detectors $DETECTORS --calibration $CALIBRATION

setup.py

@@ -153,16 +153,7 @@ setup(
         ],
         "test": [
             "coverage",
-            "nbval",
-            "pytest-asyncio",
-            "pytest-cov",
-            "pytest-subprocess",
-            "pytest>=5.4.0",
-            "testpath",
-            "unittest-xml-reporting==3.0.2",
-        ],
-        "automated_test": [
-            "coverage",
+            "deepdiff==6.7.1",
             "nbval",
             "pytest-asyncio",
             "pytest-cov",

tests/test_reference_runs/test_pre_deployment.py

@@ -13,6 +13,8 @@ from typing import Any, Dict, List, Tuple
 import h5py
 import numpy as np
 import pytest
+import yaml
+from deepdiff import DeepDiff
 
 import xfel_calibrate.calibrate as calibrate
 
@@ -164,9 +166,9 @@ def validate_hdf5_files(
         test_key (str): The test name.
         out_folder (pathlib.Path): The OUT folder for the tested data.
         reference_folder (pathlib.Path): The Reference folder for
-          the reference data to validate against
+            the reference data to validate against
         cal_type (str): The type of calibration processing.
-          e.g. dark or correct.
+            e.g. dark or correct.
     """
     print("\n--- Compare HDF5 files  ----")
     print("REF:", reference_folder)
@@ -257,6 +259,11 @@ def slurm_watcher(test_key: str, std_out: str):
     LOGGER.info(f"{test_key}'s jobs were COMPLETED")
 
 
+def load_yaml(file_path):
+    with open(file_path, 'r') as file:
+        return yaml.safe_load(file)
+
+
 @pytest.mark.manual_run
 @pytest.mark.parametrize(
     "test_key, val_dict",
@@ -317,10 +324,9 @@ def test_xfel_calibrate(
     )
 
     report_name = out_folder / f"{test_key}_{datetime.now():%y%m%d_%H%M%S}"
-
     cal_conf["report-to"] = str(report_name)
 
-    cmd = parse_config(cmd, cal_conf, out_folder)
+    cmd = parse_config(cmd, cal_conf, str(out_folder))
 
     if only_validate:
         assert validate_hdf5_files(
@@ -360,8 +366,21 @@ def test_xfel_calibrate(
             assert False, f"{test_key} failure, report doesn't exists."
     LOGGER.info("Report found.")
 
+    if cal_type.lower() == "correct":
+        # For corrections validate calibration constants
+        metadata_file = f"calibration_metadata_{cal_conf['karabo-id']}.yml"
+        # Load the relevant section from both files
+        ccvs_test = load_yaml(
+            out_folder / metadata_file).get("retrieved-constants", {})
+        ccvs_reference = load_yaml(
+            reference_folder / metadata_file).get("retrieved-constants", {})
+        ccvs_diff = DeepDiff(ccvs_test, ccvs_reference, ignore_order=True)
+        assert ccvs_diff == {}, "Found difference in the metadata for the retrieved Constants."  # noqa
+        LOGGER.info("Retrieved CCVs validated.")
+
     # Stop tests at this point, if desired.
     if not skip_numerical_validation:
         assert validate_hdf5_files(
             out_folder,  reference_folder, cal_type
         ), "HDF5 files changed - see details above"
+        LOGGER.info("H5 Files validated.")