[Tests] Validate metadata for retrieved constants before validating the corrected files

Description

This MR addresses: https://git.xfel.eu/calibration/pycalibration/-/issues/89

In case the retrieved constants are different, the tests should realize this before validating the corrected files. This help in understanding the reason for the test failure instead of spending time investigating why the files were not validated and have different values.

How Has This Been Tested?

• Tested locally after changing parameters of one test to get the wrong Calibration constants.
• CI-InProgress

Relevant Documents (optional)

Types of changes

• Test (additional or refactored tests)

Checklist:

Reviewers

@kluyvert @schmidtp

tests/test_reference_runs/test_pre_deployment.py

@@ -14,6 +14,7 @@ import h5py
 import numpy as np
 import pytest
 import yaml
+from deepdiff import DeepDiff
 
 import xfel_calibrate.calibrate as calibrate
 
@@ -258,21 +259,10 @@ def slurm_watcher(test_key: str, std_out: str):
     LOGGER.info(f"{test_key}'s jobs were COMPLETED")
 
 
-def compare_metadata_ccvs(test, reference, tested_key):
+def load_yaml(file_path):
+    with open(file_path, 'r') as file:
+        return yaml.safe_load(file)
 
-    def load_yaml(file_path):
-        with open(file_path, 'r') as file:
-            return yaml.safe_load(file)
-
-    # Load the relevant section from both files
-    ccvs_test = load_yaml(test).get(tested_key, {})
-    ccvs_reference = load_yaml(reference).get(tested_key, {})
-
-    # Compare the content directly
-    if ccvs_test == ccvs_reference:
-        return True
-    else:
-        return False
 
 @pytest.mark.manual_run
 @pytest.mark.parametrize(
@@ -375,17 +365,22 @@ def test_xfel_calibrate(
         if time_counter > time_to_wait:
             assert False, f"{test_key} failure, report doesn't exists."
     LOGGER.info("Report found.")
-    # For corrections validate same calibration constants were retrieved.
 
-    metadata_file = f"calibration_metadata_{cal_conf['karabo-id']}"
-    assert compare_metadata_ccvs(
-        out_folder / metadata_file,
-        reference_folder / metadata_file,
-        "retrieved-constants",
-    ), "Found difference in the metadata for the retrieved Constants."
+    if cal_type.lower() == "correct":
+        # For corrections validate calibration constants
+        metadata_file = f"calibration_metadata_{cal_conf['karabo-id']}.yml"
+        # Load the relevant section from both files
+        ccvs_test = load_yaml(
+            out_folder / metadata_file)["retrieved-constants"]
+        ccvs_reference = load_yaml(
+            reference_folder / metadata_file)["retrieved-constants"]
+        ccvs_diff = DeepDiff(ccvs_test, ccvs_reference, ignore_order=True)
+        assert ccvs_diff == {}, "Found difference in the metadata for the retrieved Constants."  # noqa
+        LOGGER.info("Retrieved CCVs validated.")
 
     # Stop tests at this point, if desired.
     if not skip_numerical_validation:
         assert validate_hdf5_files(
             out_folder,  reference_folder, cal_type
         ), "HDF5 files changed - see details above"
+        LOGGER.info("H5 Files validated.")