diff --git a/tests/test_reference_runs/callab_tests.py b/tests/test_reference_runs/callab_tests.py
index 8568f23264dee1fdb0a4eb1b865d9852e03fda16..2d88f7aa3c0625447aef0722517b21149a23d46b 100644
--- a/tests/test_reference_runs/callab_tests.py
+++ b/tests/test_reference_runs/callab_tests.py
@@ -1,4 +1,74 @@
automated_test_config = {
+ "SPB_DET_AGIPD1M-1-CORRECT-ADAPTIVE-HCDS": {
+ "det_type": "AGIPD",
+ "cal_type": "CORRECT",
+ "config": {
+ "blc-stripes": True,
+ "out-folder": "{}/{}/{}",
+ # "/gpfs/exfel/exp/SPB/202331/900376/raw"
+ "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ "run": "7", # Original run: "13"
+ "karabo-id-control": "SPB_IRU_AGIPD1M1",
+ "karabo-id": "SPB_DET_AGIPD1M-1",
+ "slurm-mem": "750",
+ "n-cores-files": 2,
+ "rel-gain-mode": "CS",
+ "ctrl-source-template": "{}/MDL/FPGA_COMP",
+ },
+ "reference-folder": "{}/{}/{}",
+ },
+ "SPB_DET_AGIPD1M-1-DARK-ADAPTIVE-HCDS": {
+ "det_type": "AGIPD",
+ "cal_type": "DARK",
+ "config": {
+ "out-folder": "{}/{}/{}",
+ # "/gpfs/exfel/exp/SPB/202331/900376/raw"
+ "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ # Unsorted dark runs
+ "run-high": "4", # Original run "10"
+ "run-med": "5", # Original run: "11"
+ "run-low": "6", # Original run: "12"
+ "karabo-id-control": "SPB_IRU_AGIPD1M1",
+ "karabo-id": "SPB_DET_AGIPD1M-1",
+ "ctrl-source-template": "{}/MDL/FPGA_COMP",
+ },
+ "reference-folder": "{}/{}/{}",
+ },
+ "SPB_DET_AGIPD1M-1-CORRECT-FIXED-HCDS": {
+ "det_type": "AGIPD",
+ "cal_type": "CORRECT",
+ "config": {
+ "blc-stripes": True,
+ "out-folder": "{}/{}/{}",
+ # "/gpfs/exfel/exp/SPB/202331/900376/raw"
+ "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ "run": "11", # Original run: "44"
+ "karabo-id-control": "SPB_IRU_AGIPD1M1",
+ "karabo-id": "SPB_DET_AGIPD1M-1",
+ "slurm-mem": "750",
+ "rel-gain-mode": "PC",
+ "n-cores-files": 2,
+ "ctrl-source-template": "{}/MDL/FPGA_COMP",
+ },
+ "reference-folder": "{}/{}/{}",
+ },
+ "SPB_DET_AGIPD1M-1-DARK-FIXED-HCDS": {
+ "det_type": "AGIPD",
+ "cal_type": "DARK",
+ "config": {
+ "out-folder": "{}/{}/{}",
+ # "/gpfs/exfel/exp/SPB/202331/900376/raw"
+ "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ # Unsorted dark runs
+ "run-high": "8", # Original run "39"
+ "run-med": "9", # Original run: "40"
+ "run-low": "10", # Original run: "41"
+ "karabo-id-control": "SPB_IRU_AGIPD1M1",
+ "karabo-id": "SPB_DET_AGIPD1M-1",
+ "ctrl-source-template": "{}/MDL/FPGA_COMP",
+ },
+ "reference-folder": "{}/{}/{}",
+ },
"SPB_DET_AGIPD1M-1-CORRECT-FIXED": {
"det_type": "AGIPD",
"cal_type": "CORRECT",
@@ -92,6 +162,96 @@ automated_test_config = {
},
"reference-folder": "{}/{}/{}",
},
+ # TEST FILES ARE MISSING CONTROL DATA.
+ # "MID_DET_AGIPD1M-1-CORRECT-ADAPTIVE-HCDS": {
+ # "det_type": "AGIPD",
+ # "cal_type": "CORRECT",
+ # "config": {
+ # "blc-stripes": True,
+ # "rel-gain": True,
+ # "xray-gain": True,
+ # "out-folder": "{}/{}/{}",
+ # # "/gpfs/exfel/exp/MID/202331/900385/raw"
+ # "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ # "run": "15", # Original run: "130",
+ # "karabo-id-control": "MID_EXP_AGIPD1M1",
+ # "karabo-id": "MID_DET_AGIPD1M-1",
+ # "slurm-mem": "750",
+ # "n-cores-files": 2,
+ # "ctrl-source-template": "{}/MDL/FPGA_COMP",
+ # },
+ # "reference-folder": "{}/{}/{}",
+ # },
+ # TEST FILES ARE MISSING CONTROL DATA.
+ # "MID_DET_AGIPD1M-1-DARK-ADAPTIVE-HCDS": {
+ # "det_type": "AGIPD",
+ # "cal_type": "DARK",
+ # "config": {
+ # "out-folder": "{}/{}/{}",
+ # # "//gpfs/exfel/exp/MID/202331/900385/raw"
+ # "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ # "run-high": "12", # Original run: "126"
+ # "run-med": "13", # Original run: "127"
+ # "run-low": "14", # Original run: "128"
+ # "karabo-id-control": "MID_EXP_AGIPD1M1",
+ # "karabo-id": "MID_DET_AGIPD1M-1",
+ # "ctrl-source-template": "{}/MDL/FPGA_COMP",
+ # },
+ # "reference-folder": "{}/{}/{}",
+ # },
+ "MID_DET_AGIPD1M-1-CORRECT-FIXED-HCDS": {
+ "det_type": "AGIPD",
+ "cal_type": "CORRECT",
+ "config": {
+ "blc-stripes": True,
+ "out-folder": "{}/{}/{}",
+ "only-offset": True,
+ # "/gpfs/exfel/exp/MID/202331/900385/raw"
+ "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ "run": "19", # Original run: "274",
+ "karabo-id-control": "MID_EXP_AGIPD1M1",
+ "karabo-id": "MID_DET_AGIPD1M-1",
+ "slurm-mem": "750",
+ "n-cores-files": 2,
+ "ctrl-source-template": "{}/MDL/FPGA_COMP",
+ },
+ "reference-folder": "{}/{}/{}",
+ },
+ "MID_DET_AGIPD1M-1-DARK-FIXED-HCDS": {
+ "det_type": "AGIPD",
+ "cal_type": "DARK",
+ "config": {
+ "out-folder": "{}/{}/{}",
+ # "//gpfs/exfel/exp/MID/202331/900385/raw"
+ "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ "run-high": "16", # Original run: "269"
+ "run-med": "17", # Original run: "270"
+ "run-low": "18", # Original run: "271"
+ "karabo-id-control": "MID_EXP_AGIPD1M1",
+ "karabo-id": "MID_DET_AGIPD1M-1",
+ "ctrl-source-template": "{}/MDL/FPGA_COMP",
+ },
+ "reference-folder": "{}/{}/{}",
+ },
+ "MID_DET_AGIPD1M-1-CORRECT-PHOTONIZATION": {
+ "det_type": "AGIPD",
+ "cal_type": "CORRECT",
+ "config": {
+ "blc-stripes": True,
+ "rel-gain": True,
+ "xray-gain": True,
+ "out-folder": "{}/{}/{}",
+ # proposal 3348
+ "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ "run": "20", # Original run: "304",
+ "karabo-id-control": "MID_EXP_AGIPD1M1",
+ "karabo-id": "MID_DET_AGIPD1M-1",
+ "slurm-mem": "750",
+ "n-cores-files": 2,
+ "ctrl-source-template": "{}/MDL/FPGA_COMP",
+ },
+ "reference-folder": "{}/{}/{}",
+ },
"MID_DET_AGIPD1M-1-CORRECT-SELECT_CELLS_CM": {
"det_type": "AGIPD",
"cal_type": "CORRECT",
@@ -516,6 +676,19 @@ automated_test_config = {
},
"reference-folder": "{}/{}/{}",
},
+ "MID_EXP_EPIX-1-CORRECT_NEW": {
+ "det_type": "EPIX100",
+ "cal_type": "CORRECT",
+ "config": {
+ "out-folder": "{}/{}/{}",
+ # Original proposal 4559
+ "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ "run": "25", # Original run: "80"
+ "karabo-id": "MID_EXP_EPIX-1",
+ "karabo-da": "EPIX01",
+ },
+ "reference-folder": "{}/{}/{}",
+ },
"HED_IA1_EPX100-1-CORRECT": {
"det_type": "EPIX100",
"cal_type": "CORRECT",
@@ -570,6 +743,35 @@ automated_test_config = {
},
"reference-folder": "{}/{}/{}",
},
+ "FXE_DET_LPD1MINI-DARK": {
+ "det_type": "LPDMINI",
+ "cal_type": "DARK",
+ "config": {
+ "out-folder": "{}/{}/{}",
+ # Proposal: 4936
+ "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ "run-high": "21", # Original run: 146,
+ "run-med": "22", # Original run: 147,
+ "run-low": "23", # Original run: 148,
+ "karabo-id": "FXE_DET_LPD_MINI",
+ },
+ "reference-folder": "{}/{}/{}",
+ },
+ "FXE_DET_LPD1MINI-CORRECT": {
+ "det_type": "LPDMINI",
+ "cal_type": "CORRECT",
+ "config": {
+ "out-folder": "{}/{}/{}",
+ # Proposal: 4936
+ "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ "run": "24", # Original run: 164,
+ "sequences": "0",
+ "karabo-id": "FXE_DET_LPD_MINI",
+ "slurm-mem": "750",
+ "num-workers": 4,
+ },
+ "reference-folder": "{}/{}/{}",
+ },
"FXE_DET_LPD1M-1-DARK": {
"det_type": "LPD",
"cal_type": "DARK",
@@ -658,44 +860,53 @@ automated_test_config = {
},
"reference-folder": "{}/{}/{}",
},
- # "FXE_XAD_G2XES-DARK": {
- # "det_type": "Gotthard2",
- # "cal_type": "DARK",
- # "config": {
- # "out-folder": "{}/{}/{}",
- # "in-folder": "/gpfs/exfel/exp/FXE/202231/p900298/raw",
- # "karabo-da": "GH201",
- # "run-high": "7",
- # "run-med": "8",
- # "run-low": "9",
- # "karabo-id": "FXE_XAD_G2XES",
- # },
- # "reference-folder": "{}/{}/{}",
- # },
- # "FXE_XAD_G2XES-CORRECT": {
- # "det_type": "Gotthard2",
- # "cal_type": "CORRECT",
- # "config": {
- # "out-folder": "{}/{}/{}",
- # "in-folder": "/gpfs/exfel/exp/FXE/202231/p900298/raw",
- # "karabo-da": "GH201",
- # "run": "7",
- # "karabo-id": "FXE_XAD_G2XES",
- # "no-offset-correction": True,
- # },
- # "reference-folder": "{}/{}/{}",
- # },
- # "SPB_50UM_GH2-CORRECT": {
- # "det_type": "Gotthard2",
- # "cal_type": "CORRECT",
- # "config": {
- # "out-folder": "{}/{}/{}",
- # "in-folder": "/gpfs/exfel/exp/SPB/202321/p004577/raw",
- # "karabo-da": "GH200",
- # "run": "98",
- # "sequences": "0,3,6",
- # "karabo-id": "SPB_50UM_GH2",
- # },
- # "reference-folder": "{}/{}/{}",
- # },
+ "SA1_XTD9_HIREX-DARK": {
+ "det_type": "Gotthard2",
+ "cal_type": "DARK",
+ "config": {
+ "out-folder": "{}/{}/{}",
+ # Proposal 900384
+ "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ "karabo-da": ["GH200/1", "GH200/2",],
+ "control-template": "GOTTHARD2_CTRL",
+ "receiver-affixes": ["_MASTER:daqOutput", "_SLAVE:daqOutput"],
+ "receiver-template": "GOTTHARD2{}",
+ "run-high": "28", # 82
+ "run-med": "29", # 83
+ "run-low": "30", # 84
+ "karabo-id": "SA1_XTD9_HIREX",
+ },
+ "reference-folder": "{}/{}/{}",
+ },
+ "SA1_XTD9_HIREX-CORRECT": {
+ "det_type": "Gotthard2",
+ "cal_type": "CORRECT",
+ "config": {
+ "out-folder": "{}/{}/{}",
+ # from proposal 900384 run 8
+ "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ "karabo-da": ["GH200/1", "GH200/2",],
+ "control-template": "GOTTHARD2_CTRL",
+ "receiver-affixes": ["_MASTER:daqOutput", "_SLAVE:daqOutput"],
+ "receiver-template": "GOTTHARD2{}",
+ "corr-receiver": "GOTTHARD2_RECEIVER",
+ "run": "31",
+ "karabo-id": "SA1_XTD9_HIREX",
+ "no-offset-correction": True,
+ },
+ "reference-folder": "{}/{}/{}",
+ },
+ "TIMEPIX": {
+ "det_type": "TIMEPIX",
+ "cal_type": "CORRECT",
+ "config": {
+ "out-folder": "{}/{}/{}",
+ # Proposal 900421, Run 307
+ "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+ "karabo-da": "TIMEPIX01",
+ "run": "27",
+ "karabo-id": "SQS_EXP_TIMEPIX",
+ },
+ "reference-folder": "{}/{}/{}",
+ },
}
diff --git a/tests/test_reference_runs/test_pre_deployment.py b/tests/test_reference_runs/test_pre_deployment.py
index dcda91b36b1c997e0c775e01d56d9180edd1ce23..14d9eb72a35e7d374200c7d2fc75714c14804e2f 100644
--- a/tests/test_reference_runs/test_pre_deployment.py
+++ b/tests/test_reference_runs/test_pre_deployment.py
@@ -71,6 +71,54 @@ def iter_sized_chunks(ds: h5py.Dataset, chunk_size: int):
yield slice(start, start + chunk_l)
+def equal_data(ref, test, dtype):
+ """Compare arrays based on their dtype."""
+ if (
+ np.issubdtype(dtype, np.floating) or
+ np.issubdtype(dtype, np.complexfloating)
+ ):
+ return np.allclose(ref, test, equal_nan=True)
+ else:
+ return np.array_equal(ref, test)
+
+
+def handle_differences(ref_chunk, out_chunk, ref_ds, out_ds):
+ changes = []
+ dsname = ref_ds.name
+ if out_ds.dtype.names:
+ # Handle compound datasets
+ field_differences = []
+ for field in out_ds.dtype.names:
+ ref_field_chunk = ref_chunk[field]
+ out_field_chunk = out_chunk[field]
+ if not equal_data(
+ ref_field_chunk, out_field_chunk, ref_field_chunk.dtype):
+ field_differences.append(field)
+
+ if field_differences:
+ if ref_ds.size == 1:
+ # If just 1 entry, show the values for all differing fields
+ for field in field_differences:
+ r, o = np.squeeze(
+ ref_chunk[field]), np.squeeze(out_chunk[field])
+ changes.append((
+ dsname, f"Field '{field}' Value: {r} -> {o}"))
+ else:
+ changes.append((
+ dsname, f"Fields changed: {', '.join(field_differences)}"))
+
+ else:
+ if not equal_data(ref_chunk, out_chunk, ref_ds.dtype):
+ # If just 1 entry, show the values
+ if ref_ds.size == 1:
+ r, o = np.squeeze(ref_chunk), np.squeeze(out_chunk)
+ changes.append((ref_ds.name, f"Value: {r} -> {o}"))
+ else:
+ changes.append((ref_ds.name, "Data changed"))
+
+ return changes
+
+
def validate_file(
ref_folder: pathlib.PosixPath,
out_folder: pathlib.PosixPath,
@@ -95,25 +143,15 @@ def validate_file(
dsname, f"Dtype: {ref_ds.dtype} -> {out_ds.dtype}"
))
else:
- floaty = np.issubdtype(ref_ds.dtype, np.floating) \
- or np.issubdtype(ref_ds.dtype, np.complexfloating)
-
# Compare data incrementally rather than loading it all at once;
# read in blocks of ~64 MB (arbitrary limit) along first axis.
for chunk_slice in iter_sized_chunks(ref_ds, 64 * 1024 * 1024):
ref_chunk = ref_ds[chunk_slice]
out_chunk = out_ds[chunk_slice]
- if floaty:
- eq = np.allclose(ref_chunk, out_chunk, equal_nan=True)
- else:
- eq = np.array_equal(ref_chunk, out_chunk)
- if not eq:
- # If just 1 entry, show the values
- if ref_ds.size == 1:
- r, o = np.squeeze(ref_chunk), np.squeeze(out_chunk)
- changed.append((dsname, f"Value: {r} -> {o}"))
- else:
- changed.append((dsname, "Data changed"))
+ differences = handle_differences(
+ ref_chunk, out_chunk, ref_ds, out_ds)
+ changed += differences
+ if differences:
break
return ComparisonResult(