[Test] Configure and Include new test runs

Description

Adding new test runs from this issue: https://git.xfel.eu/calibration/planning/-/issues/194

• AGIPD@SPB High CDS Adaptive and Fixed gain correct and darks.
• AGIPD@MID High CDS Fixed gain correct and darks and a photonization correction test.
• LPDMINI correct and darks
• GH2 25um correct and darks
• Timepix correction

I have also added an update to the test script for the compound data types (e.g. Timepix Centroids)

Note: I am considering reducing some of the test runs that doesn't offer a different tested case. And I welcome comments related to this

How Has This Been Tested?

Relevant Documents (optional)

Types of changes

• Test (additional or refactored tests)

Checklist:

Reviewers

@schmidtp @kluyvert

tests/test_reference_runs/callab_tests.py

 automated_test_config = {
+    "SPB_DET_AGIPD1M-1-CORRECT-ADAPTIVE-HCDS": {
+        "det_type": "AGIPD",
+        "cal_type": "CORRECT",
+        "config": {
+            "blc-stripes": True,
+            "out-folder": "{}/{}/{}",
+            # "/gpfs/exfel/exp/SPB/202331/900376/raw"
+            "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+            "run": "7",  # Original run: "13"
+            "karabo-id-control": "SPB_IRU_AGIPD1M1",
+            "karabo-id": "SPB_DET_AGIPD1M-1",
+            "slurm-mem": "750",
+            "n-cores-files": 2,
+            "rel-gain-mode": "CS",
+            "ctrl-source-template": "{}/MDL/FPGA_COMP",
+        },
+        "reference-folder": "{}/{}/{}",
+    },
+    "SPB_DET_AGIPD1M-1-DARK-ADAPTIVE-HCDS": {
+        "det_type": "AGIPD",
+        "cal_type": "DARK",
+        "config": {
+            "out-folder": "{}/{}/{}",
+            # "/gpfs/exfel/exp/SPB/202331/900376/raw"
+            "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+            # Unsorted dark runs
+            "run-high": "4",  # Original run "10"
+            "run-med": "5",  # Original run: "11"
+            "run-low": "6",  # Original run: "12"
+            "karabo-id-control": "SPB_IRU_AGIPD1M1",
+            "karabo-id": "SPB_DET_AGIPD1M-1",
+            "ctrl-source-template": "{}/MDL/FPGA_COMP",
+        },
+        "reference-folder": "{}/{}/{}",
+    },
+    "SPB_DET_AGIPD1M-1-CORRECT-FIXED-HCDS": {
+        "det_type": "AGIPD",
+        "cal_type": "CORRECT",
+        "config": {
+            "blc-stripes": True,
+            "out-folder": "{}/{}/{}",
+            # "/gpfs/exfel/exp/SPB/202331/900376/raw"
+            "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+            "run": "11",  # Original run: "44"
+            "karabo-id-control": "SPB_IRU_AGIPD1M1",
+            "karabo-id": "SPB_DET_AGIPD1M-1",
+            "slurm-mem": "750",
+            "rel-gain-mode": "PC",
+            "n-cores-files": 2,
+            "ctrl-source-template": "{}/MDL/FPGA_COMP",
+        },
+        "reference-folder": "{}/{}/{}",
+    },
+    "SPB_DET_AGIPD1M-1-DARK-FIXED-HCDS": {
+        "det_type": "AGIPD",
+        "cal_type": "DARK",
+        "config": {
+            "out-folder": "{}/{}/{}",
+            # "/gpfs/exfel/exp/SPB/202331/900376/raw"
+            "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+            # Unsorted dark runs
+            "run-high": "8",  # Original run "39"
+            "run-med": "9",  # Original run: "40"
+            "run-low": "10",  # Original run: "41"
+            "karabo-id-control": "SPB_IRU_AGIPD1M1",
+            "karabo-id": "SPB_DET_AGIPD1M-1",
+            "ctrl-source-template": "{}/MDL/FPGA_COMP",
+        },
+        "reference-folder": "{}/{}/{}",
+    },
     "SPB_DET_AGIPD1M-1-CORRECT-FIXED": {
         "det_type": "AGIPD",
         "cal_type": "CORRECT",
@@ -92,6 +162,96 @@ automated_test_config = {
         },
         "reference-folder": "{}/{}/{}",
     },
+    # TEST FILES ARE MISSING CONTROL DATA.
+    # "MID_DET_AGIPD1M-1-CORRECT-ADAPTIVE-HCDS": {
+    #     "det_type": "AGIPD",
+    #     "cal_type": "CORRECT",
+    #     "config": {
+    #         "blc-stripes": True,
+    #         "rel-gain": True,
+    #         "xray-gain": True,
+    #         "out-folder": "{}/{}/{}",
+    #         # "/gpfs/exfel/exp/MID/202331/900385/raw"
+    #         "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+    #         "run": "15", # Original run: "130",
+    #         "karabo-id-control": "MID_EXP_AGIPD1M1",
+    #         "karabo-id": "MID_DET_AGIPD1M-1",
+    #         "slurm-mem": "750",
+    #         "n-cores-files": 2,
+    #         "ctrl-source-template": "{}/MDL/FPGA_COMP",
+    #     },
+    #     "reference-folder": "{}/{}/{}",
+    # },
+    # TEST FILES ARE MISSING CONTROL DATA.
+    # "MID_DET_AGIPD1M-1-DARK-ADAPTIVE-HCDS": {
+    #     "det_type": "AGIPD",
+    #     "cal_type": "DARK",
+    #     "config": {
+    #         "out-folder": "{}/{}/{}",
+    #         # "//gpfs/exfel/exp/MID/202331/900385/raw"
+    #         "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+    #         "run-high": "12",  # Original run: "126"
+    #         "run-med": "13",  # Original run: "127"
+    #         "run-low": "14",  # Original run: "128"
+    #         "karabo-id-control": "MID_EXP_AGIPD1M1",
+    #         "karabo-id": "MID_DET_AGIPD1M-1",
+    #         "ctrl-source-template": "{}/MDL/FPGA_COMP",
+    #     },
+    #     "reference-folder": "{}/{}/{}",
+    # },
+    "MID_DET_AGIPD1M-1-CORRECT-FIXED-HCDS": {
+        "det_type": "AGIPD",
+        "cal_type": "CORRECT",
+        "config": {
+            "blc-stripes": True,
+            "out-folder": "{}/{}/{}",
+            "only-offset": True,
+            # "/gpfs/exfel/exp/MID/202331/900385/raw"
+            "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+            "run": "19", # Original run: "274",
+            "karabo-id-control": "MID_EXP_AGIPD1M1",
+            "karabo-id": "MID_DET_AGIPD1M-1",
+            "slurm-mem": "750",
+            "n-cores-files": 2,
+            "ctrl-source-template": "{}/MDL/FPGA_COMP",
+        },
+        "reference-folder": "{}/{}/{}",
+    },
+    "MID_DET_AGIPD1M-1-DARK-FIXED-HCDS": {
+        "det_type": "AGIPD",
+        "cal_type": "DARK",
+        "config": {
+            "out-folder": "{}/{}/{}",
+            # "//gpfs/exfel/exp/MID/202331/900385/raw"
+            "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+            "run-high": "16",  # Original run: "269"
+            "run-med": "17",  # Original run: "270"
+            "run-low": "18",  # Original run: "271"
+            "karabo-id-control": "MID_EXP_AGIPD1M1",
+            "karabo-id": "MID_DET_AGIPD1M-1",
+            "ctrl-source-template": "{}/MDL/FPGA_COMP",
+        },
+        "reference-folder": "{}/{}/{}",
+    },
+    "MID_DET_AGIPD1M-1-CORRECT-PHOTONIZATION": {
+        "det_type": "AGIPD",
+        "cal_type": "CORRECT",
+        "config": {
+            "blc-stripes": True,
+            "rel-gain": True,
+            "xray-gain": True,
+            "out-folder": "{}/{}/{}",
+            # proposal 3348
+            "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+            "run": "20", # Original run: "304",
+            "karabo-id-control": "MID_EXP_AGIPD1M1",
+            "karabo-id": "MID_DET_AGIPD1M-1",
+            "slurm-mem": "750",
+            "n-cores-files": 2,
+            "ctrl-source-template": "{}/MDL/FPGA_COMP",
+        },
+        "reference-folder": "{}/{}/{}",
+    },
     "MID_DET_AGIPD1M-1-CORRECT-SELECT_CELLS_CM": {
         "det_type": "AGIPD",
         "cal_type": "CORRECT",
@@ -516,6 +676,19 @@ automated_test_config = {
         },
         "reference-folder": "{}/{}/{}",
     },
+    "MID_EXP_EPIX-1-CORRECT_NEW": {
+        "det_type": "EPIX100",
+        "cal_type": "CORRECT",
+        "config": {
+            "out-folder": "{}/{}/{}",
+            # Original proposal 4559
+            "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+            "run": "25",  # Original run: "80"
+            "karabo-id": "MID_EXP_EPIX-1",
+            "karabo-da": "EPIX01",
+        },
+        "reference-folder": "{}/{}/{}",
+    },
     "HED_IA1_EPX100-1-CORRECT": {
         "det_type": "EPIX100",
         "cal_type": "CORRECT",
@@ -570,6 +743,35 @@ automated_test_config = {
         },
         "reference-folder": "{}/{}/{}",
     },
+    "FXE_DET_LPD1MINI-DARK": {
+        "det_type": "LPDMINI",
+        "cal_type": "DARK",
+        "config": {
+            "out-folder": "{}/{}/{}",
+            # Proposal: 4936
+            "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+            "run-high": "21",  # Original run: 146,
+            "run-med": "22",  # Original run: 147,
+            "run-low": "23",  # Original run: 148,
+            "karabo-id": "FXE_DET_LPD_MINI",
+        },
+        "reference-folder": "{}/{}/{}",
+    },
+    "FXE_DET_LPD1MINI-CORRECT": {
+        "det_type": "LPDMINI",
+        "cal_type": "CORRECT",
+        "config": {
+            "out-folder": "{}/{}/{}",
+            # Proposal: 4936
+            "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+            "run": "24",  # Original run: 164,
+            "sequences": "0",
+            "karabo-id": "FXE_DET_LPD_MINI",
+            "slurm-mem": "750",
+            "num-workers": 4,
+        },
+        "reference-folder": "{}/{}/{}",
+    },
     "FXE_DET_LPD1M-1-DARK": {
         "det_type": "LPD",
         "cal_type": "DARK",
@@ -658,44 +860,53 @@ automated_test_config = {
         },
         "reference-folder": "{}/{}/{}",
     },
-    # "FXE_XAD_G2XES-DARK": {
-    #     "det_type": "Gotthard2",
-    #     "cal_type": "DARK",
-    #     "config": {
-    #         "out-folder": "{}/{}/{}",
-    #         "in-folder": "/gpfs/exfel/exp/FXE/202231/p900298/raw",
-    #         "karabo-da": "GH201",
-    #         "run-high": "7",
-    #         "run-med": "8",
-    #         "run-low": "9",
-    #         "karabo-id": "FXE_XAD_G2XES",
-    #     },
-    #     "reference-folder": "{}/{}/{}",
-    # },
-    # "FXE_XAD_G2XES-CORRECT": {
-    #     "det_type": "Gotthard2",
-    #     "cal_type": "CORRECT",
-    #     "config": {
-    #         "out-folder": "{}/{}/{}",
-    #         "in-folder": "/gpfs/exfel/exp/FXE/202231/p900298/raw",
-    #         "karabo-da": "GH201",
-    #         "run": "7",
-    #         "karabo-id": "FXE_XAD_G2XES",
-    #         "no-offset-correction": True,
-    #     },
-    #     "reference-folder": "{}/{}/{}",
-    # },
-    # "SPB_50UM_GH2-CORRECT": {
-    #     "det_type": "Gotthard2",
-    #     "cal_type": "CORRECT",
-    #     "config": {
-    #         "out-folder": "{}/{}/{}",
-    #         "in-folder": "/gpfs/exfel/exp/SPB/202321/p004577/raw",
-    #         "karabo-da": "GH200",
-    #         "run": "98",
-    #         "sequences": "0,3,6",
-    #         "karabo-id": "SPB_50UM_GH2",
-    #     },
-    #     "reference-folder": "{}/{}/{}",
-    # },
+    "SA1_XTD9_HIREX-DARK": {
+        "det_type": "Gotthard2",
+        "cal_type": "DARK",
+        "config": {
+            "out-folder": "{}/{}/{}",
+            # Proposal 900384
+            "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+            "karabo-da": ["GH200/1", "GH200/2",],
+            "control-template": "GOTTHARD2_CTRL",
+            "receiver-affixes": ["_MASTER:daqOutput", "_SLAVE:daqOutput"],
+            "receiver-template": "GOTTHARD2{}",
+            "run-high": "28",  # 82
+            "run-med": "29",  # 83
+            "run-low": "30",  # 84
+            "karabo-id": "SA1_XTD9_HIREX",
+        },
+        "reference-folder": "{}/{}/{}",
+    },
+    "SA1_XTD9_HIREX-CORRECT": {
+        "det_type": "Gotthard2",
+        "cal_type": "CORRECT",
+        "config": {
+            "out-folder": "{}/{}/{}",
+            # from proposal 900384 run 8
+            "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+            "karabo-da": ["GH200/1", "GH200/2",],
+            "control-template": "GOTTHARD2_CTRL",
+            "receiver-affixes": ["_MASTER:daqOutput", "_SLAVE:daqOutput"],
+            "receiver-template": "GOTTHARD2{}",
+            "corr-receiver": "GOTTHARD2_RECEIVER",
+            "run": "31",
+            "karabo-id": "SA1_XTD9_HIREX",
+            "no-offset-correction": True,
+        },
+        "reference-folder": "{}/{}/{}",
+    },
+    "TIMEPIX": {
+        "det_type": "TIMEPIX",
+        "cal_type": "CORRECT",
+        "config": {
+            "out-folder": "{}/{}/{}",
+            # Proposal 900421, Run 307
+            "in-folder": "/gpfs/exfel/exp/CALLAB/202130/p900203/raw",
+            "karabo-da": "TIMEPIX01",
+            "run": "27",
+            "karabo-id": "SQS_EXP_TIMEPIX",
+        },
+        "reference-folder": "{}/{}/{}",
+    },
 }