add CL options for release_test

tests/conftest.py

@@ -56,6 +56,34 @@ def pytest_addoption(parser):
         default=False,
         help="Start release test for all supported calibration notebooks",
     )
+    detectors_choices = []
+    for k in ['agipd', 'jungfrau', 'pnccd', 'epix100', 'all']:
+        detectors_choices += [k, k.upper()]
+    parser.addoption(
+        "--detectors",
+        action='append',
+        choices=detectors_choices,
+    )
+
+    parser.addoption(
+        "--calibration",
+        type=str,
+        choices=['correct', 'dark', 'all'],
+        default='all',
+    )
+
+    parser.addoption(
+        "--no-numerical-validation",
+        action="store_true",
+        default=False,
+        help="Skips tests for numerical validation for produced h5files.",
+    )
+
+    parser.addoption(
+        "--picked-test",
+        type=str,
+        default=None,
+    )
 
 
 def pytest_configure(config):
@@ -79,6 +107,21 @@ def pytest_configure(config):
         "manual_run(): marks skips for tests that required to be run manually",
     )
 
+    config.addinivalue_line(
+        "markers",
+        "manual_run(): marks skips for tests that required to be run manually",
+    )
+
+
+@pytest.fixture
+def release_test_config(request):
+    detectors = request.config.getoption("--detectors")
+    calibration = request.config.getoption("--calibration")
+    skip_numerical_validation = request.config.getoption(
+        "--no-numerical-validation")
+    picked_test = request.config.getoption("--picked-test")
+    return detectors, calibration, picked_test, skip_numerical_validation
+
 
 @lru_cache()
 def server_reachable(server: str = "max-exfl-cal002"):

tests/test_pre_deployment.py

@@ -106,12 +106,38 @@ def parse_config(
 
 
 @pytest.mark.manual_run
-@pytest.mark.parametrize("calibration_test", list(callab_test_dict.items()))
-def test_xfel_calibrate(calibration_test):
-
-    test_key, val_dict = calibration_test
+@pytest.mark.parametrize(
+    "k, v",
+    list(callab_test_dict.items()),
+    ids=list(callab_test_dict.keys()),
+)
+def test_xfel_calibrate(k, v, release_test_config):
+
+    (
+        detectors, calibration,
+        picked_test, skip_numerical_validation
+    ) = release_test_config
+
+    test_key, val_dict = k, v
     cal_type = val_dict["cal_type"]
-    cmd = ["xfel-calibrate", val_dict["det_type"], cal_type]
+    det_type = val_dict["det_type"]
+
+    if picked_test is None:
+        # Skip non-selected detectors
+        if (
+            detectors != ["all"] and
+            det_type.lower() not in [d.lower() for d in detectors]
+        ):
+            pytest.skip()
+
+        # Skip non-selected calibration
+        if calibration != "all" and cal_type.lower() != calibration:
+            pytest.skip()
+    else:
+        if test_key != picked_test:
+            pytest.skip()
+
+    cmd = ["xfel-calibrate", det_type, cal_type]
     cal_conf = val_dict["config"]
     report_name = f"{test_key}_{datetime.now().strftime('%y%m%d_%H%M%S')}"
     cal_conf["report-to"] = report_name
@@ -122,6 +148,7 @@ def test_xfel_calibrate(calibration_test):
 
     reference_folder = pathlib.Path(val_dict["reference-folder"].format(
         REFERENCE_FOLDER, cal_conf["karabo-id"], test_key))
+
     cmd += ["--slurm-name", test_key]
     f = io.StringIO()
 
@@ -149,6 +176,7 @@ def test_xfel_calibrate(calibration_test):
             slurm_watcher = False
         else:
             time.sleep(0.5)
+
     # 1st check that all jobs were COMPLETED without errors.
     states = res.stdout.decode().split("\n")[2:-1]
     assert all(s.strip() == "COMPLETED" for s in states), f"{test_key} failure, calibration jobs were not completed. {jobids}: {states}"  # noqa
@@ -164,6 +192,10 @@ def test_xfel_calibrate(calibration_test):
     assert len(h5files) == len(expected_h5files), f"{test_key} failure, number of files are not as expected."  # noqa
     print(f"{test_key}'s calibration h5files numbers are as expected.")
 
+    # Stop tests at this point, if desired.
+    if skip_numerical_validation:
+        return
+
     # 4th check that h5files are exactly the same as the reference h5files.
     all_files_validated = []
     non_valid_files = []