From 89971d0570ce6d651db9ce46a05efecd9914059e Mon Sep 17 00:00:00 2001
From: Thomas Kluyver <thomas@kluyver.me.uk>
Date: Thu, 21 Oct 2021 18:47:25 +0100
Subject: [PATCH 1/3] Remove unused code to automatically select Slurm
reservation
---
src/xfel_calibrate/calibrate.py | 24 ++----------------------
src/xfel_calibrate/nb_args.py | 4 ----
tests/test_xfel_calibrate/conftest.py | 2 --
3 files changed, 2 insertions(+), 28 deletions(-)
diff --git a/src/xfel_calibrate/calibrate.py b/src/xfel_calibrate/calibrate.py
index 26e45f0fe..089a6ed2a 100755
--- a/src/xfel_calibrate/calibrate.py
+++ b/src/xfel_calibrate/calibrate.py
@@ -35,13 +35,8 @@ from .nb_args import (
from .settings import (
default_report_path,
finalize_time_limit,
- free_nodes_cmd,
launcher_command,
- max_reserved,
- preempt_nodes_cmd,
python_path,
- reservation,
- reservation_char,
sprof,
temp_path,
try_report_to_output,
@@ -276,36 +271,22 @@ def save_executed_command(run_tmp_path, version, argv):
class SlurmOptions:
def __init__(
self, job_name=None, nice=None, mem=None, partition=None, reservation=None,
- priority_group=2,
):
self.job_name = job_name or 'xfel_calibrate'
self.nice = nice
self.mem = mem
self.partition = partition
self.reservation = reservation
- self.priority_group = priority_group
def get_partition_or_reservation(self) -> List[str]:
"""Return sbatch arguments to use a partition or reservation
--reservation and --slurm-partition options have precedence.
- Otherwise, if --priority is <=1, it will use a configured reservation
- depending on how many nodes are currently free.
+ Otherwise, a default partition is used.
"""
if self.reservation:
return ['--reservation', self.reservation]
- elif self.partition:
- return ['--partition', self.partition]
- elif self.priority_group <= 1:
- auto_resvn = reservation_char if self.priority_group <= 0 else reservation
- # Use a reservation if there aren't many general nodes available to us
- free = int(check_output(free_nodes_cmd, shell=True).decode('utf8'))
- preempt = int(check_output(preempt_nodes_cmd, shell=True).decode('utf8'))
- if free + preempt < max_reserved:
- return ['--reservation', auto_resvn]
-
- # Fallback to using the configured partition (default: exfel)
- return ['--partition', sprof]
+ return ['--partition', self.partition or sprof]
def get_launcher_command(self, temp_path, after_ok=(), after_any=()) -> List[str]:
"""
@@ -836,7 +817,6 @@ def run(argv=None):
mem=args['slurm_mem'],
reservation=args['reservation'],
partition=args['slurm_partition'],
- priority_group=args['priority'],
))
errors = False
diff --git a/src/xfel_calibrate/nb_args.py b/src/xfel_calibrate/nb_args.py
index 511223ed6..edf893e00 100644
--- a/src/xfel_calibrate/nb_args.py
+++ b/src/xfel_calibrate/nb_args.py
@@ -77,10 +77,6 @@ def make_initial_parser(**kwargs):
"retrieved-constants will be copied to use for a new correction."
))
- parser.add_argument('--priority', type=int, default=2,
- help="Priority of batch jobs. If priority<=1, reserved"
- " nodes become available.")
-
parser.add_argument('--vector-figs', action="store_true", default=False,
help="Use vector graphics for figures in the report.")
diff --git a/tests/test_xfel_calibrate/conftest.py b/tests/test_xfel_calibrate/conftest.py
index 95bc80f2e..7533d2f27 100644
--- a/tests/test_xfel_calibrate/conftest.py
+++ b/tests/test_xfel_calibrate/conftest.py
@@ -35,8 +35,6 @@ class FakeProcessCalibrate(FakeProcess):
"""
super().__init__()
# Fake calls to slurm
- self.register_subprocess(settings.free_nodes_cmd, stdout=["1"])
- self.register_subprocess(settings.preempt_nodes_cmd, stdout=["1"])
self.register_subprocess(
["sbatch", self.any()], stdout=["000000"]
)
--
GitLab
From 4e93002dedc09ff34f0976e6ed4fec9b9f24f6c8 Mon Sep 17 00:00:00 2001
From: Thomas Kluyver <thomas@kluyver.me.uk>
Date: Mon, 6 Dec 2021 17:47:32 +0000
Subject: [PATCH 2/3] Remove some unused settings
---
src/xfel_calibrate/settings.py | 5 -----
1 file changed, 5 deletions(-)
diff --git a/src/xfel_calibrate/settings.py b/src/xfel_calibrate/settings.py
index 989cd1022..15067c0a6 100644
--- a/src/xfel_calibrate/settings.py
+++ b/src/xfel_calibrate/settings.py
@@ -21,11 +21,6 @@ sprof = os.environ.get("XFELCALSLURM", "exfel") # TODO: https://git.xfel.eu/git
launcher_command = "sbatch -t 24:00:00 --requeue --output {temp_path}/slurm-%j.out --parsable"
free_nodes_cmd = "sinfo -p exfel -t idle -N --noheader | wc -l"
preempt_nodes_cmd = "squeue -p all,grid --noheader | grep max-exfl | egrep -v 'max-exfl18[3-8]|max-exfl100|max-exflg' | wc -l"
-max_reserved = 8
-# "xcal" reservation value removed as ITDM
-# is giving xcal priority by default.
-reservation = ""
-reservation_char = "darks"
# Time limit for the finalize job (creates PDF report & moves files)
finalize_time_limit = "30:00"
--
GitLab
From 47db95d928147905b36b8287032801732fe0b70e Mon Sep 17 00:00:00 2001
From: Thomas Kluyver <thomas@kluyver.me.uk>
Date: Mon, 6 Dec 2021 17:53:38 +0000
Subject: [PATCH 3/3] Don't pass --priority from webservice to xfel-calibrate
---
webservice/webservice.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/webservice/webservice.py b/webservice/webservice.py
index 924455968..3c1fc1b1f 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -838,8 +838,8 @@ class ActionsServer:
:param instrument: is the instrument
:param cycle: is the facility cycle
:param proposal: is the proposal id
- :param runnr: is the run number in integer form, e.g. without leading
- "r"
+ :param runnr: the run number in integer form, i.e. without leading "r"
+ :param priority: unused, retained for compatibility
This will trigger a correction process to be launched for that run in
the given cycle and proposal.
@@ -911,8 +911,6 @@ class ActionsServer:
thisconf["out-folder"] = out_folder
thisconf["karabo-id"] = karabo_id
thisconf["run"] = runnr
- if priority:
- thisconf["priority"] = str(priority)
detectors[karabo_id] = thisconf
copy_file_set = copy_file_list.difference(corr_file_list)
--
GitLab