Launch jobs with variable nice value

Description

Based on the discussion in https://git.xfel.eu/calibration/planning/-/issues/110, this MR introduces a flexible --nice value passed to sbatch depending on the number of jobs running in the selected partition for this instrument. Originally only meant for active proposals, the code is cleaner when just using the partition value determined before and is currently changed by ITDM's script anyway shortly after submission. I will clarify this point before merging.

How Has This Been Tested?

Tests! Unit tests!

Types of changes

• New feature (non-breaking change which adds functionality)

Reviewers

@roscar @kluyvert @ahmedk

tests/test_webservice.py

@@ -14,6 +14,7 @@ from webservice.webservice import (  # noqa: import not at top of file
     run_action,
     wait_on_transfer,
     get_slurm_partition,
+    get_slurm_nice
 )
 
 
@@ -143,13 +144,15 @@ async def test_wait_on_transfer_exceptions(
         ('sim', ['DARK', '1', '2', '3', '4'], 1, "success: simulated"),
     ],
 )
-async def test_run_action(mode, cmd, retcode, expected):
+async def test_run_action(mode, cmd, retcode, expected, monkeypatch):
     job_db = mock.Mock()
 
     async def mock_run_proc_async(*args):
         return retcode, b'Submitted job: 42'
 
-    webservice.webservice.run_proc_async = mock_run_proc_async
+    monkeypatch.setattr(
+        webservice.webservice, 'run_proc_async', mock_run_proc_async
+    )
     ret = await run_action(job_db, cmd, mode, 1, 1, 1)
     assert ret.lower().startswith(expected)
 
@@ -177,3 +180,44 @@ async def test_get_slurm_partition(proposal_number,
 
     ret = await get_slurm_partition(client, action, proposal_number)
     assert ret == expected_result
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    'cycle, num_jobs, expected_result',
+    [
+        ('202201', 0, 0), ('202201', 10, 3*10**2),  # user proposal
+        ('202221', 0, 5), ('202221', 10, 5+3*10**2),  # commissioning
+    ]
+)
+async def test_get_slurm_nice_values(fp, cycle, num_jobs, expected_result):
+    """ Test get_slurm_nice values."""
+
+    fp.register(
+        ['squeue', '-h', '-o', '%.20j', '-p', 'upex-higher', '--me'],
+        stdout='\n'.join(
+            [f'correct_SPB_{i}' for i in range(num_jobs)] +
+            [f'correct_FXE_{i}' for i in range(num_jobs*2)]).encode('ascii'),
+        returncode=0)
+
+    ret = await get_slurm_nice(
+        'upex-higher', 'SPB', cycle, job_penalty=3, commissioning_penalty=5)
+    assert ret == expected_result
+
+@pytest.mark.asyncio
+async def test_get_slurm_nice_fails(fp):
+    """Test corner cases for get_slurm_nice."""
+
+    # non-zero returncode
+    fp.register(
+        ['squeue', '-h', '-o', '%.20j', '-p', 'upex-higher', '--me'],
+        stdout='', returncode=1)
+
+    assert await get_slurm_nice('upex-higher', 'SPB', '202201') == 0
+
+    # exfel is special
+    fp.register(
+        ['squeue', '-h', '-o', '%.20j', '-p', 'exfel', '--me'],
+        stdout='\n'.join([f'correct_SPB_{i}' for i in range(10)]),
+        returncode=0)
+
+    assert await get_slurm_nice('exfel', 'SPB', '202201') == 0