List which calibrations succeeded & failed when request fails overall

webservice/job_monitor.py

@@ -231,10 +231,11 @@ class JobsMonitor:
 
     def process_request_finished(self, req_id):
         """Send notifications that a request has finished"""
-        exec_successes = {r[0] for r in self.job_db.execute(
-            "SELECT success FROM executions WHERE req_id = ?", (req_id,)
+        krb_id_successes = {r[0]: r[1] for r in self.job_db.execute(
+            "SELECT karabo_id, success FROM executions WHERE req_id = ?",
+            (req_id,)
         ).fetchall()}
-        success = (exec_successes == {1})
+        success = (set(krb_id_successes.values()) == {1})
 
         r = self.job_db.execute(
             "SELECT * FROM requests WHERE req_id = ?", (req_id,)
@@ -258,15 +259,11 @@ class JobsMonitor:
         if success:
             msg = "Calibration jobs succeeded"
         else:
-            # Count failed jobs
-            job_statuses = [r[0] for r in self.job_db.execute(
-                "SELECT slurm_jobs.status FROM slurm_jobs "
-                "INNER JOIN executions USING (exec_id) "
-                "INNER JOIN requests USING (req_id) "
-                "WHERE req_id = ?", (req_id,)
-            ).fetchall()]
-            n_failed = sum(s != 'COMPLETED' for s in job_statuses)
-            msg = f"{n_failed}/{len(job_statuses)} calibration jobs failed"
+            # List success & failure by karabo_id
+            krb_ids_ok = {k for (k, v) in krb_id_successes if v == 1}
+            ok = ', '.join(sorted(krb_ids_ok)) if krb_ids_ok else 'none'
+            krb_ids_failed = {k for (k, v) in krb_id_successes if v == 0}
+            msg = f"Succeeded: {ok}; Failed: {', '.join(krb_ids_failed)}"
 
         log.debug("Update MDC for %s, %s: %s", r['action'], r['mymdc_id'], msg)