Create correction reports in MyMdC when jobs have finished

setup.py

@@ -79,7 +79,7 @@ install_requires = [
         "lxml==4.5.0",
         "markupsafe==2.0.1",
         "matplotlib==3.4.2",
-        "metadata_client==3.0.8",
+        "metadata_client==4.0.0",
         "nbclient==0.5.1",
         "nbconvert==5.6.1",
         "nbformat==5.0.7",

webservice/job_monitor.py

@@ -3,6 +3,8 @@ import argparse
 import json
 import locale
 import logging
+import os.path
+import shlex
 import signal
 import time
 from datetime import datetime, timezone
@@ -232,7 +234,7 @@ class JobsMonitor:
         ).fetchall()]
         success = set(statuses) == {'COMPLETED'}
         r = self.job_db.execute(
-            "SELECT det_type, karabo_id, req_id, proposal, run, action "
+            "SELECT det_type, karabo_id, command, req_id, proposal, run, action, mymdc_id "
             "FROM executions JOIN requests USING (req_id)"
             "WHERE exec_id = ?",
             (exec_id,)
@@ -259,6 +261,10 @@ class JobsMonitor:
                 log.warning("Error sending Kafka notification",
                             exc_info=True)
 
+            self.record_correction_report(
+                r['mymdc_id'], r['command'], r['karabo_id'], success
+            )
+
         return r['req_id']
 
     def process_request_finished(self, req_id):
@@ -331,9 +337,44 @@ class JobsMonitor:
             log.error("Failed to update MDC dark run id %s", dark_run_id)
             log.error(Errors.MDC_RESPONSE.format(response))
 
+    def record_correction_report(self, mymdc_run_id, command, karabo_id, success):
+        """Add report to MyMdC when a correction execution has finished"""
+        args = shlex.split(command)
+        try:
+            report_path = args[args.index("--report-to") + 1] + ".pdf"
+        except (ValueError, IndexError):
+            log.warning("Couldn't find report path in %r", args)
+            return
+
+        if not os.path.isfile(report_path):
+            log.warning("Jobs finished, but report file %s missing", report_path)
+            return
+
+        ts = datetime.fromtimestamp(os.stat(report_path).st_mtime, tz=timezone.utc)
+        desc = f"{karabo_id} detector corrections"
+        if not success:
+            desc += " (errors occurred)"
+
+        log.debug("Adding report file %s to MDC for run ID %s",
+                  report_path, mymdc_run_id)
+
+        response = self.mdc.create_report_api({
+            "name": os.path.basename(report_path),
+            "cal_report_path": os.path.dirname(report_path).rstrip('/') + '/',
+            "cal_report_at": ts.isoformat(),
+            "run_id": mymdc_run_id,
+            "description": desc,
+        })
+
+        if response.status_code >= 400:
+            log.error("Failed to add report to MDC for run ID %s: HTTP status %s",
+                      mymdc_run_id, response.status_code)
+
+
 def interrupted(signum, frame):
     raise KeyboardInterrupt
 
+
 def main(argv=None):
     # Ensure files are opened as UTF-8 by default, regardless of environment.
     locale.setlocale(locale.LC_CTYPE, ('en_US', 'UTF-8'))