Add script to backfill reports

webservice/add_reports.py

+"""Backfill reports to myMdC from job databases before we started injecting them"""
+import argparse
+import os.path
+import shlex
+import sys
+import sqlite3
+
+from .config import webservice as config
+from .webservice import init_md_client
+
+ap = argparse.ArgumentParser()
+ap.add_argument("db-file")
+ap.add_argument("--really", action="store_true")
+args = ap.parse_args()
+
+db_file = sys.argv[1]
+conn = sqlite3.connect(args.db_file)
+conn.row_factory = sqlite3.Row
+
+mdc = init_md_client(config)
+print("MyMdC API is:", mdc.base_api_url)
+
+rows = conn.execute(
+    "SELECT det_type, karabo_id, command, success, "
+    "req_id, proposal, run, action, mymdc_id, timestamp "
+    "FROM executions JOIN requests USING (req_id) "
+    "WHERE success IS NOT NULL"
+).fetchall()
+
+nreports = 0
+for i, r in enumerate(rows):
+    cmd_args = shlex.split(r["command"])
+    try:
+        report_path = cmd_args[cmd_args.index("--report-to") + 1] + ".pdf"
+    except (ValueError, IndexError):
+        print("Couldn't find report path in %r", cmd_args)
+        continue
+
+    if not os.path.isfile(report_path):
+        print(f"Report file {report_path} missing (p{r['proposal']}, r{r['run']}")
+        continue
+
+    desc = f"{r['karabo_id']} detector corrections"
+    if not r["success"]:
+        desc += " (errors occurred)"
+
+    nreports += 1
+    if not args.really:
+        continue
+
+    response = mdc.create_report_api(
+        {
+            "name": os.path.basename(report_path),
+            "cal_report_path": os.path.dirname(report_path).rstrip("/") + "/",
+            "cal_report_at": r["timestamp"],
+            "run_id": r["mymdc_id"],
+            "description": desc,
+        }
+    )
+
+    if response.status_code >= 400:
+        print(
+            f"Failed to add report to MDC for run ID {r['mymdc_id']}: "
+            f"HTTP status {response.status_code}",
+        )
+    if i % 20 == 0:
+        print(f"Done {i}")
+
+print(f"Found {nreports} reports from {len(rows)} executions")
+if args.really:
+    print(f"  Injected to myMdC")
+else:
+    print(f"  Re-run with --really to add to myMdC")