From 1b43c004b908c3683505db286e19f8a96d0cbd1c Mon Sep 17 00:00:00 2001
From: Steffen Hauf <steffen.hauf@xfel.eu>
Date: Mon, 26 Aug 2019 16:54:27 +0200
Subject: [PATCH] Initial working version
---
webservice/serve_overview.css | 33 +++++
webservice/serve_overview.py | 248 ++++++++++++++++++++++++++++++++++
2 files changed, 281 insertions(+)
create mode 100644 webservice/serve_overview.css
create mode 100644 webservice/serve_overview.py
diff --git a/webservice/serve_overview.css b/webservice/serve_overview.css
new file mode 100644
index 000000000..d3d2e2522
--- /dev/null
+++ b/webservice/serve_overview.css
@@ -0,0 +1,33 @@
+h2 {
+ font-family: monospace;
+}
+
+div {
+ background-color: #e5f0e4;
+ padding: 20px;
+ text-align: left;
+ border: 1px solid black;
+ font-family: monospace;
+ width: auto;
+ margin: auto;
+
+}
+
+dt {
+ font-weight: bold;
+ float: left;
+ min-width: 300px;
+ padding-right: 1em;
+ text-align: left;
+}
+
+dd {
+ text-align: left;
+}
+
+.log-out {
+ height: 200px;
+ overflow: auto;
+ background-color: white;
+
+}
diff --git a/webservice/serve_overview.py b/webservice/serve_overview.py
new file mode 100644
index 000000000..9a92ee3a2
--- /dev/null
+++ b/webservice/serve_overview.py
@@ -0,0 +1,248 @@
+import argparse
+from datetime import datetime
+import glob
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from iCalibrationDB import Detectors
+from jinja2 import Template
+import json
+import os
+import requests
+from oauthlib.oauth2 import BackendApplicationClient
+from requests_oauthlib import OAuth2Session, OAuth2
+from subprocess import check_output
+import yaml
+
+
+from xfel_calibrate.settings import free_nodes_cmd, preempt_nodes_cmd, reservation, reservation_char
+
+main_doc = """
+
+<!DOCTYPE html>
+<html lang="en">
+<head>
+ <link rel="stylesheet" href="serve_overview.css">
+ <title>Overview of offline calibration tasks</title>
+</head>
+<body>
+{{ maxwell_status }}
+{{ log_output }}
+{{ last_characterizations }}
+</body>
+</html>
+
+"""
+
+nodes_avail_res_cmd = "sinfo --nodes=`sinfo -p exfel -t idle -N --noheader -T | grep {} | awk '{{print $6}}'` --noheader -p exfel -o %A"
+
+maxwell_status = """
+<div>
+ <h2>Maxwell status</h2>
+ <dl>
+ <dt>Total jobs running</dt>
+ <dd>{{ total_jobs_running }}</dd>
+ <dt>Nodes available on general partition</dt>
+ <dd>{{ nodes_avail_general }}</dd>
+ <dt>Nodes available on general reservation</dt>
+ <dd>{{ nodes_avail_general_res }}</dd>
+ {% for res, avail in upex_reservations.items() %}
+ <dt>Nodes available on {{ res }}</dt>
+ <dd>{{ avail }}</dd>
+ {% endfor %}
+ <dt>Recommendation</dt>
+ <dd>{{ recommendation }}</dd>
+ </dl>
+</div>
+"""
+
+log_output = """
+<div>
+ <h2> Webservice log </h2>
+ <div class="log-out">
+ {{ logout }}
+ </div>
+</div>
+
+"""
+
+last_characterizations = """
+<div>
+ <h2>Last characterization runs</h2>
+ {% for instrument, data in char_runs.items() %}
+ <h3>{{ instrument }}</h3>
+ <dl>
+ <dt>Requested</dt><dd>{{ data['requested'] }}</dd>
+ <dt>Check in DB:</dt><dd><a href="https://in.xfel.eu/calibration/admin/calibration_constant_version?model_name=calibration_constant_version&utf8=%E2%9C%93&f%5Bphysical_device%5D%5B02524%5D%5Bo%5D=like&f%5Bphysical_device%5D%5B02524%5D%5Bv%5D={{ data['device_type'] }}&f%5Bcalibration_constant%5D%5B02697%5D%5Bo%5D=like&f%5Bcalibration_constant%5D%5B02697%5D%5Bv%5D=Offset&query=" target="_blank"> Open in calDB </a></dd>
+ <dt>Output path</dt><dd>{{ data['out_path'] }}</dd>
+ <dt>Input path</dt><dd>{{ data['in_path'] }}</dd>
+ <dt>Input runs</dt><dd>{{ data['runs'] }}</dd>
+ <dt>Input data size</dt><dd>{{ data['size'] }}</dd>
+ <dt>Output PDFs</dt><dd>{{ data['pdfs'] }}</dd>
+ </dl>
+ {% endfor %}
+
+</div>
+"""
+
+# HTTPRequestHandler class
+class testHTTPServer_RequestHandler(BaseHTTPRequestHandler):
+
+ # GET
+ def do_GET(self):
+ # Send response status code
+ self.send_response(200)
+ if self.path == "/serve_overview.css":
+ with open("serve_overview.css", "r") as f:
+ self.send_header('Content-type','text/css')
+ self.end_headers()
+ for s in f:
+ self.wfile.write(s.encode())
+ return
+
+ # Send headers
+ self.send_header('Content-type','text/html')
+ self.end_headers()
+
+ # Send message back to client
+ # general maxwell stats
+ free = int(check_output(free_nodes_cmd, shell=True).decode('utf8'))
+ preempt = int(check_output(
+ preempt_nodes_cmd, shell=True).decode('utf8'))
+ nodes_avail_general = free + preempt
+ nodes_avail_general_res = check_output(nodes_avail_res_cmd.format(reservation), shell=True).decode('utf8')
+ ures = nodes_avail_general_res.split("/")
+ if len(ures) == 2:
+ used, reserved = ures
+ else:
+ used = 0
+ reserved = 0
+ nodes_avail_general_res = "{}/{}".format(int(reserved)-int(used), reserved)
+ total_jobs_running = check_output("sinfo -p exfel -o %A --noheader", shell=True).decode('utf8').split("/")[0]
+
+ upex_res = [r for r in check_output("sinfo -T --noheader | awk '{print $1}'", shell=True).decode('utf8').split()
+ if "upex_" in r]
+
+ upex_reservations = {}
+ for res in upex_res:
+ nodes_avail_res = check_output(nodes_avail_res_cmd.format(res), shell=True).decode('utf8')
+ used, reserved = nodes_avail_res.split("/")
+ nodes_avail_res = "{}/{}".format(int(reserved)-int(used), reserved)
+ upex_reservations[res] = nodes_avail_res
+
+ recommendation = "DON'T SUBMIT TO RESERVATION"
+ if nodes_avail_general < int(reserved)-int(used):
+ "CONSIDER SUBMITTING TO RESERVATION"
+
+ maxwell_status_r = Template(maxwell_status).render(nodes_avail_general=nodes_avail_general,
+ nodes_avail_general_res=nodes_avail_general_res,
+ total_jobs_running=total_jobs_running,
+ upex_reservations=upex_reservations,
+ recommendation=recommendation)
+
+ last_n_lines = check_output("tail -5000 web.log", shell=True).decode('utf8').split("\n")
+ last_n_lines = [l for l in last_n_lines if "Response error from MDC" not in l]
+ log_output_r = Template(log_output).render(logout="<br>".join(last_n_lines[::-1]))
+
+
+ last_n_lines = check_output("cat web.log", shell=True).decode('utf8').split("\n")[::-1]
+ last_chars = {}
+ mappings = {"LPD": ["LPD"],
+ "AGIPD": ["AGIPD"],
+ "DSSC": ["DSSC"],
+ "PNCCD": ["PNCCD"],
+ "JUNGFRAU": ["JNGFR", "DA05", "DA04"],
+ "EPIX": ["EPIX"], "EPIX10K": ["EPIX"],
+ "FASTCCD": ["DA05"] }
+ for l in last_n_lines:
+ if "DARK" in l:
+ ls = l.split()
+ if "DARK" not in ls:
+ continue
+ detector = ls[ls.index("DARK")-1]
+ dinstance = ""
+ if "--instrument" in ls:
+ instrument = ls[ls.index("--instrument") + 1]
+ if detector == "DSSC":
+ dinstance = "DSSC1M1"
+ if detector == "AGIPD" and instrument == "SPB":
+ dinstance = "AGIPD1M1"
+ if detector == "AGIPD" and instrument == "MID":
+ dinstance = "AGIPD1M2"
+ if detector == "LPD":
+ dinstance = "LPD1M1"
+ else:
+ if "--db-module" in ls:
+ instrument = ls[ls.index("--db-module") + 1]
+ dinstance = instrument
+ elif detector == "PNCCD":
+ instrument = "SQS"
+ dinstance = "PnCCD1"
+ elif detector == "FASTCCD":
+ instrument = "SCS"
+ dinstance = "fastCCD1"
+ else:
+ instrument = ""
+
+ in_folder = ls[ls.index("--in-folder") + 1]
+ out_folder = ls[ls.index("--out-folder") + 1]
+ run_candidates = ["--run-high", "--run-med", "--run-low", "--run"]
+ runs = []
+ for rc in run_candidates:
+ if rc in ls:
+ runs.append(ls[ls.index(rc)+1])
+ if f"{instrument}-{detector}" in last_chars:
+ continue
+
+ requested = "{} {}".format(ls[0], ls[1])
+
+ pdfs = glob.glob(f"{out_folder}/*.pdf")
+ pdfs = [p.split("/")[-1] for p in pdfs]
+ if "xfel.pdf" in pdfs:
+ del pdfs[pdfs.index("xfel.pdf")]
+ if len(pdfs):
+ pdfs = ", ".join(pdfs)
+ else:
+ pdfs = "---"
+
+ tsize = 0
+
+ for run in runs:
+ run = int(run)
+ if detector not in mappings:
+ continue
+ for mp in mappings[detector]:
+ for f in glob.glob(f"{in_folder}/r{run:04d}/*{mp}*.h5"):
+ tsize += os.stat(f).st_size
+ last_injected = ""
+ constant_valid_from = ""
+
+ last_chars[f"{instrument}-{detector}"] = {"in_path": in_folder,
+ "out_path": out_folder,
+ "runs": runs,
+ "pdfs": pdfs,
+ "size": "{:0.1f} GB".format(tsize/1e9),
+ "requested": requested,
+ "device_type": detector,
+ "last_valid_from": constant_valid_from}
+ last_characterizations_r = Template(last_characterizations).render(char_runs=last_chars)
+
+ message = Template(main_doc).render(maxwell_status=maxwell_status_r,
+ log_output=log_output_r,
+ last_characterizations=last_characterizations_r)
+ # Write content as utf-8 data
+ self.wfile.write(bytes(message, "utf8"))
+ return
+
+def run(port=8008):
+ print('starting server...')
+
+ server_address = ('localhost', port)
+ httpd = HTTPServer(server_address, testHTTPServer_RequestHandler)
+ print('running server...')
+ httpd.serve_forever()
+
+parser = argparse.ArgumentParser(
+ description='Start the overview server')
+parser.add_argument('--port', type=int, default=8008)
+if __name__ == "__main__":
+ args = vars(parser.parse_args())
+ run(args["port"])
--
GitLab