A simple overview webserver

See merge request detectors/pycalibration!136

webservice/serve_overview.py

+import argparse
+import glob
+import os
+import sqlite3
+from collections import OrderedDict
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from subprocess import check_output
+from uuid import uuid4
+
+import yaml
+from jinja2 import Template
+from xfel_calibrate.settings import (free_nodes_cmd, preempt_nodes_cmd,
+                                     reservation)
+
+
+class LimitedSizeDict(OrderedDict):
+    def __init__(self, *args, **kwds):
+        self.size_limit = kwds.pop("size_limit", None)
+        OrderedDict.__init__(self, *args, **kwds)
+        self._check_size_limit()
+
+    def __setitem__(self, key, value):
+        OrderedDict.__setitem__(self, key, value)
+        self._check_size_limit()
+
+    def _check_size_limit(self):
+        if self.size_limit is not None:
+            while len(self) > self.size_limit:
+                self.popitem(last=False)
+
+
+config = None
+pdf_queue = LimitedSizeDict(size_limit=50)
+
+
+# HTTPRequestHandler class
+class RequestHandler(BaseHTTPRequestHandler):
+    conf_was_init = False
+
+    def init_config(self):
+
+        global config
+
+        self.nodes_avail_res_cmd = config["shell-commands"]["nodes-avail-res"]
+        self.total_jobs_cmd = config["shell-commands"]["total-jobs"]
+        self.upex_jobs_cmd = config["shell-commands"]["upex-jobs"]
+        self.upex_prefix = config["shell-commands"]["upex-prefix"]
+        self.tail_log_cmd = config["shell-commands"]["tail-log"]
+        self.cat_log_cmd = config["shell-commands"]["cat-log"]
+        self.mappings = config["mappings"]
+        self.run_candidates = config["run-candidates"]
+
+        self.templates = {}
+        for template, tfile in config["templates"].items():
+            with open(tfile, "r") as tf:
+                self.templates[template] = tf.read()
+        global pdf_queue
+        self.pdf_queue = pdf_queue
+        self.conf_was_init = True
+
+    def do_GET(self):
+
+        if not self.conf_was_init:
+            self.init_config()
+        # Send response status code
+        self.send_response(200)
+        if self.path == "/serve_overview.css":
+            self.send_header('Content-type', 'text/css')
+            self.end_headers()
+            for s in self.templates["css"].split("\n"):
+                self.wfile.write(s.encode())
+            return
+        if "pdf?" in self.path:
+            puuid = self.path.split("?")[1]
+            fpath = self.pdf_queue.get(puuid, None)
+            if fpath is None:
+                return
+            self.send_header('Content-type', 'application/pdf')
+            self.end_headers()
+            with open(fpath, "rb") as f:
+                self.wfile.write(f.read())
+            return
+
+        # Send headers
+        self.send_header('Content-type', 'text/html')
+        self.end_headers()
+
+        # Send message back to client
+        # general maxwell stats
+        free = int(check_output(free_nodes_cmd, shell=True).decode('utf8'))
+        preempt = int(check_output(
+            preempt_nodes_cmd, shell=True).decode('utf8'))
+        nodes_avail_general = free + preempt
+        ret = check_output(self.nodes_avail_res_cmd.format(reservation),
+                           shell=True).decode('utf8')
+        ures = ret.split("/")
+        if len(ures) == 2:
+            used, reserved = ures
+        else:
+            used = 0
+            reserved = 0
+        nodes_avail_gr = "{}/{}".format(int(reserved) - int(used),
+                                        reserved)
+        total_jobs_running = check_output(self.total_jobs_cmd, shell=True)
+        total_jobs_running = total_jobs_running.decode('utf8').split("/")[0]
+
+        upex_res = [r for r in
+                    check_output(self.upex_jobs_cmd,
+                                 shell=True).decode('utf8').split()
+                    if self.upex_prefix in r]
+
+        upex_reservations = {}
+        for res in upex_res:
+            nodes_avail_res = check_output(
+                self.nodes_avail_res_cmd.format(res),
+                shell=True).decode('utf8')
+            used, reserved = nodes_avail_res.split("/")
+            nodes_avail_res = "{}/{}".format(int(reserved) - int(used),
+                                             reserved)
+            upex_reservations[res] = nodes_avail_res
+
+        recommendation = "DON'T SUBMIT TO RESERVATION"
+        if nodes_avail_general < int(reserved) - int(used):
+            "CONSIDER SUBMITTING TO RESERVATION"
+
+        tmpl = Template(self.templates["maxwell-status"])
+        maxwell_status_r = tmpl.render(nodes_avail_general=nodes_avail_general,
+                                       nodes_avail_general_res=nodes_avail_gr,
+                                       total_jobs_running=total_jobs_running,
+                                       upex_reservations=upex_reservations,
+                                       recommendation=recommendation)
+
+        last_n_lines = check_output(self.tail_log_cmd,
+                                    shell=True).decode('utf8').split("\n")
+        last_n_lines = [l for l in last_n_lines
+                        if "Response error from MDC" not in l]
+        tmpl = Template(self.templates["log-output"])
+        log_output_r = tmpl.render(logout="<br>".join(last_n_lines[::-1]))
+
+        last_n_lines = check_output(self.cat_log_cmd,
+                                    shell=True).decode('utf8').split("\n")[
+                       ::-1]
+
+        last_chars = {}
+        for l in last_n_lines:
+            if "DARK" in l:
+                ls = l.split()
+                if "DARK" not in ls:
+                    continue
+                detector = ls[ls.index("DARK") - 1]
+
+                if "--instrument" in ls:
+                    instrument = ls[ls.index("--instrument") + 1]
+                else:
+                    if "--db-module" in ls:
+                        instrument = ls[ls.index("--db-module") + 1]
+                    elif detector == "PNCCD":
+                        instrument = "SQS"
+                    elif detector == "FASTCCD":
+                        instrument = "SCS"
+                    else:
+                        instrument = ""
+
+                in_folder = ls[ls.index("--in-folder") + 1]
+                out_folder = ls[ls.index("--out-folder") + 1]
+                runs = []
+                for rc in self.run_candidates:
+                    if rc in ls:
+                        runs.append(ls[ls.index(rc) + 1])
+
+                if f"{instrument}-{detector}" in last_chars:
+                    continue
+
+                requested = "{} {}".format(ls[0], ls[1])
+
+                pdfs = glob.glob(f"{out_folder}/*.pdf")
+                pdfs = {p.split("/")[-1]: p for p in pdfs}
+                fpdfs = []
+                if "xfel.pdf" in pdfs:
+                    del pdfs["xfel.pdf"]
+                if len(pdfs):
+                    # pdfs = ", ".join(pdfs)
+                    for pdf, p in pdfs.items():
+                        puuid = uuid4().hex
+                        self.pdf_queue[puuid] = p
+                        host = config["server-config"]["host"]
+                        port = config["server-config"]["port"]
+                        fpdfs.append(
+                            (pdf, f"http://{host}:{port}/pdf?{puuid}"))
+                pdfs = fpdfs
+                tsize = 0
+                for run in runs:
+                    run = int(run)
+                    if detector not in self.mappings:
+                        continue
+                    for mp in self.mappings[detector]:
+                        for f in glob.glob(
+                                f"{in_folder}/r{run:04d}/*{mp}*.h5"):
+                            tsize += os.stat(f).st_size
+                last_injected = ""
+                constant_valid_from = ""
+                key = f"{instrument}-{detector}"
+                last_chars[key] = {"in_path": in_folder,
+                                   "out_path": out_folder,
+                                   "runs": runs,
+                                   "pdfs": pdfs,
+                                   "size": "{:0.1f} GB".format(tsize / 1e9),
+                                   "requested": requested,
+                                   "device_type": detector,
+                                   "last_valid_from": constant_valid_from}
+
+        tmpl = self.templates["last-characterizations"]
+        last_characterizations_r = Template(tmpl).render(char_runs=last_chars)
+
+        conn = sqlite3.connect(config['web-service']['job-db']).cursor()
+        conn.execute("SELECT * FROM jobs")
+        running_jobs = {}
+        for r in conn.fetchall():
+            rid, jobid, proposal, run, flg, status = r
+            run = int(run)
+            rjobs = running_jobs.get("r{:04d}".format(run),
+                                     {'proposal': proposal, 'statii': []})
+            rjobs["statii"].append((flg, status))
+            running_jobs["r{:04d}".format(run)] = rjobs
+
+        tmpl = self.templates["running-jobs"]
+        running_jobs_r = Template(tmpl).render(running_jobs=running_jobs)
+
+        tmpl = Template(self.templates["main-doc"])
+        message = tmpl.render(maxwell_status=maxwell_status_r,
+                              log_output=log_output_r,
+                              last_characterizations=last_characterizations_r,
+                              running_jobs=running_jobs_r)
+        # Write content as utf-8 data
+        self.wfile.write(bytes(message, "utf8"))
+        return
+
+
+def run(configfile, port=8008):
+    print('reading config file')
+    with open(configfile, "r") as cf:
+        global config
+        config = yaml.load(cf.read())
+    print('starting server...')
+    sconfig = config["server-config"]
+    server_address = (sconfig["host"], sconfig["port"])
+    httpd = HTTPServer(server_address, RequestHandler)
+    print('running server...')
+    httpd.serve_forever()
+
+
+parser = argparse.ArgumentParser(
+    description='Start the overview server')
+parser.add_argument('--config', type=str, default="serve_overview.yaml")
+if __name__ == "__main__":
+    args = vars(parser.parse_args())
+    run(args["config"])

webservice/serve_overview.yaml

+templates:
+  main-doc: ./templates/main_doc.html
+  maxwell-status: ./templates/maxwell_status.html
+  log-output: ./templates/log_output.html
+  last-characterizations: ./templates/last_characterizations.html
+  running-jobs: ./templates/running_jobs.html
+  css: ./templates/serve_overview.css
+
+shell-commands:
+  nodes-avail-res: "sinfo --nodes=`sinfo -p exfel -t idle -N --noheader -T | grep {} | awk '{{print $6}}'` --noheader -p exfel -o %A"
+  total-jobs: "sinfo -p exfel -o %A --noheader"
+  upex-jobs: "sinfo -T --noheader | awk '{print $1}'"
+  upex-prefix: "upex_"
+  tail-log: "tail -5000 web.log"
+  cat-log: "cat web.log"
+
+mappings:
+  LPD: 
+    - LPD
+  AGIPD: 
+    - AGIPD
+  DSSC: 
+    - DSSC
+  PNCCD: 
+    - PNCCD
+  JUNGFRAU: 
+    - JNGFR 
+    - DA05 
+    - DA04
+  EPIX: 
+    - EPIX
+  EPIX10K: 
+    - EPIX10K
+  FASTCCD: 
+    - DA05
+
+run-candidates:
+    - "--run-high" 
+    - "--run-med"
+    - "--run-low" 
+    - "--run"
+
+server-config:
+    port: 8008
+    host: max-exfl016
+
+web-service:
+    job-db: ./webservice_jobs.sqlite
\ No newline at end of file

webservice/templates/last_characterizations.html

+<div class="block">
+   <h2>Last characterization runs</h2>
+   {% for instrument, data in char_runs.items() %}
+       <h3>{{ instrument }}</h3>
+       <dl>
+           <dt>Requested:</dt><dd>{{ data['requested'] }}</dd>
+           <dt>Check in DB:</dt><dd><a href="https://in.xfel.eu/calibration/admin/calibration_constant_version?model_name=calibration_constant_version&utf8=%E2%9C%93&f%5Bphysical_device%5D%5B02524%5D%5Bo%5D=like&f%5Bphysical_device%5D%5B02524%5D%5Bv%5D={{ data['device_type'] }}&f%5Bcalibration_constant%5D%5B02697%5D%5Bo%5D=like&f%5Bcalibration_constant%5D%5B02697%5D%5Bv%5D=Offset&query=" target="_blank"> Open in calDB </a></dd>
+           <dt>Output path:</dt><dd>{{ data['out_path'] }}</dd>
+           <dt>Input path:</dt><dd>{{ data['in_path'] }}</dd>
+           <dt>Input runs:</dt><dd>{{ data['runs'] }}</dd>
+           <dt>Input data size:</dt><dd>{{ data['size'] }}</dd>
+           <dt>Output PDFs:</dt><dd>
+           {% for pdf in data['pdfs'] %}
+               <a href={{ pdf[1] }} target="_blank">{{ pdf[0] }}</a>
+           {% endfor %}
+           &nbsp;
+           </dd>
+       </dl>
+   {% endfor %}
+
+</div>

webservice/templates/log_output.html

+<div class="block">
+   <h2> Webservice log </h2>
+   <div class="log-out">
+   {{ logout }}
+   </div>
+</div>
\ No newline at end of file

webservice/templates/main_doc.html

+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <link rel="stylesheet" href="serve_overview.css">
+    <title>Overview of offline calibration tasks</title>
+</head>
+<body>
+{{ maxwell_status }}
+{{ running_jobs }}
+{{ last_characterizations }}
+{{ log_output }}
+</body>
+</html>

webservice/templates/maxwell_status.html

+<div class="block">
+    <div class="logo">
+    <svg class="xfel-header-navigation__logo-svg" width="100px" height="100px" viewBox="0 0 100 100">
+    <use xlink:href="#icon-logo-xfel">
+    <symbol id="icon-logo-xfel" viewBox="0 0 100 100"><g fill="none" fill-rule="evenodd"><path fill="#0D1546" d="M0 0h100v100H0V0zm2 2h96v96H2V2z"></path><path fill="#0D1546" d="M14.784 80.536L5.82 94h6.336l5.724-8.892L23.496 94h6.732l-8.964-13.428 8.244-12.276h-6.192l-5.22 8.208-5.04-8.208H6.504l8.28 12.24zm17.712-12.24V94h5.652V83.38h10.764v-4.392H38.148v-5.94h12.42v-4.752H32.496zm21.098 0V94h19.512v-4.752h-13.86v-6.3h12.456v-4.392H59.246v-5.508h13.572v-4.752H53.594zm22.328 0V94h18.18v-4.752H81.574V68.296h-5.652zM6.173 49.862V62h9.214v-2.244H8.842v-2.975h5.882v-2.074H8.842v-2.601h6.409v-2.244H6.173zM25.179 62v-8.789h-2.414v4.607c0 .895-.147 1.538-.442 1.93-.295.39-.77.586-1.428.586-.578 0-.98-.178-1.207-.536-.227-.357-.34-.898-.34-1.623v-4.964h-2.414v5.406c0 .544.048 1.04.144 1.488.097.447.264.827.502 1.139.238.311.564.552.978.722.413.17.943.255 1.589.255.51 0 1.009-.113 1.496-.34.487-.227.884-.595 1.19-1.105h.051V62h2.295zm1.836-8.789V62h2.414v-3.961c0-.397.04-.765.119-1.105.08-.34.212-.637.4-.892.187-.256.433-.457.739-.604.306-.147.68-.221 1.122-.221.147 0 .3.008.459.026.159.017.295.036.408.059v-2.244a1.883 1.883 0 0 0-.527-.085c-.306 0-.6.045-.884.136a3.24 3.24 0 0 0-.799.383 3.007 3.007 0 0 0-1.122 1.351h-.034v-1.632h-2.295zm8.755 4.403c0-.351.034-.697.102-1.037.068-.34.184-.64.349-.901.164-.26.382-.473.654-.638.272-.164.612-.246 1.02-.246.408 0 .75.082 1.028.246.278.165.5.377.663.638.165.26.281.561.349.901a5.275 5.275 0 0 1 0 2.066 2.74 2.74 0 0 1-.349.9 1.835 1.835 0 0 1-.663.638c-.277.159-.62.238-1.028.238-.408 0-.748-.08-1.02-.238a1.856 1.856 0 0 1-.654-.637 2.74 2.74 0 0 1-.349-.901 5.147 5.147 0 0 1-.102-1.029zm-2.414 0c0 .703.108 1.337.323 1.904a4.13 4.13 0 0 0 .918 1.453c.397.403.873.712 1.428.927.555.215 1.179.323 1.87.323.691 0 1.317-.108 1.878-.323a4 4 0 0 0 1.437-.927 4.13 4.13 0 0 0 .918-1.453c.215-.567.323-1.201.323-1.904 0-.703-.108-1.34-.323-1.912a4.116 4.116 0 0 0-.918-1.462 4.097 4.097 0 0 0-1.437-.936c-.56-.22-1.187-.331-1.878-.331-.691 0-1.315.11-1.87.331a4.114 4.114 0 0 0-1.428.935c-.397.403-.703.89-.918 1.463a5.395 5.395 0 0 0-.323 1.912zm14.846 2.805c-.385 0-.714-.08-.986-.238a1.91 1.91 0 0 1-.654-.62 2.68 2.68 0 0 1-.357-.893 4.883 4.883 0 0 1-.111-1.037c0-.363.034-.714.102-1.054.068-.34.184-.64.349-.901.164-.26.38-.473.645-.638.267-.164.598-.246.995-.246.385 0 .711.082.977.246.267.165.485.38.655.647.17.266.292.569.365.909.074.34.111.686.111 1.037 0 .351-.034.697-.102 1.037-.068.34-.184.637-.349.892-.164.256-.38.462-.645.621-.267.159-.598.238-.995.238zm-4.437-7.208v11.866h2.414v-4.165h.034c.295.43.671.756 1.13.977.46.222.961.332 1.505.332.646 0 1.21-.125 1.691-.374a3.58 3.58 0 0 0 1.207-1.003c.324-.42.564-.901.723-1.445a6.04 6.04 0 0 0 .238-1.7c0-.623-.08-1.221-.238-1.793a4.38 4.38 0 0 0-.731-1.505 3.685 3.685 0 0 0-1.241-1.037c-.499-.26-1.094-.391-1.785-.391-.544 0-1.043.108-1.496.323-.453.215-.827.561-1.122 1.037h-.034v-1.122h-2.295zm16.303 3.451h-3.927a2.77 2.77 0 0 1 .11-.578 1.775 1.775 0 0 1 .936-1.097c.255-.13.575-.195.96-.195.59 0 1.028.159 1.318.476.289.317.49.782.603 1.394zm-3.927 1.53h6.341a6.205 6.205 0 0 0-.17-1.955 4.785 4.785 0 0 0-.773-1.666 3.902 3.902 0 0 0-1.369-1.164c-.555-.29-1.207-.434-1.955-.434-.669 0-1.278.119-1.828.357a4.32 4.32 0 0 0-1.419.977 4.309 4.309 0 0 0-.918 1.471 5.126 5.126 0 0 0-.323 1.836c0 .68.105 1.303.315 1.87.21.567.507 1.054.892 1.462.385.408.856.722 1.411.944.555.22 1.179.331 1.87.331.997 0 1.847-.227 2.55-.68.703-.453 1.224-1.207 1.564-2.261h-2.125c-.08.272-.295.53-.646.773-.351.244-.77.366-1.258.366-.68 0-1.201-.176-1.564-.527-.363-.351-.561-.918-.595-1.7zm7.417-2.278c.034-.567.176-1.037.425-1.411.25-.374.567-.674.952-.901a4.149 4.149 0 0 1 1.3-.484 7.394 7.394 0 0 1 1.454-.145c.442 0 .89.031 1.343.093.453.063.867.185 1.241.366.374.181.68.433.918.756.238.324.357.751.357 1.284v4.573c0 .397.023.776.068 1.139.045.363.125.635.238.816h-2.448a3.464 3.464 0 0 1-.17-.85 3.146 3.146 0 0 1-1.36.833 5.459 5.459 0 0 1-1.598.238c-.42 0-.81-.051-1.173-.153a2.686 2.686 0 0 1-.952-.476 2.216 2.216 0 0 1-.638-.816c-.153-.329-.229-.72-.229-1.173 0-.499.088-.91.264-1.233.175-.323.402-.58.68-.773.277-.193.594-.337.951-.434.358-.096.717-.172 1.08-.229.363-.057.72-.102 1.071-.136a6.44 6.44 0 0 0 .935-.153 1.63 1.63 0 0 0 .646-.298c.159-.13.232-.32.221-.569 0-.26-.042-.467-.127-.62a.938.938 0 0 0-.34-.358 1.337 1.337 0 0 0-.493-.17 4.034 4.034 0 0 0-.604-.042c-.476 0-.85.102-1.122.306-.272.204-.43.544-.476 1.02h-2.414zm5.576 1.785c-.102.09-.23.161-.383.212a3.907 3.907 0 0 1-.493.128c-.175.034-.36.062-.552.085-.193.023-.385.051-.578.085a4.36 4.36 0 0 0-.535.136 1.583 1.583 0 0 0-.46.23c-.13.096-.235.218-.314.365-.08.147-.119.334-.119.561 0 .215.04.397.119.544.08.147.187.263.323.349.136.085.295.144.476.178s.368.051.561.051c.476 0 .844-.08 1.105-.238.26-.159.453-.348.578-.57.125-.22.201-.444.23-.671.028-.227.042-.408.042-.544v-.901zm4.034-4.488V62h2.414v-4.607c0-.895.147-1.538.442-1.93.295-.39.77-.586 1.428-.586.578 0 .98.178 1.207.535.227.358.34.899.34 1.624V62h2.414v-5.406c0-.544-.048-1.04-.144-1.488a2.829 2.829 0 0 0-.502-1.139 2.357 2.357 0 0 0-.977-.73c-.414-.176-.944-.264-1.59-.264-.51 0-1.009.116-1.496.349-.487.232-.884.603-1.19 1.113h-.051v-1.224h-2.295zM60 6h20v7H60V6zM6 6h51v7H6V6z"></path><path fill="#F39200" d="M83 6h11v7H83"></path></g></symbol>
+    </use>
+    </svg>
+    </div>
+    <h2>Maxwell status</h2>
+
+    <div>
+    <dl>
+        <dt>Total jobs running</dt>
+        <dd>{{ total_jobs_running }}</dd>
+        <dt>Nodes available on general partition</dt>
+        <dd>{{ nodes_avail_general }}</dd>
+        <dt>Nodes available on general reservation</dt>
+        <dd>{{ nodes_avail_general_res }}</dd>
+        {% for res, avail in upex_reservations.items() %}
+            <dt>Nodes available on {{ res }}</dt>
+            <dd>{{ avail }}</dd>
+        {% endfor %}
+        <dt>Recommendation</dt>
+        <dd>{{ recommendation }}</dd>
+    </dl>
+<p><b>Note:</b> This is only a hint! Maxwell status can change quickly, so the feedback here
+   can rapidly become outdated.
+</p>
+</div>
+
+</div>

webservice/templates/running_jobs.html

+<div class="block">
+   <h2>Running calibration jobs</h2>
+    <dl>
+   {% for run, data in running_jobs.items() %}
+           <dt>{{ data['proposal'] }}/{{ run }}</dt>
+           <dd>
+           {% for status in data['statii'] %}
+               <span class="status, {{ status[0] }}">{{ status[1] }}</span>
+           {% endfor %}
+           </dd>
+   {% endfor %}
+        </dl>
+
+</div>

webservice/templates/serve_overview.css

+h2 {
+   font-family: monospace;
+}
+
+.block {
+   background-color: white;
+   padding: 20px;
+   text-align: left;
+   border: 1px solid black;
+   font-family: monospace;
+   width: auto;
+   margin-bottom: 20px;
+
+}
+
+dt {
+   font-weight: bold;
+   float: left;
+   min-width: 300px;
+   padding-right: 1em;
+   text-align: left;
+}
+
+dd {
+   text-align: left;
+}
+
+.log-out {
+   height: 200px;
+   overflow: auto;
+   background-color: white;
+
+}
+
+.status {
+    margin: 5px;
+    border: 1px solid black;
+}
+
+.R {
+    background-color: LightGreen;
+}
+
+.NA {
+    background-color: orange;
+}
+
+.A {
+    background-color: white;
+}
+
+.logo {
+    float: right;
+    width: 100px;
+
+}
\ No newline at end of file