Merge branch 'Fix/serve_overview' into 'master'

Fix: Adjust serve_overview for chnages in the webservice

See merge request detectors/pycalibration!302

webservice/serve_overview.py

@@ -49,7 +49,6 @@ class RequestHandler(BaseHTTPRequestHandler):
         self.upex_prefix = config["shell-commands"]["upex-prefix"]
         self.tail_log_cmd = config["shell-commands"]["tail-log"]
         self.cat_log_cmd = config["shell-commands"]["cat-log"]
-        self.mappings = config["mappings"]
         self.run_candidates = config["run-candidates"]
 
         self.templates = {}
@@ -164,11 +163,8 @@ class RequestHandler(BaseHTTPRequestHandler):
             for instrument, detectors in cal_config['dark'].items():
                 reports[instrument] = {}
                 for detector in detectors:
-                    det_inset = detector.replace('-', '_')
-                    tmpl = '/gpfs/exfel/exp/{}/*/*/usr/dark/*/{}'.format(
-                        instrument, det_inset)
-                    files = glob.glob(tmpl + '/*pdf')
-                    files += glob.glob(tmpl + '/*/*pdf')
+                    tmpl = f'/gpfs/exfel/d/cal_tst/caldb_store/xfel/reports/{instrument}/{detector}/dark/*pdf'
+                    files = glob.glob(tmpl)
                     files.sort(key=os.path.getmtime, reverse=True)
                     file_info = []
                     for i, file in enumerate(files):
@@ -289,28 +285,22 @@ class RequestHandler(BaseHTTPRequestHandler):
             :param l: Line to parse
             :param key: A key work: DARK or CORRECT
             :return: Detector name, Instrument name, input folder,
-            output folder, list of runs, time of request
+            output folder, path to report, list of runs, time of request
             """
             if key in l:
                 ls = l.split()
-                if key not in ls:
+                # karabo-id and report-to are expected to be
+                # in the log file
+                if any(x not in ls for x in
+                       [key, '--karabo-id', '--report-to']):
                     return None
+
                 dclass = ls[ls.index(key) - 1]
                 in_folder = ls[ls.index("--in-folder") + 1]
+                report_to = ls[ls.index("--report-to") + 1]
                 out_folder = ls[ls.index("--out-folder") + 1]
-
-                if "--db-module" in ls:
-                    detector = ls[ls.index("--db-module") + 1]
-                else:
-                    detector = dclass
-
-                if "--instrument" in ls:
-                    instrument = ls[ls.index("--instrument") + 1]
-                else:
-                    if detector == "PNCCD":
-                        instrument = "SQS"
-                    else:
-                        instrument = in_folder.split('/')[4]
+                detector = ls[ls.index("--karabo-id") + 1]
+                instrument = in_folder.split('/')[4]
 
                 runs = []
                 for rc in self.run_candidates:
@@ -320,28 +310,29 @@ class RequestHandler(BaseHTTPRequestHandler):
                 requested = "{} {}".format(ls[0], ls[1])
 
                 return [dclass, detector, instrument, in_folder, out_folder,
-                        runs, requested]
+                        report_to, runs, requested]
 
         last_chars = {}
         last_calib = {}
         host = config["server-config"]["host"]
         port = config["server-config"]["port"]
         for l in last_n_lines:
-
+            # Extract dark requests
             info = get_run_info(l, 'DARK')
             if info is not None:
-                dclass, detector, instrument, in_folder, out_folder, runs, requested = info  # noqa
+                dclass, detector, instrument, in_folder, out_folder, report_to, runs, requested = info  # noqa
 
-                if f"{instrument}-{detector}" in last_chars:
+                # Check if instrument is in detector name
+                # This is not the case for e.g. CALLAB
+                key = detector if instrument in detector else f"{instrument}-{detector}"  # noqa
+                if key in last_chars:
                     continue
 
-                pdfs = glob.glob(f"{out_folder}/*.pdf")
-                pdfs += glob.glob(f"{out_folder}/*/*.pdf")
+                pdfs = glob.glob(f"{report_to}*.pdf")
                 pdfs.sort()
                 pdfs = {p.split("/")[-1]: p for p in pdfs}
                 fpdfs = []
                 if len(pdfs):
-                    # pdfs = ", ".join(pdfs)
                     for pdf, p in pdfs.items():
                         fpdfs.append(
                             (pdf, f"http://{host}:{port}/{p}"))
@@ -349,15 +340,15 @@ class RequestHandler(BaseHTTPRequestHandler):
                 tsize = 0
                 for run in runs:
                     run = int(run)
-                    if dclass not in self.mappings:
+                    if detector not in cal_config['data-mapping']:
                         continue
-                    for mp in self.mappings[dclass]:
+                    # ToDo calculate tsize based on selected karabo-da
+                    for mp in cal_config['data-mapping'][detector]['karabo-da']:
                         for f in glob.glob(
                                 f"{in_folder}/r{run:04d}/*{mp}*.h5"):
                             tsize += os.stat(f).st_size
-                last_injected = ""
                 constant_valid_from = ""
-                key = f"{instrument}-{detector}"
+
                 last_chars[key] = {"in_path": in_folder,
                                    "out_path": out_folder,
                                    "runs": runs,
@@ -367,16 +358,17 @@ class RequestHandler(BaseHTTPRequestHandler):
                                    "device_type": detector,
                                    "last_valid_from": constant_valid_from}
 
+            # Extract correction requests
             info = get_run_info(l, 'CORRECT')
             if info is not None:
-                _, _, _, in_folder, out_folder, runs, requested = info
+                _, _, _, in_folder, _, report_to, runs, requested = info
                 instrument = in_folder.split('/')[4]
                 if instrument not in last_calib:
                     last_calib[instrument] = []
                 if len(last_calib[instrument]) > config["server-config"]["n-calib"]:   # noqa
                     continue
                 proposal = in_folder.split('/')[6]
-                pdfs = glob.glob(f"{out_folder}/*.pdf")
+                pdfs = glob.glob(f"{report_to}*.pdf")
                 pdfs.sort(key=os.path.getmtime, reverse=True)
                 pdfs = {p.split("/")[-1]: p for p in pdfs}
 

webservice/serve_overview.yaml

@@ -18,27 +18,6 @@ shell-commands:
   tail-log: "tail -5000 web.log"
   cat-log: "cat web.log"
 
-mappings:
-  LPD: 
-    - LPD
-  AGIPD: 
-    - AGIPD
-  DSSC: 
-    - DSSC
-  PNCCD: 
-    - PNCCD
-  JUNGFRAU: 
-    - JNGFR 
-    - DA05 
-    - DA04
-  EPIX: 
-    - EPIX
-  EPIX10K: 
-    - EPIX10K
-  FASTCCD: 
-    - DA05
-    - DA01
-
 run-candidates:
     - "--run-high" 
     - "--run-med"
@@ -50,8 +29,6 @@ server-config:
     host: max-exfl016
     dark-timeout: 30
     n-calib: 10
-    cal_db_interface: tcp://max-exfl016:8022
-    cal_db_timeout: 30000
 
 scrpts-cnf:
     req-dark: request_darks.py

webservice/templates/last_characterizations.html

@@ -8,7 +8,7 @@
            <dt>Output path:</dt><dd>{{ data['out_path'] }}</dd>
            <dt>Input path:</dt><dd>{{ data['in_path'] }}</dd>
            <dt>Input runs:</dt><dd>{{ data['runs'] }}</dd>
-           <dt>Input data size:</dt><dd>{{ data['size'] }}</dd>
+           <dt>Total data size:</dt><dd>{{ data['size'] }}</dd>
            <dt>Output PDFs:</dt><dd>
            {% for pdf in data['pdfs'] %}
                <a href={{ pdf[1] }} target="_blank">{{ pdf[0] }}</a>