diff --git a/webservice/serve_overview.py b/webservice/serve_overview.py
index 3863271e454400c8dad8329e6193a4a5caf7e11d..41b9396891da9300c11e20c7968a4f6423efd8ce 100644
--- a/webservice/serve_overview.py
+++ b/webservice/serve_overview.py
@@ -49,7 +49,6 @@ class RequestHandler(BaseHTTPRequestHandler):
self.upex_prefix = config["shell-commands"]["upex-prefix"]
self.tail_log_cmd = config["shell-commands"]["tail-log"]
self.cat_log_cmd = config["shell-commands"]["cat-log"]
- self.mappings = config["mappings"]
self.run_candidates = config["run-candidates"]
self.templates = {}
@@ -164,11 +163,8 @@ class RequestHandler(BaseHTTPRequestHandler):
for instrument, detectors in cal_config['dark'].items():
reports[instrument] = {}
for detector in detectors:
- det_inset = detector.replace('-', '_')
- tmpl = '/gpfs/exfel/exp/{}/*/*/usr/dark/*/{}'.format(
- instrument, det_inset)
- files = glob.glob(tmpl + '/*pdf')
- files += glob.glob(tmpl + '/*/*pdf')
+ tmpl = f'/gpfs/exfel/d/cal_tst/caldb_store/xfel/reports/{instrument}/{detector}/dark/*pdf'
+ files = glob.glob(tmpl)
files.sort(key=os.path.getmtime, reverse=True)
file_info = []
for i, file in enumerate(files):
@@ -289,28 +285,22 @@ class RequestHandler(BaseHTTPRequestHandler):
:param l: Line to parse
:param key: A key work: DARK or CORRECT
:return: Detector name, Instrument name, input folder,
- output folder, list of runs, time of request
+ output folder, path to report, list of runs, time of request
"""
if key in l:
ls = l.split()
- if key not in ls:
+ # karabo-id and report-to are expected to be
+ # in the log file
+ if any(x not in ls for x in
+ [key, '--karabo-id', '--report-to']):
return None
+
dclass = ls[ls.index(key) - 1]
in_folder = ls[ls.index("--in-folder") + 1]
+ report_to = ls[ls.index("--report-to") + 1]
out_folder = ls[ls.index("--out-folder") + 1]
-
- if "--db-module" in ls:
- detector = ls[ls.index("--db-module") + 1]
- else:
- detector = dclass
-
- if "--instrument" in ls:
- instrument = ls[ls.index("--instrument") + 1]
- else:
- if detector == "PNCCD":
- instrument = "SQS"
- else:
- instrument = in_folder.split('/')[4]
+ detector = ls[ls.index("--karabo-id") + 1]
+ instrument = in_folder.split('/')[4]
runs = []
for rc in self.run_candidates:
@@ -320,28 +310,29 @@ class RequestHandler(BaseHTTPRequestHandler):
requested = "{} {}".format(ls[0], ls[1])
return [dclass, detector, instrument, in_folder, out_folder,
- runs, requested]
+ report_to, runs, requested]
last_chars = {}
last_calib = {}
host = config["server-config"]["host"]
port = config["server-config"]["port"]
for l in last_n_lines:
-
+ # Extract dark requests
info = get_run_info(l, 'DARK')
if info is not None:
- dclass, detector, instrument, in_folder, out_folder, runs, requested = info # noqa
+ dclass, detector, instrument, in_folder, out_folder, report_to, runs, requested = info # noqa
- if f"{instrument}-{detector}" in last_chars:
+ # Check if instrument is in detector name
+ # This is not the case for e.g. CALLAB
+ key = detector if instrument in detector else f"{instrument}-{detector}" # noqa
+ if key in last_chars:
continue
- pdfs = glob.glob(f"{out_folder}/*.pdf")
- pdfs += glob.glob(f"{out_folder}/*/*.pdf")
+ pdfs = glob.glob(f"{report_to}*.pdf")
pdfs.sort()
pdfs = {p.split("/")[-1]: p for p in pdfs}
fpdfs = []
if len(pdfs):
- # pdfs = ", ".join(pdfs)
for pdf, p in pdfs.items():
fpdfs.append(
(pdf, f"http://{host}:{port}/{p}"))
@@ -349,15 +340,15 @@ class RequestHandler(BaseHTTPRequestHandler):
tsize = 0
for run in runs:
run = int(run)
- if dclass not in self.mappings:
+ if detector not in cal_config['data-mapping']:
continue
- for mp in self.mappings[dclass]:
+ # ToDo calculate tsize based on selected karabo-da
+ for mp in cal_config['data-mapping'][detector]['karabo-da']:
for f in glob.glob(
f"{in_folder}/r{run:04d}/*{mp}*.h5"):
tsize += os.stat(f).st_size
- last_injected = ""
constant_valid_from = ""
- key = f"{instrument}-{detector}"
+
last_chars[key] = {"in_path": in_folder,
"out_path": out_folder,
"runs": runs,
@@ -367,16 +358,17 @@ class RequestHandler(BaseHTTPRequestHandler):
"device_type": detector,
"last_valid_from": constant_valid_from}
+ # Extract correction requests
info = get_run_info(l, 'CORRECT')
if info is not None:
- _, _, _, in_folder, out_folder, runs, requested = info
+ _, _, _, in_folder, _, report_to, runs, requested = info
instrument = in_folder.split('/')[4]
if instrument not in last_calib:
last_calib[instrument] = []
if len(last_calib[instrument]) > config["server-config"]["n-calib"]: # noqa
continue
proposal = in_folder.split('/')[6]
- pdfs = glob.glob(f"{out_folder}/*.pdf")
+ pdfs = glob.glob(f"{report_to}*.pdf")
pdfs.sort(key=os.path.getmtime, reverse=True)
pdfs = {p.split("/")[-1]: p for p in pdfs}
diff --git a/webservice/serve_overview.yaml b/webservice/serve_overview.yaml
index e0ba7cb53e068db29be1d21191784de15f6935b7..14948daae12171a587e67b5643df5694393c5b95 100644
--- a/webservice/serve_overview.yaml
+++ b/webservice/serve_overview.yaml
@@ -18,27 +18,6 @@ shell-commands:
tail-log: "tail -5000 web.log"
cat-log: "cat web.log"
-mappings:
- LPD:
- - LPD
- AGIPD:
- - AGIPD
- DSSC:
- - DSSC
- PNCCD:
- - PNCCD
- JUNGFRAU:
- - JNGFR
- - DA05
- - DA04
- EPIX:
- - EPIX
- EPIX10K:
- - EPIX10K
- FASTCCD:
- - DA05
- - DA01
-
run-candidates:
- "--run-high"
- "--run-med"
@@ -50,8 +29,6 @@ server-config:
host: max-exfl016
dark-timeout: 30
n-calib: 10
- cal_db_interface: tcp://max-exfl016:8022
- cal_db_timeout: 30000
scrpts-cnf:
req-dark: request_darks.py
diff --git a/webservice/templates/last_characterizations.html b/webservice/templates/last_characterizations.html
index 1eefe10fe21605426ccc890c57d04c6602caed2f..1a9b893740aabbe1a7ffe80f26668a9e9276e18e 100644
--- a/webservice/templates/last_characterizations.html
+++ b/webservice/templates/last_characterizations.html
@@ -8,7 +8,7 @@
<dt>Output path:</dt><dd>{{ data['out_path'] }}</dd>
<dt>Input path:</dt><dd>{{ data['in_path'] }}</dd>
<dt>Input runs:</dt><dd>{{ data['runs'] }}</dd>
- <dt>Input data size:</dt><dd>{{ data['size'] }}</dd>
+ <dt>Total data size:</dt><dd>{{ data['size'] }}</dd>
<dt>Output PDFs:</dt><dd>
{% for pdf in data['pdfs'] %}
<a href={{ pdf[1] }} target="_blank">{{ pdf[0] }}</a>