From 449c5b4545634983ef124a63be75c42e0a2ea3ad Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Tue, 7 Apr 2020 14:04:07 +0200
Subject: [PATCH] Update timing summary
---
webservice/webservice.py | 4 +++-
webservice/webservice.yaml | 2 ++
xfel_calibrate/calibrate.py | 30 ++++++++++++++++++++++++----
xfel_calibrate/finalize.py | 40 ++++++++++++++++++++++++++++---------
4 files changed, 62 insertions(+), 14 deletions(-)
diff --git a/webservice/webservice.py b/webservice/webservice.py
index fdab52339..5722c8fc4 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -584,6 +584,7 @@ async def server_runner(config, mode):
socket.send(yaml.dump(err_msg, default_flow_style=False).encode())
if action in ['dark', 'correct']:
+ request_time = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
try:
wait_runs = []
if action == 'correct':
@@ -779,7 +780,8 @@ async def server_runner(config, mode):
cycle=cycle, proposal=proposal,
runs="_r".join(wait_runs),
time_stamp=ts,
- det_instance=karabo_id
+ det_instance=karabo_id,
+ request_time=request_time
).split()
cmd = await parse_config(cmd, dconfig)
diff --git a/webservice/webservice.yaml b/webservice/webservice.yaml
index 54d59f49f..b87d9b7d0 100644
--- a/webservice/webservice.yaml
+++ b/webservice/webservice.yaml
@@ -29,6 +29,7 @@ correct:
cmd : >
python -m xfel_calibrate.calibrate {detector} CORRECT
--slurm-scheduling {sched_prio}
+ --request-time {request_time}
--slurm-name {action}_{instrument}_{detector}_{cycle}_p{proposal}_r{runs}
--report-to {action}_{det_instance}_{time_stamp}
--cal-db-timeout 300000
@@ -41,6 +42,7 @@ dark:
python -m xfel_calibrate.calibrate {detector} DARK
--priority {priority}
--slurm-scheduling {sched_prio}
+ --request-time {request_time}
--slurm-name {action}_{instrument}_{detector}_{cycle}_p{proposal}_r{runs}
--report-to /gpfs/exfel/d/cal/caldb_store/xfel/reports/{detector}/{instrument}/{det_instance}/{action}/{action}_{proposal}_{runs}_{time_stamp}
--db-output
diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index d8088e260..bf653d981 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -2,6 +2,7 @@
import argparse
from datetime import datetime
+import dateutil.parser
import nbconvert
import nbformat
from nbparameterise import (
@@ -75,6 +76,9 @@ def make_initial_parser():
'+- 2147483645 (negative value increases '
'priority)')
+ parser.add_argument('--request-time', type=str, default='Now',
+ help='Time of request to process notebook. Iso format')
+
parser.add_argument_group('required arguments')
parser.add_argument('--reservation', type=str, default="")
@@ -523,9 +527,18 @@ def create_finalize_script(fmt_args, temp_path, job_list):
module load texlive
echo 'Running finalize script'
python3 -c "from xfel_calibrate.finalize import finalize;
- finalize({{joblist}}, $1, '{{run_path}}', '{{out_path}}',
- '{{project}}', '{{calibration}}', '{{author}}',
- '{{version}}', '{{report_to}}', '{{in_folder}}' )"
+ finalize(joblist={{joblist}},
+ finaljob=$1,
+ run_path='{{run_path}}',
+ out_path='{{out_path}}',
+ project='{{project}}',
+ calibration='{{calibration}}',
+ author='{{author}}',
+ version='{{version}}',
+ report_to='{{report_to}}',
+ data_path='{{in_folder}}',
+ request_time='{{request_time}}',
+ submission_time='{{submission_time}}')"
''')
@@ -858,6 +871,12 @@ def run():
folder = get_par_attr(parms, 'in_folder', 'value', '')
+ if args["request_time"] == "Now":
+ request_time = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
+ else:
+ request_time = args["request_time"]
+
+ submission_time = datetime.now().strftime('%Y-%m-%dT%H:%M:%S')
fmt_args = {'run_path': run_tmp_path,
'out_path': out_path,
'project': title,
@@ -865,7 +884,10 @@ def run():
'author': author,
'version': version,
'report_to': report_to,
- 'in_folder': folder}
+ 'in_folder': folder,
+ 'request_time': request_time,
+ 'submission_time': submission_time
+ }
joblist = []
if concurrency.get("parameter", None) is None:
diff --git a/xfel_calibrate/finalize.py b/xfel_calibrate/finalize.py
index c6573b036..0a54fd134 100644
--- a/xfel_calibrate/finalize.py
+++ b/xfel_calibrate/finalize.py
@@ -1,3 +1,4 @@
+from datetime import datetime
from glob import glob
from importlib.machinery import SourceFileLoader
from os import chdir, listdir, makedirs, path, remove, stat
@@ -122,21 +123,23 @@ def prepare_plots(run_path, threshold=1000000):
shell=False)
-def make_timing_summary(run_path, joblist):
+def make_timing_summary(run_path, joblist, request_time, submission_time):
"""
- Create an rst file with timing summary of executed slurm jobs
+ Create an rst file with timing summary of executed notebooks
:param run_path: Run path of the slurm job
:param joblist: List of slurm jobs
+ :param request_time: Timestamp of notebook request
+ :param submission_time: Timestamp for slurm jobs being submitted
"""
print('Prepare timing summary')
run_path = path.abspath(run_path)
pars_vals = []
- pars = 'JobID,Elapsed,Suspended'
+ pars = 'JobID,Start,End,Elapsed,Suspended,State'
pars_name = pars.split(',')
for job in joblist:
- out = check_output(['sacct', '-j', job,
+ out = check_output(['sacct', '-T', '-j', job,
'--format={}'.format(pars)],
shell=False)
lines = str(out).split('\\n')
@@ -153,17 +156,34 @@ def make_timing_summary(run_path, joblist):
==============
.. math::
- {% for line in table %}
+ {% for line in time_table %}
{{ line }}
{%- endfor %}
+
+ .. math::
+ {% for line in job_table %}
+ {{ line }}
+ {%- endfor %}
+
''')
+ time_vals = [['Time of Request', request_time],
+ ['Job submission', submission_time],
+ ['Report compilation',
+ datetime.now().strftime('%Y-%m-%dT%H:%M:%S')]]
+
with open("{}/timing_summary.rst".format(run_path), "w+") as gfile:
if len(pars_vals) > 0:
- table = tabulate.tabulate(pars_vals, tablefmt='latex',
+ job_table = tabulate.tabulate(pars_vals, tablefmt='latex',
headers=pars_name)
- gfile.write(dedent(tmpl.render(table=table.split('\n'))))
+
+ time_table = tabulate.tabulate(time_vals, tablefmt='latex',
+ headers=['Processing step',
+ 'Timestamp'])
+
+ gfile.write(dedent(tmpl.render(job_table=job_table.split('\n'),
+ time_table=time_table.split('\n'))))
def make_report(run_path, tmp_path, out_path, project, author, version,
@@ -348,7 +368,8 @@ def tex_escape(text):
def finalize(joblist, finaljob, run_path, out_path, project, calibration,
- author, version, report_to, data_path='Unknown'):
+ author, version, report_to, data_path='Unknown',
+ request_time='', submission_time=''):
print("Waiting on jobs to finish: {}".format(joblist))
while True:
found_jobs = set()
@@ -362,7 +383,8 @@ def finalize(joblist, finaljob, run_path, out_path, project, calibration,
sleep(10)
prepare_plots(run_path)
- make_timing_summary(run_path, joblist + [str(finaljob)])
+ make_timing_summary(run_path, joblist + [str(finaljob)], request_time,
+ submission_time)
sphinx_path = combine_report(run_path, calibration)
make_titlepage(sphinx_path, project, data_path, version)
make_report(sphinx_path, run_path, out_path, project, author, version,
--
GitLab