From 0a92c1c119c21c80dc0e519093fe14ffb35edff9 Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Tue, 21 May 2019 13:54:40 +0200
Subject: [PATCH] Add timing table for all slurm jobs in report
---
bin/slurm_calibrate.sh | 9 ++++-
cal_tools/cal_tools/tools.py | 69 ++++++++++++++++++++++++++++++++----
xfel_calibrate/calibrate.py | 2 +-
3 files changed, 72 insertions(+), 8 deletions(-)
diff --git a/bin/slurm_calibrate.sh b/bin/slurm_calibrate.sh
index 8a7f0cc80..c447f639f 100755
--- a/bin/slurm_calibrate.sh
+++ b/bin/slurm_calibrate.sh
@@ -23,6 +23,13 @@ echo "Jupyter path: $jupyter_path"
echo "IP-Cluster path: $ipcluster_path"
echo "Karabo activate: $krb_activate_path"
echo "IP-Cluster profile: $uuid"
+echo "notebook: $notebook"
+echo "detector: $detector"
+echo "caltype: $caltype"
+echo "final: $final"
+echo "finalize: $finalize"
+echo "cluster_cores: $cluster_cores"
+echo "job ID: $SLURM_JOB_ID"
module load anaconda/3
@@ -63,6 +70,6 @@ fi
if [ "${final}" == "FINAL" ]
then
- ${finalize}
+ ${finalize} $SLURM_JOB_ID
fi
killall -9 cal_influx_feeder.sh || true
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index 2b1e99b10..a9de20072 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -100,6 +100,61 @@ def prepare_plots(run_path, threshold=1000000):
shell=False)
+def make_timing_summary(run_path, joblist):
+ """
+ Create an rst file with timing summary of executed slurm jobs
+
+ :param run_path: Run path of the slurm job
+ :param joblist: List of slurm jobs
+ """
+ run_path = os.path.abspath(run_path)
+ pars_vals = []
+ pars = 'JobID,Elapsed,Suspended'
+ pars_name = pars.split(',')
+
+ for job in joblist:
+ print('Summary for job {}'.format(job))
+ out = check_output(['sacct', '-j', job,
+ '--format={}'.format(pars)],
+ shell=False)
+ l = str(out).split('\\n')
+ print (l)
+ for line in l[2:]:
+ s = line.split()
+ if len(s)==len(pars_name):
+ pars_vals.append(s)
+ print(s)
+ break
+
+ with open("{}/timing_summary.rst".format(run_path), "w+") as gfile:
+ gfile.write('Timing summary\n')
+ gfile.write('==============\n\n')
+
+ gfile.write('.. math::\n')
+ gfile.write(' \\begin{tabular}{ccc}\n')
+ gfile.write(' \\hline\n')
+ gfile.write(' ')
+ for i in range(len(pars_name)):
+ gfile.write(pars_name[i])
+ if i<len(pars_name)-1:
+ gfile.write(' & ')
+ else:
+ gfile.write(' \\\\ \n')
+ gfile.write(' \\hline\n')
+
+ for pars_val in pars_vals:
+ gfile.write(' ')
+ for i in range(len(pars_name)):
+ gfile.write(pars_val[i])
+ if i < len(pars_name) - 1:
+ gfile.write(' & ')
+ else:
+ gfile.write(' \\\\ \n')
+
+ gfile.write(' \\hline\n')
+ gfile.write(' \\end{tabular}\n')
+
+
def make_report(run_path, tmp_path, out_path, project, author, version, report_to):
run_path = os.path.abspath(run_path)
report_path, report_name = os.path.split(report_to)
@@ -196,8 +251,8 @@ def make_report(run_path, tmp_path, out_path, project, author, version, report_t
shutil.rmtree(tmp_path)
-def finalize(joblist, run_path, out_path, project, calibration, author, version, report_to):
-
+def finalize(joblist, finaljob, run_path, out_path, project, calibration,
+ author, version, report_to):
print("Waiting on jobs to finish: {}".format(joblist))
while True:
@@ -210,12 +265,14 @@ def finalize(joblist, run_path, out_path, project, calibration, author, version,
if len(found_jobs) == 0:
break
sleep(10)
+
prepare_plots(run_path)
+ make_timing_summary(run_path, joblist+[str(finaljob)])
sphinx_path = combine_report(run_path, calibration)
- make_report(sphinx_path, run_path, out_path, project, author, version, report_to)
-
-
-
+ make_report(sphinx_path, run_path, out_path, project, author, version,
+ report_to)
+
+
def parse_runs(runs, return_type=str):
pruns = []
if isinstance(runs, str):
diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index a56db134f..9578b33ee 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -638,7 +638,7 @@ def run():
else:
os.makedirs(out_path)
cmd = ('"from cal_tools.tools import finalize; ' +
- 'finalize({{joblist}}, \'{run_path}\', \'{out_path}\', ' +
+ 'finalize({{joblist}}, $1, \'{run_path}\', \'{out_path}\', ' +
'\'{project}\', \'{calibration}\', \'{author}\', '
'\'{version}\', \'{report_to}\')"')
--
GitLab