From a4e1e5250a9c8741e94d493017c8f92e2f7dff73 Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Tue, 18 Jun 2019 11:21:21 +0200
Subject: [PATCH] Use template. Optimise imports.
---
cal_tools/cal_tools/tools.py | 132 ++++++++++++++++++-----------------
xfel_calibrate/settings.py | 3 +
2 files changed, 70 insertions(+), 65 deletions(-)
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index 70c7e3b8f..ecad64db2 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -1,19 +1,16 @@
-import argparse
-from collections import OrderedDict
-import copy
-import glob
-from jinja2 import Template
import os
from os.path import isfile, isdir, splitext
+from time import sleep
+from subprocess import check_output, check_call, CalledProcessError
+from shutil import copy, move, rmtree, copytree
from queue import Queue
-import re
-import shutil
-from subprocess import Popen, PIPE, check_output, check_call
+from collections import OrderedDict
+import glob
+from jinja2 import Template
import textwrap
-from time import sleep
-import time
-from uuid import uuid4
-
+import tabulate
+from importlib.machinery import SourceFileLoader
+from xfel_calibrate import settings
def atoi(text):
'''
@@ -72,7 +69,7 @@ def combine_report(run_path, calibration):
gfile.write("\n\n")
if isdir("{}/{}".format(run_path, entry)):
- shutil.copytree("{}/{}".format(run_path, entry), "{}/{}".format(sphinx_path, entry))
+ copytree("{}/{}".format(run_path, entry), "{}/{}".format(sphinx_path, entry))
return sphinx_path
@@ -154,7 +151,7 @@ def make_timing_summary(run_path, joblist):
{{ line }}
{%- endfor %}
''')
- import tabulate
+
with open("{}/timing_summary.rst".format(run_path), "w+") as gfile:
table = tabulate.tabulate(pars_vals, tablefmt='latex',
@@ -162,36 +159,48 @@ def make_timing_summary(run_path, joblist):
gfile.write(textwrap.dedent(tmpl.render(table=table.split('\n'))))
-def make_report(run_path, tmp_path, out_path, project, author, version, report_to):
+def make_report(run_path, tmp_path, out_path, project, author, version,
+ report_to):
+ """
+ Create calibration report (pdf file)
+
+ Automatically generated report document results, produced by
+ jupyter-notebooks.
+
+ :param run_path: Path to sphinx run directory
+ :param tmp_path: Run path of the slurm job
+ :param out_path: Output directory for report.
+ Overwritten if path to report is given in `report_to`
+ :param project: Project title
+ :param author: Author of the notebook
+ :param version: Version of the notebook
+ :param report_to: Name or path of the report file
+ """
run_path = os.path.abspath(run_path)
report_path, report_name = os.path.split(report_to)
if report_path != '':
out_path = report_path
try:
- import subprocess
- subprocess.check_call(["sphinx-quickstart",
- "--quiet",
- "--project='{}'".format(project),
- "--author='{}'".format(author),
- "-v", str(version),
- "--suffix=.rst",
- "--master=index",
- "--ext-intersphinx",
- "--ext-mathjax",
- "--makefile",
- "--no-batchfile", run_path])
-
- except subprocess.CalledProcessError:
+ check_call(["sphinx-quickstart",
+ "--quiet",
+ "--project='{}'".format(project),
+ "--author='{}'".format(author),
+ "-v", str(version),
+ "--suffix=.rst",
+ "--master=index",
+ "--ext-intersphinx",
+ "--ext-mathjax",
+ "--makefile",
+ "--no-batchfile", run_path])
+
+ except CalledProcessError:
raise Exception("Failed to run sphinx-quickbuild. Is sphinx installed?"
"Generated simple index.rst instead")
# quickbuild went well we need to edit the index.rst and conf.py files
- from shutil import move
-
module_path = "{}".format(os.path.abspath(os.path.dirname(__file__)))
- from importlib.machinery import SourceFileLoader
conf = SourceFileLoader("conf",
"{}/conf.py".format(run_path)).load_module()
l_var = [v for v in dir(conf) if not v.startswith('__')]
@@ -200,7 +209,8 @@ def make_report(run_path, tmp_path, out_path, project, author, version, report_t
latex_elements = {'extraclassoptions': ',openany, oneside',
'maketitle': r'\input{titlepage.tex.txt}'}
mf.write("latex_elements = {}\n".format(latex_elements))
- mf.write("latex_logo = '{}/xfel.pdf'\n".format(module_path))
+ mf.write("latex_logo = '{}/{}'\n".format(module_path,
+ settings.logo_path))
mf.write("latex_additional_files = ['titlepage.tex.txt']\n")
for var in l_var:
@@ -208,12 +218,13 @@ def make_report(run_path, tmp_path, out_path, project, author, version, report_t
'latex_additional_files']:
continue
tmpl = '{} = {}\n'
- if isinstance(getattr(conf, var, "None"), str):
+ v = getattr(conf, var, None)
+ if isinstance(v, str):
+ print(var, v)
tmpl = '{} = "{}"\n'
- mf.write(tmpl.format(var, getattr(conf, var, "None")))
+ mf.write(tmpl.format(var, v))
- _ = os.getcwd()
os.remove("{}/conf.py".format(run_path))
move("{}/conf.py.tmp".format(run_path), "{}/conf.py".format(run_path))
@@ -221,7 +232,7 @@ def make_report(run_path, tmp_path, out_path, project, author, version, report_t
files_to_handle = []
for entry in direntries:
if isfile("{}/{}".format(run_path, entry)):
- name, ext = splitext("{}".format(entry))
+ name, ext = splitext("{}".format(entry))
if ext == ".rst" and "index" not in name:
files_to_handle.append(name.strip())
@@ -242,45 +253,36 @@ def make_report(run_path, tmp_path, out_path, project, author, version, report_t
# finally call the make scripts
os.chdir(run_path)
try:
- import subprocess
- subprocess.check_call(["make", "latexpdf"])
+ check_call(["make", "latexpdf"])
- except subprocess.CalledProcessError:
+ except CalledProcessError:
print("Failed to make pdf documentation")
- print("Temp files will not be deleted and "+
+ print("Temp files will not be deleted and " +
"can be inspected at: {}".format(run_path))
return
print("Moving report to final location: {}".format(out_path))
for f in glob.glob(r'{}/_build/latex/*.pdf'.format(run_path)):
print(f)
- shutil.copy(f, out_path)
+ copy(f, out_path)
print("Removing temporary files at: {}".format(tmp_path))
- shutil.rmtree(tmp_path)
+ rmtree(tmp_path)
def make_titlepage(sphinx_path, project, data_path):
- with open("{}/titlepage.tex.txt".format(sphinx_path), "w+") as gfile:
- gfile.write('\\begin{titlepage}\n')
- gfile.write(' \\centering\n')
- gfile.write(' \\vspace*{40mm}\n')
- gfile.write(' \\textbf{\\Huge {')
- gfile.write(project)
- gfile.write('}}\n')
- gfile.write(' \\vspace{0mm}\n\n')
- gfile.write(' \\begin{figure}[!h]\n')
- gfile.write(' \\centering\n')
- gfile.write(' \\includegraphics[scale=0.3]{xfel.pdf}\n')
- gfile.write(' \\end{figure}\n\n')
- gfile.write(' \\vspace{0mm}\n')
- gfile.write(' \\Large \\textbf{{Detector group}}\n\n')
- gfile.write(' Based on data sample: {}\n\n'.format(data_path))
- gfile.write(' Release : 0.1\n\n')
- gfile.write(' \\vspace*{0mm}\n')
- gfile.write(' \\today\n')
- gfile.write(' \\vfill\n')
- gfile.write(' \\textit{For internal use only}\n')
- gfile.write('\\end{titlepage}\n')
- gfile.write("\n\n")
+ """
+ Create title page for report using template
+
+ :param sphinx_path: path to sphinx run directory
+ :param project: title of the project
+ :param data_path: path to input data sample used for notebook
+ """
+ module_path = "{}".format(os.path.abspath(os.path.dirname(__file__)))
+ with open('{}/titlepage.tmpl'.format(module_path)) as file_:
+ title_tmp = Template(file_.read())
+
+ with open("{}/titlepage.tex.txt".format(sphinx_path), "w+") as mf:
+ mf.write(textwrap.dedent(title_tmp.render(project=project,
+ data_path=data_path)))
def finalize(joblist, finaljob, run_path, out_path, project, calibration,
diff --git a/xfel_calibrate/settings.py b/xfel_calibrate/settings.py
index 81fdf3c63..892412e80 100644
--- a/xfel_calibrate/settings.py
+++ b/xfel_calibrate/settings.py
@@ -24,6 +24,9 @@ report_path = "{}/calibration_reports/".format(os.getcwd())
# Also try to output the report to an out_folder defined by the notebook
try_report_to_output = True
+# logo file for calibration report (located in cal_tools/cal_tools)
+logo_path = "xfel.pdf"
+
# the command to run this concurrently. It is prepended to the actual call
sprof = os.environ.get("XFELCALSLURM", "exfel")
launcher_command = "sbatch -t 24:00:00 --mem 500G --requeue --output {temp_path}/slurm-%j.out"
--
GitLab