Merge branch 'feat/vector_graphics' into 'master'

Feat: vector graphics in the calibration report

See merge request detectors/pycalibration!70

cal_tools/cal_tools/tools.py

@@ -8,7 +8,7 @@ from os.path import isfile, isdir, splitext
 from queue import Queue
 import re
 import shutil
-from subprocess import Popen, PIPE, check_output
+from subprocess import Popen, PIPE, check_output, check_call
 import textwrap
 from time import sleep
 import time
@@ -56,6 +56,50 @@ def combine_report(run_path, calibration):
             shutil.copytree("{}/{}".format(run_path, entry), "{}/{}".format(sphinx_path, entry))
     return sphinx_path
 
+
+def prepare_plots(run_path, threshold=1000000):
+    """
+    Convert svg file to pdf or png to be used for latex
+
+    Conversion of svg to vector graphics pdf is performed using svglib3.
+    This procedure is CPU consuming. In order to speed up process
+    large svg files are converted to png format.
+
+    The links in the rst files are adapted accordingly to the
+    converted image files.
+
+    :param run_path: Run path of the slurm job
+    :param threshold: Max svg file size (in bytes) to be converted to pdf
+    """
+    print('Convert svg to pdf and png')
+    run_path = os.path.abspath(run_path)
+
+    rst_files = glob.glob('{}/*rst'.format(run_path))
+    for rst_file in rst_files:
+        rst_file_name = os.path.basename(rst_file)
+        rst_file_name = os.path.splitext(rst_file_name)[0]
+
+        svg_files = glob.glob(
+            '{}/{}_files/*svg'.format(run_path, rst_file_name))
+        for f_path in svg_files:
+            f_name = os.path.basename(f_path)
+            f_name = os.path.splitext(f_name)[0]
+
+            if (os.stat(f_path)).st_size < threshold:
+                check_call(["svg2pdf", "{}".format(f_path)], shell=False)
+                new_ext = 'pdf'
+            else:
+                check_call(["convert", "{}".format(f_path),
+                            "{}.png".format(f_name)], shell=False)
+                new_ext = 'png'
+
+            check_call(["sed",
+                        "-i",
+                        "s/{}.svg/{}.{}/g".format(f_name, f_name, new_ext),
+                        rst_file],
+                       shell=False)
+
+
 def make_report(run_path, tmp_path, out_path, project, author, version, report_to):
     run_path = os.path.abspath(run_path)
     report_path, report_name = os.path.split(report_to)
@@ -166,6 +210,7 @@ def finalize(joblist, run_path, out_path, project, calibration, author, version,
         if len(found_jobs) == 0:
             break
         sleep(10)
+    prepare_plots(run_path)
     sphinx_path = combine_report(run_path, calibration)
     make_report(sphinx_path, run_path, out_path, project, author, version, report_to)
     

xfel_calibrate/calibrate.py

@@ -53,6 +53,9 @@ def make_initial_parser():
     parser.add_argument('--priority', type=int, default=2,
                         help="Priority of batch jobs. If priority<=1, reserved nodes become available.")
 
+    parser.add_argument('--vector-figs', action="store_true", default=False,
+                        help="Use vector graphics for figures in the report.")
+
     parser.add_argument_group('required arguments')
 
     return parser
@@ -112,12 +115,7 @@ def deconsolize_args(args):
 def extract_title_author_version(nb):
     """ Tries to extract title, author and versions from markdown """
 
-    def find_first_markdown(nb):
-        for cell in nb.cells:
-            if cell.cell_type == 'markdown':
-                return cell
-
-    first_md = find_first_markdown(nb)
+    first_md = first_markdown_cell(nb)
     source = first_md["source"]
     title = re.findall(r'\#+\s*(.*)\s*\#+', source)
     author = re.findall(
@@ -130,18 +128,31 @@ def extract_title_author_version(nb):
     return title, author, version
 
 
+def get_cell_n(nb, cell_type, cell_n):
+    """
+    Return notebook cell with given number and given type
+
+    :param nb: jupyter notebook
+    :param cell_type: cell type, 'code' or 'markdown'
+    :param cell_n: cell number (count from 0)
+    :return: notebook cell
+    """
+    counter = 0
+    for cell in nb.cells:
+        if cell.cell_type == cell_type:
+            if counter == cell_n:
+                return cell
+            counter=+1
+
+
 def first_code_cell(nb):
     """ Return the first code cell of a notebook """
-    for cell in nb.cells:
-        if cell.cell_type == 'code':
-            return cell
+    return get_cell_n(nb, 'code', 0)
 
 
 def first_markdown_cell(nb):
     """ Return the first markdown cell of a notebook """
-    for cell in nb.cells:
-        if cell.cell_type == 'markdown':
-            return cell
+    return get_cell_n(nb, 'markdown', 0)
 
 
 def make_epilog(nb, caltype=None):
@@ -359,6 +370,23 @@ def flatten_list(l):
     return "_".join([str(flatten_list(v)) for v in l]) if isinstance(l, list) else l
 
 
+def set_figure_format(nb, enable_vector_format):
+    """
+    Set svg format in inline backend for figures
+
+    If parameter 'vector_figs' is set to True svg format will
+    be used for figures in the notebook rendering. Subsequently  vector
+    graphics figures will be used for report.
+
+    :param nb: jupyter notebook
+    :param param: value of corresponding parameter
+    """
+
+    if enable_vector_format:
+        cell = get_cell_n(nb, 'code', 1)
+        cell.source += "\n%config InlineBackend.figure_formats = ['svg']\n"
+
+
 def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
                    final_job=False, job_list=[], fmtcmd="", cluster_cores=8,
                    sequential=False, priority=2):
@@ -377,6 +405,7 @@ def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
     parms = extract_parameters(nb)
     params = parameter_values(parms, **args)
     new_nb = replace_definitions(nb, params, execute=False)
+    set_figure_format(new_nb, args["vector_figs"])
     base_name = nbname.replace(".ipynb", "")
     new_name = "{}__{}__{}.ipynb".format(
         os.path.basename(base_name), cparm, suffix)