diff --git a/cal_tools/cal_tools/conf_latex.patch b/cal_tools/cal_tools/conf_latex.patch
deleted file mode 100644
index e2259c2d82693c70ffb14ec31547805d72dd4c77..0000000000000000000000000000000000000000
--- a/cal_tools/cal_tools/conf_latex.patch
+++ /dev/null
@@ -1,10 +0,0 @@
---- conf_orig.py 2018-08-27 11:53:53.069138000 +0200
-+++ conf_latex.py 2018-08-27 11:53:18.472939000 +0200
-@@ -257,6 +257,7 @@
- # Latex figure (float) alignment
- #
- # 'figure_align': 'htbp',
-+ 'extraclassoptions': ',openany, oneside',
- }
-
- # Grouping the document tree into LaTeX files. List of tuples
diff --git a/cal_tools/cal_tools/conf_latex.py b/cal_tools/cal_tools/conf_latex.py
deleted file mode 100644
index 8b0ce452a6345c0766cec73cc35d7cbe465ff14f..0000000000000000000000000000000000000000
--- a/cal_tools/cal_tools/conf_latex.py
+++ /dev/null
@@ -1,347 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# 'Tutorial Calculation ' documentation build configuration file, created by
-# sphinx-quickstart on Thu Aug 23 12:23:00 2018.
-#
-# This file is execfile()d with the current directory set to its
-# containing dir.
-#
-# Note that not all possible configuration values are present in this
-# autogenerated file.
-#
-# All configuration values have a default; values that are commented out
-# serve to show the default.
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
-
-# -- General configuration ------------------------------------------------
-
-# If your documentation needs a minimal Sphinx version, state it here.
-#
-# needs_sphinx = '1.0'
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-extensions = [
- 'sphinx.ext.intersphinx',
- 'sphinx.ext.mathjax',
-]
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
-
-# The suffix(es) of source filenames.
-# You can specify multiple suffix as a list of string:
-#
-# source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
-
-# The encoding of source files.
-#
-# source_encoding = 'utf-8-sig'
-
-# The master toctree document.
-master_doc = 'index'
-
-# General information about the project.
-project = '\'Tutorial Calculation \''
-copyright = '2018, \'Astrid Muennich\''
-author = '\'Astrid Muennich\''
-
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-version = ''
-# The full version, including alpha/beta/rc tags.
-release = ''
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#
-# This is also used if you do content translation via gettext catalogs.
-# Usually you set "language" from the command line for these cases.
-language = None
-
-# There are two options for replacing |today|: either, you set today to some
-# non-false value, then it is used:
-#
-# today = ''
-#
-# Else, today_fmt is used as the format for a strftime call.
-#
-# today_fmt = '%B %d, %Y'
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
-
-# The reST default role (used for this markup: `text`) to use for all
-# documents.
-#
-# default_role = None
-
-# If true, '()' will be appended to :func: etc. cross-reference text.
-#
-# add_function_parentheses = True
-
-# If true, the current module name will be prepended to all description
-# unit titles (such as .. function::).
-#
-# add_module_names = True
-
-# If true, sectionauthor and moduleauthor directives will be shown in the
-# output. They are ignored by default.
-#
-# show_authors = False
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
-
-# A list of ignored prefixes for module index sorting.
-# modindex_common_prefix = []
-
-# If true, keep warnings as "system message" paragraphs in the built documents.
-# keep_warnings = False
-
-# If true, `todo` and `todoList` produce output, else they produce nothing.
-todo_include_todos = False
-
-
-# -- Options for HTML output ----------------------------------------------
-
-# The theme to use for HTML and HTML Help pages. See the documentation for
-# a list of builtin themes.
-#
-html_theme = 'alabaster'
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further. For a list of options available for each theme, see the
-# documentation.
-#
-# html_theme_options = {}
-
-# Add any paths that contain custom themes here, relative to this directory.
-# html_theme_path = []
-
-# The name for this set of Sphinx documents.
-# "<project> v<release> documentation" by default.
-#
-# html_title = '\'Tutorial Calculation \' v'
-
-# A shorter title for the navigation bar. Default is the same as html_title.
-#
-# html_short_title = None
-
-# The name of an image file (relative to this directory) to place at the top
-# of the sidebar.
-#
-# html_logo = None
-
-# The name of an image file (relative to this directory) to use as a favicon of
-# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
-# pixels large.
-#
-# html_favicon = None
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-
-# Add any extra paths that contain custom files (such as robots.txt or
-# .htaccess) here, relative to this directory. These files are copied
-# directly to the root of the documentation.
-#
-# html_extra_path = []
-
-# If not None, a 'Last updated on:' timestamp is inserted at every page
-# bottom, using the given strftime format.
-# The empty string is equivalent to '%b %d, %Y'.
-#
-# html_last_updated_fmt = None
-
-# If true, SmartyPants will be used to convert quotes and dashes to
-# typographically correct entities.
-#
-# html_use_smartypants = True
-
-# Custom sidebar templates, maps document names to template names.
-#
-# html_sidebars = {}
-
-# Additional templates that should be rendered to pages, maps page names to
-# template names.
-#
-# html_additional_pages = {}
-
-# If false, no module index is generated.
-#
-# html_domain_indices = True
-
-# If false, no index is generated.
-#
-# html_use_index = True
-
-# If true, the index is split into individual pages for each letter.
-#
-# html_split_index = False
-
-# If true, links to the reST sources are added to the pages.
-#
-# html_show_sourcelink = True
-
-# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#
-# html_show_sphinx = True
-
-# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#
-# html_show_copyright = True
-
-# If true, an OpenSearch description file will be output, and all pages will
-# contain a <link> tag referring to it. The value of this option must be the
-# base URL from which the finished HTML is served.
-#
-# html_use_opensearch = ''
-
-# This is the file name suffix for HTML files (e.g. ".xhtml").
-# html_file_suffix = None
-
-# Language to be used for generating the HTML full-text search index.
-# Sphinx supports the following languages:
-# 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja'
-# 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh'
-#
-# html_search_language = 'en'
-
-# A dictionary with options for the search language support, empty by default.
-# 'ja' uses this config value.
-# 'zh' user can custom change `jieba` dictionary path.
-#
-# html_search_options = {'type': 'default'}
-
-# The name of a javascript file (relative to the configuration directory) that
-# implements a search results scorer. If empty, the default will be used.
-#
-# html_search_scorer = 'scorer.js'
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'TutorialCalculationdoc'
-
-# -- Options for LaTeX output ---------------------------------------------
-
-latex_elements = {
- # The paper size ('letterpaper' or 'a4paper').
- #
- # 'papersize': 'letterpaper',
-
- # The font size ('10pt', '11pt' or '12pt').
- #
- # 'pointsize': '10pt',
-
- # Additional stuff for the LaTeX preamble.
- #
- # 'preamble': '',
-
- # Latex figure (float) alignment
- #
- # 'figure_align': 'htbp',
- 'extraclassoptions': ',openany, oneside',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-# author, documentclass [howto, manual, or own class]).
-latex_documents = [
- (master_doc, 'TutorialCalculation.tex', '\'Tutorial Calculation \' Documentation',
- '\'Astrid Muennich\'', 'manual'),
-]
-
-# The name of an image file (relative to this directory) to place at the top of
-# the title page.
-#
-# latex_logo = None
-
-# For "manual" documents, if this is true, then toplevel headings are parts,
-# not chapters.
-#
-# latex_use_parts = False
-
-# If true, show page references after internal links.
-#
-# latex_show_pagerefs = False
-
-# If true, show URL addresses after external links.
-#
-# latex_show_urls = False
-
-# Documents to append as an appendix to all manuals.
-#
-# latex_appendices = []
-
-# It false, will not define \strong, \code, itleref, \crossref ... but only
-# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added
-# packages.
-#
-# latex_keep_old_macro_names = True
-
-# If false, no module index is generated.
-#
-# latex_domain_indices = True
-
-
-# -- Options for manual page output ---------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
- (master_doc, 'tutorialcalculation', '\'Tutorial Calculation \' Documentation',
- [author], 1)
-]
-
-# If true, show URL addresses after external links.
-#
-# man_show_urls = False
-
-
-# -- Options for Texinfo output -------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-# dir menu entry, description, category)
-texinfo_documents = [
- (master_doc, 'TutorialCalculation', '\'Tutorial Calculation \' Documentation',
- author, 'TutorialCalculation', 'One line description of project.',
- 'Miscellaneous'),
-]
-
-# Documents to append as an appendix to all manuals.
-#
-# texinfo_appendices = []
-
-# If false, no module index is generated.
-#
-# texinfo_domain_indices = True
-
-# How to display URL addresses: 'footnote', 'no', or 'inline'.
-#
-# texinfo_show_urls = 'footnote'
-
-# If true, do not generate a @detailmenu in the "Top" node's menu.
-#
-# texinfo_no_detailmenu = False
-
-
-# Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'https://docs.python.org/': None}
diff --git a/cal_tools/cal_tools/conf_orig.py b/cal_tools/cal_tools/conf_orig.py
deleted file mode 100644
index 5074430ad326f5b22dff8cb009190d9033bcdbfd..0000000000000000000000000000000000000000
--- a/cal_tools/cal_tools/conf_orig.py
+++ /dev/null
@@ -1,346 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# 'Tutorial Calculation ' documentation build configuration file, created by
-# sphinx-quickstart on Thu Aug 23 12:23:00 2018.
-#
-# This file is execfile()d with the current directory set to its
-# containing dir.
-#
-# Note that not all possible configuration values are present in this
-# autogenerated file.
-#
-# All configuration values have a default; values that are commented out
-# serve to show the default.
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
-
-# -- General configuration ------------------------------------------------
-
-# If your documentation needs a minimal Sphinx version, state it here.
-#
-# needs_sphinx = '1.0'
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-extensions = [
- 'sphinx.ext.intersphinx',
- 'sphinx.ext.mathjax',
-]
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
-
-# The suffix(es) of source filenames.
-# You can specify multiple suffix as a list of string:
-#
-# source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
-
-# The encoding of source files.
-#
-# source_encoding = 'utf-8-sig'
-
-# The master toctree document.
-master_doc = 'index'
-
-# General information about the project.
-project = '\'Tutorial Calculation \''
-copyright = '2018, \'Astrid Muennich\''
-author = '\'Astrid Muennich\''
-
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-version = ''
-# The full version, including alpha/beta/rc tags.
-release = ''
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#
-# This is also used if you do content translation via gettext catalogs.
-# Usually you set "language" from the command line for these cases.
-language = None
-
-# There are two options for replacing |today|: either, you set today to some
-# non-false value, then it is used:
-#
-# today = ''
-#
-# Else, today_fmt is used as the format for a strftime call.
-#
-# today_fmt = '%B %d, %Y'
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
-
-# The reST default role (used for this markup: `text`) to use for all
-# documents.
-#
-# default_role = None
-
-# If true, '()' will be appended to :func: etc. cross-reference text.
-#
-# add_function_parentheses = True
-
-# If true, the current module name will be prepended to all description
-# unit titles (such as .. function::).
-#
-# add_module_names = True
-
-# If true, sectionauthor and moduleauthor directives will be shown in the
-# output. They are ignored by default.
-#
-# show_authors = False
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
-
-# A list of ignored prefixes for module index sorting.
-# modindex_common_prefix = []
-
-# If true, keep warnings as "system message" paragraphs in the built documents.
-# keep_warnings = False
-
-# If true, `todo` and `todoList` produce output, else they produce nothing.
-todo_include_todos = False
-
-
-# -- Options for HTML output ----------------------------------------------
-
-# The theme to use for HTML and HTML Help pages. See the documentation for
-# a list of builtin themes.
-#
-html_theme = 'alabaster'
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further. For a list of options available for each theme, see the
-# documentation.
-#
-# html_theme_options = {}
-
-# Add any paths that contain custom themes here, relative to this directory.
-# html_theme_path = []
-
-# The name for this set of Sphinx documents.
-# "<project> v<release> documentation" by default.
-#
-# html_title = '\'Tutorial Calculation \' v'
-
-# A shorter title for the navigation bar. Default is the same as html_title.
-#
-# html_short_title = None
-
-# The name of an image file (relative to this directory) to place at the top
-# of the sidebar.
-#
-# html_logo = None
-
-# The name of an image file (relative to this directory) to use as a favicon of
-# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
-# pixels large.
-#
-# html_favicon = None
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-
-# Add any extra paths that contain custom files (such as robots.txt or
-# .htaccess) here, relative to this directory. These files are copied
-# directly to the root of the documentation.
-#
-# html_extra_path = []
-
-# If not None, a 'Last updated on:' timestamp is inserted at every page
-# bottom, using the given strftime format.
-# The empty string is equivalent to '%b %d, %Y'.
-#
-# html_last_updated_fmt = None
-
-# If true, SmartyPants will be used to convert quotes and dashes to
-# typographically correct entities.
-#
-# html_use_smartypants = True
-
-# Custom sidebar templates, maps document names to template names.
-#
-# html_sidebars = {}
-
-# Additional templates that should be rendered to pages, maps page names to
-# template names.
-#
-# html_additional_pages = {}
-
-# If false, no module index is generated.
-#
-# html_domain_indices = True
-
-# If false, no index is generated.
-#
-# html_use_index = True
-
-# If true, the index is split into individual pages for each letter.
-#
-# html_split_index = False
-
-# If true, links to the reST sources are added to the pages.
-#
-# html_show_sourcelink = True
-
-# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
-#
-# html_show_sphinx = True
-
-# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
-#
-# html_show_copyright = True
-
-# If true, an OpenSearch description file will be output, and all pages will
-# contain a <link> tag referring to it. The value of this option must be the
-# base URL from which the finished HTML is served.
-#
-# html_use_opensearch = ''
-
-# This is the file name suffix for HTML files (e.g. ".xhtml").
-# html_file_suffix = None
-
-# Language to be used for generating the HTML full-text search index.
-# Sphinx supports the following languages:
-# 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja'
-# 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr', 'zh'
-#
-# html_search_language = 'en'
-
-# A dictionary with options for the search language support, empty by default.
-# 'ja' uses this config value.
-# 'zh' user can custom change `jieba` dictionary path.
-#
-# html_search_options = {'type': 'default'}
-
-# The name of a javascript file (relative to the configuration directory) that
-# implements a search results scorer. If empty, the default will be used.
-#
-# html_search_scorer = 'scorer.js'
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'TutorialCalculationdoc'
-
-# -- Options for LaTeX output ---------------------------------------------
-
-latex_elements = {
- # The paper size ('letterpaper' or 'a4paper').
- #
- # 'papersize': 'letterpaper',
-
- # The font size ('10pt', '11pt' or '12pt').
- #
- # 'pointsize': '10pt',
-
- # Additional stuff for the LaTeX preamble.
- #
- # 'preamble': '',
-
- # Latex figure (float) alignment
- #
- # 'figure_align': 'htbp',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-# author, documentclass [howto, manual, or own class]).
-latex_documents = [
- (master_doc, 'TutorialCalculation.tex', '\'Tutorial Calculation \' Documentation',
- '\'Astrid Muennich\'', 'manual'),
-]
-
-# The name of an image file (relative to this directory) to place at the top of
-# the title page.
-#
-# latex_logo = None
-
-# For "manual" documents, if this is true, then toplevel headings are parts,
-# not chapters.
-#
-# latex_use_parts = False
-
-# If true, show page references after internal links.
-#
-# latex_show_pagerefs = False
-
-# If true, show URL addresses after external links.
-#
-# latex_show_urls = False
-
-# Documents to append as an appendix to all manuals.
-#
-# latex_appendices = []
-
-# It false, will not define \strong, \code, itleref, \crossref ... but only
-# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added
-# packages.
-#
-# latex_keep_old_macro_names = True
-
-# If false, no module index is generated.
-#
-# latex_domain_indices = True
-
-
-# -- Options for manual page output ---------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
- (master_doc, 'tutorialcalculation', '\'Tutorial Calculation \' Documentation',
- [author], 1)
-]
-
-# If true, show URL addresses after external links.
-#
-# man_show_urls = False
-
-
-# -- Options for Texinfo output -------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-# dir menu entry, description, category)
-texinfo_documents = [
- (master_doc, 'TutorialCalculation', '\'Tutorial Calculation \' Documentation',
- author, 'TutorialCalculation', 'One line description of project.',
- 'Miscellaneous'),
-]
-
-# Documents to append as an appendix to all manuals.
-#
-# texinfo_appendices = []
-
-# If false, no module index is generated.
-#
-# texinfo_domain_indices = True
-
-# How to display URL addresses: 'footnote', 'no', or 'inline'.
-#
-# texinfo_show_urls = 'footnote'
-
-# If true, do not generate a @detailmenu in the "Top" node's menu.
-#
-# texinfo_no_detailmenu = False
-
-
-# Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'https://docs.python.org/': None}
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index 78947ea1c295af07f76de0ac3736911037a6b196..16af777d056699ff0df1f771f1b38c54e8c03921 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -1,42 +1,42 @@
-import argparse
from collections import OrderedDict
-import copy
-import glob
-from jinja2 import Template
-import os
-from os.path import isfile, isdir, splitext
+from glob import glob
+from importlib.machinery import SourceFileLoader
+from os import chdir, listdir, makedirs, path, remove, stat
+from os.path import isdir, isfile, splitext
from queue import Queue
-import re
-import shutil
-from subprocess import Popen, PIPE, check_output, check_call
-import textwrap
+from shutil import copy, copytree, move, rmtree
+from subprocess import CalledProcessError, check_call, check_output
+from textwrap import dedent
from time import sleep
-import time
-from uuid import uuid4
+
+from jinja2 import Template
+import tabulate
+
+from xfel_calibrate import settings
def atoi(text):
- '''
+ """
Convert string to integer is possible
:param text: string to be converted
:return: integer value or input string
- '''
+ """
return int(text) if text.isdigit() else text
def natural_keys(text):
- '''
+ """
Decompose string to list of integers and sub-strings
- '''
- return [ atoi(c) for c in re.split(r'(\d+)', text) ]
+ """
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
def combine_report(run_path, calibration):
- sphinx_path = "{}/sphinx_rep".format(os.path.abspath(run_path))
- os.makedirs(sphinx_path)
- direntries = os.listdir(run_path)
+ sphinx_path = "{}/sphinx_rep".format(path.abspath(run_path))
+ makedirs(sphinx_path)
+ direntries = listdir(run_path)
direntries.sort(key=natural_keys)
for entry in direntries:
@@ -72,7 +72,7 @@ def combine_report(run_path, calibration):
gfile.write("\n\n")
if isdir("{}/{}".format(run_path, entry)):
- shutil.copytree("{}/{}".format(run_path, entry), "{}/{}".format(sphinx_path, entry))
+ copytree("{}/{}".format(run_path, entry), "{}/{}".format(sphinx_path, entry))
return sphinx_path
@@ -91,20 +91,20 @@ def prepare_plots(run_path, threshold=1000000):
:param threshold: Max svg file size (in bytes) to be converted to pdf
"""
print('Convert svg to pdf and png')
- run_path = os.path.abspath(run_path)
+ run_path = path.abspath(run_path)
- rst_files = glob.glob('{}/*rst'.format(run_path))
+ rst_files = glob('{}/*rst'.format(run_path))
for rst_file in rst_files:
- rst_file_name = os.path.basename(rst_file)
- rst_file_name = os.path.splitext(rst_file_name)[0]
+ rst_file_name = path.basename(rst_file)
+ rst_file_name = path.splitext(rst_file_name)[0]
- svg_files = glob.glob(
+ svg_files = glob(
'{}/{}_files/*svg'.format(run_path, rst_file_name))
for f_path in svg_files:
- f_name = os.path.basename(f_path)
- f_name = os.path.splitext(f_name)[0]
+ f_name = path.basename(f_path)
+ f_name = path.splitext(f_name)[0]
- if (os.stat(f_path)).st_size < threshold:
+ if (stat(f_path)).st_size < threshold:
check_call(["svg2pdf", "{}".format(f_path)], shell=False)
new_ext = 'pdf'
else:
@@ -127,7 +127,7 @@ def make_timing_summary(run_path, joblist):
:param joblist: List of slurm jobs
"""
print('Prepare timing summary')
- run_path = os.path.abspath(run_path)
+ run_path = path.abspath(run_path)
pars_vals = []
pars = 'JobID,Elapsed,Suspended'
pars_name = pars.split(',')
@@ -136,10 +136,10 @@ def make_timing_summary(run_path, joblist):
out = check_output(['sacct', '-j', job,
'--format={}'.format(pars)],
shell=False)
- l = str(out).split('\\n')
+ lines = str(out).split('\\n')
# loop over output lines, skip first two lines with header.
- for line in l[2:]:
+ for line in lines[2:]:
s = line.split()
if len(s) == len(pars_name):
pars_vals.append(s)
@@ -154,65 +154,87 @@ def make_timing_summary(run_path, joblist):
{{ line }}
{%- endfor %}
''')
- import tabulate
+
with open("{}/timing_summary.rst".format(run_path), "w+") as gfile:
table = tabulate.tabulate(pars_vals, tablefmt='latex',
headers=pars_name)
- gfile.write(textwrap.dedent(tmpl.render(table=table.split('\n'))))
+ gfile.write(dedent(tmpl.render(table=table.split('\n'))))
-def make_report(run_path, tmp_path, out_path, project, author, version, report_to):
- run_path = os.path.abspath(run_path)
- report_path, report_name = os.path.split(report_to)
+def make_report(run_path, tmp_path, out_path, project, author, version,
+ report_to):
+ """
+ Create calibration report (pdf file)
+
+ Automatically generated report document results, produced by
+ jupyter-notebooks.
+
+ :param run_path: Path to sphinx run directory
+ :param tmp_path: Run path of the slurm job
+ :param out_path: Output directory for report.
+ Overwritten if path to report is given in `report_to`
+ :param project: Project title
+ :param author: Author of the notebook
+ :param version: Version of the notebook
+ :param report_to: Name or path of the report file
+ """
+ run_path = path.abspath(run_path)
+ report_path, report_name = path.split(report_to)
if report_path != '':
out_path = report_path
try:
- import subprocess
- subprocess.check_call(["sphinx-quickstart",
- "--quiet",
- "--project='{}'".format(project),
- "--author='{}'".format(author),
- "-v", str(version),
- "--suffix=.rst",
- "--master=index",
- "--ext-intersphinx",
- "--ext-mathjax",
- "--makefile",
- "--no-batchfile", run_path])
- # now conf.py exists in temp folder, so add latex options to remove empty pages
- patch_path = "{}".format(os.path.abspath(os.path.dirname(__file__)))
- patch_file = ("{}/conf_latex.patch".format(patch_path))
- try:
- subprocess.check_call(["patch",
- "{}/sphinx_rep/conf.py".format(tmp_path),
- patch_file])
- except:
- print("Patching latex conf failed")
+ check_call(["sphinx-quickstart",
+ "--quiet",
+ "--project='{}'".format(project),
+ "--author='{}'".format(author),
+ "-v", str(version),
+ "--suffix=.rst",
+ "--master=index",
+ "--ext-intersphinx",
+ "--ext-mathjax",
+ "--makefile",
+ "--no-batchfile", run_path])
+
+ except CalledProcessError:
+ raise Exception("Failed to run sphinx-quickbuild. Is sphinx installed?"
+ "Generated simple index.rst instead")
- try:
+ # quickbuild went well we need to edit the index.rst and conf.py files
+ module_path = "{}".format(path.abspath(path.dirname(__file__)))
- subprocess.check_call(["sed",
- "-i",
- "-e",
- "s/{}.tex/{}.tex/g".format(project.replace(" ", ""), report_name),
- "{}/sphinx_rep/conf.py".format(tmp_path)], shell=False)
- except:
- print("SEDing tex failed")
+ conf = SourceFileLoader("conf",
+ "{}/conf.py".format(run_path)).load_module()
+ l_var = [v for v in dir(conf) if not v.startswith('__')]
- except subprocess.CalledProcessError:
- raise Exception("Failed to run sphinx-quickbuild. Is sphinx installed?"
- "Generated simple index.rst instead")
+ with open("{}/conf.py.tmp".format(run_path), "w") as mf:
+ latex_elements = {'extraclassoptions': ',openany, oneside',
+ 'maketitle': r'\input{titlepage.tex.txt}'}
+ mf.write("latex_elements = {}\n".format(latex_elements))
+ mf.write("latex_logo = '{}/{}'\n".format(module_path,
+ settings.logo_path))
+ mf.write("latex_additional_files = ['titlepage.tex.txt']\n")
- # quickbuild went well we need to edit the index file
- from shutil import move
-
- direntries = os.listdir(run_path)
+ for var in l_var:
+ if var in ['latex_elements', 'latex_logo',
+ 'latex_additional_files']:
+ continue
+ tmpl = '{} = {}\n'
+ v = getattr(conf, var, None)
+ if isinstance(v, str):
+ tmpl = '{} = "{}"\n'
+
+ mf.write(tmpl.format(var, v))
+
+ remove("{}/conf.py".format(run_path))
+ move("{}/conf.py.tmp".format(run_path), "{}/conf.py".format(run_path))
+
+ direntries = listdir(run_path)
files_to_handle = []
for entry in direntries:
if isfile("{}/{}".format(run_path, entry)):
- name, ext = splitext("{}".format(entry))
+ name, ext = splitext("{}".format(entry))
if ext == ".rst" and "index" not in name:
files_to_handle.append(name.strip())
@@ -228,29 +250,44 @@ def make_report(run_path, tmp_path, out_path, project, author, version, report_t
''')
with open("{}/index.rst".format(run_path), "w+") as mf:
- mf.write(textwrap.dedent(index_tmp.render(keys=files_to_handle)))
+ mf.write(dedent(index_tmp.render(keys=files_to_handle)))
# finally call the make scripts
- os.chdir(run_path)
+ chdir(run_path)
try:
- import subprocess
- subprocess.check_call(["make", "latexpdf"])
+ check_call(["make", "latexpdf"])
- except subprocess.CalledProcessError:
+ except CalledProcessError:
print("Failed to make pdf documentation")
- print("Temp files will not be deleted and "+
+ print("Temp files will not be deleted and " +
"can be inspected at: {}".format(run_path))
return
print("Moving report to final location: {}".format(out_path))
- for f in glob.glob(r'{}/_build/latex/*.pdf'.format(run_path)):
- print(f)
- shutil.copy(f, out_path)
+ for f in glob(r'{}/_build/latex/*.pdf'.format(run_path)):
+ copy(f, out_path)
print("Removing temporary files at: {}".format(tmp_path))
- shutil.rmtree(tmp_path)
+ rmtree(tmp_path)
+
-
+def make_titlepage(sphinx_path, project, data_path):
+ """
+ Create title page for report using template
+
+ :param sphinx_path: path to sphinx run directory
+ :param project: title of the project
+ :param data_path: path to input data sample used for notebook
+ """
+ module_path = "{}".format(path.abspath(path.dirname(__file__)))
+ with open('{}/titlepage.tmpl'.format(module_path)) as file_:
+ title_tmp = Template(file_.read())
+
+ with open("{}/titlepage.tex.txt".format(sphinx_path), "w+") as mf:
+ mf.write(dedent(title_tmp.render(project=project,
+ data_path=data_path)))
+
+
def finalize(joblist, finaljob, run_path, out_path, project, calibration,
- author, version, report_to):
+ author, version, report_to, data_path='Unknown'):
print("Waiting on jobs to finish: {}".format(joblist))
while True:
@@ -267,6 +304,7 @@ def finalize(joblist, finaljob, run_path, out_path, project, calibration,
prepare_plots(run_path)
make_timing_summary(run_path, joblist+[str(finaljob)])
sphinx_path = combine_report(run_path, calibration)
+ make_titlepage(sphinx_path, project, data_path)
make_report(sphinx_path, run_path, out_path, project, author, version,
report_to)
@@ -317,7 +355,7 @@ def map_modules_from_files(filelist, file_inset, quadrants, modules_per_quad):
if file_infix in file:
module_files[name].put(file)
total_sequences += 1
- total_file_size += os.path.getsize(file)
+ total_file_size += path.getsize(file)
return module_files, mod_ids, total_sequences, total_file_size
@@ -328,19 +366,19 @@ def gain_map_files(in_folder, runs, sequences, file_inset, quadrants, modules_pe
gain_mapped_files = OrderedDict()
for gain, run in runs.items():
ginfolder = "{}/{}".format(in_folder, run)
- dirlist = os.listdir(ginfolder)
+ dirlist = listdir(ginfolder)
file_list = []
for entry in dirlist:
#only h5 file
abs_entry = "{}/{}".format(ginfolder, entry)
- if os.path.isfile(abs_entry) and os.path.splitext(abs_entry)[1] == ".h5":
+ if path.isfile(abs_entry) and path.splitext(abs_entry)[1] == ".h5":
if sequences is None:
file_list.append(abs_entry)
else:
for seq in sequences:
if "{:05d}.h5".format(seq) in abs_entry:
- file_list.append(os.path.abspath(abs_entry))
+ file_list.append(path.abspath(abs_entry))
mapped_files, mod_ids, seq, fs = map_modules_from_files(file_list, file_inset,
quadrants, modules_per_quad)
diff --git a/cal_tools/cal_tools/xfel.pdf b/cal_tools/cal_tools/xfel.pdf
new file mode 100644
index 0000000000000000000000000000000000000000..b78a8a6e5500b21fc602696785110cff81c03bff
Binary files /dev/null and b/cal_tools/cal_tools/xfel.pdf differ
diff --git a/cal_tools/setup.py b/cal_tools/setup.py
index ce0e900eb20058a4f8e2f8acfa4debc89672f288..37538c3bbf3a1e82006bee390c5ddb18ef5a204e 100644
--- a/cal_tools/setup.py
+++ b/cal_tools/setup.py
@@ -4,7 +4,6 @@ setup(
name='calTools',
version='0.1',
packages=['cal_tools',],
- package_data={'': ['conf_latex.patch']},
url='',
license='(c) European XFEL GmbH',
author='Steffen Hauf',
diff --git a/xfel_calibrate/settings.py b/xfel_calibrate/settings.py
index 81fdf3c635f4079df5e476ed959a00201098bec8..892412e80143b4e41d603b5a2d82f29f95a1445c 100644
--- a/xfel_calibrate/settings.py
+++ b/xfel_calibrate/settings.py
@@ -24,6 +24,9 @@ report_path = "{}/calibration_reports/".format(os.getcwd())
# Also try to output the report to an out_folder defined by the notebook
try_report_to_output = True
+# logo file for calibration report (located in cal_tools/cal_tools)
+logo_path = "xfel.pdf"
+
# the command to run this concurrently. It is prepended to the actual call
sprof = os.environ.get("XFELCALSLURM", "exfel")
launcher_command = "sbatch -t 24:00:00 --mem 500G --requeue --output {temp_path}/slurm-%j.out"