From da728d61e9cd90eaa79953b13e50d70985f737c4 Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Wed, 19 Jun 2019 13:35:03 +0200
Subject: [PATCH] code refactoring
---
cal_tools/cal_tools/tools.py | 83 ++++++++++++++++++------------------
1 file changed, 42 insertions(+), 41 deletions(-)
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index a54f189bd..16af777d0 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -1,12 +1,12 @@
from collections import OrderedDict
-import glob
+from glob import glob
from importlib.machinery import SourceFileLoader
-import os
-from os.path import isfile, isdir, splitext
+from os import chdir, listdir, makedirs, path, remove, stat
+from os.path import isdir, isfile, splitext
from queue import Queue
-from shutil import copy, move, rmtree, copytree
-from subprocess import check_output, check_call, CalledProcessError
-import textwrap
+from shutil import copy, copytree, move, rmtree
+from subprocess import CalledProcessError, check_call, check_output
+from textwrap import dedent
from time import sleep
from jinja2 import Template
@@ -14,28 +14,29 @@ import tabulate
from xfel_calibrate import settings
+
def atoi(text):
- '''
+ """
Convert string to integer is possible
:param text: string to be converted
:return: integer value or input string
- '''
+ """
return int(text) if text.isdigit() else text
def natural_keys(text):
- '''
+ """
Decompose string to list of integers and sub-strings
- '''
- return [ atoi(c) for c in re.split(r'(\d+)', text) ]
+ """
+ return [atoi(c) for c in re.split(r'(\d+)', text)]
def combine_report(run_path, calibration):
- sphinx_path = "{}/sphinx_rep".format(os.path.abspath(run_path))
- os.makedirs(sphinx_path)
- direntries = os.listdir(run_path)
+ sphinx_path = "{}/sphinx_rep".format(path.abspath(run_path))
+ makedirs(sphinx_path)
+ direntries = listdir(run_path)
direntries.sort(key=natural_keys)
for entry in direntries:
@@ -90,20 +91,20 @@ def prepare_plots(run_path, threshold=1000000):
:param threshold: Max svg file size (in bytes) to be converted to pdf
"""
print('Convert svg to pdf and png')
- run_path = os.path.abspath(run_path)
+ run_path = path.abspath(run_path)
- rst_files = glob.glob('{}/*rst'.format(run_path))
+ rst_files = glob('{}/*rst'.format(run_path))
for rst_file in rst_files:
- rst_file_name = os.path.basename(rst_file)
- rst_file_name = os.path.splitext(rst_file_name)[0]
+ rst_file_name = path.basename(rst_file)
+ rst_file_name = path.splitext(rst_file_name)[0]
- svg_files = glob.glob(
+ svg_files = glob(
'{}/{}_files/*svg'.format(run_path, rst_file_name))
for f_path in svg_files:
- f_name = os.path.basename(f_path)
- f_name = os.path.splitext(f_name)[0]
+ f_name = path.basename(f_path)
+ f_name = path.splitext(f_name)[0]
- if (os.stat(f_path)).st_size < threshold:
+ if (stat(f_path)).st_size < threshold:
check_call(["svg2pdf", "{}".format(f_path)], shell=False)
new_ext = 'pdf'
else:
@@ -126,7 +127,7 @@ def make_timing_summary(run_path, joblist):
:param joblist: List of slurm jobs
"""
print('Prepare timing summary')
- run_path = os.path.abspath(run_path)
+ run_path = path.abspath(run_path)
pars_vals = []
pars = 'JobID,Elapsed,Suspended'
pars_name = pars.split(',')
@@ -135,10 +136,10 @@ def make_timing_summary(run_path, joblist):
out = check_output(['sacct', '-j', job,
'--format={}'.format(pars)],
shell=False)
- l = str(out).split('\\n')
+ lines = str(out).split('\\n')
# loop over output lines, skip first two lines with header.
- for line in l[2:]:
+ for line in lines[2:]:
s = line.split()
if len(s) == len(pars_name):
pars_vals.append(s)
@@ -158,7 +159,7 @@ def make_timing_summary(run_path, joblist):
table = tabulate.tabulate(pars_vals, tablefmt='latex',
headers=pars_name)
- gfile.write(textwrap.dedent(tmpl.render(table=table.split('\n'))))
+ gfile.write(dedent(tmpl.render(table=table.split('\n'))))
def make_report(run_path, tmp_path, out_path, project, author, version,
@@ -178,8 +179,8 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
:param version: Version of the notebook
:param report_to: Name or path of the report file
"""
- run_path = os.path.abspath(run_path)
- report_path, report_name = os.path.split(report_to)
+ run_path = path.abspath(run_path)
+ report_path, report_name = path.split(report_to)
if report_path != '':
out_path = report_path
@@ -201,7 +202,7 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
"Generated simple index.rst instead")
# quickbuild went well we need to edit the index.rst and conf.py files
- module_path = "{}".format(os.path.abspath(os.path.dirname(__file__)))
+ module_path = "{}".format(path.abspath(path.dirname(__file__)))
conf = SourceFileLoader("conf",
"{}/conf.py".format(run_path)).load_module()
@@ -226,10 +227,10 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
mf.write(tmpl.format(var, v))
- os.remove("{}/conf.py".format(run_path))
+ remove("{}/conf.py".format(run_path))
move("{}/conf.py.tmp".format(run_path), "{}/conf.py".format(run_path))
- direntries = os.listdir(run_path)
+ direntries = listdir(run_path)
files_to_handle = []
for entry in direntries:
if isfile("{}/{}".format(run_path, entry)):
@@ -249,10 +250,10 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
''')
with open("{}/index.rst".format(run_path), "w+") as mf:
- mf.write(textwrap.dedent(index_tmp.render(keys=files_to_handle)))
+ mf.write(dedent(index_tmp.render(keys=files_to_handle)))
# finally call the make scripts
- os.chdir(run_path)
+ chdir(run_path)
try:
check_call(["make", "latexpdf"])
@@ -262,7 +263,7 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
"can be inspected at: {}".format(run_path))
return
print("Moving report to final location: {}".format(out_path))
- for f in glob.glob(r'{}/_build/latex/*.pdf'.format(run_path)):
+ for f in glob(r'{}/_build/latex/*.pdf'.format(run_path)):
copy(f, out_path)
print("Removing temporary files at: {}".format(tmp_path))
rmtree(tmp_path)
@@ -276,13 +277,13 @@ def make_titlepage(sphinx_path, project, data_path):
:param project: title of the project
:param data_path: path to input data sample used for notebook
"""
- module_path = "{}".format(os.path.abspath(os.path.dirname(__file__)))
+ module_path = "{}".format(path.abspath(path.dirname(__file__)))
with open('{}/titlepage.tmpl'.format(module_path)) as file_:
title_tmp = Template(file_.read())
with open("{}/titlepage.tex.txt".format(sphinx_path), "w+") as mf:
- mf.write(textwrap.dedent(title_tmp.render(project=project,
- data_path=data_path)))
+ mf.write(dedent(title_tmp.render(project=project,
+ data_path=data_path)))
def finalize(joblist, finaljob, run_path, out_path, project, calibration,
@@ -354,7 +355,7 @@ def map_modules_from_files(filelist, file_inset, quadrants, modules_per_quad):
if file_infix in file:
module_files[name].put(file)
total_sequences += 1
- total_file_size += os.path.getsize(file)
+ total_file_size += path.getsize(file)
return module_files, mod_ids, total_sequences, total_file_size
@@ -365,19 +366,19 @@ def gain_map_files(in_folder, runs, sequences, file_inset, quadrants, modules_pe
gain_mapped_files = OrderedDict()
for gain, run in runs.items():
ginfolder = "{}/{}".format(in_folder, run)
- dirlist = os.listdir(ginfolder)
+ dirlist = listdir(ginfolder)
file_list = []
for entry in dirlist:
#only h5 file
abs_entry = "{}/{}".format(ginfolder, entry)
- if os.path.isfile(abs_entry) and os.path.splitext(abs_entry)[1] == ".h5":
+ if path.isfile(abs_entry) and path.splitext(abs_entry)[1] == ".h5":
if sequences is None:
file_list.append(abs_entry)
else:
for seq in sequences:
if "{:05d}.h5".format(seq) in abs_entry:
- file_list.append(os.path.abspath(abs_entry))
+ file_list.append(path.abspath(abs_entry))
mapped_files, mod_ids, seq, fs = map_modules_from_files(file_list, file_inset,
quadrants, modules_per_quad)
--
GitLab