From 6f73fbe1429cc84e528340f934f757e00b05eb6e Mon Sep 17 00:00:00 2001
From: Karim Ahmed <karim.ahmed@xfel.eu>
Date: Mon, 21 Oct 2019 09:57:33 +0200
Subject: [PATCH] no copying if no push
---
reportservice/automatic_run.py | 2 +-
reportservice/manual_run.py | 6 +--
reportservice/report_service.py | 81 +++++++++++++++++----------------
3 files changed, 46 insertions(+), 43 deletions(-)
diff --git a/reportservice/automatic_run.py b/reportservice/automatic_run.py
index 76242dc07..b6aba1f5e 100644
--- a/reportservice/automatic_run.py
+++ b/reportservice/automatic_run.py
@@ -73,7 +73,7 @@ if __name__ == "__main__":
args = vars(arg_parser.parse_args())
conf_file = args["config_file"]
with open(conf_file, "r") as f:
- cfg = yaml.load(f.read())
+ cfg = yaml.load(f.read(), Loader=yaml.FullLoader)
logfile = args["log_file"]
fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
diff --git a/reportservice/manual_run.py b/reportservice/manual_run.py
index bf0ddf2a3..d57248838 100644
--- a/reportservice/manual_run.py
+++ b/reportservice/manual_run.py
@@ -35,11 +35,11 @@ def manual_run(request, cfg):
arg_parser = argparse.ArgumentParser(description='Manual Launch')
arg_parser.add_argument('--instrument', default=['all'], nargs='+',
help='select the requested instruments. '
- 'Default=None. "all" can be used for selecting'
+ 'Default=\"all\", which can be used for selecting'
' all instruments')
arg_parser.add_argument('--gitpush', dest='Push to git', action='store_true',
help='required for pushing the generated figures '
- 'to the DC git repository')
+ 'to the DC git repository. Default=bool(False)')
arg_parser.set_defaults(gitpush=False)
arg_parser.add_argument('--config-file', type=str,
default='./report_conf.yaml',
@@ -56,7 +56,7 @@ if __name__ == "__main__":
args = vars(arg_parser.parse_args())
conf_file = args["config_file"]
with open(conf_file, "r") as f:
- cfg = yaml.load(f.read())
+ cfg = yaml.load(f.read(), Loader=yaml.FullLoader)
logfile = args["log_file"]
fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
diff --git a/reportservice/report_service.py b/reportservice/report_service.py
index 254f85453..3d2d02a25 100644
--- a/reportservice/report_service.py
+++ b/reportservice/report_service.py
@@ -156,7 +156,7 @@ async def server_runner(conf_file):
be accessed through ReadTheDocs.
"""
with open(conf_file, "r") as f:
- config = yaml.load(f.read())
+ config = yaml.load(f.read(), Loader=yaml.FullLoader)
# perform git-dir checks and pull the project for updates.
init_config_repo(config['GLOBAL']['git'])
@@ -263,57 +263,60 @@ async def server_runner(conf_file):
logging.error('Jobs have timed-out!')
logging.error('{}/temp has not been deleted.'.format(
os.path.dirname(os.path.abspath(__file__))))
+ # Avoid copying files if no git-push is planned
+ # to avoid causing local git repository errors.
+ if git_push:
+ # Copy all plots
+ for det_name, det_conf in instrument.items():
- # Copy all plots
- for det_name, det_conf in instrument.items():
-
- out_folder = det_conf['out-folder'].format(
- instrument=instr_name,
- detector=det_name)
+ out_folder = det_conf['out-folder'].format(
+ instrument=instr_name,
+ detector=det_name)
- figures = glob.glob("{}/*png".format(out_folder))
+ figures = glob.glob("{}/*png".format(out_folder))
- det_new_files = {}
+ det_new_files = {}
- for f in figures:
- const = f.split('/')[-1].split('_')[0]
- fpath = '{}/{}/{}/{}'.format(fig_local, instr_name,
- det_name, const)
+ for f in figures:
+ const = f.split('/')[-1].split('_')[0]
+ fpath = '{}/{}/{}/{}'.format(fig_local, instr_name,
+ det_name, const)
- os.makedirs(fpath, exist_ok=True)
- det_new_files[f] = fpath
+ os.makedirs(fpath, exist_ok=True)
+ det_new_files[f] = fpath
- # Set concurrency limitation.
- # 50 have been chosen by trial
- # Note: This is not the max limitation.
- sem = asyncio.Semaphore(50)
- all_new_files.append(
- '{}/{}'.format(fpath, f.split('/')[-1]))
+ # Set concurrency limitation.
+ # 50 have been chosen by trial
+ # Note: This is not the max limitation.
+ sem = asyncio.Semaphore(50)
+ all_new_files.append(
+ '{}/{}'.format(fpath, f.split('/')[-1]))
- await asyncio.gather(*[copy_files(k, v, sem)
- for k, v in det_new_files.items()])
+ await asyncio.gather(*[copy_files(k, v, sem)
+ for k, v in det_new_files.items()])
- logging.info('{} figures of {} are copied into {}'.format(
- len(figures), det_name, fig_local))
+ logging.info('{} figures of {} are copied into {}'.format(
+ len(figures), det_name, fig_local))
- # Remove sensitive information from the config file.
- del cfg['GLOBAL']
- # Write the requested cfg.yaml before pushing all figures.
- with open('{}/report_conf.yaml'.format(
- fig_local), 'w') as outfile:
- yaml.dump(cfg, outfile, default_flow_style=False)
+ if git_push:
+ # Remove sensitive information from the config file.
+ del cfg['GLOBAL']
+ # Write the requested cfg.yaml before pushing all figures.
+ with open('{}/report_conf.yaml'.format(
+ fig_local), 'w') as outfile:
+ yaml.dump(cfg, outfile, default_flow_style=False)
- all_new_files.append('{}/report_conf.yaml'.format(fig_local))
+ all_new_files.append('{}/report_conf.yaml'.format(fig_local))
- if git_push:
asyncio.ensure_future(push_figures(local_repo, all_new_files))
- # TODO:delete out-folder
- #try:
- # asyncio.ensure_future(del_folder(out_folder))
- #except:
- #logging.error(str(e))
- logging.info('All done')
+ # TODO:delete out-folder
+ #try:
+ # asyncio.ensure_future(del_folder(out_folder))
+ #except:
+ #logging.error(str(e))
+
+ logging.info('Generating requested plots is finished!')
return
--
GitLab