diff --git a/reportservice/automatic_run.py b/reportservice/automatic_run.py
index 2387eed2ccc741c4bb7d0b033885dc77304037ac..a85fa0b299aa1d5ad109c3bae349035e59551006 100644
--- a/reportservice/automatic_run.py
+++ b/reportservice/automatic_run.py
@@ -24,6 +24,7 @@ async def auto_run(cfg, timeout=3000):
request = {}
request['req'] = ['all']
request['upload'] = True
+ request['report-fmt'] = 'html'
for i, ti in enumerate(run_time):
run_time[i] = parser.parse(ti)
diff --git a/reportservice/build_dc_report.sh b/reportservice/build_dc_report.sh
index 1a3e65cbf4e0ff9df7269b10aac967cc2ab4a1e9..cc40a6e46f41264321a39c767c32580b77269765 100755
--- a/reportservice/build_dc_report.sh
+++ b/reportservice/build_dc_report.sh
@@ -2,16 +2,16 @@
# the path to doc folder with a Makefile
dc_folder=$1
-mode=$2
+report_fmt=$2
echo "Running with the following parameters:"
echo "DC folder path: $dc_folder"
-echo "Mode is: $mode"
+echo "DC Report format should be: report_fmt"
-if [ "${mode}" == "local" ]
+if [ "${report_fmt}" == "pdf" ]
then
make latexpdf -C "${dc_folder}"
-elif [ "${mode}" == "prod" ]
+elif [ "${report_fmt}" == "html" ]
then
make html -C "${dc_folder}"
fi
diff --git a/reportservice/manual_run.py b/reportservice/manual_run.py
index a2d4e9f3a47dd5714b8a929f8c0ba0247441cc20..0566e3fba290c088fa76ec309d782a9bbdf134d9 100644
--- a/reportservice/manual_run.py
+++ b/reportservice/manual_run.py
@@ -32,19 +32,27 @@ def manual_run(request, cfg):
msg = socket.recv_pyobj()
logging.info('{} Manual Run'.format(msg))
+
arg_parser = argparse.ArgumentParser(description='Manual Launch')
arg_parser.add_argument('--instrument', default=['all'], nargs='+',
- help='select the requested instruments. '
+ help='Select the requested instruments. '
'Default=\"all\", which can be used for selecting'
- ' all instruments')
-arg_parser.add_argument('--upload', dest='upload', action='store_true',
- help='required for pushing the generated figures '
- 'to the DC git repository. Default=bool(False)')
+ ' all instruments.')
+arg_parser.add_argument('--upload', action='store_true',
+ help='Required for uploading the generated figures.'
+ 'Default=False. '
+ 'Note: THIS HAS NO EFFECT IN SIM MODE!')
+arg_parser.add_argument('--report-fmt', default='html',
+ type=str, choices=['pdf', 'html'],
+ help='If available in the report service running mode,'
+ ' this can configure the report format '
+ 'to be html or pdf. Default=html '
+ 'Note: THIS HAS NO EFFECT IN PROD AND SIM MODES!')
arg_parser.set_defaults(upload=False)
-arg_parser.add_argument('--config-file', type=str,
- default='./report_conf.yaml',
- help='path to report configuration file '
- 'Default=./report_conf.yaml')
+arg_parser.add_argument('--overwrite-conf', type=str,
+ default=None,
+ help='Path to new report configuration file. '
+ 'Default=None, type=str.')
arg_parser.add_argument('--log-file', type=str, default='./report.log',
help='The report log file path. Default=./report.log')
arg_parser.add_argument('--logging', type=str, default="INFO",
@@ -54,7 +62,7 @@ arg_parser.add_argument('--logging', type=str, default="INFO",
if __name__ == "__main__":
args = vars(arg_parser.parse_args())
- conf_file = args["config_file"]
+ conf_file = args["overwrite_conf"]
with open(conf_file, "r") as f:
cfg = yaml.load(f.read(), Loader=yaml.FullLoader)
@@ -66,13 +74,11 @@ if __name__ == "__main__":
format='%(levelname)-6s: %(asctime)s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
- request = {}
-
- request['upload'] = args["upload"]
+ request = {'upload': args["upload"],
+ 'report-fmt': args["report_fmt"]}
- if args["instrument"]:
+ if args["overwrite_conf"] is None:
request['req'] = args["instrument"]
else:
request['req'] = cfg
-
manual_run(request, cfg)
diff --git a/reportservice/report_conf.yaml b/reportservice/report_conf.yaml
index 45b6ca5764b038c71017ff0865d3afb3ee93b972..c272c6755ae1b980abc625613e65c1a9d741c125 100644
--- a/reportservice/report_conf.yaml
+++ b/reportservice/report_conf.yaml
@@ -43,6 +43,10 @@ SPB:
- 1.1
- 2.2
- 4.5
+ gain-setting:
+ - 0
+ - 1
+ - 2
photon-energy: 9.2
separate-plot:
- "gain_setting"
@@ -156,6 +160,10 @@ MID:
- 1.1
- 2.2
- 4.5
+ gain-setting:
+ - 0
+ - 1
+ - 2
photon-energy: 9.2
separate-plot:
- "gain_setting"
@@ -247,6 +255,10 @@ FXE:
- 1
- 128
- 512
+ gain-setting:
+ - 0
+ - 1
+ - 2
photon-energy: 9.2
separate-plot:
- "gain_setting"
diff --git a/reportservice/report_service.py b/reportservice/report_service.py
index 90bde34a0d472e89a0796ad9ef614221ea10d447..130885c78473355967a4724ce3e0342f05fd86c3 100644
--- a/reportservice/report_service.py
+++ b/reportservice/report_service.py
@@ -1,4 +1,3 @@
-
import argparse
import asyncio
from asyncio.subprocess import PIPE
@@ -125,16 +124,16 @@ async def copy_files(f, path, sem):
await asyncio.subprocess.create_subprocess_shell(" ".join(cmd))
-async def build_dc_report(dc_folder, mode):
+async def build_dc_report(dc_folder, report_fmt):
"""
Generating a DC report (latex or html) using maxwell nodes.
With the supported inputs a slurm job is submitted to sphinx-build
- pdflatex or html depending on mode of the report_service.
- html in production mode and pdf in local mode
+ pdf or html depending on mode of the report_service.
+ html for prod mode and pdf or html for local mode depending
+ on the chosen report_fmt
- :param dc_folder: the local DC folder path with figures and doc file
- for sphinx-build a pdf latex DC report
- :param mode: the running mode of the report service
+ :param dc_folder: the local DC folder path with figures and rst files
+ :param report_fmt: the expected report format(html or pdf)
"""
temp_path = "{}/temp/build_dc_report/".format(os.getcwd())
os.makedirs(temp_path, exist_ok=True)
@@ -149,7 +148,7 @@ async def build_dc_report(dc_folder, mode):
srun_base += [os.path.abspath("./build_dc_report.sh"),
os.path.abspath("{}/doc".format(dc_folder)),
- mode]
+ report_fmt]
logging.info("Building DC report submission: {}".format(srun_base))
output = subprocess.check_output(srun_base).decode('utf8')
@@ -161,6 +160,7 @@ async def build_dc_report(dc_folder, mode):
await wait_jobs([jobid])
asyncio.ensure_future(del_folder("{}/slurm-{}.out".format(temp_path, jobid)))
+
async def push_figures(repo_master, addf):
""" Upload new figures
@@ -220,25 +220,19 @@ async def server_runner(conf_file, mode):
while True:
response = await socket.recv_pyobj()
await socket.send_pyobj('Build DC reports through -->')
- logging.info("response: {} with git pushing: {}"
- .format(response['req'], response['upload']))
+ logging.info("response: {} with uploading: {} and report format: {}"
+ .format(response['req'],
+ response['upload'],
+ response['report-fmt']))
# Check if response is a list or a dict.
# if list, it should either have instrument names or ['all'].
- # if dict, it should acquires the details of the requested reports
+ # if dict, it should acquire the details of the requested reports
# for generation. As it will be used instead of report_conf.yaml
# reports config file
req_cfg = {}
- # No interaction with DC repo (local or remote)
- # is allowed if sim mode.
- if mode == 'sim':
- upload = False
- else:
- # boolean for pushing to DC git repo.
- upload = response['upload']
-
# Validate the type of 'requested' response.
if isinstance(response['req'], dict):
req_cfg = response['req']
@@ -258,9 +252,21 @@ async def server_runner(conf_file, mode):
logging.error(Errors.REQUEST_MALFORMED.format(response['req']))
continue
+ # No interaction with DC repository (local or remote)
+ # is allowed if sim mode.
+ if mode == 'sim':
+ req_cfg['GLOBAL']['upload'] = False
+ req_cfg['GLOBAL']['report-fmt'] = False
+ else:
+ # boolean for pushing to DC git repo.
+ req_cfg['GLOBAL']['upload'] = response['upload']
+ if mode == 'prod':
+ req_cfg['GLOBAL']['report-fmt'] = 'html'
+ else:
+ req_cfg['GLOBAL']['report-fmt'] = response['report-fmt']
logging.info('Requested Configuration: {}'.format(req_cfg))
- async def do_action(cfg, upload, mode):
+ async def do_action(cfg, service_mode):
logging.info('Run plot production')
local_repo = cfg['GLOBAL']['git']['repo-local']
@@ -301,10 +307,10 @@ async def server_runner(conf_file, mode):
for output in outputs:
if output[0]:
logging.info('Submission Output: {}'
- .format(output[0].decode('utf8')))
+ .format(output[0].decode('utf8')))
if output[1]:
logging.error('Submission Error: {}'
- .format(output[1].decode('utf8')))
+ .format(output[1].decode('utf8')))
job_list += await parse_output(output[0].decode('utf8'))
try:
@@ -318,7 +324,7 @@ async def server_runner(conf_file, mode):
# Avoid copying files if upload bool is False
# to avoid causing local git repository errors.
- if upload:
+ if cfg['GLOBAL']['upload']:
# Copy all plots
for det_name, det_conf in instrument.items():
@@ -346,29 +352,35 @@ async def server_runner(conf_file, mode):
'{}/{}'.format(fpath, f.split('/')[-1]))
await asyncio.gather(*[copy_files(k, v, sem)
- for k, v in det_new_files.items()])
+ for k, v in det_new_files.items()]) # noqa
logging.info('{} figures of {} are copied into {}'
- .format(len(figures), det_name, fig_local))
-
- if upload:
-
- # Remove sensitive information from the config file.
- del cfg['GLOBAL']
- # Write the requested cfg.yaml before pushing all figures.
- with open('{}/report_conf.yaml'.format(
- fig_local), 'w') as outfile:
- yaml.dump(cfg, outfile, default_flow_style=False)
-
- if mode == 'prod':
- # add report_con.yaml in the list of files added to the
- # new git commit before pushing to remote
- all_new_files.append('{}/report_conf.yaml'.format(fig_local))
- asyncio.ensure_future(push_figures(local_repo,
- all_new_files))
- # build either html or latexpdf depending
- # the running mode of the report_service
- asyncio.ensure_future(build_dc_report(local_repo, mode))
+ .format(len(figures), det_name,
+ fig_local))
+
+ if cfg['GLOBAL']['upload']:
+ try:
+ report_fmt = cfg['GLOBAL']['report-fmt']
+ # Remove sensitive information from the config file.
+ del cfg['GLOBAL']
+ # Write the requested cfg.yaml before pushing all figures.
+ with open('{}/report_conf.yaml'.format(
+ fig_local), 'w') as outfile:
+ yaml.dump(cfg, outfile, default_flow_style=False)
+
+ if service_mode == 'prod':
+ # add report_con.yaml in the list of files added to the
+ # new git commit before pushing to remote
+ all_new_files.append('{}/report_conf.yaml'
+ .format(fig_local))
+ asyncio.ensure_future(push_figures(local_repo,
+ all_new_files))
+ # build either html or pdf depending on the running mode
+ # of the report_service and requested report format.
+ asyncio.ensure_future(build_dc_report(local_repo,
+ report_fmt)) # noqa
+ except Exception as upload_e:
+ logging.error("upload failed: {}".format(upload_e))
# TODO:delete out-folder
#try:
@@ -383,7 +395,6 @@ async def server_runner(conf_file, mode):
try:
asyncio.ensure_future(do_action(copy.copy(req_cfg),
- copy.copy(upload),
mode))
except Exception as e: # actions that fail are only error logged
logging.error(str(e))