select pdf or html in local mode

reportservice/automatic_run.py

@@ -24,6 +24,7 @@ async def auto_run(cfg, timeout=3000):
     request = {}
     request['req'] = ['all']
     request['upload'] = True
+    request['report-fmt'] = 'html'
     for i, ti in enumerate(run_time):
         run_time[i] = parser.parse(ti)
 

reportservice/build_dc_report.sh

@@ -2,16 +2,16 @@
 
 # the path to doc folder with a Makefile
 dc_folder=$1
-mode=$2
+report_fmt=$2
 
 echo "Running with the following parameters:"
 echo "DC folder path: $dc_folder"
-echo "Mode is: $mode"
+echo "DC Report format should be: report_fmt"
 
-if [ "${mode}" == "local" ]
+if [ "${report_fmt}" == "pdf" ]
 then
     make latexpdf -C "${dc_folder}"
-elif [ "${mode}" == "prod" ]
+elif [ "${report_fmt}" == "html" ]
 then
     make html -C "${dc_folder}"
 fi

reportservice/manual_run.py

@@ -32,19 +32,27 @@ def manual_run(request, cfg):
     msg = socket.recv_pyobj()
     logging.info('{} Manual Run'.format(msg))
 
+
 arg_parser = argparse.ArgumentParser(description='Manual Launch')
 arg_parser.add_argument('--instrument', default=['all'], nargs='+',
-                        help='select the requested instruments. '
+                        help='Select the requested instruments. '
                              'Default=\"all\", which can be used for selecting'
-                             ' all instruments')
-arg_parser.add_argument('--upload', dest='upload', action='store_true',
-                        help='required for pushing the generated figures '
-                             'to the DC git repository. Default=bool(False)')
+                             ' all instruments.')
+arg_parser.add_argument('--upload', action='store_true',
+                        help='Required for uploading the generated figures.'
+                             'Default=False. '
+                             'Note: THIS HAS NO EFFECT IN SIM MODE!')
+arg_parser.add_argument('--report-fmt', default='html',
+                        type=str, choices=['pdf', 'html'],
+                        help='If available in the report service running mode,'
+                             ' this can configure the report format '
+                             'to be html or pdf. Default=html '
+                             'Note: THIS HAS NO EFFECT IN PROD AND SIM MODES!')
 arg_parser.set_defaults(upload=False)
-arg_parser.add_argument('--config-file', type=str, 
-                        default='./report_conf.yaml',
-                        help='path to report configuration file '
-                             'Default=./report_conf.yaml')
+arg_parser.add_argument('--overwrite-conf', type=str,
+                        default=None,
+                        help='Path to new report configuration file. '
+                             'Default=None, type=str.')
 arg_parser.add_argument('--log-file', type=str, default='./report.log',
                         help='The report log file path. Default=./report.log')
 arg_parser.add_argument('--logging', type=str, default="INFO", 
@@ -54,7 +62,7 @@ arg_parser.add_argument('--logging', type=str, default="INFO",
 
 if __name__ == "__main__":
     args = vars(arg_parser.parse_args())
-    conf_file = args["config_file"]
+    conf_file = args["overwrite_conf"]
     with open(conf_file, "r") as f:
         cfg = yaml.load(f.read(), Loader=yaml.FullLoader)
 
@@ -66,13 +74,11 @@ if __name__ == "__main__":
                         format='%(levelname)-6s: %(asctime)s %(message)s',
                         datefmt='%Y-%m-%d %H:%M:%S')
 
-    request = {}
-
-    request['upload'] = args["upload"]
+    request = {'upload': args["upload"],
+               'report-fmt': args["report_fmt"]}
 
-    if args["instrument"]:
+    if args["overwrite_conf"] is None:
         request['req'] = args["instrument"]
     else:
         request['req'] = cfg
-
     manual_run(request, cfg)

reportservice/report_conf.yaml

@@ -43,6 +43,10 @@ SPB:
             - 1.1
             - 2.2
             - 4.5
+        gain-setting:
+            - 0
+            - 1
+            - 2
         photon-energy: 9.2
         separate-plot:
             - "gain_setting"
@@ -156,6 +160,10 @@ MID:
             - 1.1
             - 2.2
             - 4.5
+        gain-setting:
+            - 0
+            - 1
+            - 2
         photon-energy: 9.2
         separate-plot:
             - "gain_setting"
@@ -247,6 +255,10 @@ FXE:
             - 1
             - 128
             - 512
+        gain-setting:
+            - 0
+            - 1
+            - 2
         photon-energy: 9.2
         separate-plot:
             - "gain_setting"

reportservice/report_service.py

-
 import argparse
 import asyncio
 from asyncio.subprocess import PIPE
@@ -125,16 +124,16 @@ async def copy_files(f, path, sem):
         await asyncio.subprocess.create_subprocess_shell(" ".join(cmd))
 
 
-async def build_dc_report(dc_folder, mode):
+async def build_dc_report(dc_folder, report_fmt):
     """
     Generating a DC report (latex or html) using maxwell nodes.
     With the supported inputs a slurm job is submitted to sphinx-build
-    pdflatex or html depending on mode of the report_service.
-    html in production mode and pdf in local mode
+    pdf or html depending on mode of the report_service.
+    html for prod mode and pdf or html for local mode depending
+    on the chosen report_fmt
 
-    :param dc_folder: the local DC folder path with figures and doc file
-                      for sphinx-build a pdf latex DC report
-    :param mode: the running mode of the report service
+    :param dc_folder: the local DC folder path with figures and rst files
+    :param report_fmt: the expected report format(html or pdf)
     """
     temp_path = "{}/temp/build_dc_report/".format(os.getcwd())
     os.makedirs(temp_path, exist_ok=True)
@@ -149,7 +148,7 @@ async def build_dc_report(dc_folder, mode):
 
     srun_base += [os.path.abspath("./build_dc_report.sh"),
                   os.path.abspath("{}/doc".format(dc_folder)),
-                  mode]
+                  report_fmt]
     logging.info("Building DC report submission: {}".format(srun_base))
     output = subprocess.check_output(srun_base).decode('utf8')
 
@@ -161,6 +160,7 @@ async def build_dc_report(dc_folder, mode):
     await wait_jobs([jobid])
     asyncio.ensure_future(del_folder("{}/slurm-{}.out".format(temp_path, jobid)))
 
+
 async def push_figures(repo_master, addf):
     """ Upload new figures
 
@@ -220,25 +220,19 @@ async def server_runner(conf_file, mode):
     while True:
         response = await socket.recv_pyobj()
         await socket.send_pyobj('Build DC reports through -->')
-        logging.info("response: {} with git pushing: {}"
-                     .format(response['req'], response['upload']))
+        logging.info("response: {} with uploading: {} and report format: {}"
+                     .format(response['req'],
+                             response['upload'],
+                             response['report-fmt']))
 
         # Check if response is a list or a dict.
         # if list, it should either have instrument names or ['all'].
-        # if dict, it should acquires the details of the requested reports
+        # if dict, it should acquire the details of the requested reports
         # for generation. As it will be used instead of report_conf.yaml
 
         # reports config file
         req_cfg = {}
 
-        # No interaction with DC repo (local or remote)
-        # is allowed if sim mode.
-        if mode == 'sim':
-            upload = False
-        else:
-            # boolean for pushing to DC git repo.
-            upload = response['upload']
-
         # Validate the type of 'requested' response.
         if isinstance(response['req'], dict):
             req_cfg = response['req']
@@ -258,9 +252,21 @@ async def server_runner(conf_file, mode):
             logging.error(Errors.REQUEST_MALFORMED.format(response['req']))
             continue
 
+        # No interaction with DC repository (local or remote)
+        # is allowed if sim mode.
+        if mode == 'sim':
+            req_cfg['GLOBAL']['upload'] = False
+            req_cfg['GLOBAL']['report-fmt'] = False
+        else:
+            # boolean for pushing to DC git repo.
+            req_cfg['GLOBAL']['upload'] = response['upload']
+            if mode == 'prod':
+                req_cfg['GLOBAL']['report-fmt'] = 'html'
+            else:
+                req_cfg['GLOBAL']['report-fmt'] = response['report-fmt']
         logging.info('Requested Configuration: {}'.format(req_cfg))
 
-        async def do_action(cfg, upload, mode):
+        async def do_action(cfg, service_mode):
 
             logging.info('Run plot production')
             local_repo = cfg['GLOBAL']['git']['repo-local']
@@ -301,10 +307,10 @@ async def server_runner(conf_file, mode):
                 for output in outputs:
                     if output[0]:
                         logging.info('Submission Output: {}'
-                                            .format(output[0].decode('utf8')))
+                                     .format(output[0].decode('utf8')))
                     if output[1]:
                         logging.error('Submission Error: {}'
-                                            .format(output[1].decode('utf8')))
+                                      .format(output[1].decode('utf8')))
                     job_list += await parse_output(output[0].decode('utf8'))
 
                 try:
@@ -318,7 +324,7 @@ async def server_runner(conf_file, mode):
 
                 # Avoid copying files if upload bool is False
                 # to avoid causing local git repository errors.
-                if upload:
+                if cfg['GLOBAL']['upload']:
                     # Copy all plots
                     for det_name, det_conf in instrument.items():
 
@@ -346,29 +352,35 @@ async def server_runner(conf_file, mode):
                                 '{}/{}'.format(fpath, f.split('/')[-1]))
 
                         await asyncio.gather(*[copy_files(k, v, sem)
-                                            for k, v in det_new_files.items()])
+                                               for k, v in det_new_files.items()])  # noqa
 
                         logging.info('{} figures of {} are copied into {}'
-                            .format(len(figures), det_name, fig_local))
-
-            if upload:
-
-                # Remove sensitive information from the config file.
-                del cfg['GLOBAL']
-                # Write the requested cfg.yaml before pushing all figures.
-                with open('{}/report_conf.yaml'.format(
-                                              fig_local), 'w') as outfile:
-                    yaml.dump(cfg, outfile, default_flow_style=False)
-
-                if mode == 'prod':
-                    # add report_con.yaml in the list of files added to the
-                    # new git commit before pushing to remote
-                    all_new_files.append('{}/report_conf.yaml'.format(fig_local))
-                    asyncio.ensure_future(push_figures(local_repo,
-                                                       all_new_files))
-                # build either html or latexpdf depending
-                # the running mode of the report_service
-                asyncio.ensure_future(build_dc_report(local_repo, mode))
+                                     .format(len(figures), det_name,
+                                             fig_local))
+
+            if cfg['GLOBAL']['upload']:
+                try:
+                    report_fmt = cfg['GLOBAL']['report-fmt']
+                    # Remove sensitive information from the config file.
+                    del cfg['GLOBAL']
+                    # Write the requested cfg.yaml before pushing all figures.
+                    with open('{}/report_conf.yaml'.format(
+                                                  fig_local), 'w') as outfile:
+                        yaml.dump(cfg, outfile, default_flow_style=False)
+
+                    if service_mode == 'prod':
+                        # add report_con.yaml in the list of files added to the
+                        # new git commit before pushing to remote
+                        all_new_files.append('{}/report_conf.yaml'
+                                             .format(fig_local))
+                        asyncio.ensure_future(push_figures(local_repo,
+                                                           all_new_files))
+                    # build either html or pdf depending on the running mode
+                    # of the report_service and requested report format.
+                    asyncio.ensure_future(build_dc_report(local_repo,
+                                                          report_fmt))  # noqa
+                except Exception as upload_e:
+                    logging.error("upload failed: {}".format(upload_e))
 
                 # TODO:delete out-folder
                 #try:
@@ -383,7 +395,6 @@ async def server_runner(conf_file, mode):
 
         try:
             asyncio.ensure_future(do_action(copy.copy(req_cfg),
-                                            copy.copy(upload),
                                             mode))
         except Exception as e:  # actions that fail are only error logged
             logging.error(str(e))