Merge branch 'feat/diff_mods' into 'master'

Feat/diff mods

See merge request detectors/pycalibration!204

reportservice/README.md

@@ -16,6 +16,37 @@ Configuration
 
 It is important to know the machine name and the port, where the reportservice is running, for successful connection.
 
+Starting the Service
+--------------------
+
+The reportservice is a python script that can run through:
+
+    ```bash
+    python reportservice.py
+    ```
+
+The available command line arguments are:
+
+* --report-conf: The path for the main report configuration yaml file.
+* --log-file: The path for the log file.
+* --mode: The mode for running the service. Choices are sim[simulation], local and prod[production].
+* --logging: The required logs to be written. Choices are INFO, DEBUG and Error.
+
+Modes:
+
+*prod* is the production mode working on the max-exfl016 as xcal user for generating the DC report through RTD
+and it should generate a very generalized DC report for the available detectors with information useful for most of the detector experts and users.
+
+*local* is the mode used for generating figures locally without uploading the DC report on RTD or pushing figures
+to the git repository, rather generated figures are copied to the local repository and depending on the
+given report-fmt(report format) argument an html or a pdf is generated in doc/_build/ 
+of the report service out folder (repo-local).
+
+*sim* is a simulation mode, which is mostly used for debugging purposes and tool development without generating any reports locally or over RTD.
+This mode make sure not to do any git interactions and it only works on running the notebooks for generating figures in the out-folder without further work.
+
+Report Configuration(report-conf):
+
 *report_conf.yaml* is the configuration file, which contains all the required information
 for operating and connecting to the reportservice.
 
@@ -65,25 +96,11 @@ The YAML configuration file can be modified with all the available parameters, r
             cal-db-interface: "<cal-db-host-port>"
 ```
 
-
-Starting the Service
---------------------
-
-The reportservice is a python script that can run through:
-
-    ```bash
-    python reportservice.py
-    ```
-
-The available command line arguments are:
-
-* --report-conf : The path for the main report configuration yaml file
-* --log : The logging mode (INFO, DEBUG, ERROR)
-
-Launching the service
+Triggering the service
 ---------------------
 
-The service can be launched through two processes:
+To use the service and generate a DC report corresponding to the report_conf.yaml.
+The service can be triggered through two processes:
 
 Automatic Launch:
 
@@ -92,7 +109,21 @@ Automatic Launch:
     ```bash
     python automatic_run.py
     ```
+* --config-file: The path for the configuration file* --log-file: The path for the log file.
+* --logging: The required logs to be written. Choices are INFO, DEBUG and Error
+* --log-file: The path for the log file.
 
 Manual Launch:
 
     This manual launch script is currently used for debugging purposes, only.
+    
+    The available command line arguments are:
+    
+* --config-file: The path for the configuration file
+* --instrument: A selected list of instruments to generate a report for. This instrument must be in the report_conf.yaml. The default for this argument is ['all]
+* --overwrite-conf: A bool for indicating a new report configuration file(conf-file) should be sent instead of the default report_conf.yaml,
+which is used by report_service.py from the start.
+* --log-file: The path for the log file.
+* --report-fmt: The output DC report format. Choices are pdf or html
+* --upload: A bool for uploading the figure to out-folder of the report service(repo-local) and generating a report. Default is False
+* --logging: The required log mode to be used. Choices are INFO, DEBUG and Error

reportservice/automatic_run.py

@@ -23,7 +23,8 @@ async def auto_run(cfg, timeout=3000):
     run_time = cfg['GLOBAL']['run-on']
     request = {}
     request['req'] = ['all']
-    request['gitpush'] = True
+    request['upload'] = True
+    request['report-fmt'] = 'html'
     for i, ti in enumerate(run_time):
         run_time[i] = parser.parse(ti)
 

reportservice/build_dc_report.sh

+#!/bin/bash
+
+# the path to doc folder with a Makefile
+dc_folder=$1
+report_fmt=$2
+
+echo "Running with the following parameters:"
+echo "DC folder path: $dc_folder"
+echo "DC Report format should be: report_fmt"
+
+if [ "${report_fmt}" == "pdf" ]
+then
+    make latexpdf -C "${dc_folder}"
+elif [ "${report_fmt}" == "html" ]
+then
+    make html -C "${dc_folder}"
+fi

reportservice/manual_run.py

@@ -13,7 +13,7 @@ def manual_run(request, cfg):
 
     :param request: a dictionary for generating reports for the
                      requested Instruments. This dict consists
-                     of two keys. A gitpush boolean key for defining
+                     of two keys. A upload boolean key for defining
                      the fate of the generated plots
                      (pushed to the DC repo. or staying locally)
                      and a key request which can either be a list
@@ -32,19 +32,32 @@ def manual_run(request, cfg):
     msg = socket.recv_pyobj()
     logging.info('{} Manual Run'.format(msg))
 
+
 arg_parser = argparse.ArgumentParser(description='Manual Launch')
 arg_parser.add_argument('--instrument', default=['all'], nargs='+',
-                        help='select the requested instruments. '
+                        help='Select the requested instruments. '
                              'Default=\"all\", which can be used for selecting'
-                             ' all instruments')
-arg_parser.add_argument('--gitpush', dest='gitpush', action='store_true',
-                        help='required for pushing the generated figures '
-                             'to the DC git repository. Default=bool(False)')
-arg_parser.set_defaults(gitpush=False)
-arg_parser.add_argument('--config-file', type=str, 
+                             ' all instruments.')
+arg_parser.add_argument('--config-file', type=str,
                         default='./report_conf.yaml',
                         help='path to report configuration file '
                              'Default=./report_conf.yaml')
+arg_parser.add_argument('--upload', action='store_true',
+                        help='Required for uploading the generated figures.'
+                             'Default=False. '
+                             'Note: THIS HAS NO EFFECT IN SIM MODE!')
+arg_parser.set_defaults(upload=False)
+arg_parser.add_argument('--overwrite-conf', action='store_true',
+                        help='A flag for using a different config file than'
+                             'what is used by the running report_service.'
+                             'Default=False, type=str.')
+arg_parser.set_defaults(overwrite_conf=False)
+arg_parser.add_argument('--report-fmt', default='html',
+                        type=str, choices=['pdf', 'html'],
+                        help='If available in the report service running mode,'
+                             ' this can configure the report format '
+                             'to be html or pdf. Default=html '
+                             'Note: THIS HAS NO EFFECT IN PROD AND SIM MODES!')
 arg_parser.add_argument('--log-file', type=str, default='./report.log',
                         help='The report log file path. Default=./report.log')
 arg_parser.add_argument('--logging', type=str, default="INFO", 
@@ -66,13 +79,11 @@ if __name__ == "__main__":
                         format='%(levelname)-6s: %(asctime)s %(message)s',
                         datefmt='%Y-%m-%d %H:%M:%S')
 
-    request = {}
-
-    request['gitpush'] = args["gitpush"]
+    request = {'upload': args["upload"],
+               'report-fmt': args["report_fmt"]}
 
-    if args["instrument"]:
-        request['req'] = args["instrument"]
-    else:
+    if args["overwrite_conf"]:
         request['req'] = cfg
-
+    else:
+        request['req'] = args["instrument"]
     manual_run(request, cfg)

reportservice/report_conf.yaml

@@ -20,7 +20,7 @@ SPB:
         det-type:
             - "GENERIC"
             - "STATS_FROM_DB2"
-        modules: 
+        modules:
             - "AGIPD1M1"
         start-date: "2019-01-01"
         end-date: "NOW"
@@ -42,7 +42,11 @@ SPB:
         acquisition-rate:
             - 1.1
             - 2.2
-            - 4.5 
+            - 4.5
+        gain-setting:
+            - 0
+            - 1
+            - 2
         photon-energy: 9.2
         separate-plot:
             - "gain_setting"
@@ -133,7 +137,7 @@ MID:
         det-type:
             - "GENERIC"
             - "STATS_FROM_DB2"
-        modules: 
+        modules:
             - "AGIPD1M2"
         start-date: "2019-01-01"
         end-date: "NOW"
@@ -155,7 +159,11 @@ MID:
         acquisition-rate:
             - 1.1
             - 2.2
-            - 4.5 
+            - 4.5
+        gain-setting:
+            - 0
+            - 1
+            - 2
         photon-energy: 9.2
         separate-plot:
             - "gain_setting"
@@ -228,7 +236,7 @@ FXE:
         det-type:
             - "GENERIC"
             - "STATS_FROM_DB2"
-        modules: 
+        modules:
             - "LPD1M1"
         start-date: "2019-01-01"
         end-date: "NOW"
@@ -247,6 +255,10 @@ FXE:
             - 1
             - 128
             - 512
+        gain-setting:
+            - 0
+            - 1
+            - 2
         photon-energy: 9.2
         separate-plot:
             - "gain_setting"
@@ -343,7 +355,7 @@ DETLAB:
             - "Offset"
         dclass: "CCD"
         nMemToShow: 1
-        modules: 
+        modules:
             - "fastCCD1"
         bias-voltage:
             - 79
@@ -406,7 +418,7 @@ SCS:
             - "Offset"
         dclass: "CCD"
         nMemToShow: 1
-        modules: 
+        modules:
             - "fastCCD1"
         bias-voltage:
             - 79
@@ -468,7 +480,7 @@ SQS:
             - "Offset"
         dclass: "CCD"
         nMemToShow: 1
-        modules: 
+        modules:
             - "PnCCD1"
         bias-voltage:
             - 300

reportservice/report_service.py

@@ -6,7 +6,6 @@ import glob
 import logging
 import os
 import subprocess
-from time import sleep
 
 from git import Repo, InvalidGitRepositoryError
 import yaml
@@ -15,7 +14,6 @@ import zmq.asyncio
 
 from messages import Errors
 
-
 loop = asyncio.get_event_loop()
 
 
@@ -33,9 +31,17 @@ def init_config_repo(config):
         # clone the repo.
         repo = Repo.clone_from(config['figures-remote'],
                                config['repo-local'])
-        logging.info("Clonning the repository")
-    # make sure it is updated
-    repo.remote().pull()
+        logging.info("Cloning the repository")
+    try:
+        # make sure it is updated
+        repo.remote().pull()
+    except Exception as e:
+        logging.error(e)
+        # update the head of local repository as the remote's
+        repo.remote().fetch()
+        repo.git.reset('--hard', 'origin/master')
+        # then make sure
+        repo.remote().pull()
     logging.info("Config repo is initialized")
 
 
@@ -60,7 +66,7 @@ async def wait_jobs(joblist):
                 if str(job) in line:
                     found_jobs.add(job)
         if len(found_jobs) == 0:
-            logging.info('Plot modules is done')
+            logging.info('Jobs are finished')
             break
         await asyncio.sleep(10)
         counter += 10
@@ -96,7 +102,7 @@ async def del_folder(fpath):
     """
     cmd = ["rm", "-rf", fpath]
     await asyncio.subprocess.create_subprocess_shell(" ".join(cmd))
-    logging.info('tmp file has been deleted')
+    logging.info('temp file {} has been deleted'.format(fpath))
 
 
 async def copy_files(f, path, sem):
@@ -118,6 +124,43 @@ async def copy_files(f, path, sem):
         await asyncio.subprocess.create_subprocess_shell(" ".join(cmd))
 
 
+async def build_dc_report(dc_folder, report_fmt):
+    """
+    Generating a DC report (latex or html) using maxwell nodes.
+    With the supported inputs a slurm job is submitted to sphinx-build
+    pdf or html depending on mode of the report_service.
+    html for prod mode and pdf or html for local mode depending
+    on the chosen report_fmt
+
+    :param dc_folder: the local DC folder path with figures and rst files
+    :param report_fmt: the expected report format(html or pdf)
+    """
+    temp_path = "{}/temp/build_dc_report/".format(os.getcwd())
+    os.makedirs(temp_path, exist_ok=True)
+
+    # launching a slurm job and assigning the bash script to it.
+    sprof = os.environ.get("XFELCALSLURM", "exfel")
+    launcher_command = "sbatch -t 24:00:00 --mem 500G --requeue " \
+                       "--output {temp_path}/slurm-%j.out"
+    srun_base = launcher_command.format(
+                temp_path=temp_path) + " -p {}".format(sprof)
+    srun_base = srun_base.split()
+
+    srun_base += [os.path.abspath("./build_dc_report.sh"),
+                  os.path.abspath("{}/doc".format(dc_folder)),
+                  report_fmt]
+    logging.info("Building DC report submission: {}".format(srun_base))
+    output = subprocess.check_output(srun_base).decode('utf8')
+
+    jobid = None
+    for line in output.split("\n"):
+        if "Submitted batch job " in line:
+            jobid = line.split(" ")[3]
+    logging.info("Submitted job for building a report: {}".format(jobid))
+    await wait_jobs([jobid])
+    asyncio.ensure_future(del_folder("{}/slurm-{}.out".format(temp_path, jobid)))
+
+
 async def push_figures(repo_master, addf):
     """ Upload new figures
 
@@ -137,13 +180,11 @@ async def push_figures(repo_master, addf):
             await asyncio.sleep(2)
             add_tries += 1
     repo.index.commit("Add {} new figures".format(len(addf)))
-    #TODO: create an async function for pushing new figures
-    # to avoid blocking the report service.
     repo.remote().push()
     logging.info('Pushed to git')
 
 
-async def server_runner(conf_file):
+async def server_runner(conf_file, mode):
     """
     The main server loop. After pulling the latest changes
     of the DC project, it awaits receiving configurations
@@ -159,9 +200,13 @@ async def server_runner(conf_file):
     with open(conf_file, "r") as f:
         config = yaml.load(f.read(), Loader=yaml.FullLoader)
 
-    # perform git-dir checks and pull the project for updates.
-    init_config_repo(config['GLOBAL']['git'])
-    
+    # perform git-dir checks and pull the project
+    # for updates only in production mode.
+    if mode != 'sim':
+        init_config_repo(config['GLOBAL']['git'])
+
+    logging.info("Report service started in mode: {}".format(mode))
+
     logging.info("report service port: {}:{}"
                         .format(config['GLOBAL']['report-service']['bind-to'],
                                 config['GLOBAL']['report-service']['port']))
@@ -175,20 +220,20 @@ async def server_runner(conf_file):
     while True:
         response = await socket.recv_pyobj()
         await socket.send_pyobj('Build DC reports through -->')
-        logging.info("response: {} with git pushing: {}"
-                     .format(response['req'], response['gitpush']))
+        logging.info("response: {} with uploading: {} and report format: {}"
+                     .format(response['req'],
+                             response['upload'],
+                             response['report-fmt']))
 
         # Check if response is a list or a dict.
         # if list, it should either have instrument names or ['all'].
-        # if dict, it should acquires the details of the requested reports
+        # if dict, it should acquire the details of the requested reports
         # for generation. As it will be used instead of report_conf.yaml
 
         # reports config file
         req_cfg = {}
 
-        # boolean for pushing to DC git repo.
-        git_push = response['gitpush']
-
+        # Validate the type of 'requested' response.
         if isinstance(response['req'], dict):
             req_cfg = response['req']
         elif isinstance(response['req'], list):
@@ -207,9 +252,21 @@ async def server_runner(conf_file):
             logging.error(Errors.REQUEST_MALFORMED.format(response['req']))
             continue
 
+        # No interaction with DC repository (local or remote)
+        # is allowed if sim mode.
+        if mode == 'sim':
+            req_cfg['GLOBAL']['upload'] = False
+            req_cfg['GLOBAL']['report-fmt'] = False
+        else:
+            # boolean for pushing to DC git repo.
+            req_cfg['GLOBAL']['upload'] = response['upload']
+            if mode == 'prod':
+                req_cfg['GLOBAL']['report-fmt'] = 'html'
+            else:
+                req_cfg['GLOBAL']['report-fmt'] = response['report-fmt']
         logging.info('Requested Configuration: {}'.format(req_cfg))
 
-        async def do_action(cfg, git_push):
+        async def do_action(cfg, service_mode):
 
             logging.info('Run plot production')
             local_repo = cfg['GLOBAL']['git']['repo-local']
@@ -250,10 +307,10 @@ async def server_runner(conf_file):
                 for output in outputs:
                     if output[0]:
                         logging.info('Submission Output: {}'
-                                            .format(output[0].decode('utf8')))
+                                     .format(output[0].decode('utf8')))
                     if output[1]:
                         logging.error('Submission Error: {}'
-                                            .format(output[1].decode('utf8')))
+                                      .format(output[1].decode('utf8')))
                     job_list += await parse_output(output[0].decode('utf8'))
 
                 try:
@@ -264,9 +321,10 @@ async def server_runner(conf_file):
                     logging.error('Jobs have timed-out!')
                     logging.error('{}/temp has not been deleted.'.format(
                                   os.path.dirname(os.path.abspath(__file__))))
-                # Avoid copying files if no git-push is planned
+
+                # Avoid copying files if upload bool is False
                 # to avoid causing local git repository errors.
-                if git_push:
+                if cfg['GLOBAL']['upload']:
                     # Copy all plots
                     for det_name, det_conf in instrument.items():
 
@@ -294,36 +352,50 @@ async def server_runner(conf_file):
                                 '{}/{}'.format(fpath, f.split('/')[-1]))
 
                         await asyncio.gather(*[copy_files(k, v, sem)
-                                            for k, v in det_new_files.items()])
+                                               for k, v in det_new_files.items()])  # noqa
 
-                        logging.info('{} figures of {} are copied into {}'.format(
-                                               len(figures), det_name, fig_local))
+                        logging.info('{} figures of {} are copied into {}'
+                                     .format(len(figures), det_name,
+                                             fig_local))
 
-            if git_push:
-                # Remove sensitive information from the config file.
-                del cfg['GLOBAL']
-                # Write the requested cfg.yaml before pushing all figures.
-                with open('{}/report_conf.yaml'.format(
-                                              fig_local), 'w') as outfile:
-                    yaml.dump(cfg, outfile, default_flow_style=False)
-
-                all_new_files.append('{}/report_conf.yaml'.format(fig_local))
-
-                asyncio.ensure_future(push_figures(local_repo, all_new_files))
+            if cfg['GLOBAL']['upload']:
+                try:
+                    report_fmt = cfg['GLOBAL']['report-fmt']
+                    # Remove sensitive information from the config file.
+                    del cfg['GLOBAL']
+                    # Write the requested cfg.yaml before pushing all figures.
+                    with open('{}/report_conf.yaml'.format(
+                                                  fig_local), 'w') as outfile:
+                        yaml.dump(cfg, outfile, default_flow_style=False)
+
+                    if service_mode == 'prod':
+                        # add report_con.yaml in the list of files added to the
+                        # new git commit before pushing to remote
+                        all_new_files.append('{}/report_conf.yaml'
+                                             .format(fig_local))
+                        asyncio.ensure_future(push_figures(local_repo,
+                                                           all_new_files))
+                    # build either html or pdf depending on the running mode
+                    # of the report_service and requested report format.
+                    asyncio.ensure_future(build_dc_report(local_repo,
+                                                          report_fmt))  # noqa
+                except Exception as upload_e:
+                    logging.error("upload failed: {}".format(upload_e))
 
                 # TODO:delete out-folder
                 #try:
-                    # asyncio.ensure_future(del_folder(out_folder))
+                #    asyncio.ensure_future(del_folder(out_folder))
                 #except:
-                    #logging.error(str(e))
+                #    logging.error(str(e))
 
             logging.info('Generating requested plots is finished!')
+            logging.info('=======================================')
 
             return
 
         try:
             asyncio.ensure_future(do_action(copy.copy(req_cfg),
-                                            copy.copy(git_push)))
+                                            mode))
         except Exception as e:  # actions that fail are only error logged
             logging.error(str(e))
             break
@@ -334,6 +406,7 @@ arg_parser.add_argument('--config-file', type=str,
                         help='config file path with '
                              'reportservice port. '
                              'Default=./report_conf.yaml')
+arg_parser.add_argument('--mode', type=str, default="sim", choices=['sim', 'prod', 'local'])
 arg_parser.add_argument('--log-file', type=str, default='./report.log',
                         help='The report log file path. Default=./report.log')
 arg_parser.add_argument('--logging', type=str, default="INFO",
@@ -352,7 +425,7 @@ if __name__ == "__main__":
                         level=getattr(logging, args['logging']),
                         format='%(levelname)-6s: %(asctime)s %(message)s',
                         datefmt='%Y-%m-%d %H:%M:%S')
-
+    mode = args["mode"]
     loop = asyncio.get_event_loop()
-    loop.run_until_complete(server_runner(conf_file))
+    loop.run_until_complete(server_runner(conf_file, mode))
     loop.close()