add sim, local and prod modes and ability to generate pdfs

reportservice/README.md

@@ -16,6 +16,35 @@ Configuration
 
 It is important to know the machine name and the port, where the reportservice is running, for successful connection.
 
+Starting the Service
+--------------------
+
+The reportservice is a python script that can run through:
+
+    ```bash
+    python reportservice.py
+    ```
+
+The available command line arguments are:
+
+* --report-conf: The path for the main report configuration yaml file
+* --log: The logging mode (INFO, DEBUG, ERROR)
+* --mode: The mode for running the service. Choices are sim[simulation], local and prod[production]
+* --logging: The required logs to be written. Choices are INFO, DEBUG and Error
+
+Modes:
+
+*prod* is the production mode working on the max-exfl016 as xcal user for generating the DC report through RTD
+and it should generate a very generalized DC report for the available detectors with information useful for most of the detector experts and users.
+
+*local* is the mode used for generating figures locally without uploading the DC report on RTD,
+rather a pdf in the specified out-folder.
+
+*sim* is a simulation mode, which is mostly used for debugging purposes and tool development without generating any reports locally or over RTD.
+This mode make sure not to do any git interactions and it only works on running the notebooks for generating figures in the out-folder without further work.
+
+Report Configuration(report-conf):
+
 *report_conf.yaml* is the configuration file, which contains all the required information
 for operating and connecting to the reportservice.
 
@@ -65,25 +94,11 @@ The YAML configuration file can be modified with all the available parameters, r
             cal-db-interface: "<cal-db-host-port>"
 ```
 
-
-Starting the Service
---------------------
-
-The reportservice is a python script that can run through:
-
-    ```bash
-    python reportservice.py
-    ```
-
-The available command line arguments are:
-
-* --report-conf : The path for the main report configuration yaml file
-* --log : The logging mode (INFO, DEBUG, ERROR)
-
-Launching the service
+Triggering the service
 ---------------------
 
-The service can be launched through two processes:
+To use the service and generate a DC report corresponding to the report_conf.yaml.
+The service can be triggered through two processes:
 
 Automatic Launch:
 

reportservice/build_latex.sh

+#!/bin/bash
+
+# the path to doc folder with a Makefile
+dc_folder=$1
+
+echo "Running with the following parameters:"
+echo "DC folder path: $dc_folder"
+
+make latexpdf -C "${dc_folder}"
\ No newline at end of file

reportservice/report_conf.yaml

 GLOBAL:
     git:
-        repo-local: "/gpfs/exfel/data/scratch/xcal/calibration/DetectorCharacterization/"
+        repo-local: "/gpfs/exfel/data/scratch/ahmedk/test/calibration/DetectorCharacterization/"
         figures-remote: "http://git@git.xfel.eu/gitlab/detectors/DetectorCharacterization.git"
-    server-port: "tcp://max-exfl016:5566"
+    server-port: "tcp://max-exfl014:5566"
 
     run-on:
         - Monday 08:30:00 UTC
@@ -20,7 +20,7 @@ SPB:
         det-type:
             - "GENERIC"
             - "STATS_FROM_DB2"
-        modules: 
+        modules:
             - "AGIPD1M1"
         start-date: "2019-01-01"
         end-date: "NOW"
@@ -42,7 +42,7 @@ SPB:
         acquisition-rate:
             - 1.1
             - 2.2
-            - 4.5 
+            - 4.5
         photon-energy: 9.2
         separate-plot:
             - "gain_setting"
@@ -63,7 +63,7 @@ SPB:
         sp-name: "ASICs id"
         nMemToShow: 32
         use-existing: "''"
-        out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+        out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
         cal-db-timeout: 180000
         cal-db-interface: "tcp://max-exfl016:8015#8025"
 
@@ -124,7 +124,7 @@ SPB:
         sp-name: "Supercolumn 256*64"
         photon-energy: 9.2
         use-existing: "''"
-        out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+        out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
         cal-db-timeout: 180000
         cal-db-interface: "tcp://max-exfl016:8015#8025"
 
@@ -133,7 +133,7 @@ MID:
         det-type:
             - "GENERIC"
             - "STATS_FROM_DB2"
-        modules: 
+        modules:
             - "AGIPD1M2"
         start-date: "2019-01-01"
         end-date: "NOW"
@@ -155,7 +155,7 @@ MID:
         acquisition-rate:
             - 1.1
             - 2.2
-            - 4.5 
+            - 4.5
         photon-energy: 9.2
         separate-plot:
             - "gain_setting"
@@ -176,7 +176,7 @@ MID:
         sp-name: "ASICs id"
         nMemToShow: 32
         use-existing: "''"
-        out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+        out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
         cal-db-timeout: 180000
         cal-db-interface: "tcp://max-exfl016:8015#8025"
 
@@ -218,7 +218,7 @@ MID:
             - "Integration Time"
         sp-name: "ASICs id"
         use-existing: "''"
-        out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+        out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
         cal-db-timeout: 180000
         cal-db-interface: "tcp://max-exfl016:8015#8025"
 
@@ -228,7 +228,7 @@ FXE:
         det-type:
             - "GENERIC"
             - "STATS_FROM_DB2"
-        modules: 
+        modules:
             - "LPD1M1"
         start-date: "2019-01-01"
         end-date: "NOW"
@@ -265,7 +265,7 @@ FXE:
         sp-name: "ASICs id"
         nMemToShow: 32
         use-existing: "''"
-        out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+        out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
         cal-db-timeout: 180000
         cal-db-interface: "tcp://max-exfl016:8015#8025"
 
@@ -326,7 +326,7 @@ FXE:
         sp-name: "Supercolumn 256*64"
         photon-energy: 9.2
         use-existing: "''"
-        out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+        out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
         cal-db-timeout: 180000
         cal-db-interface: "tcp://max-exfl016:8015#8025"
 
@@ -343,7 +343,7 @@ DETLAB:
             - "Offset"
         dclass: "CCD"
         nMemToShow: 1
-        modules: 
+        modules:
             - "fastCCD1"
         bias-voltage:
             - 79
@@ -388,7 +388,7 @@ DETLAB:
         sp-name: "Supercolumn 967*10"
         photon-energy: 9.2
         use-existing: "''"
-        out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+        out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
         cal-db-timeout: 180000
         cal-db-interface: "tcp://max-exfl016:8015#8025"
 
@@ -406,7 +406,7 @@ SCS:
             - "Offset"
         dclass: "CCD"
         nMemToShow: 1
-        modules: 
+        modules:
             - "fastCCD1"
         bias-voltage:
             - 79
@@ -451,7 +451,7 @@ SCS:
         sp-name: "Supercolumn 967*10"
         photon-energy: 9.2
         use-existing: "''"
-        out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+        out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
         cal-db-timeout: 180000
         cal-db-interface: "tcp://max-exfl016:8015#8025"
 
@@ -468,7 +468,7 @@ SQS:
             - "Offset"
         dclass: "CCD"
         nMemToShow: 1
-        modules: 
+        modules:
             - "PnCCD1"
         bias-voltage:
             - 300
@@ -502,7 +502,7 @@ SQS:
         sp-name: "ASICs id"
         photon-energy: 9.2
         use-existing: "''"
-        out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+        out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
         cal-db-timeout: 180000
         cal-db-interface: "tcp://max-exfl016:8015#8025"
 
@@ -545,7 +545,7 @@ HED:
             - "Integration Time"
         sp-name: "superpixel id"
         use-existing: "''"
-        out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+        out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
         cal-db-timeout: 180000
         cal-db-interface: "tcp://max-exfl016:8015#8025"
 
@@ -605,7 +605,7 @@ HED:
         sp-name: "Supercolumn 256*64"
         photon-energy: 9.2
         use-existing: "''"
-        out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+        out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
         cal-db-timeout: 180000
         cal-db-interface: "tcp://max-exfl016:8015#8025"
 

reportservice/report_service.py

+#!/usr/bin/env python
+
 import argparse
 import asyncio
 from asyncio.subprocess import PIPE
@@ -6,7 +8,6 @@ import glob
 import logging
 import os
 import subprocess
-from time import sleep
 
 from git import Repo, InvalidGitRepositoryError
 import yaml
@@ -15,7 +16,6 @@ import zmq.asyncio
 
 from messages import Errors
 
-
 loop = asyncio.get_event_loop()
 
 
@@ -33,9 +33,17 @@ def init_config_repo(config):
         # clone the repo.
         repo = Repo.clone_from(config['figures-remote'],
                                config['repo-local'])
-        logging.info("Clonning the repository")
-    # make sure it is updated
-    repo.remote().pull()
+        logging.info("Cloning the repository")
+    try:
+        # make sure it is updated
+        repo.remote().pull()
+    except Exception as e:
+        logging.error(e)
+        # update the head of local repository as the remote's
+        repo.remote().fetch()
+        repo.git.reset('--hard', 'origin/master')
+        # then make sure
+        repo.remote().pull()
     logging.info("Config repo is initialized")
 
 
@@ -118,6 +126,35 @@ async def copy_files(f, path, sem):
         await asyncio.subprocess.create_subprocess_shell(" ".join(cmd))
 
 
+async def create_pdf(dc_folder):
+    """
+    Generating a DC report pdf file using maxwell nodes. With the supported
+    inputs a slurm job is submitted to sphinx-build latexpdf.
+
+    :param dc_folder: the local DC folder path with figures and doc file
+                      for sphinx-build a pdf latex DC report
+    """
+
+    temp_path = "temp/pdf_generation/"
+    os.makedirs(temp_path, exist_ok=True)
+    sprof = os.environ.get("XFELCALSLURM", "exfel")
+    launcher_command = "sbatch -t 24:00:00 --mem 500G --requeue " \
+                       "--output {temp_path}/slurm-%j.out"
+
+    srun_base = launcher_command.format(
+                temp_path=temp_path) + " -p {}".format(sprof)
+    srun_base = srun_base.split()
+    srun_base += [os.path.abspath("./build_latex.sh"),
+                  os.path.abspath("{}/doc".format(dc_folder))
+                  ]
+    output = subprocess.check_output(srun_base).decode('utf8')
+    jobid = None
+    for line in output.split("\n"):
+        if "Submitted batch job " in line:
+            jobid = line.split(" ")[3]
+    logging.info("Submitted job for generating a pdf: {}".format(jobid))
+
+
 async def push_figures(repo_master, addf):
     """ Upload new figures
 
@@ -137,13 +174,11 @@ async def push_figures(repo_master, addf):
             await asyncio.sleep(2)
             add_tries += 1
     repo.index.commit("Add {} new figures".format(len(addf)))
-    #TODO: create an async function for pushing new figures
-    # to avoid blocking the report service.
     repo.remote().push()
     logging.info('Pushed to git')
 
 
-async def server_runner(conf_file):
+async def server_runner(conf_file, mode):
     """
     The main server loop. After pulling the latest changes
     of the DC project, it awaits receiving configurations
@@ -159,9 +194,13 @@ async def server_runner(conf_file):
     with open(conf_file, "r") as f:
         config = yaml.load(f.read(), Loader=yaml.FullLoader)
 
-    # perform git-dir checks and pull the project for updates.
-    init_config_repo(config['GLOBAL']['git'])
-    
+    # perform git-dir checks and pull the project
+    # for updates only in production mode.
+    if mode != 'sim':
+        init_config_repo(config['GLOBAL']['git'])
+
+    logging.info("Report service started in mode: {}".format(mode))
+
     logging.info("report service port: {}:{}"
                         .format(config['GLOBAL']['report-service']['bind-to'],
                                 config['GLOBAL']['report-service']['port']))
@@ -186,9 +225,15 @@ async def server_runner(conf_file):
         # reports config file
         req_cfg = {}
 
-        # boolean for pushing to DC git repo.
-        git_push = response['gitpush']
+        # No interaction with DC repo (local or remote)
+        # is allowed if sim mode.
+        if mode == 'sim':
+            git_push = False
+        else:
+            # boolean for pushing to DC git repo.
+            git_push = response['gitpush']
 
+        # Validate the type of 'requested' response.
         if isinstance(response['req'], dict):
             req_cfg = response['req']
         elif isinstance(response['req'], list):
@@ -209,7 +254,7 @@ async def server_runner(conf_file):
 
         logging.info('Requested Configuration: {}'.format(req_cfg))
 
-        async def do_action(cfg, git_push):
+        async def do_action(cfg, git_push, mode):
 
             logging.info('Run plot production')
             local_repo = cfg['GLOBAL']['git']['repo-local']
@@ -296,34 +341,39 @@ async def server_runner(conf_file):
                         await asyncio.gather(*[copy_files(k, v, sem)
                                             for k, v in det_new_files.items()])
 
-                        logging.info('{} figures of {} are copied into {}'.format(
-                                               len(figures), det_name, fig_local))
+                        logging.info('{} figures of {} are copied into {}'
+                            .format(len(figures), det_name, fig_local))
 
             if git_push:
+
                 # Remove sensitive information from the config file.
                 del cfg['GLOBAL']
                 # Write the requested cfg.yaml before pushing all figures.
                 with open('{}/report_conf.yaml'.format(
                                               fig_local), 'w') as outfile:
                     yaml.dump(cfg, outfile, default_flow_style=False)
-
-                all_new_files.append('{}/report_conf.yaml'.format(fig_local))
-
-                asyncio.ensure_future(push_figures(local_repo, all_new_files))
+                if mode == 'prod':
+                    all_new_files.append('{}/report_conf.yaml'.format(fig_local))
+                    asyncio.ensure_future(push_figures(local_repo,
+                                                       all_new_files))
+                elif mode == 'local':
+                    asyncio.ensure_future(create_pdf(local_repo))
 
                 # TODO:delete out-folder
                 #try:
-                    # asyncio.ensure_future(del_folder(out_folder))
+                #    asyncio.ensure_future(del_folder(out_folder))
                 #except:
-                    #logging.error(str(e))
+                #    logging.error(str(e))
 
             logging.info('Generating requested plots is finished!')
+            logging.info('=======================================')
 
             return
 
         try:
             asyncio.ensure_future(do_action(copy.copy(req_cfg),
-                                            copy.copy(git_push)))
+                                            copy.copy(git_push),
+                                            mode))
         except Exception as e:  # actions that fail are only error logged
             logging.error(str(e))
             break
@@ -334,6 +384,7 @@ arg_parser.add_argument('--config-file', type=str,
                         help='config file path with '
                              'reportservice port. '
                              'Default=./report_conf.yaml')
+arg_parser.add_argument('--mode', type=str, default="sim", choices=['sim', 'prod', 'local'])
 arg_parser.add_argument('--log-file', type=str, default='./report.log',
                         help='The report log file path. Default=./report.log')
 arg_parser.add_argument('--logging', type=str, default="INFO",
@@ -352,7 +403,7 @@ if __name__ == "__main__":
                         level=getattr(logging, args['logging']),
                         format='%(levelname)-6s: %(asctime)s %(message)s',
                         datefmt='%Y-%m-%d %H:%M:%S')
-
+    mode = args["mode"]
     loop = asyncio.get_event_loop()
-    loop.run_until_complete(server_runner(conf_file))
+    loop.run_until_complete(server_runner(conf_file, mode))
     loop.close()