diff --git a/reportservice/README.md b/reportservice/README.md
index fd1acab83e3aeea09fe76c2e663b411abfb4dea2..e971acde8038cd13bfe1b4261bf7b5f2b05b9d48 100644
--- a/reportservice/README.md
+++ b/reportservice/README.md
@@ -16,6 +16,35 @@ Configuration
It is important to know the machine name and the port, where the reportservice is running, for successful connection.
+Starting the Service
+--------------------
+
+The reportservice is a python script that can run through:
+
+ ```bash
+ python reportservice.py
+ ```
+
+The available command line arguments are:
+
+* --report-conf: The path for the main report configuration yaml file
+* --log: The logging mode (INFO, DEBUG, ERROR)
+* --mode: The mode for running the service. Choices are sim[simulation], local and prod[production]
+* --logging: The required logs to be written. Choices are INFO, DEBUG and Error
+
+Modes:
+
+*prod* is the production mode working on the max-exfl016 as xcal user for generating the DC report through RTD
+and it should generate a very generalized DC report for the available detectors with information useful for most of the detector experts and users.
+
+*local* is the mode used for generating figures locally without uploading the DC report on RTD,
+rather a pdf in the specified out-folder.
+
+*sim* is a simulation mode, which is mostly used for debugging purposes and tool development without generating any reports locally or over RTD.
+This mode make sure not to do any git interactions and it only works on running the notebooks for generating figures in the out-folder without further work.
+
+Report Configuration(report-conf):
+
*report_conf.yaml* is the configuration file, which contains all the required information
for operating and connecting to the reportservice.
@@ -65,25 +94,11 @@ The YAML configuration file can be modified with all the available parameters, r
cal-db-interface: "<cal-db-host-port>"
```
-
-Starting the Service
---------------------
-
-The reportservice is a python script that can run through:
-
- ```bash
- python reportservice.py
- ```
-
-The available command line arguments are:
-
-* --report-conf : The path for the main report configuration yaml file
-* --log : The logging mode (INFO, DEBUG, ERROR)
-
-Launching the service
+Triggering the service
---------------------
-The service can be launched through two processes:
+To use the service and generate a DC report corresponding to the report_conf.yaml.
+The service can be triggered through two processes:
Automatic Launch:
diff --git a/reportservice/build_latex.sh b/reportservice/build_latex.sh
new file mode 100755
index 0000000000000000000000000000000000000000..f8a3c98419a8562d7e93532b3369fe2c9328f4dc
--- /dev/null
+++ b/reportservice/build_latex.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# the path to doc folder with a Makefile
+dc_folder=$1
+
+echo "Running with the following parameters:"
+echo "DC folder path: $dc_folder"
+
+make latexpdf -C "${dc_folder}"
\ No newline at end of file
diff --git a/reportservice/report_conf.yaml b/reportservice/report_conf.yaml
index b8fbc0c3e49415474dcb3e441976f082b8516ae6..32b9dfd614395ddf51f0696e04a27afac4ea4c3e 100644
--- a/reportservice/report_conf.yaml
+++ b/reportservice/report_conf.yaml
@@ -1,8 +1,8 @@
GLOBAL:
git:
- repo-local: "/gpfs/exfel/data/scratch/xcal/calibration/DetectorCharacterization/"
+ repo-local: "/gpfs/exfel/data/scratch/ahmedk/test/calibration/DetectorCharacterization/"
figures-remote: "http://git@git.xfel.eu/gitlab/detectors/DetectorCharacterization.git"
- server-port: "tcp://max-exfl016:5566"
+ server-port: "tcp://max-exfl014:5566"
run-on:
- Monday 08:30:00 UTC
@@ -20,7 +20,7 @@ SPB:
det-type:
- "GENERIC"
- "STATS_FROM_DB2"
- modules:
+ modules:
- "AGIPD1M1"
start-date: "2019-01-01"
end-date: "NOW"
@@ -42,7 +42,7 @@ SPB:
acquisition-rate:
- 1.1
- 2.2
- - 4.5
+ - 4.5
photon-energy: 9.2
separate-plot:
- "gain_setting"
@@ -63,7 +63,7 @@ SPB:
sp-name: "ASICs id"
nMemToShow: 32
use-existing: "''"
- out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+ out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
cal-db-timeout: 180000
cal-db-interface: "tcp://max-exfl016:8015#8025"
@@ -124,7 +124,7 @@ SPB:
sp-name: "Supercolumn 256*64"
photon-energy: 9.2
use-existing: "''"
- out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+ out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
cal-db-timeout: 180000
cal-db-interface: "tcp://max-exfl016:8015#8025"
@@ -133,7 +133,7 @@ MID:
det-type:
- "GENERIC"
- "STATS_FROM_DB2"
- modules:
+ modules:
- "AGIPD1M2"
start-date: "2019-01-01"
end-date: "NOW"
@@ -155,7 +155,7 @@ MID:
acquisition-rate:
- 1.1
- 2.2
- - 4.5
+ - 4.5
photon-energy: 9.2
separate-plot:
- "gain_setting"
@@ -176,7 +176,7 @@ MID:
sp-name: "ASICs id"
nMemToShow: 32
use-existing: "''"
- out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+ out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
cal-db-timeout: 180000
cal-db-interface: "tcp://max-exfl016:8015#8025"
@@ -218,7 +218,7 @@ MID:
- "Integration Time"
sp-name: "ASICs id"
use-existing: "''"
- out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+ out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
cal-db-timeout: 180000
cal-db-interface: "tcp://max-exfl016:8015#8025"
@@ -228,7 +228,7 @@ FXE:
det-type:
- "GENERIC"
- "STATS_FROM_DB2"
- modules:
+ modules:
- "LPD1M1"
start-date: "2019-01-01"
end-date: "NOW"
@@ -265,7 +265,7 @@ FXE:
sp-name: "ASICs id"
nMemToShow: 32
use-existing: "''"
- out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+ out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
cal-db-timeout: 180000
cal-db-interface: "tcp://max-exfl016:8015#8025"
@@ -326,7 +326,7 @@ FXE:
sp-name: "Supercolumn 256*64"
photon-energy: 9.2
use-existing: "''"
- out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+ out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
cal-db-timeout: 180000
cal-db-interface: "tcp://max-exfl016:8015#8025"
@@ -343,7 +343,7 @@ DETLAB:
- "Offset"
dclass: "CCD"
nMemToShow: 1
- modules:
+ modules:
- "fastCCD1"
bias-voltage:
- 79
@@ -388,7 +388,7 @@ DETLAB:
sp-name: "Supercolumn 967*10"
photon-energy: 9.2
use-existing: "''"
- out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+ out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
cal-db-timeout: 180000
cal-db-interface: "tcp://max-exfl016:8015#8025"
@@ -406,7 +406,7 @@ SCS:
- "Offset"
dclass: "CCD"
nMemToShow: 1
- modules:
+ modules:
- "fastCCD1"
bias-voltage:
- 79
@@ -451,7 +451,7 @@ SCS:
sp-name: "Supercolumn 967*10"
photon-energy: 9.2
use-existing: "''"
- out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+ out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
cal-db-timeout: 180000
cal-db-interface: "tcp://max-exfl016:8015#8025"
@@ -468,7 +468,7 @@ SQS:
- "Offset"
dclass: "CCD"
nMemToShow: 1
- modules:
+ modules:
- "PnCCD1"
bias-voltage:
- 300
@@ -502,7 +502,7 @@ SQS:
sp-name: "ASICs id"
photon-energy: 9.2
use-existing: "''"
- out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+ out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
cal-db-timeout: 180000
cal-db-interface: "tcp://max-exfl016:8015#8025"
@@ -545,7 +545,7 @@ HED:
- "Integration Time"
sp-name: "superpixel id"
use-existing: "''"
- out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+ out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
cal-db-timeout: 180000
cal-db-interface: "tcp://max-exfl016:8015#8025"
@@ -605,7 +605,7 @@ HED:
sp-name: "Supercolumn 256*64"
photon-energy: 9.2
use-existing: "''"
- out-folder: "/gpfs/exfel/data/scratch/xcal/report_service/tmp/{instrument}/{detector}/"
+ out-folder: "/gpfs/exfel/data/scratch/ahmedk/test/report_service/tmp/{instrument}/{detector}/"
cal-db-timeout: 180000
cal-db-interface: "tcp://max-exfl016:8015#8025"
diff --git a/reportservice/report_service.py b/reportservice/report_service.py
index 808e3833743f5bcf169bf9780151663b9272165a..c3344a7f68821e7ba52863bd016e1f6a1599ffa9 100644
--- a/reportservice/report_service.py
+++ b/reportservice/report_service.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python
+
import argparse
import asyncio
from asyncio.subprocess import PIPE
@@ -6,7 +8,6 @@ import glob
import logging
import os
import subprocess
-from time import sleep
from git import Repo, InvalidGitRepositoryError
import yaml
@@ -15,7 +16,6 @@ import zmq.asyncio
from messages import Errors
-
loop = asyncio.get_event_loop()
@@ -33,9 +33,17 @@ def init_config_repo(config):
# clone the repo.
repo = Repo.clone_from(config['figures-remote'],
config['repo-local'])
- logging.info("Clonning the repository")
- # make sure it is updated
- repo.remote().pull()
+ logging.info("Cloning the repository")
+ try:
+ # make sure it is updated
+ repo.remote().pull()
+ except Exception as e:
+ logging.error(e)
+ # update the head of local repository as the remote's
+ repo.remote().fetch()
+ repo.git.reset('--hard', 'origin/master')
+ # then make sure
+ repo.remote().pull()
logging.info("Config repo is initialized")
@@ -118,6 +126,35 @@ async def copy_files(f, path, sem):
await asyncio.subprocess.create_subprocess_shell(" ".join(cmd))
+async def create_pdf(dc_folder):
+ """
+ Generating a DC report pdf file using maxwell nodes. With the supported
+ inputs a slurm job is submitted to sphinx-build latexpdf.
+
+ :param dc_folder: the local DC folder path with figures and doc file
+ for sphinx-build a pdf latex DC report
+ """
+
+ temp_path = "temp/pdf_generation/"
+ os.makedirs(temp_path, exist_ok=True)
+ sprof = os.environ.get("XFELCALSLURM", "exfel")
+ launcher_command = "sbatch -t 24:00:00 --mem 500G --requeue " \
+ "--output {temp_path}/slurm-%j.out"
+
+ srun_base = launcher_command.format(
+ temp_path=temp_path) + " -p {}".format(sprof)
+ srun_base = srun_base.split()
+ srun_base += [os.path.abspath("./build_latex.sh"),
+ os.path.abspath("{}/doc".format(dc_folder))
+ ]
+ output = subprocess.check_output(srun_base).decode('utf8')
+ jobid = None
+ for line in output.split("\n"):
+ if "Submitted batch job " in line:
+ jobid = line.split(" ")[3]
+ logging.info("Submitted job for generating a pdf: {}".format(jobid))
+
+
async def push_figures(repo_master, addf):
""" Upload new figures
@@ -137,13 +174,11 @@ async def push_figures(repo_master, addf):
await asyncio.sleep(2)
add_tries += 1
repo.index.commit("Add {} new figures".format(len(addf)))
- #TODO: create an async function for pushing new figures
- # to avoid blocking the report service.
repo.remote().push()
logging.info('Pushed to git')
-async def server_runner(conf_file):
+async def server_runner(conf_file, mode):
"""
The main server loop. After pulling the latest changes
of the DC project, it awaits receiving configurations
@@ -159,9 +194,13 @@ async def server_runner(conf_file):
with open(conf_file, "r") as f:
config = yaml.load(f.read(), Loader=yaml.FullLoader)
- # perform git-dir checks and pull the project for updates.
- init_config_repo(config['GLOBAL']['git'])
-
+ # perform git-dir checks and pull the project
+ # for updates only in production mode.
+ if mode != 'sim':
+ init_config_repo(config['GLOBAL']['git'])
+
+ logging.info("Report service started in mode: {}".format(mode))
+
logging.info("report service port: {}:{}"
.format(config['GLOBAL']['report-service']['bind-to'],
config['GLOBAL']['report-service']['port']))
@@ -186,9 +225,15 @@ async def server_runner(conf_file):
# reports config file
req_cfg = {}
- # boolean for pushing to DC git repo.
- git_push = response['gitpush']
+ # No interaction with DC repo (local or remote)
+ # is allowed if sim mode.
+ if mode == 'sim':
+ git_push = False
+ else:
+ # boolean for pushing to DC git repo.
+ git_push = response['gitpush']
+ # Validate the type of 'requested' response.
if isinstance(response['req'], dict):
req_cfg = response['req']
elif isinstance(response['req'], list):
@@ -209,7 +254,7 @@ async def server_runner(conf_file):
logging.info('Requested Configuration: {}'.format(req_cfg))
- async def do_action(cfg, git_push):
+ async def do_action(cfg, git_push, mode):
logging.info('Run plot production')
local_repo = cfg['GLOBAL']['git']['repo-local']
@@ -296,34 +341,39 @@ async def server_runner(conf_file):
await asyncio.gather(*[copy_files(k, v, sem)
for k, v in det_new_files.items()])
- logging.info('{} figures of {} are copied into {}'.format(
- len(figures), det_name, fig_local))
+ logging.info('{} figures of {} are copied into {}'
+ .format(len(figures), det_name, fig_local))
if git_push:
+
# Remove sensitive information from the config file.
del cfg['GLOBAL']
# Write the requested cfg.yaml before pushing all figures.
with open('{}/report_conf.yaml'.format(
fig_local), 'w') as outfile:
yaml.dump(cfg, outfile, default_flow_style=False)
-
- all_new_files.append('{}/report_conf.yaml'.format(fig_local))
-
- asyncio.ensure_future(push_figures(local_repo, all_new_files))
+ if mode == 'prod':
+ all_new_files.append('{}/report_conf.yaml'.format(fig_local))
+ asyncio.ensure_future(push_figures(local_repo,
+ all_new_files))
+ elif mode == 'local':
+ asyncio.ensure_future(create_pdf(local_repo))
# TODO:delete out-folder
#try:
- # asyncio.ensure_future(del_folder(out_folder))
+ # asyncio.ensure_future(del_folder(out_folder))
#except:
- #logging.error(str(e))
+ # logging.error(str(e))
logging.info('Generating requested plots is finished!')
+ logging.info('=======================================')
return
try:
asyncio.ensure_future(do_action(copy.copy(req_cfg),
- copy.copy(git_push)))
+ copy.copy(git_push),
+ mode))
except Exception as e: # actions that fail are only error logged
logging.error(str(e))
break
@@ -334,6 +384,7 @@ arg_parser.add_argument('--config-file', type=str,
help='config file path with '
'reportservice port. '
'Default=./report_conf.yaml')
+arg_parser.add_argument('--mode', type=str, default="sim", choices=['sim', 'prod', 'local'])
arg_parser.add_argument('--log-file', type=str, default='./report.log',
help='The report log file path. Default=./report.log')
arg_parser.add_argument('--logging', type=str, default="INFO",
@@ -352,7 +403,7 @@ if __name__ == "__main__":
level=getattr(logging, args['logging']),
format='%(levelname)-6s: %(asctime)s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
-
+ mode = args["mode"]
loop = asyncio.get_event_loop()
- loop.run_until_complete(server_runner(conf_file))
+ loop.run_until_complete(server_runner(conf_file, mode))
loop.close()