diff --git a/reportservice/README.md b/reportservice/README.md
index fd1acab83e3aeea09fe76c2e663b411abfb4dea2..a4e6da44a3fe67bf39f6e0622de0cd1614314d1b 100644
--- a/reportservice/README.md
+++ b/reportservice/README.md
@@ -16,6 +16,37 @@ Configuration
It is important to know the machine name and the port, where the reportservice is running, for successful connection.
+Starting the Service
+--------------------
+
+The reportservice is a python script that can run through:
+
+ ```bash
+ python reportservice.py
+ ```
+
+The available command line arguments are:
+
+* --report-conf: The path for the main report configuration yaml file.
+* --log-file: The path for the log file.
+* --mode: The mode for running the service. Choices are sim[simulation], local and prod[production].
+* --logging: The required logs to be written. Choices are INFO, DEBUG and Error.
+
+Modes:
+
+*prod* is the production mode working on the max-exfl016 as xcal user for generating the DC report through RTD
+and it should generate a very generalized DC report for the available detectors with information useful for most of the detector experts and users.
+
+*local* is the mode used for generating figures locally without uploading the DC report on RTD or pushing figures
+to the git repository, rather generated figures are copied to the local repository and depending on the
+given report-fmt(report format) argument an html or a pdf is generated in doc/_build/
+of the report service out folder (repo-local).
+
+*sim* is a simulation mode, which is mostly used for debugging purposes and tool development without generating any reports locally or over RTD.
+This mode make sure not to do any git interactions and it only works on running the notebooks for generating figures in the out-folder without further work.
+
+Report Configuration(report-conf):
+
*report_conf.yaml* is the configuration file, which contains all the required information
for operating and connecting to the reportservice.
@@ -65,25 +96,11 @@ The YAML configuration file can be modified with all the available parameters, r
cal-db-interface: "<cal-db-host-port>"
```
-
-Starting the Service
---------------------
-
-The reportservice is a python script that can run through:
-
- ```bash
- python reportservice.py
- ```
-
-The available command line arguments are:
-
-* --report-conf : The path for the main report configuration yaml file
-* --log : The logging mode (INFO, DEBUG, ERROR)
-
-Launching the service
+Triggering the service
---------------------
-The service can be launched through two processes:
+To use the service and generate a DC report corresponding to the report_conf.yaml.
+The service can be triggered through two processes:
Automatic Launch:
@@ -92,7 +109,21 @@ Automatic Launch:
```bash
python automatic_run.py
```
+* --config-file: The path for the configuration file* --log-file: The path for the log file.
+* --logging: The required logs to be written. Choices are INFO, DEBUG and Error
+* --log-file: The path for the log file.
Manual Launch:
This manual launch script is currently used for debugging purposes, only.
+
+ The available command line arguments are:
+
+* --config-file: The path for the configuration file
+* --instrument: A selected list of instruments to generate a report for. This instrument must be in the report_conf.yaml. The default for this argument is ['all]
+* --overwrite-conf: A bool for indicating a new report configuration file(conf-file) should be sent instead of the default report_conf.yaml,
+which is used by report_service.py from the start.
+* --log-file: The path for the log file.
+* --report-fmt: The output DC report format. Choices are pdf or html
+* --upload: A bool for uploading the figure to out-folder of the report service(repo-local) and generating a report. Default is False
+* --logging: The required log mode to be used. Choices are INFO, DEBUG and Error
diff --git a/reportservice/automatic_run.py b/reportservice/automatic_run.py
index b6aba1f5e20378a9d90adfa8fa2b1448abba5c65..a85fa0b299aa1d5ad109c3bae349035e59551006 100644
--- a/reportservice/automatic_run.py
+++ b/reportservice/automatic_run.py
@@ -23,7 +23,8 @@ async def auto_run(cfg, timeout=3000):
run_time = cfg['GLOBAL']['run-on']
request = {}
request['req'] = ['all']
- request['gitpush'] = True
+ request['upload'] = True
+ request['report-fmt'] = 'html'
for i, ti in enumerate(run_time):
run_time[i] = parser.parse(ti)
diff --git a/reportservice/build_dc_report.sh b/reportservice/build_dc_report.sh
new file mode 100755
index 0000000000000000000000000000000000000000..cc40a6e46f41264321a39c767c32580b77269765
--- /dev/null
+++ b/reportservice/build_dc_report.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# the path to doc folder with a Makefile
+dc_folder=$1
+report_fmt=$2
+
+echo "Running with the following parameters:"
+echo "DC folder path: $dc_folder"
+echo "DC Report format should be: report_fmt"
+
+if [ "${report_fmt}" == "pdf" ]
+then
+ make latexpdf -C "${dc_folder}"
+elif [ "${report_fmt}" == "html" ]
+then
+ make html -C "${dc_folder}"
+fi
diff --git a/reportservice/manual_run.py b/reportservice/manual_run.py
index d5ad03a14808ad707fc06461c817885cd5986a9c..caa04fcf65a2a3e0449c30ef1d1de8b58c8f48b7 100644
--- a/reportservice/manual_run.py
+++ b/reportservice/manual_run.py
@@ -13,7 +13,7 @@ def manual_run(request, cfg):
:param request: a dictionary for generating reports for the
requested Instruments. This dict consists
- of two keys. A gitpush boolean key for defining
+ of two keys. A upload boolean key for defining
the fate of the generated plots
(pushed to the DC repo. or staying locally)
and a key request which can either be a list
@@ -32,19 +32,32 @@ def manual_run(request, cfg):
msg = socket.recv_pyobj()
logging.info('{} Manual Run'.format(msg))
+
arg_parser = argparse.ArgumentParser(description='Manual Launch')
arg_parser.add_argument('--instrument', default=['all'], nargs='+',
- help='select the requested instruments. '
+ help='Select the requested instruments. '
'Default=\"all\", which can be used for selecting'
- ' all instruments')
-arg_parser.add_argument('--gitpush', dest='gitpush', action='store_true',
- help='required for pushing the generated figures '
- 'to the DC git repository. Default=bool(False)')
-arg_parser.set_defaults(gitpush=False)
-arg_parser.add_argument('--config-file', type=str,
+ ' all instruments.')
+arg_parser.add_argument('--config-file', type=str,
default='./report_conf.yaml',
help='path to report configuration file '
'Default=./report_conf.yaml')
+arg_parser.add_argument('--upload', action='store_true',
+ help='Required for uploading the generated figures.'
+ 'Default=False. '
+ 'Note: THIS HAS NO EFFECT IN SIM MODE!')
+arg_parser.set_defaults(upload=False)
+arg_parser.add_argument('--overwrite-conf', action='store_true',
+ help='A flag for using a different config file than'
+ 'what is used by the running report_service.'
+ 'Default=False, type=str.')
+arg_parser.set_defaults(overwrite_conf=False)
+arg_parser.add_argument('--report-fmt', default='html',
+ type=str, choices=['pdf', 'html'],
+ help='If available in the report service running mode,'
+ ' this can configure the report format '
+ 'to be html or pdf. Default=html '
+ 'Note: THIS HAS NO EFFECT IN PROD AND SIM MODES!')
arg_parser.add_argument('--log-file', type=str, default='./report.log',
help='The report log file path. Default=./report.log')
arg_parser.add_argument('--logging', type=str, default="INFO",
@@ -66,13 +79,11 @@ if __name__ == "__main__":
format='%(levelname)-6s: %(asctime)s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
- request = {}
-
- request['gitpush'] = args["gitpush"]
+ request = {'upload': args["upload"],
+ 'report-fmt': args["report_fmt"]}
- if args["instrument"]:
- request['req'] = args["instrument"]
- else:
+ if args["overwrite_conf"]:
request['req'] = cfg
-
+ else:
+ request['req'] = args["instrument"]
manual_run(request, cfg)
diff --git a/reportservice/report_conf.yaml b/reportservice/report_conf.yaml
index b8fbc0c3e49415474dcb3e441976f082b8516ae6..c272c6755ae1b980abc625613e65c1a9d741c125 100644
--- a/reportservice/report_conf.yaml
+++ b/reportservice/report_conf.yaml
@@ -20,7 +20,7 @@ SPB:
det-type:
- "GENERIC"
- "STATS_FROM_DB2"
- modules:
+ modules:
- "AGIPD1M1"
start-date: "2019-01-01"
end-date: "NOW"
@@ -42,7 +42,11 @@ SPB:
acquisition-rate:
- 1.1
- 2.2
- - 4.5
+ - 4.5
+ gain-setting:
+ - 0
+ - 1
+ - 2
photon-energy: 9.2
separate-plot:
- "gain_setting"
@@ -133,7 +137,7 @@ MID:
det-type:
- "GENERIC"
- "STATS_FROM_DB2"
- modules:
+ modules:
- "AGIPD1M2"
start-date: "2019-01-01"
end-date: "NOW"
@@ -155,7 +159,11 @@ MID:
acquisition-rate:
- 1.1
- 2.2
- - 4.5
+ - 4.5
+ gain-setting:
+ - 0
+ - 1
+ - 2
photon-energy: 9.2
separate-plot:
- "gain_setting"
@@ -228,7 +236,7 @@ FXE:
det-type:
- "GENERIC"
- "STATS_FROM_DB2"
- modules:
+ modules:
- "LPD1M1"
start-date: "2019-01-01"
end-date: "NOW"
@@ -247,6 +255,10 @@ FXE:
- 1
- 128
- 512
+ gain-setting:
+ - 0
+ - 1
+ - 2
photon-energy: 9.2
separate-plot:
- "gain_setting"
@@ -343,7 +355,7 @@ DETLAB:
- "Offset"
dclass: "CCD"
nMemToShow: 1
- modules:
+ modules:
- "fastCCD1"
bias-voltage:
- 79
@@ -406,7 +418,7 @@ SCS:
- "Offset"
dclass: "CCD"
nMemToShow: 1
- modules:
+ modules:
- "fastCCD1"
bias-voltage:
- 79
@@ -468,7 +480,7 @@ SQS:
- "Offset"
dclass: "CCD"
nMemToShow: 1
- modules:
+ modules:
- "PnCCD1"
bias-voltage:
- 300
diff --git a/reportservice/report_service.py b/reportservice/report_service.py
index 808e3833743f5bcf169bf9780151663b9272165a..130885c78473355967a4724ce3e0342f05fd86c3 100644
--- a/reportservice/report_service.py
+++ b/reportservice/report_service.py
@@ -6,7 +6,6 @@ import glob
import logging
import os
import subprocess
-from time import sleep
from git import Repo, InvalidGitRepositoryError
import yaml
@@ -15,7 +14,6 @@ import zmq.asyncio
from messages import Errors
-
loop = asyncio.get_event_loop()
@@ -33,9 +31,17 @@ def init_config_repo(config):
# clone the repo.
repo = Repo.clone_from(config['figures-remote'],
config['repo-local'])
- logging.info("Clonning the repository")
- # make sure it is updated
- repo.remote().pull()
+ logging.info("Cloning the repository")
+ try:
+ # make sure it is updated
+ repo.remote().pull()
+ except Exception as e:
+ logging.error(e)
+ # update the head of local repository as the remote's
+ repo.remote().fetch()
+ repo.git.reset('--hard', 'origin/master')
+ # then make sure
+ repo.remote().pull()
logging.info("Config repo is initialized")
@@ -60,7 +66,7 @@ async def wait_jobs(joblist):
if str(job) in line:
found_jobs.add(job)
if len(found_jobs) == 0:
- logging.info('Plot modules is done')
+ logging.info('Jobs are finished')
break
await asyncio.sleep(10)
counter += 10
@@ -96,7 +102,7 @@ async def del_folder(fpath):
"""
cmd = ["rm", "-rf", fpath]
await asyncio.subprocess.create_subprocess_shell(" ".join(cmd))
- logging.info('tmp file has been deleted')
+ logging.info('temp file {} has been deleted'.format(fpath))
async def copy_files(f, path, sem):
@@ -118,6 +124,43 @@ async def copy_files(f, path, sem):
await asyncio.subprocess.create_subprocess_shell(" ".join(cmd))
+async def build_dc_report(dc_folder, report_fmt):
+ """
+ Generating a DC report (latex or html) using maxwell nodes.
+ With the supported inputs a slurm job is submitted to sphinx-build
+ pdf or html depending on mode of the report_service.
+ html for prod mode and pdf or html for local mode depending
+ on the chosen report_fmt
+
+ :param dc_folder: the local DC folder path with figures and rst files
+ :param report_fmt: the expected report format(html or pdf)
+ """
+ temp_path = "{}/temp/build_dc_report/".format(os.getcwd())
+ os.makedirs(temp_path, exist_ok=True)
+
+ # launching a slurm job and assigning the bash script to it.
+ sprof = os.environ.get("XFELCALSLURM", "exfel")
+ launcher_command = "sbatch -t 24:00:00 --mem 500G --requeue " \
+ "--output {temp_path}/slurm-%j.out"
+ srun_base = launcher_command.format(
+ temp_path=temp_path) + " -p {}".format(sprof)
+ srun_base = srun_base.split()
+
+ srun_base += [os.path.abspath("./build_dc_report.sh"),
+ os.path.abspath("{}/doc".format(dc_folder)),
+ report_fmt]
+ logging.info("Building DC report submission: {}".format(srun_base))
+ output = subprocess.check_output(srun_base).decode('utf8')
+
+ jobid = None
+ for line in output.split("\n"):
+ if "Submitted batch job " in line:
+ jobid = line.split(" ")[3]
+ logging.info("Submitted job for building a report: {}".format(jobid))
+ await wait_jobs([jobid])
+ asyncio.ensure_future(del_folder("{}/slurm-{}.out".format(temp_path, jobid)))
+
+
async def push_figures(repo_master, addf):
""" Upload new figures
@@ -137,13 +180,11 @@ async def push_figures(repo_master, addf):
await asyncio.sleep(2)
add_tries += 1
repo.index.commit("Add {} new figures".format(len(addf)))
- #TODO: create an async function for pushing new figures
- # to avoid blocking the report service.
repo.remote().push()
logging.info('Pushed to git')
-async def server_runner(conf_file):
+async def server_runner(conf_file, mode):
"""
The main server loop. After pulling the latest changes
of the DC project, it awaits receiving configurations
@@ -159,9 +200,13 @@ async def server_runner(conf_file):
with open(conf_file, "r") as f:
config = yaml.load(f.read(), Loader=yaml.FullLoader)
- # perform git-dir checks and pull the project for updates.
- init_config_repo(config['GLOBAL']['git'])
-
+ # perform git-dir checks and pull the project
+ # for updates only in production mode.
+ if mode != 'sim':
+ init_config_repo(config['GLOBAL']['git'])
+
+ logging.info("Report service started in mode: {}".format(mode))
+
logging.info("report service port: {}:{}"
.format(config['GLOBAL']['report-service']['bind-to'],
config['GLOBAL']['report-service']['port']))
@@ -175,20 +220,20 @@ async def server_runner(conf_file):
while True:
response = await socket.recv_pyobj()
await socket.send_pyobj('Build DC reports through -->')
- logging.info("response: {} with git pushing: {}"
- .format(response['req'], response['gitpush']))
+ logging.info("response: {} with uploading: {} and report format: {}"
+ .format(response['req'],
+ response['upload'],
+ response['report-fmt']))
# Check if response is a list or a dict.
# if list, it should either have instrument names or ['all'].
- # if dict, it should acquires the details of the requested reports
+ # if dict, it should acquire the details of the requested reports
# for generation. As it will be used instead of report_conf.yaml
# reports config file
req_cfg = {}
- # boolean for pushing to DC git repo.
- git_push = response['gitpush']
-
+ # Validate the type of 'requested' response.
if isinstance(response['req'], dict):
req_cfg = response['req']
elif isinstance(response['req'], list):
@@ -207,9 +252,21 @@ async def server_runner(conf_file):
logging.error(Errors.REQUEST_MALFORMED.format(response['req']))
continue
+ # No interaction with DC repository (local or remote)
+ # is allowed if sim mode.
+ if mode == 'sim':
+ req_cfg['GLOBAL']['upload'] = False
+ req_cfg['GLOBAL']['report-fmt'] = False
+ else:
+ # boolean for pushing to DC git repo.
+ req_cfg['GLOBAL']['upload'] = response['upload']
+ if mode == 'prod':
+ req_cfg['GLOBAL']['report-fmt'] = 'html'
+ else:
+ req_cfg['GLOBAL']['report-fmt'] = response['report-fmt']
logging.info('Requested Configuration: {}'.format(req_cfg))
- async def do_action(cfg, git_push):
+ async def do_action(cfg, service_mode):
logging.info('Run plot production')
local_repo = cfg['GLOBAL']['git']['repo-local']
@@ -250,10 +307,10 @@ async def server_runner(conf_file):
for output in outputs:
if output[0]:
logging.info('Submission Output: {}'
- .format(output[0].decode('utf8')))
+ .format(output[0].decode('utf8')))
if output[1]:
logging.error('Submission Error: {}'
- .format(output[1].decode('utf8')))
+ .format(output[1].decode('utf8')))
job_list += await parse_output(output[0].decode('utf8'))
try:
@@ -264,9 +321,10 @@ async def server_runner(conf_file):
logging.error('Jobs have timed-out!')
logging.error('{}/temp has not been deleted.'.format(
os.path.dirname(os.path.abspath(__file__))))
- # Avoid copying files if no git-push is planned
+
+ # Avoid copying files if upload bool is False
# to avoid causing local git repository errors.
- if git_push:
+ if cfg['GLOBAL']['upload']:
# Copy all plots
for det_name, det_conf in instrument.items():
@@ -294,36 +352,50 @@ async def server_runner(conf_file):
'{}/{}'.format(fpath, f.split('/')[-1]))
await asyncio.gather(*[copy_files(k, v, sem)
- for k, v in det_new_files.items()])
+ for k, v in det_new_files.items()]) # noqa
- logging.info('{} figures of {} are copied into {}'.format(
- len(figures), det_name, fig_local))
+ logging.info('{} figures of {} are copied into {}'
+ .format(len(figures), det_name,
+ fig_local))
- if git_push:
- # Remove sensitive information from the config file.
- del cfg['GLOBAL']
- # Write the requested cfg.yaml before pushing all figures.
- with open('{}/report_conf.yaml'.format(
- fig_local), 'w') as outfile:
- yaml.dump(cfg, outfile, default_flow_style=False)
-
- all_new_files.append('{}/report_conf.yaml'.format(fig_local))
-
- asyncio.ensure_future(push_figures(local_repo, all_new_files))
+ if cfg['GLOBAL']['upload']:
+ try:
+ report_fmt = cfg['GLOBAL']['report-fmt']
+ # Remove sensitive information from the config file.
+ del cfg['GLOBAL']
+ # Write the requested cfg.yaml before pushing all figures.
+ with open('{}/report_conf.yaml'.format(
+ fig_local), 'w') as outfile:
+ yaml.dump(cfg, outfile, default_flow_style=False)
+
+ if service_mode == 'prod':
+ # add report_con.yaml in the list of files added to the
+ # new git commit before pushing to remote
+ all_new_files.append('{}/report_conf.yaml'
+ .format(fig_local))
+ asyncio.ensure_future(push_figures(local_repo,
+ all_new_files))
+ # build either html or pdf depending on the running mode
+ # of the report_service and requested report format.
+ asyncio.ensure_future(build_dc_report(local_repo,
+ report_fmt)) # noqa
+ except Exception as upload_e:
+ logging.error("upload failed: {}".format(upload_e))
# TODO:delete out-folder
#try:
- # asyncio.ensure_future(del_folder(out_folder))
+ # asyncio.ensure_future(del_folder(out_folder))
#except:
- #logging.error(str(e))
+ # logging.error(str(e))
logging.info('Generating requested plots is finished!')
+ logging.info('=======================================')
return
try:
asyncio.ensure_future(do_action(copy.copy(req_cfg),
- copy.copy(git_push)))
+ mode))
except Exception as e: # actions that fail are only error logged
logging.error(str(e))
break
@@ -334,6 +406,7 @@ arg_parser.add_argument('--config-file', type=str,
help='config file path with '
'reportservice port. '
'Default=./report_conf.yaml')
+arg_parser.add_argument('--mode', type=str, default="sim", choices=['sim', 'prod', 'local'])
arg_parser.add_argument('--log-file', type=str, default='./report.log',
help='The report log file path. Default=./report.log')
arg_parser.add_argument('--logging', type=str, default="INFO",
@@ -352,7 +425,7 @@ if __name__ == "__main__":
level=getattr(logging, args['logging']),
format='%(levelname)-6s: %(asctime)s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
-
+ mode = args["mode"]
loop = asyncio.get_event_loop()
- loop.run_until_complete(server_runner(conf_file))
+ loop.run_until_complete(server_runner(conf_file, mode))
loop.close()