From ecfce64a385d9455de9ae41f839bc0d59a4aa337 Mon Sep 17 00:00:00 2001
From: Robert Rosca <robert.rosca@xfel.eu>
Date: Wed, 14 Jul 2021 22:27:17 +0200
Subject: [PATCH] Use Dynaconf for webservice configuration
Implemented with minimal changes
---
webservice/config/__init__.py | 16 ++++++++++++++++
webservice/{ => config}/serve_overview.yaml | 0
webservice/{ => config}/webservice.yaml | 1 -
webservice/listen_kafka.py | 8 +-------
webservice/serve_overview.py | 7 +------
webservice/update_mdc.py | 5 ++---
webservice/webservice.py | 5 ++---
7 files changed, 22 insertions(+), 20 deletions(-)
create mode 100644 webservice/config/__init__.py
rename webservice/{ => config}/serve_overview.yaml (100%)
rename webservice/{ => config}/webservice.yaml (97%)
diff --git a/webservice/config/__init__.py b/webservice/config/__init__.py
new file mode 100644
index 000000000..b30c6069e
--- /dev/null
+++ b/webservice/config/__init__.py
@@ -0,0 +1,16 @@
+from dynaconf import Dynaconf
+
+webservice = Dynaconf(
+ envvar_prefix="DYNACONF",
+ settings_files=["webservice.yaml", ".webservice.secrets.yaml"],
+ environments=False,
+)
+
+serve_overview = Dynaconf(
+ envvar_prefix="DYNACONF",
+ settings_files=["serve_overview.yaml", ".serve_overview.secrets.yaml"],
+ environments=False,
+)
+
+# `envvar_prefix` = export envvars with `export DYNACONF_FOO=bar`.
+# `settings_files` = Load this files in the order.
diff --git a/webservice/serve_overview.yaml b/webservice/config/serve_overview.yaml
similarity index 100%
rename from webservice/serve_overview.yaml
rename to webservice/config/serve_overview.yaml
diff --git a/webservice/webservice.yaml b/webservice/config/webservice.yaml
similarity index 97%
rename from webservice/webservice.yaml
rename to webservice/config/webservice.yaml
index 377315d99..bad110fb5 100644
--- a/webservice/webservice.yaml
+++ b/webservice/config/webservice.yaml
@@ -15,7 +15,6 @@ metadata-client:
user-secret:
user-email:
metadata-web-app-url: 'https://in.xfel.eu/metadata'
- metadata-web-app-url: 'https://in.xfel.eu/metadata'
token-url: 'https://in.xfel.eu/metadata/oauth/token'
refresh-url: 'https://in.xfel.eu/metadata/oauth/token'
auth-url: 'https://in.xfel.eu/metadata/oauth/authorize'
diff --git a/webservice/listen_kafka.py b/webservice/listen_kafka.py
index b0dcbb42a..32287a95b 100644
--- a/webservice/listen_kafka.py
+++ b/webservice/listen_kafka.py
@@ -1,15 +1,9 @@
"""Print Kafka events sent by the webservice.
"""
import json
-import os.path as osp
-import yaml
from kafka import KafkaConsumer
-
-conf_file = osp.join(osp.dirname(__file__), 'webservice.yaml')
-
-with open(conf_file, "r") as f:
- config = yaml.safe_load(f)
+from .config import webservice as config
topic = config['kafka']['topic']
brokers = config['kafka']['brokers']
diff --git a/webservice/serve_overview.py b/webservice/serve_overview.py
index 71031e7bc..1af17e3b5 100644
--- a/webservice/serve_overview.py
+++ b/webservice/serve_overview.py
@@ -6,12 +6,12 @@ from collections import OrderedDict
from datetime import datetime, timezone
from http.server import BaseHTTPRequestHandler, HTTPServer
from subprocess import check_output
-from uuid import uuid4
import yaml
from jinja2 import Template
from xfel_calibrate.settings import free_nodes_cmd, preempt_nodes_cmd, reservation
+from .config import serve_overview as config
class LimitedSizeDict(OrderedDict):
@@ -30,7 +30,6 @@ class LimitedSizeDict(OrderedDict):
self.popitem(last=False)
-config = None
pdf_queue = LimitedSizeDict(size_limit=50)
@@ -345,10 +344,6 @@ class RequestHandler(BaseHTTPRequestHandler):
def run(configfile, port=8008):
- print('reading config file')
- with open(configfile, "r") as cf:
- global config
- config = yaml.load(cf.read(), Loader=yaml.FullLoader)
with open(config["web-service"]["cal-config"], "r") as cf:
global cal_config
cal_config = yaml.load(cf.read(), Loader=yaml.FullLoader)
diff --git a/webservice/update_mdc.py b/webservice/update_mdc.py
index 38231ee4c..c09a906b1 100644
--- a/webservice/update_mdc.py
+++ b/webservice/update_mdc.py
@@ -4,6 +4,8 @@ import os.path as osp
import yaml
from metadata_client.metadata_client import MetadataClient
+from .config import webservice as config
+
parser = argparse.ArgumentParser(
description='Update run status at MDC for a given run id.')
parser.add_argument('--conf-file', type=str, help='Path to webservice config',
@@ -20,9 +22,6 @@ rid = args['rid']
flg = args['flg']
msg = args['msg']
-with open(conf_file, "r") as f:
- config = yaml.load(f.read(), Loader=yaml.FullLoader)
-
mdconf = config['metadata-client']
client_conn = MetadataClient(client_id=mdconf['user-id'],
client_secret=mdconf['user-secret'],
diff --git a/webservice/webservice.py b/webservice/webservice.py
index 8e9bfc9a3..178f423a8 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -30,6 +30,8 @@ from kafka import KafkaProducer
from kafka.errors import KafkaError
from metadata_client.metadata_client import MetadataClient
+from .config import webservice as config
+
try:
from .messages import MDC, Errors, Success
except ImportError:
@@ -1136,9 +1138,6 @@ def main(argv: Optional[List[str]] = None):
log_level = args.log_level
mode = args.mode
- with open(config_file, "r") as f:
- config = yaml.safe_load(f.read())
-
fmt = '%(asctime)s - %(name)s - %(levelname)s - [%(filename)s:%(lineno)d] %(message)s'
logging.basicConfig(
filename=log_file,
--
GitLab