diff --git a/.gitignore b/.gitignore
index 47d66ada746a8a3f50dcaa576110574d79949d04..7e45073ed70843fde1ea1914457fdc44ccbff222 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,11 +10,12 @@
*.pkl
*.png
*.png
+*.secrets.yaml
+*.so
*.tar
*.tif
*.tiff
*.tmp
-*.so
*/slurm_tmp*
*egg*
./temp
@@ -34,4 +35,3 @@ slurm_tmp*
src/cal_tools/agipdalgs.c
webservice/*.log
webservice/*sqlite
-webservice/webservice.yaml
diff --git a/README.rst b/README.rst
index f425fdee5ac4941585012e01c63bac3331d1e7f8..28a68631cee9763ffab2c516e78ba1549972febc 100644
--- a/README.rst
+++ b/README.rst
@@ -40,13 +40,13 @@ python. This can be activated with ``source /gpfs/exfel/sw/calsoft/.pyenv/bin/ac
A quick setup would be:
-0. ``source /gpfs/exfel/sw/calsoft/.pyenv/bin/activate``
-1. ``git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git && cd pycalibration`` - clone the offline calibration package from EuXFEL GitLab
-2. ``pyenv shell 3.8.11`` - load required version of python
-3. ``python3 -m venv .venv`` - create the virtual environment
-4. ``source .venv/bin/activate`` - activate the virtual environment
-5. ``python3 -m pip install --upgrade pip`` - upgrade version of pip
-6. ``python3 -m pip install .`` - install the pycalibration package (add ``-e`` flag for editable development installation)
+1. ``source /gpfs/exfel/sw/calsoft/.pyenv/bin/activate``
+2. ``git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git && cd pycalibration`` - clone the offline calibration package from EuXFEL GitLab
+3. ``pyenv shell 3.8.11`` - load required version of python
+4. ``python3 -m venv .venv`` - create the virtual environment
+5. ``source .venv/bin/activate`` - activate the virtual environment
+6. ``python3 -m pip install --upgrade pip`` - upgrade version of pip
+7. ``python3 -m pip install .`` - install the pycalibration package (add ``-e`` flag for editable development installation)
Copy/paste script:
@@ -71,11 +71,11 @@ will downgrade/upgrade your local packages, which may cause major issues and may
**break your local environment**, it is highly recommended to use the venv
installation method instead.
-0. ``source /gpfs/exfel/sw/calsoft/.pyenv/bin/activate``
-1. ``git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git && cd pycalibration`` - clone the offline calibration package from EuXFEL GitLab
-2. ``pyenv shell 3.8.11`` - load required version of python
-3. ``pip install .`` - install the pycalibration package (add ``-e`` flag for editable development installation)
-4. ``export PATH=$HOME/.local/bin:$PATH`` - make sure that the home directory is in the PATH environment variable
+1. ``source /gpfs/exfel/sw/calsoft/.pyenv/bin/activate``
+2. ``git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git && cd pycalibration`` - clone the offline calibration package from EuXFEL GitLab
+3. ``pyenv shell 3.8.11`` - load required version of python
+4. ``pip install .`` - install the pycalibration package (add ``-e`` flag for editable development installation)
+5. ``export PATH=$HOME/.local/bin:$PATH`` - make sure that the home directory is in the PATH environment variable
Copy/paste script:
@@ -103,10 +103,142 @@ venv) activate the virtual environment first, and then run:
This can be useful for Jupyter notebook tools as https://max-jhub.desy.de/hub/login
+Offline Calibration Configuration
+*********************************
+
+The offline calibration package is configured with three configuration files:
+
+- `webservice/config/webservice.yaml` - configuration for the web service
+- `webservice/config/serve_overview.yaml` - configuration for the overview page
+- `src/cal_tools/mdc_config.yaml` - configuration for MDC access by cal tools
+
+These configuration files should not be modified directly, instead you should
+create a file `$CONFIG.secrets.yaml` (e.g. `webservice.secrets.yaml`) in the
+configuration directory, and then add any modifications, such as secrets, to
+this file.
+
+Alternatively, configurations are also searched for in
+`~/.config/pycalibration/$MODULE/$CONFIG.yaml` (e.g.
+`~/.config/pycalibration/webservice/serve_overview.yaml`), which is a useful
+place to store configurations like secrets so that they are present even if you
+delete the pycalibration directory, or if you have multiple `pycalibration`
+repos checked out, as you no longer need to copy/paste the configurations each
+time.
+
+Finally, you can use environment variables to override the configuration without
+modifying any files, which is useful for one-off changes or if you are running
+tests in a CI environment. The environment variables should be prefixed with:
+
+- `webservice/config/webservice.yaml` - `CAL_WEBSERVICE`
+- `webservice/config/serve_overview.yaml` - `CAL_SERVE_OVERVIEW`
+- `src/cal_tools/mdc_config.yaml` - `CAL_CAL_TOOLS`
+
+Followed by an underscore and the configuration key you wish to change. Nested
+keys can be accessed with two underscores, e.g.
+`CAL_WEBSERVICE_CONFIG_REPO__URL` would modify the `config-repo: url: ` value.
+
+Note that the order of priority is:
+
+- default configuration - e.g. `webservice/config/webservice.yaml`
+- local configuration - e.g. `webservice/config/webservice.secrets.yaml`
+- user configuration - e.g. `~/.config/pycalibration/webservice/webservice.yaml`
+- environment variables - e.g. `export CAL_WEBSERVICE_*=...`
+
+Examples
+========
+
+For example, `webservice/config/webservice.yaml` has:
+
+```yaml
+config-repo:
+ url: "@note add this to secrets file"
+ local-path: "@format {env[HOME]}/calibration_config"
+...
+metadata-client:
+ user-id: "@note add this to secrets file"
+ user-secret: "@note add this to secrets file"
+ user-email: "@note add this to secrets file"
+```
+
+So you would create a file `webservice/config/webservice.secrets.yaml`:
+
+```yaml
+config-repo:
+ url: "https://USERNAME:TOKEN@git.xfel.eu/gitlab/detectors/calibration_configurations.git"
+
+metadata-client:
+ user-id: "id..."
+ user-secret: "secret..."
+ user-email: "calibration@example.com"
+```
+
+Alternatively, this file could be placed at `~/.config/pycalibration/webservice/webservice.yaml`
+
+Checking Configurations
+=======================
+
+Having multiple nested configurations can get a bit confusing, so `dynaconf`
+includes a command to help view what a configuration will be resolved to. Once
+you have activated the python environment pycalibration is installed in, you
+can run the command `dynaconf -i webservice.config.webservice list` to list the
+current configuration values:
+
+```
+> dynaconf -i webservice.config.webservice list
+Working in main environment
+WEBSERVICE_DIR<PosixPath> PosixPath('/home/roscar/work/git.xfel.eu/detectors/pycalibration/webservice')
+CONFIG-REPO<dict> {'local-path': '/home/roscar/calibration_config',
+ 'url': 'https://haufs:AAABBBCCCDDDEEEFFF@git.xfel.eu/gitlab/detectors/calibration_configurations.git'}
+WEB-SERVICE<dict> {'allowed-ips': '131.169.4.197, 131.169.212.226',
+ 'bind-to': 'tcp://*',
+ 'job-db': '/home/roscar/work/git.xfel.eu/detectors/pycalibration/webservice/webservice_jobs.sqlite',
+ 'job-timeout': 3600,
+ 'job-update-interval': 60,
+ 'port': 5556}
+METADATA-CLIENT<dict> {'auth-url': 'https://in.xfel.eu/test_metadata/oauth/authorize',
+ 'base-api-url': 'https://in.xfel.eu/metadata/api/',
+ 'metadata-web-app-url': 'https://in.xfel.eu/test_metadata',
+ 'refresh-url': 'https://in.xfel.eu/test_metadata/oauth/token',
+ 'scope': '',
+ 'token-url': 'https://in.xfel.eu/test_metadata/oauth/token',
+ 'user-email': 'calibration@example.com',
+ 'user-id': 'AAABBBCCCDDDEEEFFF',
+ 'user-secret': 'AAABBBCCCDDDEEEFFF'}
+KAFKA<dict> {'brokers': ['it-kafka-broker01.desy.de',
+ 'it-kafka-broker02.desy.de',
+ 'it-kafka-broker03.desy.de'],
+ 'topic': 'xfel-test-offline-cal'}
+CORRECT<dict> {'cmd': 'python -m xfel_calibrate.calibrate {detector} CORRECT '
+ '--slurm-scheduling {sched_prio} --slurm-mem 750 --request-time '
+ '{request_time} --slurm-name '
+ '{action}_{instrument}_{detector}_{cycle}_p{proposal}_{runs} '
+ '--report-to '
+ '/gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/usr/Reports/{runs}/{det_instance}_{action}_{proposal}_{runs}_{time_stamp} '
+ '--cal-db-timeout 300000 --cal-db-interface '
+ 'tcp://max-exfl016:8015#8044',
+ 'in-folder': '/gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw',
+ 'out-folder': '/gpfs/exfel/d/proc/{instrument}/{cycle}/p{proposal}/{run}',
+ 'sched-prio': 80}
+DARK<dict> {'cmd': 'python -m xfel_calibrate.calibrate {detector} DARK --concurrency-par '
+ 'karabo_da --slurm-scheduling {sched_prio} --request-time '
+ '{request_time} --slurm-name '
+ '{action}_{instrument}_{detector}_{cycle}_p{proposal}_{runs} '
+ '--report-to '
+ '/gpfs/exfel/d/cal/caldb_store/xfel/reports/{instrument}/{det_instance}/{action}/{action}_{proposal}_{runs}_{time_stamp} '
+ '--cal-db-interface tcp://max-exfl016:8015#8044 --db-output',
+ 'in-folder': '/gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw',
+ 'out-folder': '/gpfs/exfel/u/usr/{instrument}/{cycle}/p{proposal}/dark/runs_{runs}',
+ 'sched-prio': 10}
+```
+
+And here you can see that `metadata-client: user-id: ` contains the ID now
+instead of the note "add this to secrets file", so the substitution has worked
+correctly.
+
+
Contributing
************
-
Guidelines
==========
diff --git a/pyproject.toml b/pyproject.toml
index ac3b70bf15486cd8bf285f4acb60d60bda7ad460..9b2d60a2d97e513e063f6b3b742407686218413b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -10,6 +10,9 @@ disable = "C0330, C0326"
[tool.pylint.format]
max-line-length = "88"
+[flake8]
+max-line-length = 88
+
[tool.pytest.ini_options]
norecursedirs = [
"legacy",
diff --git a/setup.py b/setup.py
index e33d55fd1b35ef982ba8d190474fe5d5fc78044b..f7792741fc29b64c81da727d11a0d2dc39f798b9 100644
--- a/setup.py
+++ b/setup.py
@@ -80,6 +80,7 @@ setup(
"astcheck==0.2.5",
"astsearch==0.2.0",
"dill==0.3.0",
+ "dynaconf==3.1.4",
"extra_data==1.4.1",
"extra_geom==1.1.1",
"gitpython==3.1.0",
diff --git a/src/cal_tools/mdc_config.py b/src/cal_tools/mdc_config.py
index 1f5f9cf2dc383e76c5a1d6cd0c7a373b89e6ae83..3c491e0ef651fc096a51623ace6d2338109fcfea 100644
--- a/src/cal_tools/mdc_config.py
+++ b/src/cal_tools/mdc_config.py
@@ -1,11 +1,15 @@
-class MDC_config:
+from pathlib import Path
- mdconf = {}
- mdconf['user-id'] = ''
- mdconf['user-secret'] = ''
- mdconf['user-email'] = 'calibration@example.com'
- mdconf['token-url'] = 'https://in.xfel.eu/metadata/oauth/token'
- mdconf['refresh-url'] = 'https://in.xfel.eu/metadata/oauth/token'
- mdconf['auth-url'] = 'https://in.xfel.eu/metadata/oauth/authorize'
- mdconf['scope'] = ''
- mdconf['base-api-url'] = 'https://in.xfel.eu/metadata/api/'
+from dynaconf import Dynaconf
+
+config_dir = Path(__file__).parent.resolve()
+
+mdc_config = Dynaconf(
+ envvar_prefix="CAL_CAL_TOOLS",
+ settings_files=[
+ config_dir / "mdc_config.yaml",
+ config_dir / "mdc_config.secrets.yaml",
+ Path("~/.config/pycalibration/cal_tools/mdc_config.yaml").expanduser(),
+ ],
+ merge_enabled=True,
+)
diff --git a/src/cal_tools/mdc_config.yaml b/src/cal_tools/mdc_config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..923407bfb14123905e58f336738bde582d97dea1
--- /dev/null
+++ b/src/cal_tools/mdc_config.yaml
@@ -0,0 +1,8 @@
+auth-url: https://in.xfel.eu/metadata/oauth/authorize
+base-api-url: https://in.xfel.eu/metadata/api/
+refresh-url: https://in.xfel.eu/metadata/oauth/token
+scope: ''
+token-url: https://in.xfel.eu/metadata/oauth/token
+user-email: calibration@example.com
+user-id: '@note add this to secrets file'
+user-secret: '@note add this to secrets file'
diff --git a/src/cal_tools/tools.py b/src/cal_tools/tools.py
index 6c743ebf21d5c2bb2706dfda7573a29f17a29a50..9855f64cb90b9ef4a6dbd3d19ea39521cffd002a 100644
--- a/src/cal_tools/tools.py
+++ b/src/cal_tools/tools.py
@@ -23,7 +23,7 @@ from metadata_client.metadata_client import MetadataClient
from notebook.notebookapp import list_running_servers
from .ana_tools import save_dict_to_hdf5
-from .mdc_config import MDC_config
+from .mdc_config import mdc_config
def parse_runs(runs, return_type=str):
@@ -215,15 +215,16 @@ def get_run_info(proposal, run):
:return: dictionary with run information
"""
- mdconf = MDC_config.mdconf
- mdc = MetadataClient(client_id=mdconf['user-id'],
- client_secret=mdconf['user-secret'],
- user_email=mdconf['user-email'],
- token_url=mdconf['token-url'],
- refresh_url=mdconf['refresh-url'],
- auth_url=mdconf['auth-url'],
- scope=mdconf['scope'],
- base_api_url=mdconf['base-api-url'])
+ mdc = MetadataClient(
+ client_id=mdc_config['user-id'],
+ client_secret=mdc_config['user-secret'],
+ user_email=mdc_config['user-email'],
+ token_url=mdc_config['token-url'],
+ refresh_url=mdc_config['refresh-url'],
+ auth_url=mdc_config['auth-url'],
+ scope=mdc_config['scope'],
+ base_api_url=mdc_config['base-api-url'],
+ )
runs = mdc.get_proposal_runs(proposal_number=proposal,
run_number=run)
diff --git a/webservice/__init__.py b/webservice/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/webservice/config/__init__.py b/webservice/config/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..8eb56e99b0360810fa654873616da207e10e3f91
--- /dev/null
+++ b/webservice/config/__init__.py
@@ -0,0 +1,33 @@
+from pathlib import Path
+
+from dynaconf import Dynaconf
+
+config_dir = Path(__file__).parent.resolve().expanduser()
+webservice_dir = config_dir.parent
+
+webservice = Dynaconf(
+ envvar_prefix="CAL_WEBSERVICE",
+ settings_files=[
+ config_dir / "webservice.yaml",
+ config_dir / "webservice.secrets.yaml",
+ Path("~/.config/pycalibration/webservice/webservice.yaml").expanduser(),
+ ],
+ environments=False,
+ merge_enabled=True,
+ webservice_dir=webservice_dir,
+)
+
+serve_overview = Dynaconf(
+ envvar_prefix="CAL_SERVE_OVERVIEW",
+ settings_files=[
+ config_dir / "serve_overview.yaml",
+ config_dir / "serve_overview.secrets.yaml",
+ Path("~/.config/pycalibration/webservice/serve_overview.yaml").expanduser(),
+ ],
+ environments=False,
+ merge_enabled=True,
+ webservice_dir=webservice_dir,
+)
+
+# `envvar_prefix` = export envvars with `export DYNACONF_FOO=bar`.
+# `settings_files` = Load this files in the order.
diff --git a/webservice/config/serve_overview.yaml b/webservice/config/serve_overview.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0e9262ae5fde526454a7844655f3b1ee63bc58f6
--- /dev/null
+++ b/webservice/config/serve_overview.yaml
@@ -0,0 +1,33 @@
+templates:
+ main-doc: "@format {this.webservice_dir}/templates/main_doc.html"
+ maxwell-status: "@format {this.webservice_dir}/templates/maxwell_status.html"
+ log-output: "@format {this.webservice_dir}/templates/log_output.html"
+ last-characterizations: "@format {this.webservice_dir}/templates/last_characterizations.html"
+ last-correction: "@format {this.webservice_dir}/templates/last_correction.html"
+ running-jobs: "@format {this.webservice_dir}/templates/running_jobs.html"
+ dark-overview: "@format {this.webservice_dir}/templates/dark_overview.html"
+ css: "@format {this.webservice_dir}/templates/serve_overview.css"
+
+shell-commands:
+ nodes-avail-res: "sinfo --nodes=`sinfo -p exfel -t idle -N --noheader -T | grep {} | awk '{{print $6}}'` --noheader -p exfel -o %A"
+ total-jobs: "sinfo -p exfel -o %A --noheader"
+ upex-jobs: "sinfo -T --noheader | awk '{print $1}'"
+ upex-prefix: "upex_"
+ tail-log: "tail -5000 web.log"
+ cat-log: "cat web.log"
+
+run-candidates:
+ - "--run-high"
+ - "--run-med"
+ - "--run-low"
+ - "--run"
+
+server-config:
+ port: 8008
+ host: max-exfl016.desy.de
+ dark-timeout: 30
+ n-calib: 10
+
+web-service:
+ job-db: "@format {this.webservice_dir}/webservice_jobs.sqlite"
+ cal-config: "@format {env[HOME]}/calibration_config/default.yaml"
diff --git a/webservice/webservice.yaml b/webservice/config/webservice.yaml
similarity index 71%
rename from webservice/webservice.yaml
rename to webservice/config/webservice.yaml
index 377315d99b6d7361850480913979a67190eadda7..e0cdd1b509783b0f2c5c2772ace3fff40e8f4d68 100644
--- a/webservice/webservice.yaml
+++ b/webservice/config/webservice.yaml
@@ -1,26 +1,25 @@
config-repo:
- url:
- local-path: /home/xcal/calibration_config/
+ url: "@note add this to secrets file"
+ local-path: "@format {env[HOME]}/calibration_config"
web-service:
port: 5555
bind-to: tcp://*
allowed-ips:
- job-db: ./webservice_jobs.sqlite
+ job-db: "@format {this.webservice_dir}/webservice_jobs.sqlite"
job-update-interval: 60
job-timeout: 3600
metadata-client:
- user-id:
- user-secret:
- user-email:
- metadata-web-app-url: 'https://in.xfel.eu/metadata'
- metadata-web-app-url: 'https://in.xfel.eu/metadata'
- token-url: 'https://in.xfel.eu/metadata/oauth/token'
- refresh-url: 'https://in.xfel.eu/metadata/oauth/token'
- auth-url: 'https://in.xfel.eu/metadata/oauth/authorize'
- scope: ''
- base-api-url: 'https://in.xfel.eu/metadata/api/'
+ user-id: "@note add this to secrets file"
+ user-secret: "@note add this to secrets file"
+ user-email: "@note add this to secrets file"
+ metadata-web-app-url: "https://in.xfel.eu/metadata"
+ token-url: "https://in.xfel.eu/metadata/oauth/token"
+ refresh-url: "https://in.xfel.eu/metadata/oauth/token"
+ auth-url: "https://in.xfel.eu/metadata/oauth/authorize"
+ scope: ""
+ base-api-url: "https://in.xfel.eu/metadata/api/"
kafka:
brokers:
@@ -33,7 +32,7 @@ correct:
in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw
out-folder: /gpfs/exfel/d/proc/{instrument}/{cycle}/p{proposal}/{run}
sched-prio: 80
- cmd : >
+ cmd : >-
python -m xfel_calibrate.calibrate {detector} CORRECT
--slurm-scheduling {sched_prio}
--slurm-mem 750
@@ -47,7 +46,7 @@ dark:
in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw
out-folder: /gpfs/exfel/u/usr/{instrument}/{cycle}/p{proposal}/dark/runs_{runs}
sched-prio: 10
- cmd: >
+ cmd: >-
python -m xfel_calibrate.calibrate {detector} DARK
--concurrency-par karabo_da
--slurm-scheduling {sched_prio}
diff --git a/webservice/listen_kafka.py b/webservice/listen_kafka.py
index b0dcbb42a72841d96f6e6135bdaca11f45a8a929..4fed88fce43ee721b3ed50f2298e28b070f6c790 100644
--- a/webservice/listen_kafka.py
+++ b/webservice/listen_kafka.py
@@ -1,15 +1,10 @@
"""Print Kafka events sent by the webservice.
"""
import json
-import os.path as osp
-import yaml
from kafka import KafkaConsumer
-conf_file = osp.join(osp.dirname(__file__), 'webservice.yaml')
-
-with open(conf_file, "r") as f:
- config = yaml.safe_load(f)
+from .config import webservice as config
topic = config['kafka']['topic']
brokers = config['kafka']['brokers']
diff --git a/webservice/serve_overview.py b/webservice/serve_overview.py
index 71031e7bc758f2547de3b36d8647620f4c5c4028..c1e0a4f903beec6ab633494638cb13eebd59a3fe 100644
--- a/webservice/serve_overview.py
+++ b/webservice/serve_overview.py
@@ -5,13 +5,16 @@ import sqlite3
from collections import OrderedDict
from datetime import datetime, timezone
from http.server import BaseHTTPRequestHandler, HTTPServer
+from pathlib import Path
from subprocess import check_output
-from uuid import uuid4
+from typing import Optional
import yaml
from jinja2 import Template
-from xfel_calibrate.settings import free_nodes_cmd, preempt_nodes_cmd, reservation
+from xfel_calibrate.settings import free_nodes_cmd, preempt_nodes_cmd, reservation # noqa: E501
+
+from .config import serve_overview as config
class LimitedSizeDict(OrderedDict):
@@ -30,7 +33,6 @@ class LimitedSizeDict(OrderedDict):
self.popitem(last=False)
-config = None
pdf_queue = LimitedSizeDict(size_limit=50)
@@ -40,7 +42,6 @@ class RequestHandler(BaseHTTPRequestHandler):
def init_config(self):
- global config
global cal_config
self.nodes_avail_res_cmd = config["shell-commands"]["nodes-avail-res"]
@@ -344,11 +345,10 @@ class RequestHandler(BaseHTTPRequestHandler):
return
-def run(configfile, port=8008):
- print('reading config file')
- with open(configfile, "r") as cf:
- global config
- config = yaml.load(cf.read(), Loader=yaml.FullLoader)
+def run(config_file: Optional[str] = None):
+ if config_file is not None:
+ config.configure(includes_for_dynaconf=[Path(config_file).absolute()])
+
with open(config["web-service"]["cal-config"], "r") as cf:
global cal_config
cal_config = yaml.load(cf.read(), Loader=yaml.FullLoader)
@@ -362,7 +362,7 @@ def run(configfile, port=8008):
parser = argparse.ArgumentParser(
description='Start the overview server')
-parser.add_argument('--config', type=str, default="serve_overview.yaml")
+parser.add_argument('--config', type=str, default=None,)
if __name__ == "__main__":
args = vars(parser.parse_args())
run(args["config"])
diff --git a/webservice/serve_overview.yaml b/webservice/serve_overview.yaml
deleted file mode 100644
index d9de4339c3deb1f9c1a2a99f611eed24eff4d658..0000000000000000000000000000000000000000
--- a/webservice/serve_overview.yaml
+++ /dev/null
@@ -1,33 +0,0 @@
-templates:
- main-doc: ./templates/main_doc.html
- maxwell-status: ./templates/maxwell_status.html
- log-output: ./templates/log_output.html
- last-characterizations: ./templates/last_characterizations.html
- last-correction: ./templates/last_correction.html
- running-jobs: ./templates/running_jobs.html
- dark-overview: ./templates/dark_overview.html
- css: ./templates/serve_overview.css
-
-shell-commands:
- nodes-avail-res: "sinfo --nodes=`sinfo -p exfel -t idle -N --noheader -T | grep {} | awk '{{print $6}}'` --noheader -p exfel -o %A"
- total-jobs: "sinfo -p exfel -o %A --noheader"
- upex-jobs: "sinfo -T --noheader | awk '{print $1}'"
- upex-prefix: "upex_"
- tail-log: "tail -5000 web.log"
- cat-log: "cat web.log"
-
-run-candidates:
- - "--run-high"
- - "--run-med"
- - "--run-low"
- - "--run"
-
-server-config:
- port: 8008
- host: max-exfl016.desy.de
- dark-timeout: 30
- n-calib: 10
-
-web-service:
- job-db: ./webservice_jobs.sqlite
- cal-config: /home/xcal/calibration_config/default.yaml
diff --git a/webservice/update_mdc.py b/webservice/update_mdc.py
index 38231ee4c9074fab44228b01c421bf75e2358a30..fd68354adaeb3ed3565b032709acf6c3ccb119ed 100644
--- a/webservice/update_mdc.py
+++ b/webservice/update_mdc.py
@@ -1,13 +1,15 @@
import argparse
-import os.path as osp
+from pathlib import Path
-import yaml
from metadata_client.metadata_client import MetadataClient
+from .config import webservice as config
+
parser = argparse.ArgumentParser(
description='Update run status at MDC for a given run id.')
+# TODO: unify configuration argument names across the project
parser.add_argument('--conf-file', type=str, help='Path to webservice config',
- default=osp.join(osp.dirname(__file__), 'webservice.yaml'))
+ default=None)
parser.add_argument('--flg', type=str, choices=["NA", "R", "A"], required=True,
help='Status flag for MDC request: NA - not available, R - running, A - available.') # noqa
parser.add_argument('--rid', type=int, help='Run id from MDC')
@@ -20,8 +22,8 @@ rid = args['rid']
flg = args['flg']
msg = args['msg']
-with open(conf_file, "r") as f:
- config = yaml.load(f.read(), Loader=yaml.FullLoader)
+if conf_file is not None:
+ config.configure(includes_for_dynaconf=[Path(conf_file).absolute()])
mdconf = config['metadata-client']
client_conn = MetadataClient(client_id=mdconf['user-id'],
diff --git a/webservice/webservice.py b/webservice/webservice.py
index 8e9bfc9a343f8c288cd7b058c6a30cf688b6a3bb..c2907d30d18ba216f651d3a247ed65df4a5ea767 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -30,6 +30,8 @@ from kafka import KafkaProducer
from kafka.errors import KafkaError
from metadata_client.metadata_client import MetadataClient
+from .config import webservice as config
+
try:
from .messages import MDC, Errors, Success
except ImportError:
@@ -1123,11 +1125,11 @@ def main(argv: Optional[List[str]] = None):
parser = argparse.ArgumentParser(
description='Start the calibration webservice'
)
- parser.add_argument('--config-file', type=str, default='./webservice.yaml')
- parser.add_argument('--mode', type=str, default="sim", choices=['sim', 'prod'])
+ parser.add_argument('--config-file', type=str, default=None)
+ parser.add_argument('--mode', type=str, default="sim", choices=['sim', 'prod']) # noqa
parser.add_argument('--log-file', type=str, default='./web.log')
parser.add_argument(
- '--log-level', type=str, default="INFO", choices=['INFO', 'DEBUG', 'ERROR']
+ '--log-level', type=str, default="INFO", choices=['INFO', 'DEBUG', 'ERROR'] # noqa
)
args = parser.parse_args(argv)
@@ -1136,10 +1138,10 @@ def main(argv: Optional[List[str]] = None):
log_level = args.log_level
mode = args.mode
- with open(config_file, "r") as f:
- config = yaml.safe_load(f.read())
+ if config_file is not None:
+ config.configure(includes_for_dynaconf=[Path(config_file).absolute()])
- fmt = '%(asctime)s - %(name)s - %(levelname)s - [%(filename)s:%(lineno)d] %(message)s'
+ fmt = '%(asctime)s - %(name)s - %(levelname)s - [%(filename)s:%(lineno)d] %(message)s' # noqa
logging.basicConfig(
filename=log_file,
level=getattr(logging, log_level),