diff --git a/.gitignore b/.gitignore index e0414e68b011b661a5effeff635a2b45120e1750..bd850f309e83e3260c0866586288d0e525541d94 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,6 @@ docs/source/_notebooks/* docs/source/test_rsts/* docs/source/test_results.rst docs/source/_static/reports +webservice/webservice.yaml +webservice/*.log +webservice/*sqlite diff --git a/webservice/README.md b/webservice/README.md index d18fb0cc6333b0390385c30dddd2223a71ac099b..501e7f9c522eee42904d37209b09d3b91235ec98 100644 --- a/webservice/README.md +++ b/webservice/README.md @@ -53,16 +53,8 @@ Additionally, the *xfel-calibrate* environment needs to be installed: mkdir pydetlib cd pydetlib git clone https://git.xfel.eu/gitlab/karaboDevices/pyDetLib.git . - pip install --user --no-index --find-links ./external_dependencies/pandas/wheelhouse pandas - pip install --user --no-index --find-links ./external_dependencies/pandoc/wheelhouse pandoc - pip install --user --no-index --find-links ./external_dependencies/seaborn/wheelhouse seaborn - pip install --user --no-index --find-links ./external_dependencies/brewer2mpl/wheelhouse brewer2mpl - pip install --user --no-index --find-links ./external_dependencies/dill/wheelhouse dill - pip install --user --no-index --find-links ./external_dependencies/sphinx-napoleon/wheelhouse sphinxcontrib-napoleon - pip install --user --no-index --find-links ./external_dependencies/coverage/wheelhouse coverage - pip install --user --no-index --find-links ./external_dependencies/pypandoc/wheelhouse pypandoc - pip install --user --no-index --find-links ./external_dependencies/pycuda pycuda - pip install --user ./external_dependencies/iminuit + pip install --user ./lib/requirements.txt + pip install --user pycuda pip install --user ./lib/ cd .. @@ -72,5 +64,94 @@ Additionally, the *xfel-calibrate* environment needs to be installed: cd webservice pip install --user -r requirements.txt ``` + +6. install the metadata_client library, according to instructions at + + https://git.xfel.eu/gitlab/ITDM/metadata_client + + +You are now good to go. + +Configuration +------------- + +Configuration is done through the *webservice.yaml* file in the webservice directory. +On a new installation you will likely need to change the following parameters. + +In the **config-repo** section, the configuration repository needs to be configured: + +``` YAML +config-repo: + url: https://git.xfel.eu/gitlab/detectors/calibration_configurations.git + local-path: /home/haufs/calibration_config/ +``` +Here you should prepend the *url* entry with a gitlab access token, that provides access +to the calibration_configurations repository. + +In the **web-service** section, the webservice itself is configured: + +``` YAML + +web-service: + port: 5555 + bind-to: tcp://* + allowed-ips: '111.222.222.111', ''111.222.222.112' + job-db: ./webservice_jobs.sqlite + job-update-interval: 30 + job-timeout: 3600 +``` + +In case you want to use authentication, add a list of *allowed-ips*. + +In the **metadata-client** section, the client interface to the MDC is configured: + +``` YAML + +metadata-client: + user-id: + user-secret: + user-email: + metadata-web-app-url: 'https://in.xfel.eu/metadata' + metadata-web-app-url: 'https://in.xfel.eu/metadata' + token-url: 'https://in.xfel.eu/metadata/oauth/token' + refresh-url: 'https://in.xfel.eu/metadata/oauth/token' + auth-url: 'https://in.xfel.eu/metadata/oauth/authorize' + scope: '' + base-api-url: 'https://in.xfel.eu/metadata/api/' +``` - \ No newline at end of file +Here, user-ids, secrets, email etc as provided by ITDM need to be entered. + +Finally, sections for the individual *actions* the service knows can be configured. +Currently, the only action is *correct*: + +``` YAML + +correct: + in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw + out-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/proc +``` +It expects templates for the input and output file paths to be provided. + +Starting the Service +-------------------- + +The webservice can be started as a normal python process: + + + ``` bash + python webservice.py --mode [prod | prod-auth | sim] + ``` +The available modes are: + +* prod: production mode +* prod-auth: production mode with authentication on ZMQ +* sim: simulation mode, no actual *xfel-calibrate* jobs are launched. + +Use + + ``` bash + python webservice.py --help + ``` + +to display a list of available options. \ No newline at end of file diff --git a/webservice/requirements.txt b/webservice/requirements.txt index 2985a603454ea799b665e5a6f98fe32bd0177ead..2f4cce2235476a13e65de3604c37d0168f6c5383 100644 --- a/webservice/requirements.txt +++ b/webservice/requirements.txt @@ -1,3 +1,2 @@ gitpython zmq -git+https://git.xfel.eu/gitlab/ITDM/metadata_client.git \ No newline at end of file diff --git a/webservice/webservice.py b/webservice/webservice.py index 00d4d2066dc35b09a0d71ff7e0ba393a205ff7bb..72c5ea0e4e9e91fd6571f4b2287f3c69aaf67c02 100644 --- a/webservice/webservice.py +++ b/webservice/webservice.py @@ -151,7 +151,6 @@ async def update_job_db(config): for r in c.fetchall(): rid, jobid, proposal, run, flg, status = r - print(jobid, status) if jobid in statii: slstatus, runtime = statii[jobid] query = "UPDATE jobs SET status='{status} - {runtime}' WHERE id LIKE '{jobid}'" # noqa diff --git a/webservice/webservice.yaml b/webservice/webservice.yaml index f4c33a7bdc79edafe77c0ca593b9734746666d56..498a306c0b77f0e9da97e6e0b65c4cc547c7a060 100644 --- a/webservice/webservice.yaml +++ b/webservice/webservice.yaml @@ -1,5 +1,5 @@ config-repo: - url: + url: https://git.xfel.eu/gitlab/detectors/calibration_configurations.git local-path: /home/haufs/calibration_config/ web-service: