diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
index 03b9eafb7d1ac8151ee55f2849eb2ed7265b237c..a02c88ca8b9c9746ebc81c2e5c8f059abe18b3a9 100644
--- a/.git-blame-ignore-revs
+++ b/.git-blame-ignore-revs
@@ -8,3 +8,6 @@
# fix/pre-commit-whitespace - Whitespace fixes
e7dfadaf4e189ef0e0f67798e8984695111257e3
+
+# fix/requirements-into-setuppy - Reformat setup.py without content changes
+31b402966a4a741d08cb1e89db96f252fbb41c54
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c55f4dfec1e2e1ecb2c8f127edebecbc7550b049..a1a86985769caa5e56f22521d980db6765b42a33 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,27 +2,45 @@ stages:
- check
- test
+.before_script: &before_script
+ before_script:
+ - eval $(ssh-agent -s)
+ - echo "$SSH_PRIVATE_KEY_GITLAB" | tr -d '\r' | ssh-add -
+ # Our self-hosted runners have persistent home directories, so here we store
+ # the known_hosts file in the temporary project dir and tell git to use a ssh
+ # command that uses this hosts file during operations
+ - export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=$CI_PROJECT_DIR/.ssh/known_hosts"
+ - mkdir -p $CI_PROJECT_DIR/.ssh
+ - chmod 700 $CI_PROJECT_DIR/.ssh
+ - ssh-keyscan -p 10022 git.xfel.eu > $CI_PROJECT_DIR/.ssh/known_hosts
+ - ls $CI_PROJECT_DIR/.ssh
+ - echo $GIT_SSH_COMMAND
+ - python3 -m venv .venv
+ - source .venv/bin/activate
+ - python3 -m pip install --upgrade pip
+
checks:
stage: check
only: [merge_requests]
allow_failure: true
+ <<: *before_script
script:
- export PATH=/home/gitlab-runner/.local/bin:$PATH
- # We'd like to run the pre-commit hooks only on files that are being
- # modified by this merge request, however
- # `CI_MERGE_REQUEST_TARGET_BRANCH_SHA` is a 'premium' feature according to
- # GitLab... so this is a workaround for extracting the hash
+ # We'd like to run the pre-commit hooks only on files that are being
+ # modified by this merge request, however
+ # `CI_MERGE_REQUEST_TARGET_BRANCH_SHA` is a 'premium' feature according to
+ # GitLab... so this is a workaround for extracting the hash
- export CI_MERGE_REQUEST_TARGET_BRANCH_SHA=$(git ls-remote origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME | cut -d$'\t' -f1)
- - export FILES=$(git diff $CI_COMMIT_SHA $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --name-only | tr '\n' ' ')
- - python3 -m pip install --user -r requirements.txt
- - echo "Running pre-commit on diff from $CI_COMMIT_SHA to $CI_MERGE_REQUEST_TARGET_BRANCH_SHA ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME)"
+ - export FILES=$(git diff $CI_COMMIT_SHA $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --name-only | tr '\n' ' ')
+ - python3 -m pip install ".[test,dev]"
+ - echo "Running pre-commit on diff from $CI_COMMIT_SHA to $CI_MERGE_REQUEST_TARGET_BRANCH_SHA ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME)"
# Pass list of modified files to pre-commit so that it only checks them
- echo $FILES | xargs pre-commit run --color=always --files
pytest:
stage: test
only: [merge_requests]
+ <<: *before_script
script:
- - python3 -m pip install --user -r requirements.txt
- - python3 -m pip install --user 'pytest>=5.4.0' pytest-asyncio testpath
- - pytest -vv tests/test_*
+ - python3 -m pip install ".[test]"
+ - python3 -m pytest --cov=cal_tools --cov=xfel_calibrate --ignore=tests/legacy
diff --git a/README.rst b/README.rst
index 4a9cbb379f4f12463b5170ce5061f9a1a420bd8b..167a2ece7c628d8b783f7bd04065851ea7b26bf6 100644
--- a/README.rst
+++ b/README.rst
@@ -24,9 +24,7 @@ Installation using python virtual environment - recommended
3. ``python3 -m venv .venv`` - create the virtual environment
4. ``source .venv/bin/activate`` - activate the virtual environment
5. ``python3 -m pip install --upgrade pip`` - upgrade version of pip
-6. ``python3 -m pip install -r requirements.txt`` - install dependencies
-7. ``python3 -m pip install .`` - install the pycalibration package (add ``-e`` flag for editable development installation)
-8. ``pip install "git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git#egg=XFELDetectorAnalysis&subdirectory=lib"``
+6. ``python3 -m pip install .`` - install the pycalibration package (add ``-e`` flag for editable development installation)
Copy/paste script:
@@ -38,19 +36,22 @@ Copy/paste script:
python3 -m venv .venv
source .venv/bin/activate
python3 -m pip install --upgrade pip
- python3 -m pip install -r requirements.txt
- python3 -m pip install . # `-e` flag for editable install
- python3 -m pip install "git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git#egg=XFELDetectorAnalysis&subdirectory=lib/"
+ python3 -m pip install . # `-e` flag for editable install, e.g. `pip install -e .`
Installation into user home directory
=====================================
+This is not recommended as `pycalibration` has pinned dependencies for
+stability, if you install it directly into you users home environment then it
+will down/upgrade your local packages, which may cause major issues and may
+**break your local environment**, it is highly recommended to use the venv
+installation method instead.
+
1. ``git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git && cd pycalibration`` - clone the offline calibration package from EuXFEL GitLab
2. ``module load anaconda/3`` - load the anaconda/3 environment. If installing into other python environments, this step can be skipped
-3. ``pip install -r requirements.txt`` - install all requirements of this tool chain in your home directory
-4. ``pip install .`` - install the pycalibration package (add ``-e`` flag for editable development installation)
-5. ``export PATH=$HOME/.local/bin:$PATH`` - make sure that the home directory is in the PATH environment variable
+3. ``pip install .`` - install the pycalibration package (add ``-e`` flag for editable development installation)
+4. ``export PATH=$HOME/.local/bin:$PATH`` - make sure that the home directory is in the PATH environment variable
Copy/paste script:
@@ -59,8 +60,7 @@ Copy/paste script:
git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git
cd pycalibration
module load anaconda/3
- pip install -r requirements.txt --user
- pip install . # `-e` flag for editable install, e.g. `pip install -e .`
+ pip install --user . # `-e` flag for editable install, e.g. `pip install -e .`
export PATH=$HOME/.local/bin:$PATH
@@ -89,6 +89,11 @@ Development guidelines can be found on the GitLab Wiki page here: https://git.xf
Basics
======
+If you are installing the package for development purposes then you should
+install the optional dependencies as well. Follow the instructions as above, but
+instead of ``pip install .`` use ``pip install ".[test,dev]"`` to install both
+the extras.
+
The installation instructions above assume that you have set up SSH keys for use
with GitLab to allow for passwordless clones from GitLab, this way it's possible
to run ``pip install git+ssh...`` commands and install packages directly from
@@ -191,8 +196,8 @@ yourself first:
salloc -p exfel/upex -t 01:00:00
-where `-p` gives the partition to use: exfel or upex and `-t` the duration the
-node should be allocated. Then `ssh` onto that node.
+where `-p` gives the partition to use: exfel **or** upex and `-t` the duration
+the node should be allocated. Then ``ssh`` onto that node.
Then activate your environment as described above (or just continue if you are
not using a venv).
@@ -216,19 +221,79 @@ Finally run the script:
.. code::
- python3 calibrate.py --input /gpfs/exfel/exp/SPB/201701/p002012/raw/r0100 \
- --output ../../test_out --mem-cells 30 --detector AGIPD --sequences 0,1
+ python3 calibrate.py --input /gpfs/exfel/exp/SPB/201701/p002012/raw/r0100 \
+ --output ../../test_out --mem-cells 30 --detector AGIPD --sequences 0,1
-Here `--input` should point to a directory of `RAW` files for the detector you
-are calibrating. They will be output into the folder specified by `--output`,
-which will have the run number or the last folder in the hierarchy of the input
-appended. Additionally, you need to specify the number of `--mem-cells` used for
-the run, as well as the `--detector`. Finally, you can optionally specify to
-only process certain `--sequences` of files, matching the sequence numbers of
-the `RAW` input. These should be given as a comma-separated list.
+Here ``--input`` should point to a directory of ``RAW`` files for the detector
+you are calibrating. They will be output into the folder specified by
+``--output``, which will have the run number or the last folder in the hierarchy
+of the input appended. Additionally, you need to specify the number of
+``--mem-cells`` used for the run, as well as the ``--detector``. Finally, you
+can optionally specify to only process certain ``--sequences`` of files,
+matching the sequence numbers of the `RAW` input. These should be given as a
+comma-separated list.
-Finally, there is a `--no-relgain` option, which disables relative gain
+Finally, there is a ``--no-relgain`` option, which disables relative gain
correction. This can be useful while we still further characterize the detectors
to provide accurate relative gain correction constants.
You'll get a series of plots in the output directory as well.
+
+Appendix
+********
+
+Important information that doesn't really fit in as part of the readme.
+
+TODO: Place this into the docs? Also, improve docs (out of scope for PR !437)
+
+GitLab Access for ``xcaltst`` and ``xcal``
+==========================================
+
+To make it easier to work with and deploy software via ``xcaltst``/``xcal``, we
+have created an xcal account for gitlab with the following details:
+
+- Full Name: ReadOnly Gitlab Calibration External
+- User ID: 423
+- Username: ``xcalgitlab``
+- Password: ask Robert Rosca
+
+This account is intended to be used as a read only account which can be given
+access to certain repos to make it easier to clone them when using our
+functional accounts on Maxwell.
+
+The ``xcaltst`` account has an ed25519 keypair under ``~/.ssh/gitlab/``, the
+public key has been added to the ``xcalgitlab``'s approved SSH keys.
+
+Additionally this block has been added to ``~/.ssh/config``:
+
+.. code::
+
+ # Special flags for gitlab over SSH
+ Host git.xfel.eu
+ User git
+ Port 10022
+ ForwardX11 no
+ IdentityFile ~/.ssh/gitlab/id_ed25519
+
+Now any repository that ``xcalgitlab`` has read access to, e.g. if it is added as
+a reporter, can be cloned on Maxwell without having to enter a password.
+
+For example, ``xcalgitlab`` is a reporter on the pycalibration
+https://git.xfel.eu/gitlab/detectors/pycalibration repository, so now
+``xcalgitlab`` can do passwordless clones with SSH:
+
+.. code::
+
+ [xcaltst@max-exfl017 tmp]$ git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git
+ Cloning into 'pycalibration'...
+ remote: Enumerating objects: 9414, done.
+ remote: Counting objects: 100% (9414/9414), done.
+ remote: Compressing objects: 100% (2858/2858), done.
+ remote: Total 9414 (delta 6510), reused 9408 (delta 6504)
+ Receiving objects: 100% (9414/9414), 611.81 MiB | 54.87 MiB/s, done.
+ Resolving deltas: 100% (6510/6510), done.
+
+References:
+
+- Redmine ticket: https://in.xfel.eu/redmine/issues/83954
+- Original issue: https://git.xfel.eu/gitlab/detectors/calibration_workshop/issues/121
diff --git a/requirements.txt b/requirements.txt
deleted file mode 100644
index d66d4c5d2dbe422bc19b65d443ea9ebe450681b5..0000000000000000000000000000000000000000
--- a/requirements.txt
+++ /dev/null
@@ -1,44 +0,0 @@
-git+file:///gpfs/exfel/sw/calsoft/git/cal_db_interactive@2.0.1
-git+file:///gpfs/exfel/sw/calsoft/git/nbparameterise@0.3
-git+file:///gpfs/exfel/sw/calsoft/git/pyDetLib@2.5.6-2.10.0#subdirectory=lib
-Cython == 0.29.21
-Jinja2 == 2.11.2
-astcheck == 0.2.5
-astsearch == 0.1.3
-dill == 0.3.0
-extra_data == 1.2.0
-extra_geom == 1.1.1
-fabio == 0.9.0
-gitpython == 3.1.0
-h5py == 2.10.0
-iminuit == 1.3.8
-ipykernel == 5.1.4
-ipyparallel == 6.2.4
-ipython == 7.12.0
-ipython_genutils == 0.2.0
-jupyter-core == 4.6.1
-jupyter_client == 6.1.7
-jupyter_console == 6.1.0
-karabo_data == 0.7.0
-lxml == 4.5.0
-metadata_client == 3.0.8
-nbclient == 0.5.1
-nbconvert == 5.6.1
-nbformat == 5.0.7
-notebook == 6.1.5
-numpy == 1.19.1
-pre-commit == 2.10.0
-prettytable == 0.7.2
-princess == 0.2
-pypandoc == 1.4
-python-dateutil == 2.8.1
-pyyaml == 5.3
-pyzmq == 19.0.0
-requests==2.22.0
-scikit-learn == 0.22.2.post1
-sharedmem == 0.3.8
-sphinx == 1.8.5
-tabulate == 0.8.6
-traitlets == 4.3.3
-unittest-xml-reporting == 3.0.2
-.
diff --git a/setup.py b/setup.py
index fc9ead08dfc7c9a90354c5bcebc689759ebd36ec..e12fac28af282b16625e6ad92c6bb7ddc1968697 100644
--- a/setup.py
+++ b/setup.py
@@ -1,46 +1,29 @@
-import sys
from distutils.command.build import build
from distutils.extension import Extension
-from subprocess import check_call, check_output
+from subprocess import check_output
import numpy
from Cython.Distutils import build_ext
from setuptools import setup
-from setuptools.command.install import install
-extensions = [Extension("cal_tools.cython.agipdalgs",
- ['cal_tools/cython/agipdalgs.pyx'],
- include_dirs=[numpy.get_include()],
- extra_compile_args=['-fopenmp', '-march=native'],
- extra_link_args=['-fopenmp'], ),
- ]
-
-class PostInstallCommand(install):
- """Post-installation for installation mode."""
- def run(self):
- install.run(self)
- # check if this is a karabo installation
- python_path = sys.executable
- if "karabo" in python_path:
- print("Karabo installation detected, checking for PyDetLib installation")
- try:
- import XFELDetAna
- print("...found!")
- return
- except:
- "No PyDetLib installation found, attempting to install"
- check_call("karabo -g https://in.xfel.eu/gitlab install pyDetLib master".split())
- else:
- print("Python environment seems to not be a Karabo environment. "+
- "Please install PyDetLib manually.")
+extensions = [
+ Extension(
+ "cal_tools.cython.agipdalgs",
+ ["cal_tools/cython/agipdalgs.pyx"],
+ include_dirs=[numpy.get_include()],
+ extra_compile_args=["-fopenmp", "-march=native"],
+ extra_link_args=["-fopenmp"],
+ ),
+]
class PreInstallCommand(build):
"""Pre-installation for installation mode."""
+
def run(self):
- version = check_output(['git', 'describe', '--tag']).decode('utf8')
+ version = check_output(["git", "describe", "--tag"]).decode("utf8")
version = version.replace("\n", "")
- file = open('xfel_calibrate/VERSION.py', 'w')
+ file = open("xfel_calibrate/VERSION.py", "w")
file.write('__version__="{}"'.format(version))
file.close()
@@ -57,33 +40,115 @@ for ctypes in notebooks.values():
data_files += nb.get("pre_notebooks", [])
setup(
- name='European XFEL Offline Calibration',
+ name="European XFEL Offline Calibration",
version="1.0",
- packages=['cal_tools', 'xfel_calibrate'],
- package_dir={'cal_tools': 'cal_tools/cal_tools',
- 'xfel_calibrate': 'xfel_calibrate',
- 'xfel_calibrate.notebooks': 'xfel_calibrate/notebooks',
- },
+ author="Steffen Hauf",
+ author_email="steffen.hauf@xfel.eu",
+ maintainer="EuXFEL Calibration Team",
+ url="",
+ description="",
+ long_description="",
+ long_description_content_type="text/markdown",
+ # TODO: find licence, assuming this will be open sourced eventually
+ license="(c) European XFEL GmbH 2018",
+ packages=[
+ "cal_tools",
+ "xfel_calibrate",
+ ], # TODO: use setuptools.find_packages(), need to resolve issue
+ package_dir={
+ "cal_tools": "cal_tools/cal_tools",
+ "xfel_calibrate": "xfel_calibrate",
+ "xfel_calibrate.notebooks": "xfel_calibrate/notebooks",
+ },
package_data={
- 'xfel_calibrate': ['bin/*.sh'] + data_files + ['titlepage.tmpl',
- 'xfel.pdf']
+ "xfel_calibrate": [
+ "bin/*.sh",
+ "titlepage.tmpl",
+ "xfel.pdf",
+ ]
+ + data_files
+ },
+ entry_points={
+ "console_scripts": [
+ "xfel-calibrate = xfel_calibrate.calibrate:run",
+ ],
},
-
cmdclass={
- 'build' : PreInstallCommand,
- 'install': PostInstallCommand,
- 'build_ext': build_ext
+ "build": PreInstallCommand,
+ "build_ext": build_ext,
},
- url='',
- license='(c) European XFEL GmbH 2018',
- author='Steffen Hauf',
- author_email='steffen.hauf@xfel.eu',
- description='',
- entry_points = {
- 'console_scripts': [
- 'xfel-calibrate = xfel_calibrate.calibrate:run',
- ],
- },
- ext_modules=extensions
-
+ ext_modules=extensions,
+ install_requires=[
+ "iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.0.1", # noqa
+ "nbparameterise @ git+ssh://git@git.xfel.eu:10022/detectors/nbparameterise.git@0.3", # noqa
+ "XFELDetectorAnalysis @ git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git@2.5.6-2.10.0#subdirectory=lib", # noqa
+ "Cython==0.29.21",
+ "Jinja2==2.11.2",
+ "astcheck==0.2.5",
+ "astsearch==0.1.3",
+ "dill==0.3.0",
+ "extra_data==1.2.0",
+ "extra_geom==1.1.1",
+ "fabio==0.9.0",
+ "gitpython==3.1.0",
+ "h5py==2.10.0",
+ "iminuit==1.3.8",
+ "ipykernel==5.1.4",
+ "ipyparallel==6.2.4",
+ "ipython==7.12.0",
+ "ipython_genutils==0.2.0",
+ "jupyter-core==4.6.1",
+ "jupyter_client==6.1.7",
+ "jupyter_console==6.1.0",
+ "karabo_data==0.7.0",
+ "lxml==4.5.0",
+ "metadata_client==3.0.8",
+ "nbclient==0.5.1",
+ "nbconvert==5.6.1",
+ "nbformat==5.0.7",
+ "notebook==6.1.5",
+ "numpy==1.19.1",
+ "prettytable==0.7.2",
+ "princess==0.2",
+ "pypandoc==1.4",
+ "python-dateutil==2.8.1",
+ "pyyaml==5.3",
+ "pyzmq==19.0.0",
+ "requests==2.22.0",
+ "scikit-learn==0.22.2.post1",
+ "sharedmem==0.3.8",
+ "tabulate==0.8.6",
+ "traitlets==4.3.3",
+ ],
+ extras_require={
+ "docs": [
+ "nbsphinx",
+ "sphinx==1.8.5",
+ ],
+ "test": [
+ "coverage",
+ "nbval",
+ "pytest-asyncio",
+ "pytest-cov",
+ "pytest>=5.4.0",
+ "testpath",
+ "unittest-xml-reporting==3.0.2",
+ ],
+ "dev": [
+ "nbqa[toolchain]",
+ "pre-commit",
+ ],
+ },
+ python_requires=">=3.6",
+ classifiers=[
+ "Development Status :: 5 - Production/Stable",
+ "Environment :: Console",
+ "Intended Audience :: Developers",
+ "Intended Audience :: Science/Research",
+ # "License :: OSI Approved :: BSD License", # TODO: put license here
+ "Operating System :: POSIX :: Linux",
+ "Programming Language :: Python :: 3",
+ "Topic :: Scientific/Engineering :: Information Analysis",
+ "Topic :: Scientific/Engineering :: Physics",
+ ],
)