Reformat setup.py to install from git ssh sources instead of gpfs

.git-blame-ignore-revs

@@ -8,3 +8,6 @@
 
 #  fix/pre-commit-whitespace - Whitespace fixes
 e7dfadaf4e189ef0e0f67798e8984695111257e3
+
+#  fix/requirements-into-setuppy - Reformat setup.py without content changes
+31b402966a4a741d08cb1e89db96f252fbb41c54

.gitlab-ci.yml

@@ -2,27 +2,45 @@ stages:
   - check
   - test
 
+.before_script: &before_script
+  before_script:
+    - eval $(ssh-agent -s)
+    - echo "$SSH_PRIVATE_KEY_GITLAB" | tr -d '\r' | ssh-add -
+    #  Our self-hosted runners have persistent home directories, so here we store
+    #  the known_hosts file in the temporary project dir and tell git to use a ssh
+    #  command that uses this hosts file during operations
+    - export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=$CI_PROJECT_DIR/.ssh/known_hosts"
+    - mkdir -p $CI_PROJECT_DIR/.ssh
+    - chmod 700 $CI_PROJECT_DIR/.ssh
+    - ssh-keyscan -p 10022 git.xfel.eu > $CI_PROJECT_DIR/.ssh/known_hosts
+    - ls  $CI_PROJECT_DIR/.ssh
+    - echo $GIT_SSH_COMMAND
+    - python3 -m venv .venv
+    - source .venv/bin/activate
+    - python3 -m pip install --upgrade pip
+
 checks:
   stage: check
   only: [merge_requests]
   allow_failure: true
+  <<: *before_script
   script:
     - export PATH=/home/gitlab-runner/.local/bin:$PATH
-    # We'd like to run the pre-commit hooks only on files that are being
-    # modified by this merge request, however
-    # `CI_MERGE_REQUEST_TARGET_BRANCH_SHA` is a 'premium' feature according to
-    # GitLab... so this is a workaround for extracting the hash
+    #  We'd like to run the pre-commit hooks only on files that are being
+    #  modified by this merge request, however
+    #  `CI_MERGE_REQUEST_TARGET_BRANCH_SHA` is a 'premium' feature according to
+    #  GitLab... so this is a workaround for extracting the hash
     - export CI_MERGE_REQUEST_TARGET_BRANCH_SHA=$(git ls-remote origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME | cut -d$'\t' -f1)
-    - export FILES=$(git diff $CI_COMMIT_SHA  $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --name-only | tr '\n' ' ')
-    - python3 -m pip install --user -r requirements.txt
-    - echo "Running pre-commit on diff from  $CI_COMMIT_SHA to $CI_MERGE_REQUEST_TARGET_BRANCH_SHA ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME)"
+    - export FILES=$(git diff $CI_COMMIT_SHA $CI_MERGE_REQUEST_TARGET_BRANCH_SHA --name-only | tr '\n' ' ')
+    - python3 -m pip install ".[test,dev]"
+    - echo "Running pre-commit on diff from $CI_COMMIT_SHA to $CI_MERGE_REQUEST_TARGET_BRANCH_SHA ($CI_MERGE_REQUEST_TARGET_BRANCH_NAME)"
     #  Pass list of modified files to pre-commit so that it only checks them
     - echo $FILES | xargs pre-commit run --color=always --files
 
 pytest:
   stage: test
   only: [merge_requests]
+  <<: *before_script
   script:
-    - python3 -m pip install --user -r requirements.txt
-    - python3 -m pip install --user 'pytest>=5.4.0' pytest-asyncio testpath
-    - pytest -vv tests/test_*
+    - python3 -m pip install ".[test]"
+    - python3 -m pytest --cov=cal_tools --cov=xfel_calibrate  --ignore=tests/legacy

README.rst

@@ -24,9 +24,7 @@ Installation using python virtual environment - recommended
 3. ``python3 -m venv .venv`` - create the virtual environment
 4. ``source .venv/bin/activate`` - activate the virtual environment
 5. ``python3 -m pip install --upgrade pip`` - upgrade version of pip
-6. ``python3 -m pip install -r requirements.txt`` - install dependencies
-7. ``python3 -m pip install .`` - install the pycalibration package (add ``-e`` flag for editable development installation)
-8. ``pip install "git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git#egg=XFELDetectorAnalysis&subdirectory=lib"``
+6. ``python3 -m pip install .`` - install the pycalibration package (add ``-e`` flag for editable development installation)
 
 Copy/paste script:
 
@@ -38,19 +36,22 @@ Copy/paste script:
   python3 -m venv .venv
   source .venv/bin/activate
   python3 -m pip install --upgrade pip
-  python3 -m pip install -r requirements.txt
-  python3 -m pip install .  # `-e` flag for editable install
-  python3 -m pip install "git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git#egg=XFELDetectorAnalysis&subdirectory=lib/"
+  python3 -m pip install .  # `-e` flag for editable install, e.g. `pip install -e .`
 
 
 Installation into user home directory
 =====================================
 
+This is not recommended as `pycalibration` has pinned dependencies for
+stability, if you install it directly into you users home environment then it
+will down/upgrade your local packages, which may cause major issues and may
+**break your local environment**, it is highly recommended to use the venv
+installation method instead.
+
 1. ``git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git && cd pycalibration`` - clone the offline calibration package from EuXFEL GitLab
 2. ``module load anaconda/3`` - load the anaconda/3 environment. If installing into other python environments, this step can be skipped
-3. ``pip install -r requirements.txt`` - install all requirements of this tool chain in your home directory
-4. ``pip install .`` - install the pycalibration package (add ``-e`` flag for editable development installation)
-5. ``export PATH=$HOME/.local/bin:$PATH`` - make sure that the home directory is in the PATH environment variable
+3. ``pip install .`` - install the pycalibration package (add ``-e`` flag for editable development installation)
+4. ``export PATH=$HOME/.local/bin:$PATH`` - make sure that the home directory is in the PATH environment variable
 
 Copy/paste script:
 
@@ -59,8 +60,7 @@ Copy/paste script:
   git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git
   cd pycalibration
   module load anaconda/3
-  pip install -r requirements.txt --user
-  pip install .  # `-e` flag for editable install, e.g. `pip install -e .`
+  pip install --user .  # `-e` flag for editable install, e.g. `pip install -e .`
   export PATH=$HOME/.local/bin:$PATH
 
 
@@ -89,6 +89,11 @@ Development guidelines can be found on the GitLab Wiki page here: https://git.xf
 Basics
 ======
 
+If you are installing the package for development purposes then you should
+install the optional dependencies as well. Follow the instructions as above, but
+instead of ``pip install .`` use ``pip install ".[test,dev]"`` to install both
+the extras.
+
 The installation instructions above assume that you have set up SSH keys for use
 with GitLab to allow for passwordless clones from GitLab, this way it's possible
 to run ``pip install git+ssh...`` commands and install packages directly from
@@ -191,8 +196,8 @@ yourself first:
 
   salloc -p exfel/upex -t 01:00:00
 
-where `-p` gives the partition to use: exfel or upex and `-t` the duration the
-node should be allocated. Then `ssh` onto that node.
+where `-p` gives the partition to use: exfel **or** upex and `-t` the duration
+the node should be allocated. Then ``ssh`` onto that node.
 
 Then activate your environment as described above (or just continue if you are
 not using a venv).
@@ -216,19 +221,79 @@ Finally run the script:
 
 .. code::
 
-    python3 calibrate.py --input /gpfs/exfel/exp/SPB/201701/p002012/raw/r0100 \
-      --output ../../test_out --mem-cells 30 --detector AGIPD --sequences 0,1
+  python3 calibrate.py --input /gpfs/exfel/exp/SPB/201701/p002012/raw/r0100 \
+    --output ../../test_out --mem-cells 30 --detector AGIPD --sequences 0,1
 
-Here `--input` should point to a directory of `RAW` files for the detector you
-are calibrating. They will be output into the folder specified by `--output`,
-which will have the run number or the last folder in the hierarchy of the input
-appended. Additionally, you need to specify the number of `--mem-cells` used for
-the run, as well as the `--detector`. Finally, you can optionally specify to
-only process certain `--sequences` of files, matching the sequence numbers of
-the `RAW` input. These should be given as a comma-separated list.
+Here ``--input`` should point to a directory of ``RAW`` files for the detector
+you are calibrating. They will be output into the folder specified by
+``--output``, which will have the run number or the last folder in the hierarchy
+of the input appended. Additionally, you need to specify the number of
+``--mem-cells`` used for the run, as well as the ``--detector``. Finally, you
+can optionally specify to only process certain ``--sequences`` of files,
+matching the sequence numbers of the `RAW` input. These should be given as a
+comma-separated list.
 
-Finally, there is a `--no-relgain` option, which disables relative gain
+Finally, there is a ``--no-relgain`` option, which disables relative gain
 correction. This can be useful while we still further characterize the detectors
 to provide accurate relative gain correction constants.
 
 You'll get a series of plots in the output directory as well.
+
+Appendix
+********
+
+Important information that doesn't really fit in as part of the readme.
+
+TODO: Place this into the docs? Also, improve docs (out of scope for PR !437)
+
+GitLab Access for ``xcaltst`` and ``xcal``
+==========================================
+
+To make it easier to work with and deploy software via ``xcaltst``/``xcal``, we
+have created an xcal account for gitlab with the following details:
+
+- Full Name: ReadOnly Gitlab Calibration External
+- User ID: 423
+- Username: ``xcalgitlab``
+- Password: ask Robert Rosca
+
+This account is intended to be used as a read only account which can be given
+access to certain repos to make it easier to clone them when using our
+functional accounts on Maxwell.
+
+The ``xcaltst`` account has an ed25519 keypair under ``~/.ssh/gitlab/``, the
+public key has been added to the ``xcalgitlab``'s approved SSH keys.
+
+Additionally this block has been added to ``~/.ssh/config``:
+
+.. code::
+
+  # Special flags for gitlab over SSH
+  Host git.xfel.eu
+      User git
+      Port 10022
+      ForwardX11 no
+      IdentityFile ~/.ssh/gitlab/id_ed25519
+
+Now any repository that ``xcalgitlab`` has read access to, e.g. if it is added as
+a reporter, can be cloned on Maxwell without having to enter a password.
+
+For example, ``xcalgitlab`` is a reporter on the pycalibration
+https://git.xfel.eu/gitlab/detectors/pycalibration repository, so now
+``xcalgitlab`` can do passwordless clones with SSH:
+
+.. code::
+
+  [xcaltst@max-exfl017 tmp]$ git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git
+  Cloning into 'pycalibration'...
+  remote: Enumerating objects: 9414, done.
+  remote: Counting objects: 100% (9414/9414), done.
+  remote: Compressing objects: 100% (2858/2858), done.
+  remote: Total 9414 (delta 6510), reused 9408 (delta 6504)
+  Receiving objects: 100% (9414/9414), 611.81 MiB | 54.87 MiB/s, done.
+  Resolving deltas: 100% (6510/6510), done.
+
+References:
+
+- Redmine ticket: https://in.xfel.eu/redmine/issues/83954
+- Original issue: https://git.xfel.eu/gitlab/detectors/calibration_workshop/issues/121

requirements.txt

-git+file:///gpfs/exfel/sw/calsoft/git/cal_db_interactive@2.0.1
-git+file:///gpfs/exfel/sw/calsoft/git/nbparameterise@0.3
-git+file:///gpfs/exfel/sw/calsoft/git/pyDetLib@2.5.6-2.10.0#subdirectory=lib
-Cython == 0.29.21
-Jinja2 == 2.11.2
-astcheck == 0.2.5
-astsearch == 0.1.3
-dill == 0.3.0
-extra_data == 1.2.0
-extra_geom == 1.1.1
-fabio == 0.9.0
-gitpython == 3.1.0
-h5py == 2.10.0
-iminuit == 1.3.8
-ipykernel == 5.1.4
-ipyparallel == 6.2.4
-ipython == 7.12.0
-ipython_genutils == 0.2.0
-jupyter-core == 4.6.1
-jupyter_client == 6.1.7
-jupyter_console == 6.1.0
-karabo_data == 0.7.0
-lxml == 4.5.0
-metadata_client == 3.0.8
-nbclient == 0.5.1
-nbconvert == 5.6.1
-nbformat == 5.0.7
-notebook == 6.1.5
-numpy == 1.19.1
-pre-commit == 2.10.0
-prettytable == 0.7.2
-princess == 0.2
-pypandoc == 1.4
-python-dateutil == 2.8.1
-pyyaml == 5.3
-pyzmq == 19.0.0
-requests==2.22.0
-scikit-learn == 0.22.2.post1
-sharedmem == 0.3.8
-sphinx == 1.8.5
-tabulate == 0.8.6
-traitlets == 4.3.3
-unittest-xml-reporting == 3.0.2
-.

setup.py

-import sys
 from distutils.command.build import build
 from distutils.extension import Extension
-from subprocess import check_call, check_output
+from subprocess import check_output
 
 import numpy
 from Cython.Distutils import build_ext
 from setuptools import setup
-from setuptools.command.install import install
 
-extensions = [Extension("cal_tools.cython.agipdalgs",
-                        ['cal_tools/cython/agipdalgs.pyx'],
-                        include_dirs=[numpy.get_include()],
-                        extra_compile_args=['-fopenmp', '-march=native'],
-                        extra_link_args=['-fopenmp'], ),
-              ]
-
-class PostInstallCommand(install):
-    """Post-installation for installation mode."""
-    def run(self):
-        install.run(self)
-        # check if this is a karabo installation
-        python_path = sys.executable
-        if "karabo" in python_path:
-            print("Karabo installation detected, checking for PyDetLib installation")
-            try:
-                import XFELDetAna
-                print("...found!")
-                return
-            except:
-                "No PyDetLib installation found, attempting to install"
-                check_call("karabo -g https://in.xfel.eu/gitlab install pyDetLib master".split())
-        else:
-            print("Python environment seems to not be a Karabo environment. "+
-                  "Please install PyDetLib manually.")
+extensions = [
+    Extension(
+        "cal_tools.cython.agipdalgs",
+        ["cal_tools/cython/agipdalgs.pyx"],
+        include_dirs=[numpy.get_include()],
+        extra_compile_args=["-fopenmp", "-march=native"],
+        extra_link_args=["-fopenmp"],
+    ),
+]
 
 
 class PreInstallCommand(build):
     """Pre-installation for installation mode."""
+
     def run(self):
-        version = check_output(['git', 'describe', '--tag']).decode('utf8')
+        version = check_output(["git", "describe", "--tag"]).decode("utf8")
         version = version.replace("\n", "")
-        file = open('xfel_calibrate/VERSION.py', 'w')
+        file = open("xfel_calibrate/VERSION.py", "w")
         file.write('__version__="{}"'.format(version))
         file.close()
 
@@ -57,33 +40,115 @@ for ctypes in notebooks.values():
         data_files += nb.get("pre_notebooks", [])
 
 setup(
-    name='European XFEL Offline Calibration',
+    name="European XFEL Offline Calibration",
     version="1.0",
-    packages=['cal_tools', 'xfel_calibrate'],
-    package_dir={'cal_tools': 'cal_tools/cal_tools',
-                 'xfel_calibrate': 'xfel_calibrate',
-                 'xfel_calibrate.notebooks': 'xfel_calibrate/notebooks',
-                 },
+    author="Steffen Hauf",
+    author_email="steffen.hauf@xfel.eu",
+    maintainer="EuXFEL Calibration Team",
+    url="",
+    description="",
+    long_description="",
+    long_description_content_type="text/markdown",
+    # TODO: find licence, assuming this will be open sourced eventually
+    license="(c) European XFEL GmbH 2018",
+    packages=[
+        "cal_tools",
+        "xfel_calibrate",
+    ],  # TODO: use setuptools.find_packages(), need to resolve issue
+    package_dir={
+        "cal_tools": "cal_tools/cal_tools",
+        "xfel_calibrate": "xfel_calibrate",
+        "xfel_calibrate.notebooks": "xfel_calibrate/notebooks",
+    },
     package_data={
-        'xfel_calibrate': ['bin/*.sh'] + data_files + ['titlepage.tmpl',
-                                                       'xfel.pdf']
+        "xfel_calibrate": [
+            "bin/*.sh",
+            "titlepage.tmpl",
+            "xfel.pdf",
+        ]
+        + data_files
+    },
+    entry_points={
+        "console_scripts": [
+            "xfel-calibrate = xfel_calibrate.calibrate:run",
+        ],
     },
-
     cmdclass={
-        'build' : PreInstallCommand,
-        'install': PostInstallCommand,
-        'build_ext': build_ext
+        "build": PreInstallCommand,
+        "build_ext": build_ext,
     },
-    url='',
-    license='(c) European XFEL GmbH 2018',
-    author='Steffen Hauf',
-    author_email='steffen.hauf@xfel.eu',
-    description='',
-    entry_points = {
-              'console_scripts': [
-                  'xfel-calibrate = xfel_calibrate.calibrate:run',
-              ],
-          },
-    ext_modules=extensions
-
+    ext_modules=extensions,
+    install_requires=[
+        "iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.0.1", # noqa
+        "nbparameterise @ git+ssh://git@git.xfel.eu:10022/detectors/nbparameterise.git@0.3", # noqa
+        "XFELDetectorAnalysis @ git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git@2.5.6-2.10.0#subdirectory=lib", # noqa
+        "Cython==0.29.21",
+        "Jinja2==2.11.2",
+        "astcheck==0.2.5",
+        "astsearch==0.1.3",
+        "dill==0.3.0",
+        "extra_data==1.2.0",
+        "extra_geom==1.1.1",
+        "fabio==0.9.0",
+        "gitpython==3.1.0",
+        "h5py==2.10.0",
+        "iminuit==1.3.8",
+        "ipykernel==5.1.4",
+        "ipyparallel==6.2.4",
+        "ipython==7.12.0",
+        "ipython_genutils==0.2.0",
+        "jupyter-core==4.6.1",
+        "jupyter_client==6.1.7",
+        "jupyter_console==6.1.0",
+        "karabo_data==0.7.0",
+        "lxml==4.5.0",
+        "metadata_client==3.0.8",
+        "nbclient==0.5.1",
+        "nbconvert==5.6.1",
+        "nbformat==5.0.7",
+        "notebook==6.1.5",
+        "numpy==1.19.1",
+        "prettytable==0.7.2",
+        "princess==0.2",
+        "pypandoc==1.4",
+        "python-dateutil==2.8.1",
+        "pyyaml==5.3",
+        "pyzmq==19.0.0",
+        "requests==2.22.0",
+        "scikit-learn==0.22.2.post1",
+        "sharedmem==0.3.8",
+        "tabulate==0.8.6",
+        "traitlets==4.3.3",
+    ],
+    extras_require={
+        "docs": [
+            "nbsphinx",
+            "sphinx==1.8.5",
+        ],
+        "test": [
+            "coverage",
+            "nbval",
+            "pytest-asyncio",
+            "pytest-cov",
+            "pytest>=5.4.0",
+            "testpath",
+            "unittest-xml-reporting==3.0.2",
+        ],
+        "dev": [
+            "nbqa[toolchain]",
+            "pre-commit",
+        ],
+    },
+    python_requires=">=3.6",
+    classifiers=[
+        "Development Status :: 5 - Production/Stable",
+        "Environment :: Console",
+        "Intended Audience :: Developers",
+        "Intended Audience :: Science/Research",
+        # "License :: OSI Approved :: BSD License",  # TODO: put license here
+        "Operating System :: POSIX :: Linux",
+        "Programming Language :: Python :: 3",
+        "Topic :: Scientific/Engineering :: Information Analysis",
+        "Topic :: Scientific/Engineering :: Physics",
+    ],
 )