Fix/requirements into setuppy

Description

• Removes the requirements.txt file
• Move requirements into setup.py
• Reformat setup.py
• Update README to reflect no longer using requirements.txt
• Update CI to use new setup
  • Generates venv for tests each time to ensure runner local env is not used
  • Added --ignore=tests/legacy flag to pytest to ignore legacy test dir
  • Added --cov=cal_tools and --cov=xfel_calibrate to get coverage output for both modules
  • Added ssh key variable to CI, added .setup-ssh-euxfel-gitlab template to CI
• Change internal dependencies to use SSH instead of files (this is now possible as xsoft has a keypair on gitlab)

These changes make the installation of the package a bit more standard, removes the step of having to run two install commands (one for requirements.txt, and one for .), and let us use extras to define additional optional dependencies (e.g. test, dev, etc...).

How Has This Been Tested?

Tested locally by running pip install without extras, with test, with dev, and with test,dev, then ran the pytests to check that everything was still working correctly.

Also updated the CI definition to use the new setup style, so the tests also executed with the pipeline.

Relevant Documents (optional)

Types of changes

• Bug fix (non-breaking change which fixes an issue)

Checklist:

• My code follows the code style of this project.
• My change requires a change to the documentation.
• I have updated the documentation accordingly.

Reviewers

@danilevc @ahmedk @hammerd

Notes:

For our local dependencies the name of the package doesn't match the name of the repository, so I have to explicitly write out the name, and then use the git ssh url as a target, e.g. iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.0.1. @kluyvert does this look correct?

setup.py

-import sys
 from distutils.command.build import build
 from distutils.extension import Extension
-from subprocess import check_call, check_output
+from subprocess import check_output
 
 import numpy
 from Cython.Distutils import build_ext
 from setuptools import setup
-from setuptools.command.install import install
 
-extensions = [Extension("cal_tools.cython.agipdalgs",
-                        ['cal_tools/cython/agipdalgs.pyx'],
-                        include_dirs=[numpy.get_include()],
-                        extra_compile_args=['-fopenmp', '-march=native'],
-                        extra_link_args=['-fopenmp'], ),
-              ]
-
-class PostInstallCommand(install):
-    """Post-installation for installation mode."""
-    def run(self):
-        install.run(self)
-        # check if this is a karabo installation
-        python_path = sys.executable
-        if "karabo" in python_path:
-            print("Karabo installation detected, checking for PyDetLib installation")
-            try:
-                import XFELDetAna
-                print("...found!")
-                return
-            except:
-                "No PyDetLib installation found, attempting to install"
-                check_call("karabo -g https://in.xfel.eu/gitlab install pyDetLib master".split())
-        else:
-            print("Python environment seems to not be a Karabo environment. "+
-                  "Please install PyDetLib manually.")
+extensions = [
+    Extension(
+        "cal_tools.cython.agipdalgs",
+        ["cal_tools/cython/agipdalgs.pyx"],
+        include_dirs=[numpy.get_include()],
+        extra_compile_args=["-fopenmp", "-march=native"],
+        extra_link_args=["-fopenmp"],
+    ),
+]
 
 
 class PreInstallCommand(build):
     """Pre-installation for installation mode."""
+
     def run(self):
-        version = check_output(['git', 'describe', '--tag']).decode('utf8')
+        version = check_output(["git", "describe", "--tag"]).decode("utf8")
         version = version.replace("\n", "")
-        file = open('xfel_calibrate/VERSION.py', 'w')
+        file = open("xfel_calibrate/VERSION.py", "w")
         file.write('__version__="{}"'.format(version))
         file.close()
 
@@ -57,33 +40,115 @@ for ctypes in notebooks.values():
         data_files += nb.get("pre_notebooks", [])
 
 setup(
-    name='European XFEL Offline Calibration',
+    name="European XFEL Offline Calibration",
     version="1.0",
-    packages=['cal_tools', 'xfel_calibrate'],
-    package_dir={'cal_tools': 'cal_tools/cal_tools',
-                 'xfel_calibrate': 'xfel_calibrate',
-                 'xfel_calibrate.notebooks': 'xfel_calibrate/notebooks',
-                 },
+    author="Steffen Hauf",
+    author_email="steffen.hauf@xfel.eu",
+    maintainer="EuXFEL Calibration Team",
+    url="",
+    description="",
+    long_description="",
+    long_description_content_type="text/markdown",
+    # TODO: find licence, assuming this will be open sourced eventually
+    license="(c) European XFEL GmbH 2018",
+    packages=[
+        "cal_tools",
+        "xfel_calibrate",
+    ],  # TODO: use setuptools.find_packages(), need to resolve issue
+    package_dir={
+        "cal_tools": "cal_tools/cal_tools",
+        "xfel_calibrate": "xfel_calibrate",
+        "xfel_calibrate.notebooks": "xfel_calibrate/notebooks",
+    },
     package_data={
-        'xfel_calibrate': ['bin/*.sh'] + data_files + ['titlepage.tmpl',
-                                                       'xfel.pdf']
+        "xfel_calibrate": [
+            "bin/*.sh",
+            "titlepage.tmpl",
+            "xfel.pdf",
+        ]
+        + data_files
+    },
+    entry_points={
+        "console_scripts": [
+            "xfel-calibrate = xfel_calibrate.calibrate:run",
+        ],
     },
-
     cmdclass={
-        'build' : PreInstallCommand,
-        'install': PostInstallCommand,
-        'build_ext': build_ext
+        "build": PreInstallCommand,
+        "build_ext": build_ext,
     },
-    url='',
-    license='(c) European XFEL GmbH 2018',
-    author='Steffen Hauf',
-    author_email='steffen.hauf@xfel.eu',
-    description='',
-    entry_points = {
-              'console_scripts': [
-                  'xfel-calibrate = xfel_calibrate.calibrate:run',
-              ],
-          },
-    ext_modules=extensions
-
+    ext_modules=extensions,
+    install_requires=[
+        "iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.0.1", # noqa
+        "nbparameterise @ git+ssh://git@git.xfel.eu:10022/detectors/nbparameterise.git@0.3", # noqa
+        "XFELDetectorAnalysis @ git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git@2.5.6-2.10.0#subdirectory=lib", # noqa
+        "Cython==0.29.21",
+        "Jinja2==2.11.2",
+        "astcheck==0.2.5",
+        "astsearch==0.1.3",
+        "dill==0.3.0",
+        "extra_data==1.2.0",
+        "extra_geom==1.1.1",
+        "fabio==0.9.0",
+        "gitpython==3.1.0",
+        "h5py==2.10.0",
+        "iminuit==1.3.8",
+        "ipykernel==5.1.4",
+        "ipyparallel==6.2.4",
+        "ipython==7.12.0",
+        "ipython_genutils==0.2.0",
+        "jupyter-core==4.6.1",
+        "jupyter_client==6.1.7",
+        "jupyter_console==6.1.0",
+        "karabo_data==0.7.0",
+        "lxml==4.5.0",
+        "metadata_client==3.0.8",
+        "nbclient==0.5.1",
+        "nbconvert==5.6.1",
+        "nbformat==5.0.7",
+        "notebook==6.1.5",
+        "numpy==1.19.1",
+        "prettytable==0.7.2",
+        "princess==0.2",
+        "pypandoc==1.4",
+        "python-dateutil==2.8.1",
+        "pyyaml==5.3",
+        "pyzmq==19.0.0",
+        "requests==2.22.0",
+        "scikit-learn==0.22.2.post1",
+        "sharedmem==0.3.8",
+        "tabulate==0.8.6",
+        "traitlets==4.3.3",
+    ],
+    extras_require={
+        "docs": [
+            "nbsphinx",
+            "sphinx==1.8.5",
+        ],
+        "test": [
+            "coverage",
+            "nbval",
+            "pytest-asyncio",
+            "pytest-cov",
+            "pytest>=5.4.0",
+            "testpath",
+            "unittest-xml-reporting==3.0.2",
+        ],
+        "dev": [
+            "nbqa[toolchain]",
+            "pre-commit",
+        ],
+    },
+    python_requires=">=3.6",
+    classifiers=[
+        "Development Status :: 5 - Production/Stable",
+        "Environment :: Console",
+        "Intended Audience :: Developers",
+        "Intended Audience :: Science/Research",
+        # "License :: OSI Approved :: BSD License",  # TODO: put license here
+        "Operating System :: POSIX :: Linux",
+        "Programming Language :: Python :: 3",
+        "Topic :: Scientific/Engineering :: Information Analysis",
+        "Topic :: Scientific/Engineering :: Physics",
+    ],
 )