CI: Build python packages and upload to PyPI

 - Also build docs and upload to gitlab Pages.

.gitlab-ci.yml

@@ -9,12 +9,13 @@ cache:
   paths:
     - .cache/pip
 
-build:
-  script:
-    - pip install -r requirements.txt
-    - invoke build
+stages:
+  - test
+  - build
+  - deploy
 
 test:
+  stage: test
   script:
     - pip install -r requirements.txt
     - invoke build-ext
@@ -31,3 +32,75 @@ test:
       junit: unittest.xml
       cobertura: coverage.xml
       codequality: codeclimate.json
+
+build-wheels:
+  stage: build
+  image: python:3.8-buster
+  # make a docker daemon available for cibuildwheel to use
+  services:
+    - name: docker:dind
+      entrypoint: ["env", "-u", "DOCKER_HOST"]
+      command: ["dockerd-entrypoint.sh"]
+  variables:
+    DOCKER_HOST: tcp://docker:2375/
+    DOCKER_DRIVER: overlay2
+    # See https://github.com/docker-library/docker/pull/166
+    DOCKER_TLS_CERTDIR: ""
+    CIBW_PLATFORM: "linux"
+    CIBW_BUILD: "cp36-manylinux_x86_64 cp37-manylinux_x86_64 cp38-manylinux_x86_64 cp39-manylinux_x86_64"
+  script:
+    - curl -sSL https://get.docker.com/ | sh
+    - python -m pip install cibuildwheel==2.1.2
+    - python -m pip config set global.timeout 100
+    - cibuildwheel --output-dir packages
+  artifacts:
+    paths:
+      - packages/*.whl
+  only:
+    - tags
+    - main
+
+build-sdist:
+  stage: build
+  script:
+    - python -m pip --timeout 100 install numpy cython
+    - python setup.py sdist
+    - mkdir -p packages/ && mv dist/*.tar.gz packages/
+  artifacts:
+    paths:
+      - packages/*.tar.gz
+
+pages:
+  stage: build
+  script:
+    - pip install -r requirements.txt
+    - invoke docs
+    - mv docs/_build/ public/
+  artifacts:
+    paths:
+      - public
+  only:
+    - tags
+    - main
+
+deploy-to-gitlab:
+  stage: deploy
+  needs: ["build-wheels", "build-sdist"]
+  script:
+    - ls -l packages/
+    - pip install twine
+    - TWINE_PASSWORD=${CI_JOB_TOKEN} TWINE_USERNAME=gitlab-ci-token python -m twine upload --repository-url https://gitlab.desy.de/api/v4/projects/${CI_PROJECT_ID}/packages/pypi packages/*
+  only:
+    - tags
+    - main
+
+deploy-to-pypi:
+  stage: deploy
+  needs: ["build-wheels", "build-sdist"]
+  script:
+    - ls -l packages/
+    - pip install twine
+    - TWINE_PASSWORD="$PYPI" TWINE_USERNAME=cfelsc python -m twine upload packages/*
+  only:
+    - tags
+    - main

CONTRIBUTING.md

----
----
 
 # Guidelines for Contributors
 
 The project is hosted at the [GitLab instance](https://gitlab.desy.de) of Deutsches Elektronen Synchrotron (DESY).
 Contributions to the project are welcome.
 Please feel free to submit a [merge request](https://docs.gitlab.com/ee/user/project/merge_requests/)
-to <https://gitlab.desy.de/cfel-sc/cfelpyutils/>.
+to <https://gitlab.desy.de/cfel-sc-public/cfelpyutils/>.
 
 ## Version Control
 
@@ -22,7 +20,7 @@ automatically runs unit tests and package builds.
 The CFELPyUtils library is mainly developed in
 [Python](https://www.python.org).
 
-- Starting with CFELPyUtils version 2.0 support for Python 2.7 is dropped.
+- From CFELPyUtils version 2.0 Python 2.7 is no longer supported.
 - All code in the library must run with the currently supported
     versions of Python 3. At the time of this writing this is:
     - Python 3
@@ -30,11 +28,6 @@ The CFELPyUtils library is mainly developed in
         - 3.7
         - 3.8
         - 3.9
-- The Python coding style should follow for the most part the [Google
-  Python Coding
-  Style](https://github.com/google/styleguide/blob/gh-pages/pyguide.md).
-- All docstrings should be written following the [Google
-  Style](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html).
 - [Pylint](https://www.pylint.org) should be run on the code before
   submission, as stated in the Google Python Coding Style Guide. In
   the root folder of the CFELPyUtils repository, contributors can find

README.md

 # The CFELPyUtils Library
 
-
 ## Introduction
 
 CFELPyUtils is a utility library written in Python and developed at the Center For Free
 Electron Laser Science (CFEL) in Hamburg. It contains several functions and classes
-that perform various tasks related to the processing of x-ray imaging data (currently,
-mostly reading and applying geometry information to x-ray detector data). It is used by
+that perform various tasks related to the processing of x-ray imaging data. It is used by
 several internal and released CFEL software projects.
 
+### Features
+
+- Read and apply geometry information to x-ray detector data
+- peak finding algorithm (new in v2.0)
+- calibration tool for the AGIPD 1M detector (new in v2.0)
+- module for reading [CrystFEL](https://www.desy.de/~twhite/crystfel/) streams (new in v2.0)
+
+Please see the [migration guide](#migration-guide) below before upgrading from CFELPyUtils 0.x or 1.x.
 
 ## Installation
 
 The CFELPyUtils library is available on the [Python Package Index](https://pypi.org/)
 (PyPI). It can be installed using the 'pip' command:
 
-    python3 -m pip install cfelpyutils
+    pip install cfelpyutils
 
-Or, for Python 2:
+For Python 2, please use the 1.0 version.
 
-    python2 -m pip install cfelpyutils
+    pip install cfelpyutils==1.0.1
 
-The library can also be installed manually by checking out the repository and running:
+The library can also be installed manually by checking out this repository and running:
 
     python setup.py install
 
+### Migration Guide
+
+The following table lists functions for which the module affiliation has changes in v2.0.
+Please replace all old occurances with the new ones.
+
+| `cfelpyutils-0.x,-1.x` | `cfelpytuils-2` |
+| ---------------------- | --------------- |
+| `geometry_utils.compute_pix_maps()` | `geometry.compute_pix_maps()` |
+| `geometry_utils.compute_visualization_pix_maps()` | `geometry.compute_visualization_pix_maps()` |
+| `geometry_utils.apply_geometry_to_data()` | `geometry.apply_geometry_to_data()` |
+| `crystfel_utils.load_crystfel_geometry()` | `geometry.load_crystfel_geometry()` |
 
 ## Contributors
 
@@ -46,14 +63,14 @@ to the project:
 
 If you have any questions or want to contribute, please [contact us](mailto:sc@cfel.de).
 
-
 ## Documentation
 
-Documentation for the CFELPyUtils is available on [ReadTheDocs](https://readthedocs.org/projects/cfelpyutils/)
-and in the 'docs' folder of the repository.
-
+Documentation for the CFELPyUtils is available
+on [ReadTheDocs](https://readthedocs.org/projects/cfelpyutils/),
+on [GitLab at DESY](https://cfel-sc-public.pages.desy.de/cfelpyutils/) and
+in the [docs/](docs) of the [repository](https://gitlab.desy.de/cfel-sc-public/cfelpyutils).
 
 ## Source Code
 
 The source code of the CFELPyUtils library can be found on the DESY GitLab instance:
-https://gitlab.desy.de/cfel-sc/cfelpyutils
+https://gitlab.desy.de/cfel-sc-public/cfelpyutils

cfelpyutils/__init__.py

@@ -13,6 +13,8 @@
 # Copyright 2014-2021 Deutsches Elektronen-Synchrotron DESY,
 # a research centre of the Helmholtz Association.
 """
-The main CFELPyUtils package, which contains the whole library.
+This is the CFELPyUtils package, which contains all modules of the library.
+
+Please refer to the individual modules to learn more about their function.
 """
-__version__ = "2.0.0-pre"
+__version__ = "2.0.0"

docs/cfelpyutils.crystfel_stream.rst

-The calibration Module
-======================
+The crystfel_stream Module
+==========================
 
 .. automodule:: cfelpyutils.crystfel_stream
     :members:

docs/cfelpyutils.rst

-The cfelpyutils Package
-=======================
+API Reference
+=============
 
 .. automodule:: cfelpyutils
     :members:

docs/conf.py

@@ -15,8 +15,11 @@
 #
 import os
 import sys
+
 sys.path.insert(0, os.path.abspath('../'))
 
+from cfelpyutils import __version__
+
 # -- Project information -----------------------------------------------------
 
 project = "CFELPyUtils"
@@ -24,9 +27,9 @@ copyright = "2021, CFEL, DESY"
 author = "CFEL1-SC and collaborators"
 
 # The short X.Y version
-version = "1.0.0"
+version = __version__
 # The full version, including alpha/beta/rc tags
-release = "1.0.0"
+release = __version__
 
 
 # -- General configuration ---------------------------------------------------
@@ -78,7 +81,7 @@ pygments_style = None
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-html_theme = "alabaster"
+html_theme = "furo"
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -99,10 +102,11 @@ html_static_path = []
 # default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
 # 'searchbox.html']``.
 #
-html_sidebars = {
-    "index": ["localtoc.html", "searchbox.html"],
-    "documentation*": ["localtoc.html", "searchbox.html"],
-}
+# html_sidebars = {
+#     "index": ["localtoc.html", "searchbox.html"],
+#     "documentation*": ["localtoc.html", "searchbox.html"],
+# }
+# html_sidebars = { '**': ['globaltoc.html', 'relations.html', 'sourcelink.html', 'searchbox.html'] }
 
 # -- Options for HTMLHelp output ---------------------------------------------
 

docs/index.rst

@@ -3,24 +3,14 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-.. toctree::
-   :hidden:
-
-   cfelpyutils
-
-.. include README.md
-
 .. mdinclude:: ../README.md
 
+Contents
+--------
 
-Code Documentation
-------------------
-
-Code documentation for the CFELPyUtils library can be found :doc:`here <cfelpyutils>`.
-
-
-Guidelines for Contributors
----------------------------
+.. toctree::
+   :maxdepth: 2
 
-Contributions to the projects are welcome. Please see the guidelines for contributors
-:doc:`here <guidelines_for_contributors>`.
+   readme
+   cfelpyutils
+   guidelines_for_contributors

docs/readme.rst

+:orphan:
+
+.. mdinclude:: ../README.md

pyproject.toml

+# This file is part of CFELPyUtils.
+#
+# CFELPyUtils is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software Foundation,
+# either version 3 of the License, or (at your option) any later version.
+#
+# CFELPyUtils is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with CFELPyUtils. If
+# not, see <http://www.gnu.org/licenses/>.
+#
+# Copyright 2014-2021 Deutsches Elektronen-Synchrotron DESY,
+# a research centre of the Helmholtz Association.
+
+[build-system]
+requires = ["setuptools>=42.0", "wheel", "cython", "numpy"]
+build-backend = "setuptools.build_meta"

requirements.txt

 black
 coverage>=5.4<6
 cython>=0.29.21
+furo
 h5py>=2.9.0
 invoke>=1.5
 m2r2>=0.2.7
-mypy>=0.812
 mypy-extensions>=0.4.3
+mypy>=0.812
 numpy>=1.16.4
 pandas>=1.2.2
 pylint-gitlab>=0.2

setup.cfg

@@ -18,4 +18,37 @@
 inplace=1
 
 [metadata]
+name = cfelpyutils
 version = attr: cfelpyutils.__version__
+description = Utility functions and classes for CFEL software projects
+long_description = file: README.md
+author = Coherent Imaging Division, CFEL
+author_email = sc@cfel.de
+url = https://gitlab.desy.de/cfel-sc-public/cfelpyutils
+license = GNU General Public License v3.0
+license_file = LICENSE
+platforms = unix, linux, osx, cygwin, win32
+classifiers =
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
+    Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)
+    Natural Language :: English
+    Intended Audience :: Science/Research
+
+[options]
+zip_safe = False
+include_package_data = True
+packages = find:
+install_requires =
+    h5py>=2.9.0
+    mypy-extensions>=0.4.3
+    numpy>=1.16.4
+    pandas>=1.2.2
+    typing>=3.6.4;python_version<'3.7'
+platforms = any
+setup_requires =
+    cython

setup.py

@@ -18,12 +18,6 @@ import numpy
 from setuptools import Extension, setup
 from Cython.Build import cythonize
 
-
-def long_description():
-    with open("README.md", "r") as fh:
-        return fh.read()
-
-
 peakfinder8_ext = Extension(
     name="cfelpyutils.peakfinding.lib.peakfinder8_extension",
     include_dirs=[numpy.get_include()],
@@ -35,37 +29,6 @@ peakfinder8_ext = Extension(
     language="c++",
 )
 
-setup(
-    name="cfelpyutils",
-    url="https://gitlab.desy.de/cfel-sc/cfelpyutils",
-    license="GNU General Public License v3.0",
-    author="Coherent Imaging Division, CFEL",
-    author_email="sc@cfel.de",
-    description="Utility functions and classes for CFEL software projects",
-    long_description=long_description(),
-    install_requires=[
-        "h5py>=2.9.0",
-        "mypy-extensions>=0.4.3",
-        "numpy>=1.16.4",
-        "pandas>=1.2.2",
-        "typing>=3.6.4;python_version<'3.7'",
-    ],
-    ext_modules=cythonize(peakfinder8_ext),
-    zip_safe=False,
-    packages=["cfelpyutils", "cfelpyutils.peakfinding.lib"],
-    package_dir={
-        "cfelpyutils": "cfelpyutils",
-        "cfelpyutils.peakfinding.lib": "cfelpyutils/peakfinding/lib",
-    },
-    include_package_data=True,
-    platforms="any",
-    classifiers=[
-        "Programming Language :: Python",
-        "Programming Language :: Python :: 3",
-        "Operating System :: OS Independent",
-        "Topic :: Software Development :: Libraries :: Python Modules",
-        "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
-        "Natural Language :: English",
-        "Intended Audience :: Science/Research",
-    ],
-)
+
+if __name__ == "__main__":
+    setup(ext_modules=cythonize(peakfinder8_ext))