From 2daa7ae2cbcba2515569aad905c9fbf6a6327c7a Mon Sep 17 00:00:00 2001
From: Robert Rosca <robert.rosca@xfel.eu>
Date: Wed, 7 Apr 2021 00:03:33 +0200
Subject: [PATCH] Fix/editable install cython build
---
.gitignore | 53 +++++++++---------
.gitlab-ci.yml | 12 +++-
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 4 +-
notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb | 6 +-
pyproject.toml | 14 +++++
setup.py | 34 +++++------
{cal_tools => src}/cal_tools/__init__.py | 0
.../cython => src/cal_tools}/agipdalgs.pyx | 0
{cal_tools => src}/cal_tools/agipdlib.py | 2 +-
{cal_tools => src}/cal_tools/agipdutils.py | 0
{cal_tools => src}/cal_tools/agipdutils_ff.py | 0
{cal_tools => src}/cal_tools/ana_tools.py | 0
{cal_tools => src}/cal_tools/dssclib.py | 0
{cal_tools => src}/cal_tools/enums.py | 0
{cal_tools => src}/cal_tools/lpdlib.py | 0
{cal_tools => src}/cal_tools/mdc_config.py | 0
{cal_tools => src}/cal_tools/metrology.py | 0
{cal_tools => src}/cal_tools/plotting.py | 0
{cal_tools => src}/cal_tools/pnccdlib.py | 0
{cal_tools => src}/cal_tools/step_timing.py | 0
{cal_tools => src}/cal_tools/tools.py | 0
.../cython => src/xfel_calibrate}/__init__.py | 0
{xfel_calibrate => src/xfel_calibrate}/bin | 0
.../xfel_calibrate}/calibrate.py | 0
.../xfel_calibrate}/finalize.py | 0
.../xfel_calibrate}/notebooks | 0
.../xfel_calibrate}/notebooks.py | 0
.../xfel_calibrate}/settings.py | 0
.../xfel_calibrate}/titlepage.tmpl | 0
.../xfel_calibrate}/xfel.pdf | Bin
tests/test_agipdalgs.py | 2 +
xfel_calibrate/__init__.py | 0
32 files changed, 71 insertions(+), 56 deletions(-)
rename {cal_tools => src}/cal_tools/__init__.py (100%)
rename {cal_tools/cython => src/cal_tools}/agipdalgs.pyx (100%)
rename {cal_tools => src}/cal_tools/agipdlib.py (99%)
rename {cal_tools => src}/cal_tools/agipdutils.py (100%)
rename {cal_tools => src}/cal_tools/agipdutils_ff.py (100%)
rename {cal_tools => src}/cal_tools/ana_tools.py (100%)
rename {cal_tools => src}/cal_tools/dssclib.py (100%)
rename {cal_tools => src}/cal_tools/enums.py (100%)
rename {cal_tools => src}/cal_tools/lpdlib.py (100%)
rename {cal_tools => src}/cal_tools/mdc_config.py (100%)
rename {cal_tools => src}/cal_tools/metrology.py (100%)
rename {cal_tools => src}/cal_tools/plotting.py (100%)
rename {cal_tools => src}/cal_tools/pnccdlib.py (100%)
rename {cal_tools => src}/cal_tools/step_timing.py (100%)
rename {cal_tools => src}/cal_tools/tools.py (100%)
rename {cal_tools/cython => src/xfel_calibrate}/__init__.py (100%)
rename {xfel_calibrate => src/xfel_calibrate}/bin (100%)
rename {xfel_calibrate => src/xfel_calibrate}/calibrate.py (100%)
rename {xfel_calibrate => src/xfel_calibrate}/finalize.py (100%)
rename {xfel_calibrate => src/xfel_calibrate}/notebooks (100%)
rename {xfel_calibrate => src/xfel_calibrate}/notebooks.py (100%)
rename {xfel_calibrate => src/xfel_calibrate}/settings.py (100%)
rename {xfel_calibrate => src/xfel_calibrate}/titlepage.tmpl (100%)
rename {xfel_calibrate => src/xfel_calibrate}/xfel.pdf (100%)
create mode 100644 tests/test_agipdalgs.py
delete mode 100644 xfel_calibrate/__init__.py
diff --git a/.gitignore b/.gitignore
index fe665f186..a5ddc11c7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,36 +1,35 @@
-*/*.out
-notebooks/*/.ipynb_checkpoints
-notebooks/*/.ipynb_checkpoints/*
+**/*.out
+**/.ipynb_checkpoints/**
+**/__pycache__/**
+*.fits
*.gif
-*.tmp
-*.png
-*/__pycache__
-*/__pychache__/*
-*/slurm_tmp*
-*Copy*.ipynb
-*.mp4
-LPD/results
-*.tar
*.h5
-__pycache__
-Test
+*.mp4
*.npy
-*.tiff
-*.tif
-*.png
-*.fits
-*.pkl
*.out
-slurm_tmp*
-./temp
+*.pkl
+*.png
+*.png
+*.tar
+*.tif
+*.tiff
+*.tmp
+*/slurm_tmp*
*egg*
-docs/source/available_notebooks.rst
+./temp
+.coverage
+LPD/results
+Test
+build
+cal_tools/agipdalgs.c*
docs/build
-docs/source/_notebooks/*
-docs/source/test_rsts/*
-docs/source/test_results.rst
+docs/source/_notebooks
docs/source/_static/reports
-webservice/webservice.yaml
+docs/source/available_notebooks.rst
+docs/source/test_results.rst
+docs/source/test_rsts
+reportservice/*.log
+slurm_tmp*
webservice/*.log
webservice/*sqlite
-reportservice/*.log
+webservice/webservice.yaml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a1a869857..2b56d9b29 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -15,7 +15,7 @@ stages:
- ssh-keyscan -p 10022 git.xfel.eu > $CI_PROJECT_DIR/.ssh/known_hosts
- ls $CI_PROJECT_DIR/.ssh
- echo $GIT_SSH_COMMAND
- - python3 -m venv .venv
+ - python3 -m venv .venv --clear
- source .venv/bin/activate
- python3 -m pip install --upgrade pip
@@ -43,4 +43,12 @@ pytest:
<<: *before_script
script:
- python3 -m pip install ".[test]"
- - python3 -m pytest --cov=cal_tools --cov=xfel_calibrate --ignore=tests/legacy
+ - python3 -m pytest --verbose --cov=cal_tools --cov=xfel_calibrate
+
+cython-editable-install-test:
+ stage: test
+ only: [merge_requests]
+ <<: *before_script
+ script:
+ - python3 -m pip install -e ".[test]"
+ - python3 -m pytest ./tests/test_agipdalgs.py
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 27de08239..8ce400170 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -133,6 +133,7 @@
"\n",
"import cal_tools\n",
"import seaborn as sns\n",
+ "from cal_tools import agipdalgs as calgs\n",
"from cal_tools.agipdlib import (\n",
" AgipdCorrections,\n",
" get_acq_rate,\n",
@@ -141,8 +142,7 @@
" get_num_cells,\n",
")\n",
"from cal_tools.ana_tools import get_range\n",
- "from cal_tools.cython import agipdalgs as calgs\n",
- "from cal_tools.enums import AgipdGainMode, BadPixels\n",
+ "from cal_tools.enums import BadPixels\n",
"from cal_tools.step_timing import StepTimer\n",
"\n",
"sns.set()\n",
diff --git a/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb b/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb
index f136170e0..9324d0a31 100644
--- a/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb
+++ b/notebooks/AGIPD/AGIPD_FF_Histogramming.ipynb
@@ -61,8 +61,8 @@
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"import sharedmem\n",
+ "from cal_tools import agipdalgs as calgs\n",
"from cal_tools.ana_tools import save_dict_to_hdf5\n",
- "from cal_tools.cython import agipdalgs as calgs\n",
"from XFELDetAna.plotting.heatmap import heatmapPlot\n",
"\n",
"%matplotlib inline\n",
@@ -555,9 +555,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "scrolled": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"x = np.linspace(0, n_cells, n_cells)\n",
diff --git a/pyproject.toml b/pyproject.toml
index caca0f782..070d67401 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -3,3 +3,17 @@ requires = ["cython==0.29.21", "numpy==1.19.1", "setuptools>=40.8.0", "wheel"]
[tool.isort]
profile = "black"
+
+[tool.pytest.ini_options]
+norecursedirs = [
+ "legacy",
+ "*.egg",
+ ".*",
+ "_darcs",
+ "build",
+ "CVS",
+ "dist",
+ "node_modules",
+ "venv",
+ "{arch}"
+]
diff --git a/setup.py b/setup.py
index 402cb3769..e506c721b 100644
--- a/setup.py
+++ b/setup.py
@@ -3,13 +3,16 @@ from distutils.extension import Extension
from subprocess import check_output
import numpy
+from Cython.Build import cythonize
from Cython.Distutils import build_ext
-from setuptools import setup
+from setuptools import setup, find_packages
-extensions = [
+from src.xfel_calibrate.notebooks import notebooks
+
+ext_modules = [
Extension(
- "cal_tools.cython.agipdalgs",
- ["cal_tools/cython/agipdalgs.pyx"],
+ "cal_tools.agipdalgs",
+ ["src/cal_tools/agipdalgs.pyx"],
include_dirs=[numpy.get_include()],
extra_compile_args=["-fopenmp", "-march=native"],
extra_link_args=["-fopenmp"],
@@ -23,15 +26,13 @@ class PreInstallCommand(build):
def run(self):
version = check_output(["git", "describe", "--tag"]).decode("utf8")
version = version.replace("\n", "")
- file = open("xfel_calibrate/VERSION.py", "w")
+ file = open("src/xfel_calibrate/VERSION.py", "w")
file.write('__version__="{}"'.format(version))
file.close()
build.run(self)
-from xfel_calibrate.notebooks import notebooks
-
data_files = []
for ctypes in notebooks.values():
for nb in ctypes.values():
@@ -51,15 +52,8 @@ setup(
long_description_content_type="text/markdown",
# TODO: find licence, assuming this will be open sourced eventually
license="(c) European XFEL GmbH 2018",
- packages=[
- "cal_tools",
- "xfel_calibrate",
- ], # TODO: use setuptools.find_packages(), need to resolve issue
- package_dir={
- "cal_tools": "cal_tools/cal_tools",
- "xfel_calibrate": "xfel_calibrate",
- "xfel_calibrate.notebooks": "xfel_calibrate/notebooks",
- },
+ packages=find_packages("src"),
+ package_dir={"": "src"},
package_data={
"xfel_calibrate": [
"bin/*.sh",
@@ -77,11 +71,11 @@ setup(
"build": PreInstallCommand,
"build_ext": build_ext,
},
- ext_modules=extensions,
+ ext_modules=cythonize(ext_modules),
install_requires=[
- "iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.0.5", # noqa
- "nbparameterise @ git+ssh://git@git.xfel.eu:10022/detectors/nbparameterise.git@0.3", # noqa
- "XFELDetectorAnalysis @ git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git@2.5.6-2.10.0#subdirectory=lib", # noqa
+ "iCalibrationDB @ git+ssh://git@git.xfel.eu:10022/detectors/cal_db_interactive.git@2.0.5", # noqa
+ "nbparameterise @ git+ssh://git@git.xfel.eu:10022/detectors/nbparameterise.git@0.3", # noqa
+ "XFELDetectorAnalysis @ git+ssh://git@git.xfel.eu:10022/karaboDevices/pyDetLib.git@2.5.6-2.10.0#subdirectory=lib", # noqa
"Cython==0.29.21",
"Jinja2==2.11.2",
"astcheck==0.2.5",
diff --git a/cal_tools/cal_tools/__init__.py b/src/cal_tools/__init__.py
similarity index 100%
rename from cal_tools/cal_tools/__init__.py
rename to src/cal_tools/__init__.py
diff --git a/cal_tools/cython/agipdalgs.pyx b/src/cal_tools/agipdalgs.pyx
similarity index 100%
rename from cal_tools/cython/agipdalgs.pyx
rename to src/cal_tools/agipdalgs.pyx
diff --git a/cal_tools/cal_tools/agipdlib.py b/src/cal_tools/agipdlib.py
similarity index 99%
rename from cal_tools/cal_tools/agipdlib.py
rename to src/cal_tools/agipdlib.py
index 1b8da40f5..4a078dc45 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/src/cal_tools/agipdlib.py
@@ -19,7 +19,7 @@ from cal_tools.enums import AgipdGainMode, BadPixels, SnowResolution
from cal_tools.tools import get_constant_from_db_and_time
from iCalibrationDB import Conditions, Constants
-from cal_tools.cython import agipdalgs as calgs
+from cal_tools import agipdalgs as calgs
def get_num_cells(fname, loc, module):
diff --git a/cal_tools/cal_tools/agipdutils.py b/src/cal_tools/agipdutils.py
similarity index 100%
rename from cal_tools/cal_tools/agipdutils.py
rename to src/cal_tools/agipdutils.py
diff --git a/cal_tools/cal_tools/agipdutils_ff.py b/src/cal_tools/agipdutils_ff.py
similarity index 100%
rename from cal_tools/cal_tools/agipdutils_ff.py
rename to src/cal_tools/agipdutils_ff.py
diff --git a/cal_tools/cal_tools/ana_tools.py b/src/cal_tools/ana_tools.py
similarity index 100%
rename from cal_tools/cal_tools/ana_tools.py
rename to src/cal_tools/ana_tools.py
diff --git a/cal_tools/cal_tools/dssclib.py b/src/cal_tools/dssclib.py
similarity index 100%
rename from cal_tools/cal_tools/dssclib.py
rename to src/cal_tools/dssclib.py
diff --git a/cal_tools/cal_tools/enums.py b/src/cal_tools/enums.py
similarity index 100%
rename from cal_tools/cal_tools/enums.py
rename to src/cal_tools/enums.py
diff --git a/cal_tools/cal_tools/lpdlib.py b/src/cal_tools/lpdlib.py
similarity index 100%
rename from cal_tools/cal_tools/lpdlib.py
rename to src/cal_tools/lpdlib.py
diff --git a/cal_tools/cal_tools/mdc_config.py b/src/cal_tools/mdc_config.py
similarity index 100%
rename from cal_tools/cal_tools/mdc_config.py
rename to src/cal_tools/mdc_config.py
diff --git a/cal_tools/cal_tools/metrology.py b/src/cal_tools/metrology.py
similarity index 100%
rename from cal_tools/cal_tools/metrology.py
rename to src/cal_tools/metrology.py
diff --git a/cal_tools/cal_tools/plotting.py b/src/cal_tools/plotting.py
similarity index 100%
rename from cal_tools/cal_tools/plotting.py
rename to src/cal_tools/plotting.py
diff --git a/cal_tools/cal_tools/pnccdlib.py b/src/cal_tools/pnccdlib.py
similarity index 100%
rename from cal_tools/cal_tools/pnccdlib.py
rename to src/cal_tools/pnccdlib.py
diff --git a/cal_tools/cal_tools/step_timing.py b/src/cal_tools/step_timing.py
similarity index 100%
rename from cal_tools/cal_tools/step_timing.py
rename to src/cal_tools/step_timing.py
diff --git a/cal_tools/cal_tools/tools.py b/src/cal_tools/tools.py
similarity index 100%
rename from cal_tools/cal_tools/tools.py
rename to src/cal_tools/tools.py
diff --git a/cal_tools/cython/__init__.py b/src/xfel_calibrate/__init__.py
similarity index 100%
rename from cal_tools/cython/__init__.py
rename to src/xfel_calibrate/__init__.py
diff --git a/xfel_calibrate/bin b/src/xfel_calibrate/bin
similarity index 100%
rename from xfel_calibrate/bin
rename to src/xfel_calibrate/bin
diff --git a/xfel_calibrate/calibrate.py b/src/xfel_calibrate/calibrate.py
similarity index 100%
rename from xfel_calibrate/calibrate.py
rename to src/xfel_calibrate/calibrate.py
diff --git a/xfel_calibrate/finalize.py b/src/xfel_calibrate/finalize.py
similarity index 100%
rename from xfel_calibrate/finalize.py
rename to src/xfel_calibrate/finalize.py
diff --git a/xfel_calibrate/notebooks b/src/xfel_calibrate/notebooks
similarity index 100%
rename from xfel_calibrate/notebooks
rename to src/xfel_calibrate/notebooks
diff --git a/xfel_calibrate/notebooks.py b/src/xfel_calibrate/notebooks.py
similarity index 100%
rename from xfel_calibrate/notebooks.py
rename to src/xfel_calibrate/notebooks.py
diff --git a/xfel_calibrate/settings.py b/src/xfel_calibrate/settings.py
similarity index 100%
rename from xfel_calibrate/settings.py
rename to src/xfel_calibrate/settings.py
diff --git a/xfel_calibrate/titlepage.tmpl b/src/xfel_calibrate/titlepage.tmpl
similarity index 100%
rename from xfel_calibrate/titlepage.tmpl
rename to src/xfel_calibrate/titlepage.tmpl
diff --git a/xfel_calibrate/xfel.pdf b/src/xfel_calibrate/xfel.pdf
similarity index 100%
rename from xfel_calibrate/xfel.pdf
rename to src/xfel_calibrate/xfel.pdf
diff --git a/tests/test_agipdalgs.py b/tests/test_agipdalgs.py
new file mode 100644
index 000000000..6296eb049
--- /dev/null
+++ b/tests/test_agipdalgs.py
@@ -0,0 +1,2 @@
+def test_import():
+ from cal_tools import agipdalgs # noqa
diff --git a/xfel_calibrate/__init__.py b/xfel_calibrate/__init__.py
deleted file mode 100644
index e69de29bb..000000000
--
GitLab