From 0dce2c56a653c02f38cc3ac832d5f4680fa71208 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Tue, 23 Nov 2021 16:10:43 +0100
Subject: [PATCH] update installation and how it works instructions
---
docs/source/conf.py | 5 +-
docs/source/how_it_works.rst | 7 +-
docs/source/installation.rst | 175 ++++++++++++++++++-----------------
3 files changed, 99 insertions(+), 88 deletions(-)
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 5c865886a..2c7d4e270 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -403,6 +403,9 @@ with open("available_notebooks.rst", "w") as f:
"""))
for detector in sorted(notebooks.notebooks.keys()):
+ # Avoid having test notebooks in detector notebooks documentations.
+ if "TEST" in detector.upper():
+ continue
values = notebooks.notebooks[detector]
f.write("{}\n".format(detector))
f.write("{}\n".format("-"*len(detector)))
@@ -410,8 +413,6 @@ with open("available_notebooks.rst", "w") as f:
for caltype in sorted(values.keys()):
data = values[caltype]
- if data.get("notebook", None) is None:
- continue
nbpath = os.path.abspath("{}/../../../{}".format(__file__, data["notebook"]))
with open(nbpath, "r") as nf:
nb = nbformat.read(nf, as_version=4)
diff --git a/docs/source/how_it_works.rst b/docs/source/how_it_works.rst
index 090ebae9f..80c86b028 100644
--- a/docs/source/how_it_works.rst
+++ b/docs/source/how_it_works.rst
@@ -2,8 +2,11 @@
How it Works
============
+The European XFEL Offline Calibration is a package that consists of different services,
+responsible for applying most of the offline calibration and characterization
+for the detectors.
-The European XFEL Offline Calibration utilizes the tools nbconvert_ and nbparameterise_
+It utilizes the tools nbconvert_ and nbparameterise_
to expose Jupyter_ notebooks to a command line interface. In the process reports are generated
from these notebooks. The general interface is::
@@ -20,6 +23,6 @@ for details on how to use as detector group staff.
If you would like to integrate additional notebooks please see the :ref:`development_workflow`.
.. _nbparameterise: https://github.com/takluyver/nbparameterise
-.. _nbconver: https://github.com/jupyter/nbconvert
+.. _nbconvet: https://github.com/jupyter/nbconvert
.. _jupyter: http://jupyter.org/
.. _SLURM: https://slurm.schedmd.com
diff --git a/docs/source/installation.rst b/docs/source/installation.rst
index 58f3f7226..90bce9c18 100644
--- a/docs/source/installation.rst
+++ b/docs/source/installation.rst
@@ -1,118 +1,125 @@
+************
Installation
-============
+************
-Installation of the European XFEL Offline Calibration tools is best
-done into an anaconda/3 environment using Maxwell_ cluster. However, there are no explicit
-dependencies, so any Python3 environment is capable of running the tool.
+It's recommended to install the offline calibration (pycalibration) package on
+maxwell, using the anaconda/3 environment.
-A more detailed step-by-step instruction can be found in the :ref:`tutorial`.
+The following instructions clone from the EuXFEL GitLab instance using SSH
+remote URLs, this assumes that you have set up SSH keys for use with GitLab
+already. If you have not then read the appendix section on `SSH Key Setup for
+GitLab`_ for instructions on how to do this .
-Package Requirements
---------------------
-Requirements can be categorized into three types:
+Installation using python virtual environment - recommended
+===========================================================
-1. Those required to run the the general tool chain,
-2. Those required by the notebooks themselves,
-3. Those required to run notebooks concurrently on a HPC cluster.
+`pycalibration` uses the same version of Python as Karabo, which in June 2021
+updated to use Python 3.8. Currently the default python installation on Maxwell
+is still Python 3.6.8, so Python 3.8 needs to be loaded from a different
+location.
-Categories 1+2 are usually Python modules, as given in the `requirements.txt`
-file found in the root directory. Additionally, PyDetLib_ is required.
-It can be installed either as a Karabo dependency or manually.
+One option is to use the Maxwell Spack installation, running `module load
+maxwell` will activate the test Spack instance from DESY, then you can use
+`module load python-3.8.6-gcc-10.2.0-622qtxd` to Python 3.8. Note that this Spack
+instance is currently a trial phase and may not be stable.
-Parallel execution is currently supported in a `SLURM_` environment.
-However, sequential execution is available using the `--no-cluster-job`
-parameter when executing `xfel-calibrate`.
+Another option is to use `pyenv`, we provide a pyenv installation at
+`/gpfs/exfel/sw/calsoft/.pyenv` which we use to manage different versions of
+python. This can be activated with ``source /gpfs/exfel/sw/calsoft/.pyenv/bin/activate``
-.. _Python: http://www.python.org/
-.. _Karabo: https://in.xfel.eu/readthedocs/docs/karabo/en/latest/
-.. _SLURM: https://slurm.schedmd.com
-.. _PyDetLib: https://in.xfel.eu/readthedocs/docs/pydetlib/en/latest/
-.. _Maxwell: https://confluence.desy.de/display/IS/Running+Jobs+on+Maxwell
+A quick setup would be:
-Installation using Anaconda
----------------------------
+1. ``source /gpfs/exfel/sw/calsoft/.pyenv/bin/activate``
+2. ``git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git && cd pycalibration`` - clone the offline calibration package from EuXFEL GitLab
+3. ``pyenv shell 3.8.11`` - load required version of python
+4. ``python3 -m venv .venv`` - create the virtual environment
+5. ``source .venv/bin/activate`` - activate the virtual environment
+6. ``python3 -m pip install --upgrade pip`` - upgrade version of pip
+7. ``python3 -m pip install .`` - install the pycalibration package (add ``-e`` flag for editable development installation)
-First you need to load the anaconda/3 environment through::
+Copy/paste script:
- 1. module load anaconda/3
+.. code:: bash
-If installing into other python enviroments, this step can be skipped.
+ source /gpfs/exfel/sw/calsoft/.pyenv/bin/activate
+ git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git
+ cd pycalibration
+ pyenv shell 3.8.11
+ python3 -m venv .venv
+ source .venv/bin/activate
+ python3 -m pip install --upgrade pip
+ python3 -m pip install . # `-e` flag for editable install, e.g. `pip install -e .`
-Then the package for the offline calibration can be obtained from the git repository::
- 2. git clone https://git.xfel.eu/gitlab/detectors/pycalibration.git
+Installation into user home directory
+=====================================
-Home directory
-++++++++++++++
+This is not recommended as `pycalibration` has pinned dependencies for
+stability, if you install it directly into you users home environment then it
+will downgrade/upgrade your local packages, which may cause major issues and may
+**break your local environment**, it is highly recommended to use the venv
+installation method instead.
-You can then install all requirements of this tool chain in your home directory by running::
+1. ``source /gpfs/exfel/sw/calsoft/.pyenv/bin/activate``
+2. ``git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git && cd pycalibration`` - clone the offline calibration package from EuXFEL GitLab
+3. ``pyenv shell 3.8.11`` - load required version of python
+4. ``pip install .`` - install the pycalibration package (add ``-e`` flag for editable development installation)
+5. ``export PATH=$HOME/.local/bin:$PATH`` - make sure that the home directory is in the PATH environment variable
- 3. pip install -r requirements.txt . --user
+Copy/paste script:
-in pycalibration's root directory.
+.. code:: bash
-After installation, you should make sure that the home directory is in the PATH environment variable::
+ source /gpfs/exfel/sw/calsoft/.pyenv/bin/activate
+ git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git
+ pyenv shell 3.8.11
+ cd pycalibration
+ pip install --user . # `-e` flag for editable install, e.g. `pip install -e .`
+ export PATH=$HOME/.local/bin:$PATH
- 4. PATH=/home/<username>/.local/bin:$PATH
-Preferred directory
-+++++++++++++++++++
+Creating an ipython kernel for virtual environments
+===================================================
-Alternatively, you can install all requirements in a directory of your preference by::
+To create an ipython kernel with pycalibration available you should (if using a
+venv) activate the virtual environment first, and then run:
- 3. mkdir /gpfs/exfel/data/scratch/<username>/<directory-name>
- pip install --target=/gpfs/exfel/data/scratch/<username>/<directory-name> -r requirements.txt .
+.. code:: bash
-and it is important to make sure that the installed requirements are in the PATH environment::
+ python3 -m pip install ipykernel # If not using a venv add `--user` flag
+ python3 -m ipykernel install --user --name pycalibration --display-name "pycalibration" # If not using a venv pick different name
- 4. PATH=/gpfs/exfel/data/scratch/<username>/<directory-name>/bin:$PATH
+This can be useful for Jupyter notebook tools as https://max-jhub.desy.de/hub/login
-After this make sure that you adjust the :ref:`configuration` and settings in the xfel-calibrate
-folder to match your environment.
+SSH Key Setup for GitLab
+========================
-The tool-chain is then available via the::
+It is highly recommended to set up SSH keys for access to GitLab as this
+simplifies the setup process for all of our internal software present on GitLab.
- xfel-calibrate
+To set up the keys:
-command.
+1. Connect to Maxwell
+2. Generate a new keypair with ``ssh-keygen -o -a 100 -t ed25519``, you can
+ either leave this in the default location (``~/.ssh/id_ed25519``) or place it
+ into a separate directory to make management of keys easier if you already
+ have multiple ones. If you are using a password for your keys please check
+ this page to learn how to manage them: https://docs.github.com/en/github/authenticating-to-github/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent#adding-your-ssh-key-to-the-ssh-agent
+3. Add the public key (``id_ed25519.pub``) to your account on GitLab: https://git.xfel.eu/gitlab/profile/keys
+4. Add the following to your ``~/.ssh/config`` file
+.. code::
-Installation using karabo
-+++++++++++++++++++++++++
+ # Special flags for gitlab over SSH
+ Host git.xfel.eu
+ User git
+ Port 10022
+ ForwardX11 no
+ IdentityFile ~/.ssh/id_ed25519
-If required, one can install into karabo environment. The difference would be to
-first source activate the karabo envrionment::
-
- 1. source karabo/activate
-
-then after cloning the offline calibration package from git, the requirements can be installed through::
-
- 3. pip install -r requirements.txt .
-
-Development Installation
-------------------------
-
-For a development installation in your home directory, which automatically
-picks up (most) changes, first install the dependencies as above,
-but then install the tool-chain separately in development mode::
-
- pip install -e . --user
-
-.. note:: Using "- -target" for development installation in a preferred directory can lead to errors.
-
-.. note:: For development installation in karabo environment "- -user" is not needed.
-
-Installation of New Notebooks
------------------------------
-
-To install new, previously untracked notebooks in the home directory,
-repeat the installation of the the tool-chain, without requirments,
-from the package base directory::
-
- pip install --upgrade . --user
-
-Or, in case you are actively developing::
-
- pip install -e . --user
+Once this is done you can clone repositories you have access to from GitLab
+without having to enter your password each time. As ``pycalibration``
+requirements are installed from SSH remote URLs having SSH keys set up is a
+requirement for installing pycalibration.
--
GitLab