From 83e2c65fde6c77c9537efbf7f67ef0baba00bc52 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Wed, 24 Nov 2021 17:14:10 +0100
Subject: [PATCH] start exposing functions
---
docs/source/cal_tools_algorithms.rst | 22 ++++++++++
docs/source/changelog.rst | 31 ++++++++++++++
docs/source/conf.py | 2 +-
docs/source/configuration.rst | 33 +++++++--------
docs/source/how_it_works.rst | 27 ------------
docs/source/index.rst | 57 +++++++++++++++++++++++--
docs/source/test_conf.py | 63 ++++++++++++++++++++++++++++
docs/source/xfel_calibrate_conf.rst | 6 +++
8 files changed, 191 insertions(+), 50 deletions(-)
create mode 100644 docs/source/cal_tools_algorithms.rst
create mode 100644 docs/source/changelog.rst
create mode 100644 docs/source/test_conf.py
create mode 100644 docs/source/xfel_calibrate_conf.rst
diff --git a/docs/source/cal_tools_algorithms.rst b/docs/source/cal_tools_algorithms.rst
new file mode 100644
index 000000000..3698e0ede
--- /dev/null
+++ b/docs/source/cal_tools_algorithms.rst
@@ -0,0 +1,22 @@
+cal_tools
+=========
+
+.. module:: cal_tools.agipdlib
+
+.. class:: AgipdCorrections
+
+ .. attribute:: read_file
+
+ .. attribute:: write_file
+
+ .. attribute:: cm_correction
+
+ .. attribute:: mask_zero_std
+
+ .. attribute:: offset_correction
+
+ .. attribute:: baseline_correction
+
+ .. attribute:: gain_correction
+
+ .. attribute:: get_valid_image_idx
\ No newline at end of file
diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst
new file mode 100644
index 000000000..5cd240c18
--- /dev/null
+++ b/docs/source/changelog.rst
@@ -0,0 +1,31 @@
+Release Notes
+=============
+
+3.4.0
+-----
+
+- Update to Python 3.8
+- Update report upload parameter key
+- Try to make pre-commit pass
+- [AGIPD][DARK] Fix / Avoid processing empty sequence files
+- Bump numpy to 1.20.3 and remove fabio
+- Fix/filename lineno webservice logs
+- Override locale to always use UTF-8
+- Fix plotting-related warnings
+- Fix user notebook path for REMI correct notebook provisionally
+- Fix data paths in LPD notebook
+- Test update config
+- Test get_from_db and send_to_db
+- Fix typo in log message
+- [AGIPD][JF]Fix: Use all available sequences for processing darks
+- add common mode correction to epix correction notebook
+- Parallelise gain/mask compression for writing corrected AGIPD files
+- remove PyQT dependency
+- Disable dark requests from serve overview
+- Remove line that creates a run folder within the run output folder
+- Show clearer messages when running webservice in sim mode
+- Use full hostname for webservice overview
+- Use argparse only if name is main, call main with args dict
+- Assorted cleanup of xfel-calibrate
+- [AGIPD][DARK] Correctly retrieve old constants for comparison
+- [AGIPD][PC][FF] FIX/Update pc and ff notebooks with new calcat mapping
\ No newline at end of file
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 2c7d4e270..e48276188 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -150,7 +150,7 @@ todo_include_todos = True
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
-# html_theme = ''
+html_theme = 'sphinx_rtd_theme'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst
index 241070cbd..d7be92673 100644
--- a/docs/source/configuration.rst
+++ b/docs/source/configuration.rst
@@ -3,16 +3,14 @@
Configuration
=============
-The European XFEL Offline Calibration tools are configure using the `settings.py`
-and `notebooks.py` files. Both can be found in the root directory. The `settings.py`
-file configures the tools to the environment. The `notebook.py` file configures the
-notebooks which should be exposed to the command line.
+The European XFEL Offline Calibration is configured through `settings.py`
+and `notebooks.py` files. Both can be found in `xfel_calibrate` source directory. The `notebook.py` file exposes and configures the
+the notebooks by the `xfel-calibrate` command line interface.
Settings
--------
-The `settings.py` file configures the enviroment the tools are run in. It is a normal
-python file of the form::
+The `settings.py` is a python configuration file, which configures the tool's environment.::
# path into which temporary files from each run are placed
temp_path = "{}/temp/".format(os.getcwd())
@@ -35,9 +33,9 @@ A comment is given for the meaning of each configuration parameter.
Notebooks
---------
-The `xfel-calibrate` tool will expose any notebooks that are configured here to the
+The `xfel-calibrate` tool exposes configured notebooks to the
command line by automatically parsing the parameters given in the notebooks first cell.
-The configuration is to be given in form of a python directory::
+The configuration is given in the form of a python dictionary::
notebooks = {
"AGIPD": {
@@ -63,22 +61,20 @@ The configuration is to be given in form of a python directory::
}
}
-The first key is the detector that the calibration may be used for, here AGIPD. The second
-key level gives the name of the task being performed (here: DARK and PC). For each of these
-entries, a path to the notebook and a concurrency hint should be given. In the concurrency
-hint the first entry specifies which parameter of the notebook expects a list whose integer
-entries, can be concurrently run (here "modules"). The second parameter state with which range
-to fill this parameter if it is not given by the user. In the example a `range(16):=0,1,2,...15`
-would be passed onto the notebook, which is run as 16 concurrent jobs, each processing one module.
-Finally, a hint for the number of cluster cores to be started should be given. This value should
-be derived e.g. by profiling memory usage per core, run times, etc.
+The first key is the detector, e.g. AGIPD. The second key is the calibration action name, e.g. DARK or PC.
+A dictionary is expected for each action with a notebook path and concurrency configuration.
+For the concurrency three values are expected. Key `parameter` with a value name of type list, which is defined in the first notebook cell.
+The key `default concurrency` to define the range of values for `parameter` in each concurrent notebook, if it is not defined by the user.
+e.g. `"default concurrency": 16` leads to running 16 concurrent jobs, each processing one module with values of [0,1,2,...,15].
+Finally, a hint for the number of `cluster cores` that is used if the notebook is using ipcluster parallelization, only.
+This value should be derived e.g. by profiling memory usage per core, run times, etc.
.. note::
It is good practice to name command line enabled notebooks with an `_NBC` suffix as shown in
the above example.
-The `CORRECT` notebook (last notebook in the example) makes use of a concurrency generating function
+The AGIPD `CORRECT` notebook (last notebook in the example) makes use of a concurrency generating function
by setting the `use function` parameter. This function must be defined in a code cell in the notebook,
its parameters should be named like other exposed parameters. It should return a list of of parameters
to be inserted into the concurrently run notebooks. The example given e.g. defines the `balance_sequences`
@@ -100,6 +96,7 @@ function::
else:
return sequences
+TESTING: :func:`balance_sequences`
.. note::
diff --git a/docs/source/how_it_works.rst b/docs/source/how_it_works.rst
index 80c86b028..8b1378917 100644
--- a/docs/source/how_it_works.rst
+++ b/docs/source/how_it_works.rst
@@ -1,28 +1 @@
-.. _how_it_works:
-How it Works
-============
-The European XFEL Offline Calibration is a package that consists of different services,
-responsible for applying most of the offline calibration and characterization
-for the detectors.
-
-It utilizes the tools nbconvert_ and nbparameterise_
-to expose Jupyter_ notebooks to a command line interface. In the process reports are generated
-from these notebooks. The general interface is::
-
- % xfel-calibrate DETECTOR TYPE
-
-where `DETECTOR` and `TYPE` specify the task to be performed.
-
-Additionally, it leverages the DESY/XFEL Maxwell cluster to run these jobs in parallel
-via SLURM_.
-
-Here is a list of :ref:`available_notebooks`. See the :ref:`advanced_topics` if you are
-for details on how to use as detector group staff.
-
-If you would like to integrate additional notebooks please see the :ref:`development_workflow`.
-
-.. _nbparameterise: https://github.com/takluyver/nbparameterise
-.. _nbconvet: https://github.com/jupyter/nbconvert
-.. _jupyter: http://jupyter.org/
-.. _SLURM: https://slurm.schedmd.com
diff --git a/docs/source/index.rst b/docs/source/index.rst
index 5512e8a00..7deab968a 100644
--- a/docs/source/index.rst
+++ b/docs/source/index.rst
@@ -3,15 +3,44 @@
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
-Welcome to European XFEL Offline Calibration's documentation!
-=============================================================
+European XFEL Offline Calibration
+=================================================
-Contents:
+The European XFEL Offline Calibration (pyCalibration) is a python package that consists of
+different services, responsible for applying most of the offline calibration
+and characterization for the detectors.
+
+Running a calibration
+---------------------
+
+The tool utilizes tools such as nbconvert_ and nbparameterise_
+to expose Jupyter_ notebooks to a command line interface.
+In the process reports are generated from these notebooks.
+The general interface is::
+
+ % xfel-calibrate DETECTOR TYPE
+
+where `DETECTOR` and `TYPE` specify the task to be performed.
+
+Additionally, it leverages the DESY/XFEL Maxwell cluster to run these jobs in parallel
+via SLURM_.
+
+Here is a list of :ref:`available_notebooks`. See the :ref:`advanced_topics` if you are
+for details on how to use as detector group staff.
+
+If you would like to integrate additional notebooks please see the :ref:`development_workflow`.
+
+.. _nbparameterise: https://github.com/takluyver/nbparameterise
+.. _nbconvert: https://github.com/jupyter/nbconvert
+.. _jupyter: http://jupyter.org/
+.. _SLURM: https://slurm.schedmd.com
+
+
+Documentation contents:
.. toctree::
:maxdepth: 2
- how_it_works
installation
configuration
workflow
@@ -20,3 +49,23 @@ Contents:
tutorial
_notebooks/index
testing
+
+.. toctree::
+ :caption: Reference
+ :maxdepth: 2
+
+ xfel_calibrate_conf
+ cal_tools_algorithms
+
+.. toctree::
+ :caption: Development
+ :maxdepth: 2
+
+ changelog
+ architecture
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`search`
\ No newline at end of file
diff --git a/docs/source/test_conf.py b/docs/source/test_conf.py
new file mode 100644
index 000000000..267c48fb5
--- /dev/null
+++ b/docs/source/test_conf.py
@@ -0,0 +1,63 @@
+import os
+from subprocess import check_output
+from textwrap import dedent, indent
+
+import nbformat
+from nbconvert import RSTExporter
+
+# generate the list of available notebooks
+from xfel_calibrate import notebooks
+
+rst_exporter = RSTExporter()
+with open("available_notebooks.rst", "w") as f:
+ f.write(dedent("""
+ .. _available_notebooks:
+
+ Available Notebooks
+ ===================
+
+ The following notebooks are currently integrated into the European XFEL
+ Offline Calibration tool chain.
+
+
+ """))
+
+ for detector in sorted(notebooks.notebooks.keys()):
+ values = notebooks.notebooks[detector]
+ f.write("{}\n".format(detector))
+ f.write("{}\n".format("-"*len(detector)))
+ f.write("\n")
+
+ for caltype in sorted(values.keys()):
+ data = values[caltype]
+
+ nbpath = os.path.abspath("{}/../../../{}".format(__file__, data["notebook"]))
+ with open(nbpath, "r") as nf:
+ nb = nbformat.read(nf, as_version=4)
+ def first_markdown_cell(nb):
+ for cell in nb.cells:
+ if cell.cell_type == 'markdown':
+ return cell
+ mdcell = first_markdown_cell(nb)
+ nb.cells = [mdcell] # we only want this single cell
+ body, _ = rst_exporter.from_notebook_node(nb)
+ adjusted = []
+ # adjust titles
+ for line in body.split("\n"):
+ if line.startswith("=="):
+ line = line.replace("=", "+")
+ if line.startswith("--"):
+ line = line.replace("-", "~")
+ adjusted.append(line)
+ f.write("\n".join(adjusted))
+ f.write("\n")
+
+ f.write("To invoke this notebook and display help use:\n\n")
+ f.write(".. code-block:: bash\n\n")
+ f.write(" xfel-calibrate {} {} --help\n\n".format(detector, caltype))
+ f.write("The full parameter list of this notebook (with defaults is): \n\n")
+ f.write(".. code-block:: bash\n\n")
+ nb_help = ["xfel-calibrate", detector, caltype, "--help"]
+ output = check_output(nb_help).decode('utf8')
+ f.write(indent(output.replace("DETECTOR", detector).replace("TYPE", caltype), " "*4))
+ f.write("\n\n")
\ No newline at end of file
diff --git a/docs/source/xfel_calibrate_conf.rst b/docs/source/xfel_calibrate_conf.rst
new file mode 100644
index 000000000..54e6211c6
--- /dev/null
+++ b/docs/source/xfel_calibrate_conf.rst
@@ -0,0 +1,6 @@
+xfel_calibrate
+==============
+
+.. module:: xfel_calibrate.calibrate
+
+.. autofunction:: balance_sequences
\ No newline at end of file
--
GitLab