.gitignore

@@ -32,6 +32,7 @@ docs/source/test_results.rst
 docs/source/test_rsts
 reportservice/*.log
 slurm_tmp*
-src/cal_tools/agipdalgs.c
+src/cal_tools/*.c
+src/cal_tools/*/*.c
 webservice/*.log
 webservice/*sqlite

.gitlab-ci.yml

@@ -57,4 +57,4 @@ cython-editable-install-test:
   <<: *before_script
   script:
     - python3 -m pip install -e ".[test]"
-    - python3 -m pytest --color yes --verbose ./tests/test_agipdalgs.py
+    - python3 -m pytest --color yes --verbose ./tests/test_cythonalgs.py 

.readthedocs.yaml

+# .readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/source/conf.py
+  fail_on_warning: false
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+  version: 3.8
+  install:
+    - requirements: docs/requirements.txt
+    - method: pip
+      path: .
+      extra_requirements:
+        - docs

README.rst

@@ -110,7 +110,7 @@ The offline calibration package is configured with three configuration files:
 
 - `webservice/config/webservice.yaml` - configuration for the web service
 - `webservice/config/serve_overview.yaml` - configuration for the overview page
-- `src/cal_tools/mdc_config.yaml` - configuration for MDC access by cal tools
+- `src/cal_tools/restful_config.yaml` - configuration for RESTful access to myMDC and CalCat by cal tools
 
 These configuration files should not be modified directly, instead you should
 create a file `$CONFIG.secrets.yaml` (e.g. `webservice.secrets.yaml`) in the
@@ -131,7 +131,7 @@ tests in a CI environment. The environment variables should be prefixed with:
 
 - `webservice/config/webservice.yaml` - `CAL_WEBSERVICE`
 - `webservice/config/serve_overview.yaml` - `CAL_SERVE_OVERVIEW`
-- `src/cal_tools/mdc_config.yaml` - `CAL_CAL_TOOLS`
+- `src/cal_tools/restful_config.yaml` - `CAL_CAL_TOOLS`
 
 Followed by an underscore and the configuration key you wish to change. Nested
 keys can be accessed with two underscores, e.g.

bin/slurm_calibrate.sh

@@ -43,9 +43,14 @@ then
     sleep 15
 fi
 
-
 echo "Running notebook"
-${python_path} -m princess ${nb_path} --save
+if [ "$caltype" == "CORRECT" ]
+then
+  # calparrot stores and repeats calcat queries
+  ${python_path} -m calparrot -- ${python_path} -m princess ${nb_path} --save
+else
+  ${python_path} -m princess ${nb_path} --save
+fi
 
 # stop the cluster if requested
 if [ "${ipcluster_profile}" != "NO_CLUSTER" ]

docs/requirements.txt

+iCalibrationDB @ git+https://xcalgitlab:${GITHUB_TOKEN}@git.xfel.eu/gitlab/detectors/cal_db_interactive.git@2.2.0

docs/source/cal_tools_algorithms.rst

+cal_tools
+=========
+
+.. module:: cal_tools.agipdlib
+
+.. class:: AgipdCorrections
+
+    .. attribute:: read_file
+
+    .. attribute:: write_file
+
+    .. attribute:: cm_correction
+
+    .. attribute:: mask_zero_std
+
+    .. attribute:: offset_correction
+
+    .. attribute:: baseline_correction
+
+    .. attribute:: gain_correction
+
+    .. attribute:: get_valid_image_idx
\ No newline at end of file

docs/source/calcat_interface.rst

+CALCAT Interface
+================
+
+.. module:: cal_tools.calcat_interface
+
+.. class:: CalCatError
+
+.. class:: CalibrationData
+
+    .. attribute:: metadata
+
+    .. attribute:: ndarray
+
+    .. attribute:: ndarray_map
+
+.. class:: SplitConditionCalibrationData
+
+.. class:: LPD_CalibrationData
+
+.. class:: DSSC_CalibrationData
+
+.. class:: JUNGFRAU_CalibrationData
+
+.. class:: PNCCD_CalibrationData
+
+.. class:: EPIX100_CalibrationData
+
+.. class:: GOTTHARD2_CalibrationData
+

docs/source/changelog.rst

+Release Notes
+=============
+
+3.7.5
+-----
+
+16-11-2022
+
+- [JUNGFRAU] Fix index location for ROI instrument output https://git.xfel.eu/detectors/pycalibration/-/merge_requests/765
+
+3.7.4
+-----
+
+15-11-2022
+
+- [Jungfrau] Only emit a warning if all sequences of a job are empty
+- [Jungfrau] Fix storing multiple ROIs for a single module
+- [Jungfrau] Replicate a raw dataset in processed data for legacy analysis code
+
+3.7.3
+-----
+
+10-11-2022
+
+- [DataFile] Add support for older INDEX and METADATA versions.
+- [LPD] Fix histogram plotting if selected cell's data is identical for trains.
+- [Jungfrau] Avoid unclear error messages when the run has no trains.
+- [AGIPD] Change notebook-default setting of max-task-per-worker to 1.
+- [REMI] Allow pulse separation in REMI/DLD reconstruction to ignore FEL pulses.
+- [webservice] Fix timestamps in the overview page.
+- [Webservice] Check run type more directly in myMdC response.
+
+3.7.2
+-----
+
+28-10-2022
+
+- [Jungfrau] Add support for strixel sensors
+- [Jungfrau] Shorter summary notebook for darks by default
+- [Jungfrau] Always align corrected data properly
+- [Jungfrau] Harden against 0 adc data in dark characterization
+- [REMI] Add support for quad nodes and various pulse separation improvements
+- [AGIPD] Allow manual photon energy input for photonization and add consistency plots
+- Grant priority partitions for set-up time
+
+3.7.1
+-----
+
+14-10-2022
+
+- [EPIX100] Add Common mode corrections to dark characterization
+- [EPIX100] Use a single value for temperature
+- [JUNGFRAU] Fix bad pixel masking
+- [JUNGFRAU] Limit number of plotted trains to 500 during corrections
+- [AGIPD] Don't fail when there are no trains in sequence file
+- [REMI] Use DataFile API and support asymmetric pump-probe patterns
+- Skip corrections for runs which match specific types
+
+3.7.0
+-----
+
+07-09-2022
+
+- [JUNGFRAU] Mask double sized pixels for Jungfrau https://git.xfel.eu/detectors/pycalibration/-/merge_requests/689
+- [LPD] Storing retrieved calibration constants in calibration_metadata.yml.
+- Storing the running logs in the report folder path instead of a `temp` folder wherever the processing ran from.
+- Selecting partitions based on beamtime dates. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/725
+- Creation time is not read from myMDC anymore.
+
+3.6.4
+-----
+
+20-08-2022
+
+- [AGIPD] Significant improvements to AGIPG LitFrameFinder implementation, including offline support.
+- [AGIPD] Fix misalignment of data and gain/mask when common mode is used with manual pulse slicing.
+
+3.6.3
+-----
+
+23-08-2022
+
+- [AGIPD] litframe finder configs in update_config https://git.xfel.eu/detectors/pycalibration/-/merge_requests/723
+
+3.6.2
+-----
+
+15-08-2022
+
+- [EPIX100][Jungfrau][pnCCD] Retrieve constants precorrection notebooks.
+- [AGIPD] skip sanitization https://git.xfel.eu/detectors/pycalibration/-/merge_requests/654
+- [AGIPD] New PC notebook https://git.xfel.eu/detectors/pycalibration/-/merge_requests/604
+- [JUNGFRAU] Allow selecting ROIs to save 1D projections  https://git.xfel.eu/detectors/pycalibration/-/merge_requests/699
+3.6.1
+-----
+
+26-07-2022
+
+- [GOTTHARD2] Correction and Dark notebooks. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/658, https://git.xfel.eu/detectors/pycalibration/-/merge_requests/684, https://git.xfel.eu/detectors/pycalibration/-/merge_requests/707
+- pin `xarray` to 2022.3.0 https://git.xfel.eu/detectors/pycalibration/-/merge_requests/706
+3.6.0
+-----
+
+20-07-2022
+
+- [LPD][Dark] Save LPD bad pixel dark data as uint32, not float64. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/698
+- [LPD][Correct] Add option to disable Oauth when using CalCat proxy. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/681
+- [AGIPD][LPD][DSSC][DARK] Avoid printing errors for expected missing constant files. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/664
+- [TESTs] Refactor and split `AGIPDCtrl` methods (old and new RAW data versions.) and add unit tests https://git.xfel.eu/detectors/pycalibration/-/merge_requests/651, https://git.xfel.eu/detectors/pycalibration/-/merge_requests/646
+- [Webservice] Monitor SLURM jobs in a separate process. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/668
+- [Webservice] Restructure the SLURM database to give more meaningful success/failure information. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/683
+- [XFEL-CALIBRATE] Recreate environments on demand for xfel-calibrate-repeat. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/660
+- Upgrade the Jupyter packages to the latest version. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/697/diffs
+- Update Extra-data to 1.12: https://git.xfel.eu/detectors/pycalibration/-/merge_requests/693
+  - [EXtra-data] Disable `_use_voview` to be able to use `.files` or avoid using `.files` https://git.xfel.eu/detectors/pycalibration/-/merge_requests/682, https://git.xfel.eu/detectors/pycalibration/-/merge_requests/688
+
+
+3.5.5
+-----
+
+15-06-2022
+
+- [AGIPD][CORRECT] Expose max tasks per pool worker.
+
+3.5.4
+-----
+
+13-06-2022
+
+- [AGIPD] Convert bias_voltage parameter condition to integer in cal_tools.
+- [LPD] Fix correcting a single pulse.
+- [LPD] VCXI require 4 modules.
+
+3.5.3
+-----
+
+19-05-2022
+
+- [LPD][CORRECT] Optionally create virtual CXI files
+- [LPD][CORRECT] Expose max-nodes parameter
+- [AGIPD] Replace gain_choose_int by fused types
+- Fix missing install of restful_config.yaml
+- Fix use of xfel-calibrate --skip-report
+
+3.5.2
+-----
+
+16.05.2022
+
+- [LPD][CORRECT] New correction notebook for LPD
+- New `files` module to write European XFEL HDF5 corrected data files.
+
+3.5.1
+-----
+
+05-04-2022
+
+- Calibration Constant version's new `Variant` file attribute. To indicate method of handling the constant post retrieval. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/619
+- Epix100 dark Badpixels Map. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/637
+- `skip-plots` flag to finish correction before plotting. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/635
+- First trainId's timestamp as RAW data creation_time, if there is myMDC connection. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/630
+- AGIPD correction can correct one cellId without plotting errors. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/642
+- Fixed mode relative gain constants in Jungfrau can be retrieved. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/639
+- Only instrument source is selected to check number of trains to dark process. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/636
+- AGIPD trains for dark processing is selected for each module individually. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/653
+- Produce report after trying to correct AGIPD run with no images. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/641
+- AGIPD's bias voltage for AGIPD1M is read from slow data. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/647
+- Removed psutil dependency. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/653
+- Update Pasha to 0.1.1 https://git.xfel.eu/detectors/pycalibration/-/merge_requests/638
+
+3.5.0
+-----
+
+01-03-2022
+
+- Updating Correction and dark notebooks for JUNGFRAU: https://git.xfel.eu/detectors/pycalibration/-/merge_requests/518
+- Updating Correction and dark notebooks for AGIPD: https://git.xfel.eu/detectors/pycalibration/-/merge_requests/535
+- Updating Correction and dark notebooks for PnCCD: https://git.xfel.eu/detectors/pycalibration/-/merge_requests/559
+- Updating Correction and dark notebooks for ePix100: https://git.xfel.eu/detectors/pycalibration/-/merge_requests/500
+
+  * EXtra-data is integrated to read files in pycalibration for AGIPD, JUNGFRAU, ePix100, and PnCCD. Dark and Correction notebooks.
+  * Pasha is now used for processing data for JUNGFRAU, ePix100 and PnCCD.
+  * pyDetLib correction functions were removed (except for common-mode correction).
+  * `db-module` is useless now for JUNGFRAU, ePix100 and PnCCD. Some parameters were updated in dark and correction notebooks for the mentioned detectors.
+
+- `gain_mode` and burst mode are now available for JUNGFRAU. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/591
+- JUNGFRAU has now a new badpixel value, `WRONG_GAIN_VALUE`. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/591
+- Pass through available for testing in-progress ORCA service. https://git.xfel.eu/detectors/pycalibration/-/merge_requests?scope=all&state=merged&search=orca
+- Non-calibrated RAW h5files are no longer copied.
+- High priority partitions (`upex-high`and `upex-middle`) are used for runs from ACTIVE and READY proposals, only. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/628
+- Supporting to disable LPD Correction through the webservice. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/629
+- Compatibility for old DAQ files for REMI is added. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/607
+- server-overview refactors. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/593 https://git.xfel.eu/detectors/pycalibration/-/merge_requests/589
+- AGIPD correction notebook support AgipdLitFrameFinder device. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/603
+- Parsing code arguments in xfel-calibrate is refactored. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/575
+- skip-plots option for AGIPD. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/581
+- Native implementation for transposition of constants AGIPD. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/580
+- Trains for AGIPD can be selected for correction. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/574
+- Skip report flag in xfel-calibrate. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/579
+- Fix ReadTheDocs. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/448
+- Fix error reporting for re-injecting the same CCV. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/627
+- Fix AGIPD for legacy runs without `gain_mode`. https://git.xfel.eu/detectors/pycalibration/-/merge_requests/617 https://git.xfel.eu/detectors/pycalibration/-/merge_requests/626
+- Pinning markupsafe version 2.0.1 https://git.xfel.eu/detectors/pycalibration/-/merge_requests/631
+- Pinning psutil 5.9.0 https://git.xfel.eu/detectors/pycalibration/-/merge_requests/535
+- Updating Extra-data to 1.9.1 https://git.xfel.eu/detectors/pycalibration/-/merge_requests/535
+- Updating h5py to 3.5.0 https://git.xfel.eu/detectors/pycalibration/-/merge_requests/602
+
+3.4.3
+-----
+
+20-10-2021
+
+- Update pyDetLib tag.
+- Add explicit dependencies on matplotlib, scipy.
+- Remove outdated matplotlib rcParams setting.
+- Update EXtra-geom to 1.6.
+- Remove cluster_profile parameter from notebooks which don't use it.
+- Fix checking availability for the concurrency parameter.
+- Fix launching work directly (not via Slurm).
+- Fix `sphinx-rep` temp folder recreation, if sphinx-rep already existed.
+- Fix missing string conversion for slurm-scheduling argument.
+- Fix title reports for multiple detectors per run folder.
+- Append to .out files for preemptable finalize job.
+- [AGIPD] [Correct] Reuse previously found constants.
+- [AGIPD] Fix missing memory cell index in SlopesPC constant sanitization.
+- [AGIPD] Only use bad pixels from darks in agipdutils.baseline_correct_via_stripes.
+- [AGIPD] [Dark] Use function to get list of karabo_da from run for making Slurm jobs.
+- [EPIX100][CORRECT] Set absolute_gain to false if relative gain was not retrieved.
+- [JUNGFRAU] Fix running for multiple modules and flip logic for do_relative_gain.
+- [JUNGFRAU] Style changes for Dark and Correct notebooks.
+- [REMI] Add notebook to reconstruct detector hits from raw data.
+- [webservice] Check run migration status using MyMDC.
+- Resolve "Skip ZMQ tests if zmq connection for calibration DB not available".
+- Reproducibility, step 1.
+
+3.4.2
+-----
+
+17-09-2021
+
+- Remove driver=core from all notebook
+- [webservice] Make use of Dynaconf for managing secrets.
+- [webservice] Make use of dedicated slurm partitions.
+- [webservice] Handle missing migration information (missing user.status fattr).
+- [webservice] Implement, raise, and catch, migration errors to send mdc messages.
+- [webservice] Simplify handling of user notebook paths.
+- [webservice] Update princess to 0.4 (use Unix sockets).
+- [webservice] Update MyMDC with begin and end times.
+- [webservice] create output folder before copying slow data.
+- [AGIPD] [CORRECT] read acq_rate from slow data.
+- [AGIPD][CORRECT] Set default memory cells to 352.
+- [AGIPD] [CORRECT] Set maximum pulses to correct based on file content.
+- [AGIPD] [FF] Correctly label legends in figures.
+- [AGIPD] [FF] Add HIBEF AGIPD500K and fix some issue with retrieval of conditions.
+- [Jungfrau] Add Gain setting to Jungfrau notebooks.
+- [Jungfrau] Fix max gain plot in LPD correct notebook
+- [JUNGFRAU] [DARK] Clearer error message for Jungfrau Dark notebooks no suitable files are found
+- [LPD] [CORRECT] Fix max gain plot.
+- [EPIX100] [CORRECT] Solve conflict between gain correction and clustering
+
+
+3.4.1
+-----
+
+16-07-2021
+
+- Update h5py to 3.3
+- Stop execution on notebook errors
+- [AGIPD] Add integration time as operating condition to all notebooks
+- [webservice] Add blocklist pattern when copying untouched files in webservice.
+- [webservice] Expose dark configurations in update_config.py
+- Fix MetadataClient.get_proposal_runs arguments call.
+- Fix Use snapshot for injecting constants for old PDU mappings
+- Fix the old time-summary (creation time for retrieved constants)
+- Update documentation notes on venv installation
+- Ignore all .so files in gitignore
+
+
+3.4.0
+-----
+
+28-06-2021
+
+- Update to Python 3.8.
+- Bump numpy to 1.20.3 and remove fabio.
+- remove PyQT dependency.
+- Disable dark requests from serve overview.
+- Update report upload parameter key.
+- Override locale to always use UTF-8.
+- Assorted cleanup of xfel-calibrate.
+- Fix pre-commit.
+- Use argparse only if name is main, call main with args dict.
+- [webservice] Use full hostname for webservice overview.
+- [webservice] Show clearer messages when running webservice in sim mode.
+- [webservice] Fix filename lineno and typos in webservice logs.
+- [webservice] Fix creating an extra run folder in run output folder.
+- [AGIPD] Parallelize gain/mask compression for writing corrected AGIPD files.
+- [AGIPD][DARK] Fix processing empty sequence files.
+- [AGIPD][PC][FF] Update notebooks with new CALCAT mapping.
+- [AGIPD][JUNGFRAU] Use all available sequences for processing darks for AGIPD and Jungfrau.
+- [AGIPD][LPD][DSSC] Fix retrieve old constants for comparison for modular detectors.
+- [LPD] Fix data paths in LPD notebook.
+- [REMI] Fix user notebook path for REMI correct notebook provisionally.
+- [EPIX][CORRECT] Add Common mode correction.
+- Fix plotting-related warnings.
+- Test update config.
+- Test get_from_db and send_to_db.

docs/source/conf.py

@@ -13,6 +13,8 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
+import glob
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
@@ -20,6 +22,23 @@
 # import os
 # import sys
 # sys.path.insert(0, os.path.abspath('.'))
+import os
+import shutil
+import sys
+import textwrap
+from datetime import datetime
+from subprocess import Popen, check_output
+from textwrap import dedent, indent
+from uuid import uuid4
+
+import nbformat
+import tabulate
+from dateutil.parser import parse
+from lxml import etree
+from nbconvert import RSTExporter
+
+# generate the list of available notebooks
+from xfel_calibrate import notebooks
 
 # -- General configuration ------------------------------------------------
 
@@ -39,11 +58,6 @@ extensions = [
     'sphinx.ext.viewcode',
 ]
 
-import glob
-import os
-import sys
-from subprocess import Popen
-
 sys.path.append(os.path.abspath("../pycalibration/"))
 p = Popen(["./makeAllDocs.sh"])
 p.communicate()
@@ -136,7 +150,7 @@ todo_include_todos = True
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-# html_theme = ''
+html_theme = 'sphinx_rtd_theme'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -374,18 +388,7 @@ except:
     check_call(["wget", pandoc_url])
     check_call(["dpkg", "-i", pandoc_pack])
 
-import os
-from subprocess import check_output
-from textwrap import dedent, indent
-
-import nbformat
-from nbconvert import RSTExporter
-
-# generate the list of available notebooks
-from xfel_calibrate import notebooks
-
 rst_exporter = RSTExporter()
-
 with open("available_notebooks.rst", "w") as f:
     f.write(dedent("""
             .. _available_notebooks:
@@ -400,6 +403,9 @@ with open("available_notebooks.rst", "w") as f:
             """))
 
     for detector in sorted(notebooks.notebooks.keys()):
+        # Avoid having test notebooks in detector notebooks documentations. 
+        if "TEST" in detector.upper():
+            continue
         values = notebooks.notebooks[detector]
         f.write("{}\n".format(detector))
         f.write("{}\n".format("-"*len(detector)))
@@ -407,7 +413,6 @@ with open("available_notebooks.rst", "w") as f:
 
         for caltype in sorted(values.keys()):
             data = values[caltype]
-
             nbpath = os.path.abspath("{}/../../../{}".format(__file__, data["notebook"]))
             with open(nbpath, "r") as nf:
                 nb = nbformat.read(nf, as_version=4)
@@ -438,18 +443,8 @@ with open("available_notebooks.rst", "w") as f:
             output = check_output(nb_help).decode('utf8')
             f.write(indent(output.replace("DETECTOR", detector).replace("TYPE", caltype), " "*4))
             f.write("\n\n")
-
 # add test results
-test_artefact_dir = os.path.realpath("../../tests/artefacts")
-
-import shutil
-import textwrap
-from datetime import datetime
-from uuid import uuid4
-
-import tabulate
-from dateutil.parser import parse
-from lxml import etree
+test_artefact_dir = os.path.realpath("../../tests/legacy/artefacts")
 
 
 def xml_to_rst_report(xml, git_tag, reports=[]):
@@ -575,7 +570,6 @@ Contents:
 """
 if not os.path.exists("./test_rsts"):
     os.makedirs("./test_rsts")
-
 with open("test_results.rst", "w") as f:
     f.write(header)
     for commit, modtime in sorted_dir(test_artefact_dir):
@@ -610,5 +604,6 @@ with open("test_results.rst", "w") as f:
                     fr.write(rst)
             f.write("   test_rsts/{}\n".format(commit))
 
+
 def setup(app):
     app.add_stylesheet('css/test_decorators.css')

docs/source/configuration.rst

@@ -3,16 +3,14 @@
 Configuration
 =============
 
-The European XFEL Offline Calibration tools are configure using the `settings.py`
-and `notebooks.py` files. Both can be found in the root directory. The `settings.py`
-file configures the tools to the environment. The `notebook.py` file configures the
-notebooks which should be exposed to the command line.
+The European XFEL Offline Calibration is configured through `settings.py`
+and `notebooks.py` files. Both can be found in `xfel_calibrate` source directory. The `notebook.py` file exposes and configures the
+the notebooks by the `xfel-calibrate` command line interface.
 
 Settings
 --------
 
-The `settings.py` file configures the enviroment the tools are run in. It is a normal
-python file of the form::
+The `settings.py` is a python configuration file, which configures the tool's environment.::
 
     # path into which temporary files from each run are placed
     temp_path = "{}/temp/".format(os.getcwd())
@@ -35,9 +33,9 @@ A comment is given for the meaning of each configuration parameter.
 Notebooks
 ---------
 
-The `xfel-calibrate` tool will expose any notebooks that are configured here to the
+The `xfel-calibrate` tool exposes configured notebooks to the
 command line by automatically parsing the parameters given in the notebooks first cell.
-The configuration is to be given in form of a python directory::
+The configuration is given in the form of a python dictionary::
 
     notebooks = {
         "AGIPD": {
@@ -63,55 +61,23 @@ The configuration is to be given in form of a python directory::
          }
      }
 
-The first key is the detector that the calibration may be used for, here AGIPD. The second
-key level gives the name of the task being performed (here: DARK and PC). For each of these
-entries, a path to the notebook and a concurrency hint should be given. In the concurrency
-hint the first entry specifies which parameter of the notebook expects a list whose integer
-entries, can be concurrently run (here "modules"). The second parameter state with which range
-to fill this parameter if it is not given by the user. In the example a `range(16):=0,1,2,...15`
-would be passed onto the notebook, which is run as 16 concurrent jobs, each processing one module.
-Finally, a hint for the number of cluster cores to be started should be given. This value should
-be derived e.g. by profiling memory usage per core, run times, etc.
+The first key is the detector, e.g. AGIPD. The second key is the calibration type name, e.g. DARK or PC.
+A dictionary is expected for each calibration type with a notebook path and concurrency configuration.
+For the concurrency three values are expected. Key `parameter` with a value name of type list, which is defined in the first notebook cell.
+The key `default concurrency` to define the range of values for `parameter` in each concurrent notebook, if it is not defined by the user.
+e.g. `"default concurrency": 16` leads to running 16 concurrent jobs, each processing one module with values of [0,1,2,...,15].
+Finally, a hint for the number of `cluster cores` that is used if the notebook is using ipcluster parallelization, only.
+This value should be derived e.g. by profiling memory usage per core, run times, etc.
 
 .. note::
 
     It is good practice to name command line enabled notebooks with an `_NBC` suffix as shown in
     the above example.
 
-The `CORRECT` notebook (last notebook in the example) makes use of a concurrency generating function
-by setting the `use function` parameter. This function must be defined in a code cell in the notebook,
-its parameters should be named like other exposed parameters. It should return a list of of parameters
-to be inserted into the concurrently run notebooks. The example given e.g. defines the `balance_sequences`
-function::
-
-    def balance_sequences(in_folder, run, sequences, sequences_per_node):
-        import glob
-        import re
-        import numpy as np
-        if sequences_per_node != 0:
-            sequence_files = glob.glob("{}/r{:04d}/*-S*.h5".format(in_folder, run))
-            seq_nums = set()
-            for sf in sequence_files:
-                seqnum = re.findall(r".*-S([0-9]*).h5", sf)[0]
-                seq_nums.add(int(seqnum))
-            seq_nums -= set(sequences)
-            return [l.tolist() for l in np.array_split(list(seq_nums),
-                                                       len(seq_nums)//sequences_per_node+1)]
-        else:
-            return sequences
-
-
-.. note::
-
-    Note how imports are inlined in the definition. This is necessary, as only the function code,
-    not the entire notebook is executed.
-
-which requires as exposed parameters e.g. ::
-
-    in_folder = "/gpfs/exfel/exp/SPB/201701/p002038/raw/" # the folder to read data from, required
-    run = 239 # runs to process, required
-    sequences = [-1] # sequences to correct, set to -1 for all, range allowed
-    sequences_per_node = 2 # number of sequence files per cluster node if run as slurm job, set to 0 to not run SLURM parallel
+The AGIPD `CORRECT` notebook (last notebook in the example) makes use of a concurrency generating function
+by setting the `use function` parameter. This function must be defined in the first cell in the notebook
+its given arguments should be named as the first cell notebook parameters. It is expected to return a list of parameters
+to concurrently run notebooks. Above the used function is :func:`xfel_calibrate.calibrate.balance_sequences`.
 
 .. note::
 

docs/source/how_it_works.rst

-.. _how_it_works:
-
-How it Works
-============
-
-The European XFEL Offline Calibration utilizes the tools nbconvert_ and nbparameterise_
-to expose Jupyter_ notebooks to a command line interface. In the process reports are generated
-from these notebooks. The general interface is::
-
-    % xfel-calibrate DETECTOR TYPE
-
-where `DETECTOR` and `TYPE` specify the task to be performed.
-
-Additionally, it leverages the DESY/XFEL Maxwell cluster to run these jobs in parallel
-via SLURM_.
-
-Here is a list of :ref:`available_notebooks`. See the :ref:`advanced_topics` if you are
-for details on how to use as detector group staff.
-
-If you would like to integrate additional notebooks please see the :ref:`development_workflow`.
-
-.. _nbparameterise: https://github.com/takluyver/nbparameterise
-.. _nbconver: https://github.com/jupyter/nbconvert
-.. _jupyter: http://jupyter.org/
-.. _SLURM: https://slurm.schedmd.com

docs/source/index.rst

@@ -3,15 +3,45 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Welcome to European XFEL Offline Calibration's documentation!
-=============================================================
+European XFEL Offline Calibration
+=================================
 
-Contents:
+The European XFEL Offline Calibration (pyCalibration) is a python package that consists of
+different services, responsible for applying most of the offline calibration
+and characterization for the detectors.
+
+Running a calibration
+---------------------
+
+It utilizes tools such as nbconvert_ and nbparameterise_
+to expose Jupyter_ notebooks to a command line interface.
+In the process reports are generated from these notebooks.
+
+The general interface is::
+
+    % xfel-calibrate DETECTOR TYPE
+
+where `DETECTOR` and `TYPE` specify the task to be performed.
+
+Additionally, it leverages the DESY/XFEL Maxwell cluster to run these jobs in parallel
+via SLURM_.
+
+Here is a list of :ref:`available_notebooks`. See the :ref:`advanced_topics` if you are looking
+for details on how to use as detector group staff.
+
+If you would like to integrate additional notebooks please see the :ref:`development_workflow`.
+
+.. _nbparameterise: https://github.com/takluyver/nbparameterise
+.. _nbconvert: https://github.com/jupyter/nbconvert
+.. _jupyter: http://jupyter.org/
+.. _SLURM: https://slurm.schedmd.com
+
+
+Documentation contents:
 
 .. toctree::
    :maxdepth: 2
 
-   how_it_works
    installation
    configuration
    workflow
@@ -20,3 +50,24 @@ Contents:
    tutorial
    _notebooks/index
    testing
+
+.. toctree::
+   :caption: Reference
+   :maxdepth: 2
+
+   xfel_calibrate_conf
+   cal_tools_algorithms
+   calcat_interface
+
+.. toctree::
+   :caption: Development
+   :maxdepth: 2
+
+   changelog
+   architecture
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`search`
\ No newline at end of file

docs/source/installation.rst

-Installation
-============
-
-Installation of the European XFEL Offline Calibration tools is best
-done into an anaconda/3 environment using Maxwell_ cluster. However, there are no explicit
-dependencies, so any Python3 environment is capable of running the tool.
-
-A more detailed step-by-step instruction can be found in the :ref:`tutorial`.
-
-Package Requirements
---------------------
-
-Requirements can be categorized into three types:
-
-1. Those required to run the the general tool chain,
-2. Those required by the notebooks themselves,
-3. Those required to run notebooks concurrently on a HPC cluster.
-
-Categories 1+2 are usually Python modules, as given in the `requirements.txt`
-file found in the root directory. Additionally, PyDetLib_ is required.
-It can be installed either as a Karabo dependency or manually.
-
-Parallel execution is currently supported in a `SLURM_` environment.
-However, sequential execution is available using the `--no-cluster-job`
-parameter when executing `xfel-calibrate`.
-
-.. _Python: http://www.python.org/
-.. _Karabo: https://in.xfel.eu/readthedocs/docs/karabo/en/latest/
-.. _SLURM: https://slurm.schedmd.com
-.. _PyDetLib: https://in.xfel.eu/readthedocs/docs/pydetlib/en/latest/
-.. _Maxwell: https://confluence.desy.de/display/IS/Running+Jobs+on+Maxwell
-
-Installation using Anaconda
----------------------------
-
-First you need to load the anaconda/3 environment through::
+.. _installation:
 
-    1. module load anaconda/3
-
-If installing into other python enviroments, this step can be skipped.
-
-Then the package for the offline calibration can be obtained from the git repository::
-
-    2. git clone https://git.xfel.eu/gitlab/detectors/pycalibration.git
-
-Home directory
-++++++++++++++
-
-You can then install all requirements of this tool chain in your home directory by running::
-
-    3. pip install -r requirements.txt . --user
-
-in pycalibration's root directory.
-
-After installation, you should make sure that the home directory is in the PATH environment variable::
-
-    4. PATH=/home/<username>/.local/bin:$PATH
+************
+Installation
+************
 
-Preferred directory
-+++++++++++++++++++
+It's recommended to install the offline calibration (pycalibration) package on
+maxwell, using the anaconda/3 environment.
 
-Alternatively, you can install all requirements in a directory of your preference by::
+The following instructions clone from the EuXFEL GitLab instance using SSH
+remote URLs, this assumes that you have set up SSH keys for use with GitLab
+already. If you have not then read the appendix section on `SSH Key Setup for
+GitLab`_ for instructions on how to do this .
 
-    3. mkdir /gpfs/exfel/data/scratch/<username>/<directory-name>
-       pip install --target=/gpfs/exfel/data/scratch/<username>/<directory-name> -r requirements.txt .
 
-and it is important to make sure that the installed requirements are in the PATH environment::
+Installation using python virtual environment - recommended
+===========================================================
 
-    4. PATH=/gpfs/exfel/data/scratch/<username>/<directory-name>/bin:$PATH
+`pycalibration` uses the same version of Python as Karabo, which in June 2021
+updated to use Python 3.8. Currently the default python installation on Maxwell
+is still Python 3.6.8, so Python 3.8 needs to be loaded from a different
+location.
 
+One option is to use the Maxwell Spack installation, running `module load
+maxwell` will activate the test Spack instance from DESY, then you can use
+`module load python-3.8.6-gcc-10.2.0-622qtxd` to Python 3.8. Note that this Spack
+instance is currently a trial phase and may not be stable.
 
-After this make sure that you adjust the :ref:`configuration` and settings in the xfel-calibrate
-folder to match your environment.
+Another option is to use `pyenv`, we provide a pyenv installation at
+`/gpfs/exfel/sw/calsoft/.pyenv` which we use to manage different versions of
+python. This can be activated with ``source /gpfs/exfel/sw/calsoft/.pyenv/bin/activate``
 
-The tool-chain is then available via the::
+A quick setup would be:
 
-    xfel-calibrate
+1. ``source /gpfs/exfel/sw/calsoft/.pyenv/bin/activate``
+2. ``git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git && cd pycalibration`` - clone the offline calibration package from EuXFEL GitLab
+3. ``pyenv shell 3.8.11`` - load required version of python
+4. ``python3 -m venv .venv`` - create the virtual environment
+5. ``source .venv/bin/activate`` - activate the virtual environment
+6. ``python3 -m pip install --upgrade pip`` - upgrade version of pip
+7. ``python3 -m pip install .`` - install the pycalibration package (add ``-e`` flag for editable development installation)
 
-command.
+Copy/paste script:
 
+.. code:: bash
 
-Installation using karabo
-+++++++++++++++++++++++++
+  source /gpfs/exfel/sw/calsoft/.pyenv/bin/activate
+  git clone ssh://git@git.xfel.eu:10022/detectors/pycalibration.git
+  cd pycalibration
+  pyenv shell 3.8.11
+  python3 -m venv .venv
+  source .venv/bin/activate
+  python3 -m pip install --upgrade pip
+  python3 -m pip install .  # `-e` flag for editable install, e.g. `pip install -e .`
 
-If required, one can install into karabo environment. The difference would be to
-first source activate the karabo envrionment::
 
-    1. source karabo/activate
+Creating an ipython kernel for virtual environments
+===================================================
 
-then after cloning the offline calibration package from git, the requirements can be installed through::
+To create an ipython kernel with pycalibration available you should (if using a
+venv) activate the virtual environment first, and then run:
 
-    3. pip install -r requirements.txt .
+.. code:: bash
 
-Development Installation
-------------------------
+  python3 -m pip install ipykernel  # If not using a venv add `--user` flag
+  python3 -m ipykernel install --user --name pycalibration --display-name "pycalibration"  # If not using a venv pick different name
 
-For a development installation in your home directory, which automatically
-picks up (most) changes, first install the dependencies as above,
-but then install the tool-chain separately in development mode::
+This can be useful for Jupyter notebook tools as https://max-jhub.desy.de/hub/login
 
-   pip install -e . --user
 
-.. note:: Using "- -target" for development installation in a preferred directory can lead to errors.
+SSH Key Setup for GitLab
+========================
 
-.. note:: For development installation in karabo environment "- -user" is not needed.
+It is highly recommended to set up SSH keys for access to GitLab as this
+simplifies the setup process for all of our internal software present on GitLab.
 
-Installation of New Notebooks
------------------------------
+To set up the keys:
 
-To install new, previously untracked notebooks in the home directory,
-repeat the installation of the the tool-chain, without requirments,
-from the package base directory::
+1. Connect to Maxwell
+2. Generate a new keypair with ``ssh-keygen -o -a 100 -t ed25519``, you can
+   either leave this in the default location (``~/.ssh/id_ed25519``) or place it
+   into a separate directory to make management of keys easier if you already
+   have multiple ones. If you are using a password for your keys please check
+   this page to learn how to manage them: https://docs.github.com/en/github/authenticating-to-github/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent#adding-your-ssh-key-to-the-ssh-agent
+3. Add the public key (``id_ed25519.pub``) to your account on GitLab: https://git.xfel.eu/gitlab/profile/keys
+4. Add the following to your ``~/.ssh/config`` file
 
-    pip install --upgrade . --user
+.. code::
 
-Or, in case you are actively developing::
+  # Special flags for gitlab over SSH
+  Host git.xfel.eu
+      User git
+      Port 10022
+      ForwardX11 no
+      IdentityFile ~/.ssh/id_ed25519
 
-    pip install -e . --user
+Once this is done you can clone repositories you have access to from GitLab
+without having to enter your password each time. As ``pycalibration``
+requirements are installed from SSH remote URLs having SSH keys set up is a
+requirement for installing pycalibration.

docs/source/tutorial.rst

@@ -25,41 +25,7 @@ This will open a jupyter kernel running in your browser where you can then open
   ipcluster start --n=4 --profile=tutorial
 
 you can step through the cells and run them.
-If you run this notebook using the xfel-calibrate command as explaind at the end of this tutorial you do not need to start the cluster yourself, it will be done by the framework.
-
-
-Installation and Configuration
-------------------------------
-
-The offline calibration tool-chain is optimised to run on the maxwell cluster.
-For more information refer to the Maxwell_ documentation.
-
-.. _Maxwell: https://confluence.desy.de/display/IS/Running+Jobs+on+Maxwell
-
-In order to use the offline calibration tool a few steps need to be carried out
-to install the necessary packages and setup the environment:
-
-1. Log into max-exfl with you own user name/account.
-
-2. Install karabo in your home directory or under /gpfs/exfel/data/scratch/username
-   by typing the following commands on you shell::
-
-     wget http://exflserv05.desy.de/karabo/karaboFramework/tags/2.2.4/karabo-2.2.4-Release-CentOS-7-x86_64.sh
-
-     chmod +x karabo-2.2.4-Release-CentOS-7-x86_64.sh
-
-     ./karabo-2.2.4-Release-CentOS-7-x86_64.sh
-
-     source karabo/activate
-
-3. Get the package pycalibration which contains the offline calibration tool-chain::
-
-     git clone https://git.xfel.eu/gitlab/detectors/pycalibration.git
-
-4. Install the necessary requirements and the package itself::
-
-     cd pycalibration
-     pip install -r requirements.txt .
+If you run this notebook using the xfel-calibrate command as explained at the end of this tutorial you do not need to start the cluster yourself, it will be done by the framework.
 
 
 Create your own notebook
@@ -87,7 +53,7 @@ Running the notebook
 
    You can see your job in the queue with::
 
-     squeue -u username
+     squeue --me
 
 3. Look at the generated report in the chosen output folder.
 4. More information on the job run on the cluster can be found in the temp folder.

docs/source/workflow.rst

@@ -4,7 +4,7 @@ Development Workflow
 ====================
 
 The following walkthrough will guide you through a possible workflow
-when developing new offline calibration tools.
+when developing new notebooks for offline calibration.
 
 Fresh Start
 -----------
@@ -12,7 +12,7 @@ Fresh Start
 If you are starting a blank notebook from scratch you should first
 think about a few preconsiderations:
 
-* Will the notebook performan a headless task, or will it also be
+* Will the notebook perform a headless task, or will it also be
   an important interface for evaluating the results in form of a
   report.
 * Do you need to run concurrently? Is concurrency handled internally,
@@ -25,7 +25,7 @@ cells in the notebook. You should also structure it into appropriate
 subsections.
 
 If you plan on running concurrently on the cluster, identify which variable
-should be mapped to concurent runs. For autofilling it an integer list is
+should be mapped to concurrent runs. For autofilling it an integer list is
 needed.
 
 Once you've clarified the above points, you should create a new notebook,
@@ -139,7 +139,7 @@ to the following parameters being exposed via the command line::
 
 .. note::
 
-    Nbparameterise can only parse the mentioned subset of variable types. An expression
+    nbparameterise_ can only parse the mentioned subset of variable types. An expression
     that evaluates to such a type will note be recognized: e.g. `a = list(range(3))` will
     not work!
 
@@ -170,59 +170,41 @@ Best Coding Practices
 In principle there a not restrictions other than that parameters that are exposed to the
 command line need to be defined in the first code cell of the notebook.
 
-However, a few guidelines should be observered to make notebook useful for display as
-reports and usage by other.
+However, a few guidelines should be observed to make notebook useful for display as
+reports and usage by others.
 
 External Libraries
 ~~~~~~~~~~~~~~~~~~
 
-You may use a wide variaty of libraries available in Python, but keep in mind that others
+You may use a wide variety of libraries available in Python, but keep in mind that others
 wanting to run the tool will need to install these requirements as well. Thus,
 
-* do not use a specialized tool if an accepted alternative exists. Plots e.g. should usually
+* Do not use a specialized tool if an accepted alternative exists. Plots e.g. should usually
   be created using matplotlib_ and numerical processing should be done in numpy_.
 
-* keep runtimes and library requirements in mind. A library doing its own parallelism either
-  needs to programatically be able to set this up, or automatically do so. If you need to
+* Keep runtime and library requirements in mind. A library doing its own parallelism either
+  needs to programmatically be able to set this up, or automatically do so. If you need to
   start something from the command line first, things might be tricky as you will likely
   need to run this via `POpen` commands with appropriate environment variable.
 
+* Reading out RAW data should be done using extra_data_. It helps in accessing the HDF5 data
+  structures efficiently. It reduces the complexity of accessing the RAW or CORRECTED datasets,
+  and it provides different methods to select and filter the trains, cells, or pixels of interest.
+
 Writing out data
 ~~~~~~~~~~~~~~~~
 
 If your notebook produces output data, consider writing data out as early as possible,
 such that it is available as soon as possible. Detailed plotting and inspection can
-possibly done later on in a notebook.
+be done later on in the notebook.
 
-Also consider using HDF5 via h5py_ as your output format. If you correct or calibrated
-input data, which adhears to the XFEL naming convention, you should maintain the convention
+Also use HDF5 via h5py_ as your output format. If you correct or calibrate
+input data, which adheres to the XFEL naming convention, you should maintain the convention
 in your output data. You should not touch any data that you do not actively work on and
-should assure that the `INDEX` and identifier entries are syncronized with respect to
+should assure that the `INDEX` and identifier entries are synchronized with respect to
 your output data. E.g. if you remove pulses from a train, the `INDEX/.../count` section
 should reflect this.
 
-Finally, XFEL RAW data can contain filler data from the DAQ. One possible way of identifying
-this data is the following::
-
-    datapath = "/INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/cellId".format(channel)
-
-    count = np.squeeze(infile[datapath])
-    first = np.squeeze(infile[datapath])
-    if np.count_nonzero(count != 0) == 0:  # filler data has counts of 0
-        print("File {} has no valid counts".format(infile))
-        return
-    valid = count != 0
-    idxtrains = np.squeeze(infile["/INDEX/trainId"])
-    medianTrain = np.nanmedian(idxtrains)  # protect against freak train ids
-    valid &= (idxtrains > medianTrain - 1e4) & (idxtrains < medianTrain + 1e4)
-
-    # index ranges in which non-filler data exists
-    last_index = int(first[valid][-1]+count[valid][-1])
-    first_index = int(first[valid][0])
-
-    # access these indices
-    cellIds = np.squeeze(np.array(infile[datapath][first_index:last_index, ...]))
-
 
 Plotting
 ~~~~~~~~
@@ -233,10 +215,10 @@ a context. Make sure to label your axes.
 
 Also make sure the plots are readable on an A4-sized PDF page; this is the format the notebook
 will be rendered to for report outputs. Specifically, this means that figure sizes should not
-exeed approx 15x15 inches.
+exceed approx 15x15 inches.
 
 The report will contain 150 dpi png images of your plots. If you need higher quality output
-of individual plot files you should save these separetly, e.g. via `fig.savefig(...)` yourself.
+of individual plot files you should save these separately, e.g. via `fig.savefig(...)` yourself.
 
 
 Calibration Database Interaction
@@ -245,7 +227,7 @@ Calibration Database Interaction
 Tasks which require calibration constants or produce such should do this by interacting with
 the European XFEL calibration database.
 
-In terms of developement workflow it is usually easier to work with file-based I/O first and
+In terms of development workflow it is usually easier to work with file-based I/O first and
 only switch over to the database after the algorithmic part of the notebook has matured.
 Reasons for this include:
 
@@ -261,7 +243,7 @@ documentation.
 Testing
 -------
 
-The most important test is that your notebook completes flawlessy outside any special
+The most important test is that your notebook completes flawlessly outside any special
 tool chain feature. After all, the tool chain will only replace parameters, and then
 launch a concurrent job and generate a report out of notebook. If it fails to run in the
 normal Jupyter notebook environment, it will certainly fail in the tool chain environment.
@@ -274,11 +256,11 @@ Specifically, you should verify that all arguments are parsed correctly, e.g. by
 
     xfel-calibrate DETECTOR NOTEBOOK_TYPE --help
 
-From then on, check include if parallel slurm jobs are exectuted correctly and if a report
+From then on, check include if parallel slurm jobs are executed correctly and if a report
 is generated at the end.
 
 Finally, you should verify that the report contains the information you'd like to convey and
-is inteligable to people other than you.
+is intelligible to people other than you.
 
 .. note::
 
@@ -298,4 +280,5 @@ documentation.
 .. _matplotlib: https://matplotlib.org/
 .. _numpy: http://www.numpy.org/
 .. _h5py: https://www.h5py.org/
-.. _iCalibrationDB: https://in.xfel.eu/readthedocs/docs/icalibrationdb/en/latest/
+.. _iCalibrationDB: https://git.xfel.eu/detectors/cal_db_interactive
+.. _extra_data: https://extra-data.readthedocs.io/en/latest/
\ No newline at end of file

docs/source/xfel_calibrate_conf.rst

+xfel_calibrate
+==============
+
+.. module:: xfel_calibrate.calibrate
+
+.. autofunction:: balance_sequences

notebooks/AGIPD/AGIPD_Correct_and_Verify_Summary_NBC.ipynb

 %% Cell type:markdown id: tags:
 
 # Summary of the AGIPD offline correction #
 
 %% Cell type:code id: tags:
 
 ``` python
 run = 11 # runs to process, required
 out_folder = "/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_Corr" # path to output to, required
+metadata_folder = ""  # Directory containing calibration_metadata.yml when run by xfel-calibrate
 karabo_id = "SPB_DET_AGIPD1M-1" # karabo karabo_id
 modules = [-1]
 karabo_da = ['-1']  # a list of data aggregators names, Default [-1] for selecting all data aggregators
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 import re
 import warnings
 from pathlib import Path
 
 import dateutil.parser
 import numpy as np
 import yaml
 
 warnings.filterwarnings('ignore')
 
 import matplotlib.pyplot as plt
 
 %matplotlib inline
 import tabulate
 from cal_tools.tools import CalibrationMetadata
 from IPython.display import Latex, Markdown, display
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 out_folder = Path(out_folder)
-metadata = CalibrationMetadata(out_folder)
+metadata = CalibrationMetadata(metadata_folder or out_folder)
 const_dict = metadata.setdefault("retrieved-constants", {})
 time_dict = const_dict.setdefault("time-summary", {})
 
 # Extracting Instrument string
 instrument = karabo_id.split("_")[0]
 # Evaluate detector instance for mapping
 if instrument == "SPB":
     dinstance = "AGIPD1M1"
     nmods = 16
 elif instrument == "MID":
     dinstance = "AGIPD1M2"
     nmods = 16
 elif instrument == "HED":
     dinstance = "AGIPD500K"
     nmods = 8
 
 if karabo_da[0] == '-1':
     if modules[0] == -1:
         modules = list(range(nmods))
     karabo_da = ["AGIPD{:02d}".format(i) for i in modules]
 else:
     modules = [int(x[-2:]) for x in karabo_da]
 
 # This is needed only if AGIPD Correction notebook had no precorrection notebooks for retrieving constants
 # gather all generated sequence yml files for time summary of retrieved constant under retrieved-constants in metadata.yml
 for fn in sorted(out_folder.glob("retrieved_constants_*.yml")):
     with fn.open("r") as fd:
         fdict = yaml.safe_load(fd)
     # append different sequences' time summary to the main yaml
     time_dict.update(fdict["time-summary"])
     fn.unlink()
 
 metadata.save()
 ```
 
 %% Cell type:code id: tags:
 
 ``` python
 def print_const_table(const):
     print(f"{const} constants were injected on:")
     table_data = {}
     for seq, mod_data in time_dict.items():
         for mod, const_data in mod_data.items():
             timestamp = const_data[const]
             table_data.setdefault(timestamp, []).append(f"{seq}:{mod}")
     table = []
-    if len(table_data) == 1:
+    if not len(table_data):
+        table.append(["No constants retrieved"])
+    elif len(table_data) == 1:
         table.append([[*table_data][0], "All modules"])
     else:
         for timestamp, seqmod in table_data.items():
             table.append([timestamp, seqmod[0]])
             for further_module in seqmod[1:]:
                 table.append(["", further_module])
     display(Latex(tabulate.tabulate(table,
                  tablefmt="latex",
                  headers=["Timestamps", "Modules and sequences"])))
 
 for const in ['Offset', 'SlopesPC', 'SlopesFF']:
     print_const_table(const)
 ```
-
-%% Cell type:code id: tags:
-
-``` python
-```