Autogenerate available notebook docs

.gitignore

@@ -24,3 +24,6 @@ Test
 slurm_tmp*
 ./temp
 *egg*
+docs/source/available_notebooks.rst
+docs/build
+

docs/source/available_notebooks.rst

-.. _available_notebooks:
-
-Available Notebooks
-===================
-
-The following notebooks are currently integrated into the European XFEL
-Offline Calibration tool chain.
-
-
-LPD
-----
-
-Dark Images Evaluation
-~~~~~~~~~~~~~~~~~~~~~~
-
-This notebook produces offset, noise and derived bad pixel data from LPD 
-dark images for one capacitor setting.
-
-.. code::
-
-    % python calibrate.py LPD dark --help
-    
-
-
-Pulse Capacitor Evaluation
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This notebook produces per-pixel relative gain, normalizing all memory cells
-for a given pixel w.r.t. the three gain stages for a given capacitor setting.
-It requires offset maps as input.
-
-.. code::
-
-    % python calibrate.py LPD PC --help  
-    
-
-Correction
-~~~~~~~~~~
-
-Corrects an LPD run with calibration constants from the database and/or files.
-
-.. code::
-
-    % python calibrate.py LPD correct --help
-    
-
-AGIPD
------
-
-Dark Images Evaluation
-~~~~~~~~~~~~~~~~~~~~~~
-
-This notebook produces offset, noise and derived bad pixel data from AGIPD 
-dark images.
-
-.. code::
-
-    % python calibrate.py AGIPD dark --help
-    
-
-
-Pulse Capacitor Evaluation
-~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-This notebook produces per-pixel relative gain, normalizing all memory cells
-for a given pixel w.r.t. the three gain stages.
-
-.. code::
-
-    % python calibrate.py AGIPD PC --help  
-    
-
-Flat Field Evaluation
-~~~~~~~~~~~~~~~~~~~~~
-
-This notebook produces flat field deduced correction constants, which are used
-to normalize the pulse capacitor data.
-
-.. code::
-
-    % python calibrate.py AGIPD FF --help  
-    
-
-Combine
-~~~~~~~
-
-This notebook combines inputs from the other characterization notebooks into 
-a consolidated, normalized set of correction constants.
-
-.. code::
-
-    % python calibrate.py AGIPD combine --help  
-    
-
-Correction
-~~~~~~~~~~
-
-Corrects an AGIPD run with calibration constants from the database and/or files.
-
-.. code::
-
-    % python calibrate.py AGIPD correct --help
-    

docs/source/conf.py

@@ -346,3 +346,64 @@ texinfo_documents = [
 
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {'https://docs.python.org/': None}
+
+# generate the list of available notebooks
+from xfel_calibrate import notebooks
+from subprocess import check_output
+from textwrap import dedent, indent
+from nbconvert import RSTExporter
+import os
+import nbformat
+rst_exporter = RSTExporter()
+with open("available_notebooks.rst", "w") as f:
+    f.write(dedent("""
+            .. _available_notebooks:
+
+            Available Notebooks
+            ===================
+
+            The following notebooks are currently integrated into the European XFEL
+            Offline Calibration tool chain.
+            
+            
+            """))
+
+    for detector in sorted(notebooks.notebooks.keys()):
+        values = notebooks.notebooks[detector]
+        f.write("{}\n".format(detector))
+        f.write("{}\n".format("-"*len(detector)))
+        f.write("\n")
+        
+        for caltype in sorted(values.keys()):
+            data = values[caltype]
+            
+            nbpath = os.path.abspath("{}/../../../{}".format(__file__, data["notebook"]))
+            with open(nbpath, "r") as nf:
+                nb = nbformat.read(nf, as_version=4)
+                def first_markdown_cell(nb):
+                    for cell in nb.cells:
+                        if cell.cell_type == 'markdown':
+                            return cell
+                mdcell = first_markdown_cell(nb)
+                nb.cells = [mdcell]  # we only want this single cell
+                body, _ = rst_exporter.from_notebook_node(nb)
+                adjusted = []
+                # adjust titles 
+                for line in body.split("\n"):
+                    if line.startswith("=="):
+                        line = line.replace("=", "+")
+                    if line.startswith("--"):
+                        line = line.replace("-", "~")
+                    adjusted.append(line)
+                f.write("\n".join(adjusted))                
+                f.write("\n")
+            
+            f.write("To invoke this notebook and display help use:\n\n")
+            f.write(".. code-block:: bash\n\n")
+            f.write("    xfel-calibrate {} {} --help\n\n".format(detector, caltype))
+            f.write("The full parameter list of this notebook (with defaults is): \n\n")
+            f.write(".. code-block:: bash\n\n")
+            nb_help = ["xfel-calibrate", detector, caltype, "--help"]
+            output = check_output(nb_help).decode('utf8')
+            f.write(indent(output.replace("DETECTOR", detector).replace("TYPE", caltype), " "*4))
+            f.write("\n\n")

xfel_calibrate/calibrate.py

@@ -161,7 +161,8 @@ if len(sys.argv) == 3 and "-h" in sys.argv[2]:
     # The information is extracted from the first markdown cell of
     # the notebook.
     for caltype, notebook in det_notebooks.items():
-        with open(os.path.abspath(notebook["notebook"]), "r") as f:
+        nbpath = os.path.abspath("{}/../../{}".format(__file__, notebook["notebook"]))
+        with open(nbpath, "r") as f:
             nb = nbformat.read(f, as_version=4)
             msg += make_epilog(nb, caltype=caltype)
     
@@ -176,7 +177,7 @@ elif len(sys.argv) >= 3:
     detector = sys.argv[1].upper()
     caltype = sys.argv[2].upper()
     try:
-        notebook = os.path.abspath(notebooks[detector][caltype]["notebook"])
+        notebook = os.path.abspath("{}/../../{}".format(__file__, notebooks[detector][caltype]["notebook"]))
         cvar = notebooks[detector][caltype].get("concurrency",
                                                 {"parameter": None,
                                                  "default concurrency": None,