Merge branch 'fewer-paths' into 'master'

Simplify how batch jobs find Python tools

Closes #26

See merge request detectors/pycalibration!350

README.rst

@@ -50,14 +50,6 @@ Clone from git::
     git clone https://git.xfel.eu/gitlab/detectors/pycalibration.git
     cd pycalibration
 
-Edit path to the python environment in the bin/activate.sh file. Change::
-
-    export PATH=/home/${USER}/.local/bin:$PATH
-
-to::
-
-    export PATH=/path/to/new/virtual/environment/bin:$PATH
-
 Install the package::
 
     pip install -r requirements.txt

bin/activate.sh

-source /etc/profile.d/modules.sh
-module load anaconda/3
-module load texlive/2019
-# export path to python environment
-export PATH=$HOME/.local/bin:$PATH

bin/slurm_calibrate.sh

@@ -3,25 +3,17 @@
 # set paths to use
 nb_path=$1
 python_path=$2
-ipython_path=$3
-jupyter_path=$4
-ipcluster_path="${5}"
-activate_path=$6
-uuid=$7
-notebook=$8
-detector=$9
-caltype=${10}
-final=${11}
-finalize=${12}
-cluster_cores=${13}
+uuid=$3
+notebook=$4
+detector=$5
+caltype=$6
+final=$7
+finalize=$8
+cluster_cores=$9
 
 echo "Running with the following parameters:"
 echo "Notebook path: $nb_path"
 echo "Python path: $python_path"
-echo "IPython path: $ipython_path"
-echo "Jupyter path: $jupyter_path"
-echo "IP-Cluster path: $ipcluster_path"
-echo "Environment activate: $activate_path"
 echo "IP-Cluster profile: $uuid"
 echo "notebook: $notebook"
 echo "detector: $detector"
@@ -34,7 +26,9 @@ echo "job ID: $SLURM_JOB_ID"
 export CAL_NOTEBOOK_NAME=$notebook
 
 # set-up enviroment
-source ${activate_path}
+source /etc/profile.d/modules.sh
+module load anaconda/3
+module load texlive/2019
 
 echo "Starting influx feeder"
 ./cal_influx_feeder.sh $notebook $detector $caltype > /dev/null 2>&1  &
@@ -45,26 +39,26 @@ export MPLBACKEND=AGG
 # start an ip cluster if requested
 if [ "${uuid}" != "NO_CLUSTER" ]
 then
-    ${ipython_path} profile create ${uuid} --parallel
-    ${ipcluster_path} start --n=${cluster_cores} --profile=${uuid} --daemon &
+    ${python_path} -m IPython profile create ${uuid} --parallel
+    ${python_path} -m ipyparallel.cluster start --n=${cluster_cores} --profile=${uuid} --daemon &
     sleep 15
 fi
 
 
 echo "Running script"
-${jupyter_path} nbconvert --to rst --ExecutePreprocessor.timeout=36000 --ExecutePreprocessor.allow_errors=True --TemplateExporter.exclude_input=True --execute ${nb_path}
+${python_path} -m nbconvert --to rst --ExecutePreprocessor.timeout=36000 --ExecutePreprocessor.allow_errors=True --TemplateExporter.exclude_input=True --execute ${nb_path}
 
 # stop the cluster if requested
 if [ "${uuid}" != "NO_CLUSTER" ]
 then
-    ${ipcluster_path} stop --profile=${uuid}
-    profile_path=`${ipython_path} locate profile ${uuid}`
+    ${python_path} -m ipyparallel.cluster stop --profile=${uuid}
+    profile_path=$(${python_path} -m IPython locate profile ${uuid})
     echo "Removing cluster profile from: $profile_path"
     rm -rf $profile_path
 fi
 
 if [ "${final}" == "FINAL" ]
 then
-   ${finalize} $SLURM_JOB_ID
+   ${python_path} ${finalize} $SLURM_JOB_ID
 fi
 killall -9 cal_influx_feeder.sh || true

docs/source/configuration.rst

@@ -20,18 +20,6 @@ python file of the form::
     # Path to use for calling Python. If the environment is correctly set, simply the command
     python_path = "python"
 
-    # Path to use for calling iPython. If the environment is correctly set, simply the command
-    ipython_path = "ipython"
-
-    # Path to use for calling Jupyter. If the environment is correctly set, simply the command
-    jupyter_path = "jupyter"
-
-    # Path to find the Karabo activate script at, set to an empty string if not running in a Karabo environemnt
-    karabo_activate_path = "/opt/karabo/activate"
-
-    # Path to use for calling ipcluster. If the environment is correctly set, simply the command
-    ipcluster_path = "/opt/karabo/extern/bin/ipcluster"
-
     # Path to store reports in
     report_path = "{}/calibration_reports/".format(os.getcwd())
 

docs/source/tutorial.rst

@@ -61,9 +61,6 @@ to install the necessary packages and setup the environment:
      cd pycalibration
      pip install -r requirements.txt .
 
-5. Adjust xfel_calibrate/settings.py by changing "karabo_activate_path"
-   and "ipcluster_path according" to where you installed karabo.
-
 
 Create your own notebook
 ------------------------

xfel_calibrate/calibrate.py

@@ -500,17 +500,15 @@ def create_finalize_script(fmt_args, temp_path, job_list):
     """
     Create a finalize script to produce output report
     :param fmt_args: Dictionary of fmt arguments
-    :param temp_path: Path to temopary folder to run slurm job
+    :param temp_path: Path to temporary folder to run slurm job
     :param job_list: List of slurm jobs
     """
     tmpl = Template('''
-                    #!/bin/tcsh
-                    source /etc/profile.d/modules.sh
-                    module load texlive/2019
-                    echo 'Running finalize script'
-                    python3 -c "from xfel_calibrate.finalize import finalize; 
+                    import sys
+                    from xfel_calibrate.finalize import finalize
+
                     finalize(joblist={{joblist}}, 
-                             finaljob=$1,
+                             finaljob=sys.argv[1],
                              run_path='{{run_path}}',
                              out_path='{{out_path}}',
                              project='{{project}}', 
@@ -520,12 +518,12 @@ def create_finalize_script(fmt_args, temp_path, job_list):
                              report_to='{{report_to}}',
                              data_path='{{in_folder}}',
                              request_time='{{request_time}}',
-                             submission_time='{{submission_time}}')"
+                             submission_time='{{submission_time}}')
                     
                     ''')
 
     fmt_args['joblist'] = job_list
-    f_name = os.path.join(temp_path, "finalize.sh")
+    f_name = os.path.join(temp_path, "finalize.py")
     with open(f_name, "w") as finfile:
         finfile.write(textwrap.dedent(tmpl.render(**fmt_args)))
 
@@ -653,17 +651,12 @@ def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
     srun_base += [os.path.join(PKG_DIR, "bin", "slurm_calibrate.sh"),  # path to helper sh
                   os.path.abspath(nbpath),  # path to notebook
                   python_path,  # path to python
-                  ipython_path,  # path to ipython
-                  jupyter_path,  # path to jupyter
-                  ipcluster_path,  # path to ipcluster
-                  # python activate path
-                  activate_path if activate_path!="" else os.path.join(PKG_DIR, "bin", "activate.sh"),
                   args.get("cluster_profile", "NO_CLUSTER"),
                   '"{}"'.format(base_name.upper()),
                   '"{}"'.format(args["detector"].upper()),
                   '"{}"'.format(args["type"].upper()),
                   "FINAL" if final_job else "NONFINAL",
-                  ". {}/finalize.sh".format(os.path.abspath(temp_path)),
+                  "{}/finalize.py".format(os.path.abspath(temp_path)),
                   str(cluster_cores)]
 
     output = check_output(srun_base).decode('utf8')

xfel_calibrate/finalize.py

@@ -209,7 +209,7 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
         out_path = report_path
 
     try:
-        check_call(["sphinx-quickstart",
+        check_call([sys.executable, "-m", "sphinx.quickstart",
                     "--quiet",
                     "--project='{}'".format(project),
                     "--author='{}'".format(author),
@@ -222,7 +222,7 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
                     "--no-batchfile", run_path])
 
     except CalledProcessError:
-        raise Exception("Failed to run sphinx-quickbuild. Is sphinx installed?"
+        raise Exception("Failed to run sphinx-quickstart. Is sphinx installed?"
                         "Generated simple index.rst instead")
 
     # quickbuild went well we need to edit the index.rst and conf.py files
@@ -299,7 +299,7 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
     # finally call the make scripts
     chdir(run_path)
     try:
-        check_call(["make", "latexpdf"])
+        check_call(["make", f"SPHINXBUILD={sys.executable} -m sphinx", "latexpdf"])
 
     except CalledProcessError:
         print("Failed to make pdf documentation")

xfel_calibrate/settings.py

 import os
+import sys
 
 # path into which temporary files from each run are placed
 temp_path = os.path.abspath("temp/")
 
-# Path to use for calling Python. If the environment is correctly set, simply the command
-python_path = "python"
-
-# Path to use for calling iPython. If the environment is correctly set, simply the command
-ipython_path = "ipython"
-
-# Path to use for calling Jupyter. If the environment is correctly set, simply the command
-jupyter_path = "jupyter"
-
-# Path to find activate script
-activate_path = ""
-
-# Path to use for calling ipcluster. If the environment is correctly set, simply the command
-ipcluster_path = "ipcluster"
+# Path to use for calling Python. By default, the Python xfel-calibrate is run in.
+python_path = sys.executable
 
 # Path to store reports in
 report_path = "{}/calibration_reports/".format(os.getcwd())