diff --git a/README.rst b/README.rst
index b62b5285b4c552585ee96c4cb35a3dd09d1f4336..6328ee6186d55220d620f7704c1967fbf829ef1a 100644
--- a/README.rst
+++ b/README.rst
@@ -50,14 +50,6 @@ Clone from git::
git clone https://git.xfel.eu/gitlab/detectors/pycalibration.git
cd pycalibration
-Edit path to the python environment in the bin/activate.sh file. Change::
-
- export PATH=/home/${USER}/.local/bin:$PATH
-
-to::
-
- export PATH=/path/to/new/virtual/environment/bin:$PATH
-
Install the package::
pip install -r requirements.txt
diff --git a/bin/activate.sh b/bin/activate.sh
deleted file mode 100644
index 8c8a0500e9d400a6094aaf05192ea40fbc5bde96..0000000000000000000000000000000000000000
--- a/bin/activate.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-source /etc/profile.d/modules.sh
-module load anaconda/3
-module load texlive/2019
-# export path to python environment
-export PATH=$HOME/.local/bin:$PATH
diff --git a/bin/slurm_calibrate.sh b/bin/slurm_calibrate.sh
index 44be0e93e511c491fe6171badb0e0f49c708d1f2..9f54463439b4f32676afe5c91073cd048ec3005e 100755
--- a/bin/slurm_calibrate.sh
+++ b/bin/slurm_calibrate.sh
@@ -3,25 +3,17 @@
# set paths to use
nb_path=$1
python_path=$2
-ipython_path=$3
-jupyter_path=$4
-ipcluster_path="${5}"
-activate_path=$6
-uuid=$7
-notebook=$8
-detector=$9
-caltype=${10}
-final=${11}
-finalize=${12}
-cluster_cores=${13}
+uuid=$3
+notebook=$4
+detector=$5
+caltype=$6
+final=$7
+finalize=$8
+cluster_cores=$9
echo "Running with the following parameters:"
echo "Notebook path: $nb_path"
echo "Python path: $python_path"
-echo "IPython path: $ipython_path"
-echo "Jupyter path: $jupyter_path"
-echo "IP-Cluster path: $ipcluster_path"
-echo "Environment activate: $activate_path"
echo "IP-Cluster profile: $uuid"
echo "notebook: $notebook"
echo "detector: $detector"
@@ -34,7 +26,9 @@ echo "job ID: $SLURM_JOB_ID"
export CAL_NOTEBOOK_NAME=$notebook
# set-up enviroment
-source ${activate_path}
+source /etc/profile.d/modules.sh
+module load anaconda/3
+module load texlive/2019
echo "Starting influx feeder"
./cal_influx_feeder.sh $notebook $detector $caltype > /dev/null 2>&1 &
@@ -45,26 +39,26 @@ export MPLBACKEND=AGG
# start an ip cluster if requested
if [ "${uuid}" != "NO_CLUSTER" ]
then
- ${ipython_path} profile create ${uuid} --parallel
- ${ipcluster_path} start --n=${cluster_cores} --profile=${uuid} --daemon &
+ ${python_path} -m IPython profile create ${uuid} --parallel
+ ${python_path} -m ipyparallel.cluster start --n=${cluster_cores} --profile=${uuid} --daemon &
sleep 15
fi
echo "Running script"
-${jupyter_path} nbconvert --to rst --ExecutePreprocessor.timeout=36000 --ExecutePreprocessor.allow_errors=True --TemplateExporter.exclude_input=True --execute ${nb_path}
+${python_path} -m nbconvert --to rst --ExecutePreprocessor.timeout=36000 --ExecutePreprocessor.allow_errors=True --TemplateExporter.exclude_input=True --execute ${nb_path}
# stop the cluster if requested
if [ "${uuid}" != "NO_CLUSTER" ]
then
- ${ipcluster_path} stop --profile=${uuid}
- profile_path=`${ipython_path} locate profile ${uuid}`
+ ${python_path} -m ipyparallel.cluster stop --profile=${uuid}
+ profile_path=$(${python_path} -m IPython locate profile ${uuid})
echo "Removing cluster profile from: $profile_path"
rm -rf $profile_path
fi
if [ "${final}" == "FINAL" ]
then
- ${finalize} $SLURM_JOB_ID
+ ${python_path} ${finalize} $SLURM_JOB_ID
fi
killall -9 cal_influx_feeder.sh || true
diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst
index a618c2414ca8dbc643496c212848b1bdc044d71f..8db9435689f61adef714f01daa23725a1204be7e 100644
--- a/docs/source/configuration.rst
+++ b/docs/source/configuration.rst
@@ -20,18 +20,6 @@ python file of the form::
# Path to use for calling Python. If the environment is correctly set, simply the command
python_path = "python"
- # Path to use for calling iPython. If the environment is correctly set, simply the command
- ipython_path = "ipython"
-
- # Path to use for calling Jupyter. If the environment is correctly set, simply the command
- jupyter_path = "jupyter"
-
- # Path to find the Karabo activate script at, set to an empty string if not running in a Karabo environemnt
- karabo_activate_path = "/opt/karabo/activate"
-
- # Path to use for calling ipcluster. If the environment is correctly set, simply the command
- ipcluster_path = "/opt/karabo/extern/bin/ipcluster"
-
# Path to store reports in
report_path = "{}/calibration_reports/".format(os.getcwd())
diff --git a/docs/source/tutorial.rst b/docs/source/tutorial.rst
index 037bcf2f86e57131844156a6b4d6fe32a14d0518..89607fdb77bb3637f7c42f0aee746d8470a6a5ed 100644
--- a/docs/source/tutorial.rst
+++ b/docs/source/tutorial.rst
@@ -61,9 +61,6 @@ to install the necessary packages and setup the environment:
cd pycalibration
pip install -r requirements.txt .
-5. Adjust xfel_calibrate/settings.py by changing "karabo_activate_path"
- and "ipcluster_path according" to where you installed karabo.
-
Create your own notebook
------------------------
diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index 8bdbdbe15f2e31e43f464ae62d845cc4d71494ea..1f5453f0b5aa7b53d07ca52788ed624d3fb8bcbf 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -500,17 +500,15 @@ def create_finalize_script(fmt_args, temp_path, job_list):
"""
Create a finalize script to produce output report
:param fmt_args: Dictionary of fmt arguments
- :param temp_path: Path to temopary folder to run slurm job
+ :param temp_path: Path to temporary folder to run slurm job
:param job_list: List of slurm jobs
"""
tmpl = Template('''
- #!/bin/tcsh
- source /etc/profile.d/modules.sh
- module load texlive/2019
- echo 'Running finalize script'
- python3 -c "from xfel_calibrate.finalize import finalize;
+ import sys
+ from xfel_calibrate.finalize import finalize
+
finalize(joblist={{joblist}},
- finaljob=$1,
+ finaljob=sys.argv[1],
run_path='{{run_path}}',
out_path='{{out_path}}',
project='{{project}}',
@@ -520,12 +518,12 @@ def create_finalize_script(fmt_args, temp_path, job_list):
report_to='{{report_to}}',
data_path='{{in_folder}}',
request_time='{{request_time}}',
- submission_time='{{submission_time}}')"
+ submission_time='{{submission_time}}')
''')
fmt_args['joblist'] = job_list
- f_name = os.path.join(temp_path, "finalize.sh")
+ f_name = os.path.join(temp_path, "finalize.py")
with open(f_name, "w") as finfile:
finfile.write(textwrap.dedent(tmpl.render(**fmt_args)))
@@ -653,17 +651,12 @@ def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
srun_base += [os.path.join(PKG_DIR, "bin", "slurm_calibrate.sh"), # path to helper sh
os.path.abspath(nbpath), # path to notebook
python_path, # path to python
- ipython_path, # path to ipython
- jupyter_path, # path to jupyter
- ipcluster_path, # path to ipcluster
- # python activate path
- activate_path if activate_path!="" else os.path.join(PKG_DIR, "bin", "activate.sh"),
args.get("cluster_profile", "NO_CLUSTER"),
'"{}"'.format(base_name.upper()),
'"{}"'.format(args["detector"].upper()),
'"{}"'.format(args["type"].upper()),
"FINAL" if final_job else "NONFINAL",
- ". {}/finalize.sh".format(os.path.abspath(temp_path)),
+ "{}/finalize.py".format(os.path.abspath(temp_path)),
str(cluster_cores)]
output = check_output(srun_base).decode('utf8')
diff --git a/xfel_calibrate/finalize.py b/xfel_calibrate/finalize.py
index 53bc72912ff3cbe105f9732fa0b7b78c8dd4d596..5c96202fb0802a9b561eb47e52fca6441b7c6207 100644
--- a/xfel_calibrate/finalize.py
+++ b/xfel_calibrate/finalize.py
@@ -209,7 +209,7 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
out_path = report_path
try:
- check_call(["sphinx-quickstart",
+ check_call([sys.executable, "-m", "sphinx.quickstart",
"--quiet",
"--project='{}'".format(project),
"--author='{}'".format(author),
@@ -222,7 +222,7 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
"--no-batchfile", run_path])
except CalledProcessError:
- raise Exception("Failed to run sphinx-quickbuild. Is sphinx installed?"
+ raise Exception("Failed to run sphinx-quickstart. Is sphinx installed?"
"Generated simple index.rst instead")
# quickbuild went well we need to edit the index.rst and conf.py files
@@ -299,7 +299,7 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
# finally call the make scripts
chdir(run_path)
try:
- check_call(["make", "latexpdf"])
+ check_call(["make", f"SPHINXBUILD={sys.executable} -m sphinx", "latexpdf"])
except CalledProcessError:
print("Failed to make pdf documentation")
diff --git a/xfel_calibrate/settings.py b/xfel_calibrate/settings.py
index cbe4c21b66bca70ed8db0d5902a7ca4e15c5365b..a6bbc597940d3b902394461ea13e372cd776d091 100644
--- a/xfel_calibrate/settings.py
+++ b/xfel_calibrate/settings.py
@@ -1,22 +1,11 @@
import os
+import sys
# path into which temporary files from each run are placed
temp_path = os.path.abspath("temp/")
-# Path to use for calling Python. If the environment is correctly set, simply the command
-python_path = "python"
-
-# Path to use for calling iPython. If the environment is correctly set, simply the command
-ipython_path = "ipython"
-
-# Path to use for calling Jupyter. If the environment is correctly set, simply the command
-jupyter_path = "jupyter"
-
-# Path to find activate script
-activate_path = ""
-
-# Path to use for calling ipcluster. If the environment is correctly set, simply the command
-ipcluster_path = "ipcluster"
+# Path to use for calling Python. By default, the Python xfel-calibrate is run in.
+python_path = sys.executable
# Path to store reports in
report_path = "{}/calibration_reports/".format(os.getcwd())