Improving SLURM based notebooks and scripts

• add printing of python path for SLURM notebooks since the metadata kernel name is not necessarily the kernel that was used to run the notebook via papermill
• slurm scripts now uses named option to pass arguments
• slurm scripts now correctly specify the kernel name for papermill
• slurm scripts now convert string to int or float, removing the need to convert them in the notebook
• if the kernel is not defined, slurm scripts will try to define one based on the proposal number

Close #47 (closed)

doc/scripts/boz_parameters_job.sh

@@ -5,14 +5,29 @@
 #SBATCH --mail-type=END,FAIL
 #SBATCH --output=logs/%j-%x.out
 
-PROPOSAL=2784
-DARK=$1
-RUN=$2
-GAIN=$3
-ROISTH=${4:-1}
-SATLEVEL=${5:-500}
-MODULE=15
+ROISTH='1'
+SATLEVEL='500'
+MODULE='15'
 
+while getopts ":p:d:r:k:g:t:s:m:" option
+do
+    case $option in
+        p) PROPOSAL="$OPTARG";;
+        d) DARK="$OPTARG";;
+        r) RUN="$OPTARG";;
+        k) KERNEL="$OPTARG";;
+        g) GAIN="$OPTARG";;
+        t) ROISTH="$OPTARG";;
+        s) SATLEVEL="$OPTARG";;
+        m) MODULE="$OPTARG";;
+        \?) echo "Unknown option"
+            exit 1;;
+        :) echo "Missing option for input flag"
+           exit 1;;
+    esac
+done
+
+# Load xfel environment
 source /etc/profile.d/modules.sh
 module load exfel
 module load exfel_anaconda3/1.1
@@ -24,20 +39,20 @@ RDIR="$PDIR/usr/processed_runs/r$(printf '%04d' $RUN)"
 mkdir $RDIR
 
 NB='BOZ analysis part I.a Correction determination.ipynb'
-KERNEL="SCS Toolbox ($PPROPOSAL)"
 
-#activate the proposal environment
-ACTIVATE="$PDIR/usr/Software/envs/toolbox_$PPROPOSAL/bin/activate"
-source $ACITVATE
+# kernel list can be seen from 'jupyter kernelspec list'
+if [ -z "${KERNEL}" ]; then
+	KERNEL="toolbox_$PPROPOSAL"
+fi
 
 python -c "import papermill as pm; pm.execute_notebook(\
   '$NB', \
-	'$RDIR/output.ipynb', \
-  parameters=dict(proposal='$PROPOSAL', \
-                  darkrun='$DARK', \
-                  run='$RUN', \
-                  module='$MODULE', \
-                  gain='$GAIN', \
-                  rois_th='$ROISTH', \
-                  sat_level='$SATLEVEL', \
-                  kernel='$KERNEL'))"
+  '$RDIR/output.ipynb', \
+  kernel_name='$KERNEL', \
+  parameters=dict(proposal=int('$PROPOSAL'), \
+                  darkrun=int('$DARK'), \
+                  run=int('$RUN'), \
+                  module=int('$MODULE'), \
+                  gain=float('$GAIN'), \
+                  rois_th=float('$ROISTH'), \
+                  sat_level=int('$SATLEVEL')))"