Improving SLURM based notebooks and scripts

• add printing of python path for SLURM notebooks since the metadata kernel name is not necessarily the kernel that was used to run the notebook via papermill
• slurm scripts now uses named option to pass arguments
• slurm scripts now correctly specify the kernel name for papermill
• slurm scripts now convert string to int or float, removing the need to convert them in the notebook
• if the kernel is not defined, slurm scripts will try to define one based on the proposal number

Close #47 (closed)

doc/scripts/bin_dssc_module_job.sh

@@ -5,21 +5,49 @@
 #SBATCH --mail-type=END,FAIL
 #SBATCH --output=logs/%j-%x.out
 
-PROPOSAL=2719
-DARK=$1
-RUN=$2
-MODULE_GROUP=$3
+while getopts ":p:d:r:k:g:t:s:m:" option
+do
+	  case $option in
+		    p) PROPOSAL="$OPTARG";;
+		    d) DARK="$OPTARG";;
+		    r) RUN="$OPTARG";;
+		    k) KERNEL="$OPTARG";;
+		    m) MODULE_GROUP="$OPTARG";;
+        x) XAXIS="$OPTARG";;
+        b) BINWIDTH="$OPTARG";;
+		    \?) echo "Unknown option"
+				    exit 1;;
+		    :) echo "Missing option for input flag"
+			     exit 1;;
+	  esac
+done
 
+# Load xfel environment
 source /etc/profile.d/modules.sh
 module load exfel
 module load exfel_anaconda3/1.1
 
 echo processing run $RUN
-OUTPATH=/gpfs/exfel/exp/SCS/202002/p002719/scratch/tests
-mkdir $OUTPATH/r$RUN
+PDIR=$(find-proposal $PROPOSAL)
+PPROPOSAL="p$(printf '%06d' $PROPOSAL)"
+RDIR="$PDIR/usr/processed_runs/r$(printf '%04d' $RUN)"
+mkdir $RDIR
 
-# Run papermill
-python3 -c "import papermill as pm; pm.execute_notebook('Dask DSSC module binning.ipynb', \
-'$OUTPATH/r$RUN/output$MODULE_GROUP.ipynb', \
-parameters=dict(proposalNB='$PROPOSAL', dark_runNB='$DARK', runNB='$RUN', module_group='$MODULE_GROUP', \
-path='$OUTPATH/r$RUN/', kernel='xfel'))" 
+NB='Dask DSSC module binning.ipynb'
+
+# kernel list can be seen from 'jupyter kernelspec list'
+if [ -z "${KERNEL}" ]; then
+	  KERNEL="toolbox_$PPROPOSAL"
+fi
+
+python -c "import papermill as pm; pm.execute_notebook(\
+  '$NB', \
+  '$RDIR/output$MODULE_GROUP.ipynb', \
+  kernel_name='$KERNEL', \
+  parameters=dict(proposalNB=int('$PROPOSAL'), \
+                  dark_runNB=int('$DARK'), \
+                  runNB=int('$RUN'), \
+                  module_group=int('$MODULE_GROUP'), \
+                  path='$RDIR', \
+                  xaxis='$XAXIS', \
+                  bin_width=float('$BINWIDTH')))"