diff --git a/bin/slurm_calibrate.sh b/bin/slurm_calibrate.sh index 917aa6fd659b8cfe568dd5d64d5fad49e7bbefee..457f71145688cc6f0e7cb6468a418bd59b3bc0b7 100755 --- a/bin/slurm_calibrate.sh +++ b/bin/slurm_calibrate.sh @@ -7,24 +7,23 @@ ipcluster_profile=$3 notebook=$4 detector=$5 caltype=$6 -final=$7 -finalize=$8 -cluster_cores=$9 -notebook_python_path=${10} +finalize=$7 +cluster_cores=$8 +cal_python_path=$9 echo "Running with the following parameters:" echo "Notebook path: $nb_path" echo "Python path: $python_path" +echo "Calibration Python: $cal_python_path" echo "IP-Cluster profile: $ipcluster_profile" echo "notebook: $notebook" echo "detector: $detector" echo "caltype: $caltype" -echo "final: $final" echo "finalize: $finalize" echo "cluster_cores: $cluster_cores" echo "job ID: $SLURM_JOB_ID" -export CAL_NOTEBOOK_NAME=$notebook +export CAL_NOTEBOOK_NAME="$notebook" # set-up enviroment source /etc/profile.d/modules.sh @@ -44,8 +43,8 @@ fi echo "Running notebook" -${notebook_python_path} -m princess ${nb_path} --save --on-error-resume-next -${notebook_python_path} -m nbconvert --to rst --TemplateExporter.exclude_input=True ${nb_path} +${python_path} -m princess ${nb_path} --save --on-error-resume-next +${cal_python_path} -m nbconvert --to rst --TemplateExporter.exclude_input=True ${nb_path} # stop the cluster if requested if [ "${ipcluster_profile}" != "NO_CLUSTER" ] @@ -56,7 +55,7 @@ then rm -rf $profile_path fi -if [ "${final}" == "FINAL" ] +if [ -n "${finalize}" ] then - ${python_path} ${finalize} $SLURM_JOB_ID + ${cal_python_path} ${finalize} $SLURM_JOB_ID fi diff --git a/src/xfel_calibrate/calibrate.py b/src/xfel_calibrate/calibrate.py index bba98fc0b78e09b428d500c99a9bef9cf9d9e296..2e351242e417053b2ee78bc9dd0f99fc8827d7cc 100755 --- a/src/xfel_calibrate/calibrate.py +++ b/src/xfel_calibrate/calibrate.py @@ -580,20 +580,21 @@ def set_figure_format(nb, enable_vector_format): cell.source += "\n%config InlineBackend.figure_formats = ['svg']\n" -def create_finalize_script(fmt_args, temp_path, job_list): +def create_finalize_script(fmt_args, temp_path, job_list) -> str: """ Create a finalize script to produce output report :param fmt_args: Dictionary of fmt arguments :param temp_path: Path to temporary folder to run slurm job :param job_list: List of slurm jobs + :return: The path of the created script """ tmpl = Template("""\ #!/usr/bin/env python3 - import sys + import os from xfel_calibrate.finalize import finalize finalize(joblist={{joblist}}, - finaljob=sys.argv[1], + finaljob=os.environ.get('SLURM_JOB_ID', ''), run_path='{{run_path}}', out_path='{{out_path}}', project='{{project}}', @@ -616,6 +617,7 @@ def create_finalize_script(fmt_args, temp_path, job_list): # executed and writable for user, readable for user, group and others all_stats = stat.S_IXUSR | stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH os.chmod(f_name, all_stats) + return f_name def save_executed_command(run_tmp_path, version): @@ -742,7 +744,9 @@ def concurrent_run( # add finalization to the last job if final_job: - create_finalize_script(fmt_args, temp_path, job_list) + finalize_script = create_finalize_script(fmt_args, temp_path, job_list) + else: + finalize_script = '' # then run an sbatch job srun_base = [] @@ -752,19 +756,21 @@ def concurrent_run( if user_venv: print(f"Running job in user venv at {user_venv}\n") + python = str(user_venv / 'bin' / 'python') + else: + python = python_path # From settings.py, default is sys.executable srun_base += [ os.path.join(PKG_DIR, "bin", "slurm_calibrate.sh"), # path to helper sh os.path.abspath(nbpath), # path to notebook - python_path, # path to python + python, # path to python to run notebook (& ipcluster) cluster_profile, - '"{}"'.format(base_name.upper()), - '"{}"'.format(args["detector"].upper()), - '"{}"'.format(args["type"].upper()), - "FINAL" if final_job else "NONFINAL", - "{}/finalize.py".format(os.path.abspath(temp_path)), + base_name.upper(), + args["detector"].upper(), + args["type"].upper(), + finalize_script, str(cluster_cores), - user_venv + "/bin/python" if user_venv else python_path # used for nb execution + sys.executable, # Python for calib machinery (nbconvert, finalize) ] output = check_output(srun_base).decode('utf8') diff --git a/src/xfel_calibrate/finalize.py b/src/xfel_calibrate/finalize.py index 9f3fd19d516c83e1df35921cad2b319e5a964f85..de865afe99a0096e57969131e38d8b9c7006506f 100644 --- a/src/xfel_calibrate/finalize.py +++ b/src/xfel_calibrate/finalize.py @@ -403,7 +403,8 @@ def finalize(joblist, finaljob, run_path, out_path, project, calibration, prepare_plots(run_path) # Archiving files in slurm_tmp - joblist.append(str(finaljob)) + if finaljob: + joblist.append(str(finaljob)) metadata = cal_tools.tools.CalibrationMetadata(out_path) job_time_fmt = 'JobID,Start,End,Elapsed,Suspended,State'.split(',') job_time_summary = get_job_info(joblist, job_time_fmt)