diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py index ebfae99317ddde50db7534a91633c26f854d2f46..fdccb8b90666beef7d5703e0dd75921943889e0b 100755 --- a/xfel_calibrate/calibrate.py +++ b/xfel_calibrate/calibrate.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import argparse -import copy from datetime import datetime import nbconvert import nbformat @@ -9,12 +8,10 @@ from nbparameterise import ( extract_parameters, replace_definitions, parameter_values ) import os -from os import chdir import pprint import re -from subprocess import Popen, PIPE, check_output +from subprocess import check_output import sys -from uuid import uuid4 import warnings from .settings import * from .notebooks import notebooks @@ -685,13 +682,12 @@ def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None, return jobid -def make_par_table(parms, temp_path, run_uuid): +def make_par_table(parms, run_tmp_path): """ - Create a table with input parameters if the notebook + Create a table with input parameters of the notebook :param parms: parameters of the notebook - :param temp_path: path to temporary directory for job outputs - :param run_uuid: inset of folder name containing job output + :param run_tmp_path: path to temporary directory for running job outputs """ # Add space in long strings without line breakers ` ,-/` to @@ -747,7 +743,7 @@ def make_par_table(parms, temp_path, run_uuid): \end{longtable} ''') - f_name = "{}/slurm_tmp_{}/InputParameters.rst".format(temp_path, run_uuid) + f_name = "{}/InputParameters.rst".format(run_tmp_path) with open(f_name, "w") as finfile: finfile.write(textwrap.dedent(tmpl.render(p=col_type, lines=l_parms))) @@ -818,7 +814,7 @@ def run(): title = title.rstrip() - run_uuid = uuid4() + run_uuid = f"t{datetime.now().strftime('%y%m%d_%H%M%S')}" # check if concurrency parameter is given and we run concurrently if not has_parm(parms, concurrency["parameter"]) and concurrency["parameter"] is not None: @@ -834,12 +830,12 @@ def run(): cluster_profile = "slurm_prof_{}".format(run_uuid) # create a temporary output directory to work in - run_tmp_path = "{}/slurm_tmp_{}".format(temp_path, run_uuid) + run_tmp_path = "{}/slurm_out_{}_{}_{}".format(temp_path, detector, caltype, run_uuid) os.makedirs(run_tmp_path) # Write all input parameters to rst file to be included to final report parms = parameter_values(parms, **args) - make_par_table(parms, temp_path, run_uuid) + make_par_table(parms, run_tmp_path) save_executed_command(run_tmp_path, version) # wait on all jobs to run and then finalize the run by creating a report from the notebooks diff --git a/xfel_calibrate/finalize.py b/xfel_calibrate/finalize.py index d37b83f6cfa6403852378181a62af8a7fad5c06b..fd099370b079b8c4c31bf419c14ada252e889ffa 100644 --- a/xfel_calibrate/finalize.py +++ b/xfel_calibrate/finalize.py @@ -273,8 +273,19 @@ def make_report(run_path, tmp_path, out_path, project, author, version, return print("Moving report to final location: {}".format(out_path)) copy('{}/_build/latex/{}.pdf'.format(run_path, report_name), out_path) - print("Removing temporary files at: {}".format(tmp_path)) - rmtree(tmp_path) + + temp_dirs = glob(f'{tmp_path}/*/') + # Remove folders with figures and sphinx files. + print(f"Removing directories [{temp_dirs}] in temp folder: {tmp_path}") + for dtmp in temp_dirs: + rmtree(f'{dtmp}/') + + # Moving temporary files to out-folder after successful execution + # This helps in keeping elements needed for re-producibility. + print(f"Moving temporary files to final location" + f": {out_path}/{os.path.basename(tmp_path)} with name: " + f"slurm_out_{report_name}") + move(tmp_path, f"{out_path}/slurm_out_{report_name}") def make_titlepage(sphinx_path, project, data_path, version):