Skip to content
Snippets Groups Projects

keep, rename and move temp folder with report

Merged Karim Ahmed requested to merge feat/move_tmp_folder_with_report into master
Files
2
#!/usr/bin/env python
#!/usr/bin/env python
import argparse
import argparse
import copy
from datetime import datetime
from datetime import datetime
import nbconvert
import nbconvert
import nbformat
import nbformat
@@ -9,12 +8,10 @@ from nbparameterise import (
@@ -9,12 +8,10 @@ from nbparameterise import (
extract_parameters, replace_definitions, parameter_values
extract_parameters, replace_definitions, parameter_values
)
)
import os
import os
from os import chdir
import pprint
import pprint
import re
import re
from subprocess import Popen, PIPE, check_output
from subprocess import check_output
import sys
import sys
from uuid import uuid4
import warnings
import warnings
from .settings import *
from .settings import *
from .notebooks import notebooks
from .notebooks import notebooks
@@ -685,13 +682,12 @@ def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
@@ -685,13 +682,12 @@ def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
return jobid
return jobid
def make_par_table(parms, temp_path, run_uuid):
def make_par_table(parms, run_tmp_path):
"""
"""
Create a table with input parameters if the notebook
Create a table with input parameters of the notebook
:param parms: parameters of the notebook
:param parms: parameters of the notebook
:param temp_path: path to temporary directory for job outputs
:param run_tmp_path: path to temporary directory for running job outputs
:param run_uuid: inset of folder name containing job output
"""
"""
# Add space in long strings without line breakers ` ,-/` to
# Add space in long strings without line breakers ` ,-/` to
@@ -747,7 +743,7 @@ def make_par_table(parms, temp_path, run_uuid):
@@ -747,7 +743,7 @@ def make_par_table(parms, temp_path, run_uuid):
\end{longtable}
\end{longtable}
''')
''')
f_name = "{}/slurm_tmp_{}/InputParameters.rst".format(temp_path, run_uuid)
f_name = "{}/InputParameters.rst".format(run_tmp_path)
with open(f_name, "w") as finfile:
with open(f_name, "w") as finfile:
finfile.write(textwrap.dedent(tmpl.render(p=col_type, lines=l_parms)))
finfile.write(textwrap.dedent(tmpl.render(p=col_type, lines=l_parms)))
@@ -818,7 +814,7 @@ def run():
@@ -818,7 +814,7 @@ def run():
title = title.rstrip()
title = title.rstrip()
run_uuid = uuid4()
run_uuid = f"t{datetime.now().strftime('%y%m%d_%H%M%S')}"
# check if concurrency parameter is given and we run concurrently
# check if concurrency parameter is given and we run concurrently
if not has_parm(parms, concurrency["parameter"]) and concurrency["parameter"] is not None:
if not has_parm(parms, concurrency["parameter"]) and concurrency["parameter"] is not None:
@@ -834,12 +830,12 @@ def run():
@@ -834,12 +830,12 @@ def run():
cluster_profile = "slurm_prof_{}".format(run_uuid)
cluster_profile = "slurm_prof_{}".format(run_uuid)
# create a temporary output directory to work in
# create a temporary output directory to work in
run_tmp_path = "{}/slurm_tmp_{}".format(temp_path, run_uuid)
run_tmp_path = "{}/slurm_out_{}_{}_{}".format(temp_path, detector, caltype, run_uuid)
os.makedirs(run_tmp_path)
os.makedirs(run_tmp_path)
# Write all input parameters to rst file to be included to final report
# Write all input parameters to rst file to be included to final report
parms = parameter_values(parms, **args)
parms = parameter_values(parms, **args)
make_par_table(parms, temp_path, run_uuid)
make_par_table(parms, run_tmp_path)
save_executed_command(run_tmp_path, version)
save_executed_command(run_tmp_path, version)
# wait on all jobs to run and then finalize the run by creating a report from the notebooks
# wait on all jobs to run and then finalize the run by creating a report from the notebooks
Loading