diff --git a/reportservice/report_conf.yaml b/reportservice/report_conf.yaml index 8c4570a8c97b27a640b3fb3e0dfe5f08d7a2db23..2fc336768a8131196c7da0fc17d2060314d05600 100644 --- a/reportservice/report_conf.yaml +++ b/reportservice/report_conf.yaml @@ -1,6 +1,5 @@ GLOBAL: git: - figures-local: "/gpfs/exfel/data/scratch/xcal/calibration/DetectorCharacterization/figures" repo-local: "/gpfs/exfel/data/scratch/xcal/calibration/DetectorCharacterization/" figures-remote: "http://git@git.xfel.eu/gitlab/detectors/DetectorCharacterization.git" server-port: "tcp://max-exfl016:5566" diff --git a/reportservice/report_service.py b/reportservice/report_service.py index f1177cc35c475ab664da23bac429eceabc79fce4..36056374e5c61722b573500c430f1a7ef0b4d427 100644 --- a/reportservice/report_service.py +++ b/reportservice/report_service.py @@ -168,10 +168,6 @@ async def server_runner(conf_file): socket.bind("{}:{}".format(config['GLOBAL']['report-service']['bind-to'], config['GLOBAL']['report-service']['port'])) - asyncio.ensure_future(copy_files(conf_file, - config['GLOBAL']['git']['figures-local'], - sem=None)) - while True: response = await socket.recv_pyobj() await socket.send_pyobj('Build DC reports through -->') @@ -204,17 +200,15 @@ async def server_runner(conf_file): continue logging.info('Requested Configuration: {}'.format(req_cfg)) - + async def do_action(cfg): - logging.info('Run plot production') + logging.info('Run plot production') + local_repo = cfg['GLOBAL']['git']['repo-local'] + fig_local = '{}/figures'.format(local_repo) jobs_timeout = cfg['GLOBAL']['report-service'].get('job-timeout', 3600) all_new_files = [] - all_new_files.append('{}/{}'.format(config['GLOBAL'] - ['git'] - ['figures-local'], - conf_file.split('/')[-1])) for instr_name, instrument in cfg.items(): @@ -238,7 +232,6 @@ async def server_runner(conf_file): logging.info('Submission information: {}:' .format(run_base)) - except Exception as e: logging.error('Submission failed: {}'.format(e)) exit(1) @@ -258,12 +251,11 @@ async def server_runner(conf_file): try: await asyncio.wait_for(wait_jobs(job_list), timeout=jobs_timeout) - logging.info('All jobs are finished') - except asyncio.TimeoutError: - asyncio.ensure_future(del_folder('./tmp')) - logging.error('Jobs timeout!') + logging.error('Jobs have timed-out!') + logging.error('{}/temp has not been deleted.'.format( + os.path.dirname(os.path.abspath(__file__)))) # Copy all plots for det_name, det_conf in instrument.items(): @@ -271,19 +263,19 @@ async def server_runner(conf_file): out_folder = det_conf['out-folder'].format( instrument=instr_name, detector=det_name) - path = cfg['GLOBAL']['git']['figures-local'] + figures = glob.glob("{}/*png".format(out_folder)) det_new_files = {} for f in figures: const = f.split('/')[-1].split('_')[0] - fpath = '{}/{}/{}/{}'.format(path, instr_name, - det_name, const) + fpath = '{}/{}/{}/{}'.format(fig_local, instr_name, + det_name, const) os.makedirs(fpath, exist_ok=True) det_new_files[f] = fpath - + # Set concurrency limitation. # 50 have been chosen by trial # Note: This is not the max limitation. @@ -295,17 +287,23 @@ async def server_runner(conf_file): for k, v in det_new_files.items()]) logging.info('{} figures of {} are copied into {}'.format( - len(figures), det_name, path)) + len(figures), det_name, fig_local)) + + # Remove sensitive information from the config file. + del cfg['GLOBAL'] + # Write the requested cfg.yaml before pushing all figures. + with open('{}/report_conf.yaml'.format( + fig_local), 'w') as outfile: + yaml.dump(cfg, outfile, default_flow_style=False) + + all_new_files.append('{}/report_conf.yaml'.format(fig_local)) - asyncio.ensure_future( - push_figures(cfg['GLOBAL']['git']['repo-local'], - all_new_files)) + asyncio.ensure_future(push_figures(local_repo, all_new_files)) # TODO:delete out-folder #try: # asyncio.ensure_future(del_folder(out_folder)) #except: #logging.error(str(e)) - logging.info('All done') return