Merge branch 'fix/cpy_conf_after_req' into 'master'

fix/cpy_conf_after_req

See merge request detectors/pycalibration!130

reportservice/report_conf.yaml

 GLOBAL:
     git:
-        figures-local: "/gpfs/exfel/data/scratch/xcal/calibration/DetectorCharacterization/figures"
         repo-local: "/gpfs/exfel/data/scratch/xcal/calibration/DetectorCharacterization/"
         figures-remote: "http://git@git.xfel.eu/gitlab/detectors/DetectorCharacterization.git"
     server-port: "tcp://max-exfl016:5566"

reportservice/report_service.py

@@ -168,10 +168,6 @@ async def server_runner(conf_file):
     socket.bind("{}:{}".format(config['GLOBAL']['report-service']['bind-to'],
                                config['GLOBAL']['report-service']['port']))
 
-    asyncio.ensure_future(copy_files(conf_file,
-                                     config['GLOBAL']['git']['figures-local'],
-                                     sem=None))
-
     while True:
         response = await socket.recv_pyobj()
         await socket.send_pyobj('Build DC reports through -->')
@@ -204,17 +200,15 @@ async def server_runner(conf_file):
             continue
 
         logging.info('Requested Configuration: {}'.format(req_cfg))
-        
+
         async def do_action(cfg):
-            logging.info('Run plot production')
 
+            logging.info('Run plot production')
+            local_repo = cfg['GLOBAL']['git']['repo-local']
+            fig_local = '{}/figures'.format(local_repo)
             jobs_timeout = cfg['GLOBAL']['report-service'].get('job-timeout',
                                                                3600)
             all_new_files = []
-            all_new_files.append('{}/{}'.format(config['GLOBAL']
-                                                      ['git']
-                                                      ['figures-local'],
-                                                conf_file.split('/')[-1]))
 
             for instr_name, instrument in cfg.items():
 
@@ -238,7 +232,6 @@ async def server_runner(conf_file):
 
                         logging.info('Submission information: {}:'
                                                     .format(run_base))
-
                     except Exception as e:
                         logging.error('Submission failed: {}'.format(e))
                         exit(1)
@@ -258,12 +251,11 @@ async def server_runner(conf_file):
                 try:
                     await asyncio.wait_for(wait_jobs(job_list),
                                            timeout=jobs_timeout)
-
                     logging.info('All jobs are finished')
-
                 except asyncio.TimeoutError:
-                    asyncio.ensure_future(del_folder('./tmp'))
-                    logging.error('Jobs timeout!')
+                    logging.error('Jobs have timed-out!')
+                    logging.error('{}/temp has not been deleted.'.format(
+                                  os.path.dirname(os.path.abspath(__file__))))
 
                 # Copy all plots
                 for det_name, det_conf in instrument.items():
@@ -271,19 +263,19 @@ async def server_runner(conf_file):
                     out_folder = det_conf['out-folder'].format(
                                           instrument=instr_name,
                                           detector=det_name)
-                    path = cfg['GLOBAL']['git']['figures-local']
+
                     figures = glob.glob("{}/*png".format(out_folder))
 
                     det_new_files = {}
 
                     for f in figures:
                         const = f.split('/')[-1].split('_')[0]
-                        fpath = '{}/{}/{}/{}'.format(path, instr_name,
-                                                  det_name, const)
+                        fpath = '{}/{}/{}/{}'.format(fig_local, instr_name,
+                                                     det_name, const)
 
                         os.makedirs(fpath, exist_ok=True)
                         det_new_files[f] = fpath
-                        
+
                         # Set concurrency limitation.
                         # 50 have been chosen by trial 
                         # Note: This is not the max limitation.
@@ -295,17 +287,23 @@ async def server_runner(conf_file):
                                         for k, v in det_new_files.items()])
 
                     logging.info('{} figures of {} are copied into {}'.format(
-                                                len(figures), det_name, path))
+                                           len(figures), det_name, fig_local))
+
+            # Remove sensitive information from the config file.
+            del cfg['GLOBAL']
+            # Write the requested cfg.yaml before pushing all figures.
+            with open('{}/report_conf.yaml'.format(
+                                          fig_local), 'w') as outfile:
+                yaml.dump(cfg, outfile, default_flow_style=False)
+
+            all_new_files.append('{}/report_conf.yaml'.format(fig_local))
 
-            asyncio.ensure_future(
-                           push_figures(cfg['GLOBAL']['git']['repo-local'],
-                                        all_new_files))
+            asyncio.ensure_future(push_figures(local_repo, all_new_files))
             # TODO:delete out-folder
             #try:
                 # asyncio.ensure_future(del_folder(out_folder))
             #except:
                 #logging.error(str(e))
-        
             logging.info('All done')
 
             return