Modify output path

webservice/webservice.py

@@ -226,16 +226,15 @@ async def update_job_db(config):
         await asyncio.sleep(int(config['web-service']['job-update-interval']))
 
 
-async def copy_untouched_files(file_list, out_folder, run):
+async def copy_untouched_files(file_list, out_folder):
     """ Copy those files whicha are not touched by calibration to outpot dir
 
     :param file_list: The list of files to copy
     :param out_folder: The output folder
-    :param run: The run which is being handled
 
     Copying is done via an asyncio subprocess call
     """
-    os.makedirs("{}/r{:04d}".format(out_folder, int(run)), exist_ok=True)
+    os.makedirs(out_folder, exist_ok=True)
     for f in file_list:
         of = f.replace("raw", "proc").replace("RAW", "CORR")
         cmd = ["rsync", "-av", f, of]
@@ -503,6 +502,8 @@ async def server_runner(config, mode):
                             "{}/RAW-*{}*.h5".format(rpath, dconfig["inset"]))
                         if len(fl):
                             thisconf = copy.copy(dconfig)
+                            out_folder = '_'.join((out_folder,
+                                                   dconfig["inset"]))
                             thisconf["in-folder"] = in_folder
                             thisconf["out-folder"] = out_folder
 
@@ -547,7 +548,8 @@ async def server_runner(config, mode):
                 rpath = "{}/r{:04d}/".format(in_folder, int(runnr))
 
                 out_folder = config[action]['out-folder'].format(
-                    instrument=instrument, cycle=cycle, proposal=proposal)
+                    instrument=instrument, cycle=cycle, proposal=proposal,
+                    run='{:04d}'.format(int(runnr)) )
                 corr_file_list = set()
                 copy_file_list = set(glob.glob("{}/*.h5".format(rpath)))
                 detectors = {}
@@ -567,7 +569,7 @@ async def server_runner(config, mode):
                 copy_file_list = copy_file_list.difference(corr_file_list)
                 print(detectors)
                 asyncio.ensure_future(
-                    copy_untouched_files(copy_file_list, out_folder, runnr))
+                    copy_untouched_files(copy_file_list, out_folder))
                 if len(detectors) == 0:
                     logging.warn(Errors.NOTHING_TO_DO.format(rpath))
                     msg = "Nothing to calibrate for this run, copied raw data only"

webservice/webservice.yaml

@@ -24,7 +24,7 @@ metadata-client:
 
 correct:
     in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw
-    out-folder: /gpfs/exfel/d/proc/{instrument}/{cycle}/p{proposal}
+    out-folder: /gpfs/exfel/d/proc/{instrument}/{cycle}/p{proposal}/r{run}
 
 dark:
     in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw

xfel_calibrate/calibrate.py

@@ -667,12 +667,6 @@ def run():
         if try_report_to_output:
             if "out_folder" in args:
                 out_path = os.path.abspath(args["out_folder"])
-                if "run" in args:
-                    rr = args["run"]
-                    if isinstance(rr, int):
-                        out_path = "{}/r{:04d}/".format(out_path, rr)
-                    else:
-                        out_path = "{}/{}/".format(out_path, rr)
             else:
                 print("No 'out_folder' defined as argument, outputting to '{}' instead.".format(
                     out_path))