diff --git a/webservice/webservice.py b/webservice/webservice.py index 76223ce414f5a740968d3ca037ea49776325f6f1..99cfa04d522f8ed60e4da985a64719d460262c5c 100644 --- a/webservice/webservice.py +++ b/webservice/webservice.py @@ -301,16 +301,15 @@ async def update_job_db(config): await asyncio.sleep(int(config['web-service']['job-update-interval'])) -async def copy_untouched_files(file_list, out_folder, run): +async def copy_untouched_files(file_list, out_folder): """ Copy those files whicha are not touched by calibration to outpot dir :param file_list: The list of files to copy :param out_folder: The output folder - :param run: The run which is being handled Copying is done via an asyncio subprocess call """ - os.makedirs("{}/r{:04d}".format(out_folder, int(run)), exist_ok=True) + os.makedirs(out_folder, exist_ok=True) for f in file_list: of = f.replace("raw", "proc").replace("RAW", "CORR") cmd = ["rsync", "-av", f, of] @@ -591,6 +590,8 @@ async def server_runner(config, mode): "{}/RAW-*{}*.h5".format(rpath, dconfig["inset"])) if len(fl): thisconf = copy.copy(dconfig) + out_folder = '_'.join((out_folder, + dconfig["inset"])) thisconf["in-folder"] = in_folder thisconf["out-folder"] = out_folder @@ -635,7 +636,8 @@ async def server_runner(config, mode): rpath = "{}/r{:04d}/".format(in_folder, int(runnr)) out_folder = config[action]['out-folder'].format( - instrument=instrument, cycle=cycle, proposal=proposal) + instrument=instrument, cycle=cycle, proposal=proposal, + run='{:04d}'.format(int(runnr)) ) corr_file_list = set() copy_file_list = set(glob.glob("{}/*.h5".format(rpath))) detectors = {} @@ -655,7 +657,7 @@ async def server_runner(config, mode): copy_file_list = copy_file_list.difference(corr_file_list) print(detectors) asyncio.ensure_future( - copy_untouched_files(copy_file_list, out_folder, runnr)) + copy_untouched_files(copy_file_list, out_folder)) if len(detectors) == 0: logging.warn(Errors.NOTHING_TO_DO.format(rpath)) msg = "Nothing to calibrate for this run, copied raw data only" diff --git a/webservice/webservice.yaml b/webservice/webservice.yaml index 6e8eef82a41321fb28d1246833a5c784fec3ec7b..2dd9fb1c48b6cf68c49d0e699bcefdb968976795 100644 --- a/webservice/webservice.yaml +++ b/webservice/webservice.yaml @@ -24,7 +24,7 @@ metadata-client: correct: in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw - out-folder: /gpfs/exfel/d/proc/{instrument}/{cycle}/p{proposal} + out-folder: /gpfs/exfel/d/proc/{instrument}/{cycle}/p{proposal}/r{run} dark: in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py index 8e73594a2bc088ae604d043c95bff1d3d15bfb09..c2da555db225b3d5a12e10dabd17d7dd803d0a42 100755 --- a/xfel_calibrate/calibrate.py +++ b/xfel_calibrate/calibrate.py @@ -667,12 +667,6 @@ def run(): if try_report_to_output: if "out_folder" in args: out_path = os.path.abspath(args["out_folder"]) - if "run" in args: - rr = args["run"] - if isinstance(rr, int): - out_path = "{}/r{:04d}/".format(out_path, rr) - else: - out_path = "{}/{}/".format(out_path, rr) else: print("No 'out_folder' defined as argument, outputting to '{}' instead.".format( out_path))