From 1362249839d5868d9a1b44451d77b55b5dcd9d10 Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Tue, 17 Sep 2019 17:46:21 +0200
Subject: [PATCH] Modify output path
---
webservice/webservice.py | 12 +++++++-----
webservice/webservice.yaml | 2 +-
xfel_calibrate/calibrate.py | 6 ------
3 files changed, 8 insertions(+), 12 deletions(-)
diff --git a/webservice/webservice.py b/webservice/webservice.py
index d046b067f..ba6a17616 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -226,16 +226,15 @@ async def update_job_db(config):
await asyncio.sleep(int(config['web-service']['job-update-interval']))
-async def copy_untouched_files(file_list, out_folder, run):
+async def copy_untouched_files(file_list, out_folder):
""" Copy those files whicha are not touched by calibration to outpot dir
:param file_list: The list of files to copy
:param out_folder: The output folder
- :param run: The run which is being handled
Copying is done via an asyncio subprocess call
"""
- os.makedirs("{}/r{:04d}".format(out_folder, int(run)), exist_ok=True)
+ os.makedirs(out_folder, exist_ok=True)
for f in file_list:
of = f.replace("raw", "proc").replace("RAW", "CORR")
cmd = ["rsync", "-av", f, of]
@@ -503,6 +502,8 @@ async def server_runner(config, mode):
"{}/RAW-*{}*.h5".format(rpath, dconfig["inset"]))
if len(fl):
thisconf = copy.copy(dconfig)
+ out_folder = '_'.join((out_folder,
+ dconfig["inset"]))
thisconf["in-folder"] = in_folder
thisconf["out-folder"] = out_folder
@@ -547,7 +548,8 @@ async def server_runner(config, mode):
rpath = "{}/r{:04d}/".format(in_folder, int(runnr))
out_folder = config[action]['out-folder'].format(
- instrument=instrument, cycle=cycle, proposal=proposal)
+ instrument=instrument, cycle=cycle, proposal=proposal,
+ run='{:04d}'.format(int(runnr)) )
corr_file_list = set()
copy_file_list = set(glob.glob("{}/*.h5".format(rpath)))
detectors = {}
@@ -567,7 +569,7 @@ async def server_runner(config, mode):
copy_file_list = copy_file_list.difference(corr_file_list)
print(detectors)
asyncio.ensure_future(
- copy_untouched_files(copy_file_list, out_folder, runnr))
+ copy_untouched_files(copy_file_list, out_folder))
if len(detectors) == 0:
logging.warn(Errors.NOTHING_TO_DO.format(rpath))
msg = "Nothing to calibrate for this run, copied raw data only"
diff --git a/webservice/webservice.yaml b/webservice/webservice.yaml
index 6e8eef82a..2dd9fb1c4 100644
--- a/webservice/webservice.yaml
+++ b/webservice/webservice.yaml
@@ -24,7 +24,7 @@ metadata-client:
correct:
in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw
- out-folder: /gpfs/exfel/d/proc/{instrument}/{cycle}/p{proposal}
+ out-folder: /gpfs/exfel/d/proc/{instrument}/{cycle}/p{proposal}/r{run}
dark:
in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw
diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index 8e73594a2..c2da555db 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -667,12 +667,6 @@ def run():
if try_report_to_output:
if "out_folder" in args:
out_path = os.path.abspath(args["out_folder"])
- if "run" in args:
- rr = args["run"]
- if isinstance(rr, int):
- out_path = "{}/r{:04d}/".format(out_path, rr)
- else:
- out_path = "{}/{}/".format(out_path, rr)
else:
print("No 'out_folder' defined as argument, outputting to '{}' instead.".format(
out_path))
--
GitLab