Merge branch 'fix/output_path' into 'master'

Fix: inset of runs should match one in notebooks

See merge request detectors/pycalibration!156

webservice/serve_overview.py

@@ -185,6 +185,7 @@ class RequestHandler(BaseHTTPRequestHandler):
                 requested = "{} {}".format(ls[0], ls[1])
 
                 pdfs = glob.glob(f"{out_folder}/*.pdf")
+                pdfs += glob.glob(f"{out_folder}/*/*.pdf")
                 pdfs = {p.split("/")[-1]: p for p in pdfs}
                 fpdfs = []
                 if "xfel.pdf" in pdfs:

webservice/webservice.py

@@ -301,7 +301,7 @@ async def update_job_db(config):
         await asyncio.sleep(int(config['web-service']['job-update-interval']))
 
 
-async def copy_untouched_files(file_list, out_folder):
+async def copy_untouched_files(file_list, out_folder, run):
     """ Copy those files whicha are not touched by calibration to outpot dir
 
     :param file_list: The list of files to copy
@@ -309,7 +309,7 @@ async def copy_untouched_files(file_list, out_folder):
 
     Copying is done via an asyncio subprocess call
     """
-    os.makedirs(out_folder, exist_ok=True)
+    os.makedirs("{}/r{:04d}".format(out_folder, int(run)), exist_ok=True)
     for f in file_list:
         of = f.replace("raw", "proc").replace("RAW", "CORR")
         cmd = ["rsync", "-av", f, of]
@@ -636,8 +636,7 @@ async def server_runner(config, mode):
                 rpath = "{}/r{:04d}/".format(in_folder, int(runnr))
 
                 out_folder = config[action]['out-folder'].format(
-                    instrument=instrument, cycle=cycle, proposal=proposal,
-                    run='{:04d}'.format(int(runnr)) )
+                    instrument=instrument, cycle=cycle, proposal=proposal)
                 corr_file_list = set()
                 copy_file_list = set(glob.glob("{}/*.h5".format(rpath)))
                 detectors = {}
@@ -657,7 +656,7 @@ async def server_runner(config, mode):
                 copy_file_list = copy_file_list.difference(corr_file_list)
                 print(detectors)
                 asyncio.ensure_future(
-                    copy_untouched_files(copy_file_list, out_folder))
+                    copy_untouched_files(copy_file_list, out_folder, runnr))
                 if len(detectors) == 0:
                     logging.warn(Errors.NOTHING_TO_DO.format(rpath))
                     msg = "Nothing to calibrate for this run, copied raw data only"

webservice/webservice.yaml

@@ -24,7 +24,7 @@ metadata-client:
 
 correct:
     in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw
-    out-folder: /gpfs/exfel/d/proc/{instrument}/{cycle}/p{proposal}/r{run}
+    out-folder: /gpfs/exfel/d/proc/{instrument}/{cycle}/p{proposal}
 
 dark:
     in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw

xfel_calibrate/calibrate.py

@@ -667,6 +667,12 @@ def run():
         if try_report_to_output:
             if "out_folder" in args:
                 out_path = os.path.abspath(args["out_folder"])
+                if "run" in args:
+                    rr = args["run"]
+                    if isinstance(rr, int):
+                        out_path = "{}/r{:04d}/".format(out_path, rr)
+                    else:
+                        out_path = "{}/{}/".format(out_path, rr)
             else:
                 print("No 'out_folder' defined as argument, outputting to '{}' instead.".format(
                     out_path))