From 6f208d6c649733a360b055bb86b1760b2f3bf8df Mon Sep 17 00:00:00 2001
From: Karim Ahmed <karim.ahmed@xfel.eu>
Date: Mon, 14 Oct 2019 17:40:30 +0200
Subject: [PATCH 1/6] enable turning a true bool to false

---
 webservice/update_config.py | 16 +++++++++++++---
 webservice/webservice.py    | 15 ++++++++++++---
 2 files changed, 25 insertions(+), 6 deletions(-)

diff --git a/webservice/update_config.py b/webservice/update_config.py
index 77f6ffe25..33fdc1347 100644
--- a/webservice/update_config.py
+++ b/webservice/update_config.py
@@ -10,7 +10,8 @@ available_options = {
               "blc-noise": bool,
               "dont-zero-nans": bool,
               "dont-zero-orange": bool,
-              "max-pulses": int},
+              "max-pulses": str,
+              "calfile": str},
 }
 
 parser = argparse.ArgumentParser(
@@ -25,7 +26,6 @@ required_args.add_argument('--instrument', type=str,
                              "DETLAB"], help='The instrument')  # noqa
 required_args.add_argument('--cycle', type=str, help='The facility cycle')
 parser.add_argument('--apply', action='store_true')
-
 # remove help calls as they will cause the argument parser to exit
 add_help = False
 if "-h" in sys.argv:
@@ -35,6 +35,14 @@ if "--help" in sys.argv:
     sys.argv.remove("--help")
     add_help = True
 
+for det, val in available_options.items():
+    bool_keys = []
+    for k, v in val.items():
+        if v == bool:
+            bool_keys.append(k)
+    for b in bool_keys:
+        available_options[det]['no-{}'.format(b)] = bool
+
 known, remaining = parser.parse_known_args()
 args = vars(known)
 detector = args["detector"]
@@ -61,6 +69,8 @@ new_conf = {task: {instrument: {detector: {}}}}
 for key, value in args.items():
     key = key.replace("_", "-")
     if key in available_options[detector] and value is not None:
+        if 'no-' in  key and isinstance(value, bool):
+            new_conf[task][instrument][detector][key.replace('no-','')] = False
         new_conf[task][instrument][detector][key] = value
 
 pyaml = yaml.dump(new_conf, default_flow_style=False)
@@ -76,7 +86,7 @@ print(f"Sending the following update: \n {pyaml}")
 print("-" * 80)
 con = zmq.Context()
 socket = con.socket(zmq.REQ)
-con = socket.connect("tcp://max-exfl016:5555")
+con = socket.connect("tcp://max-exfl001:5555")
 msg = "','".join(["update_conf", "SASEX", args["instrument"], args["cycle"],
                   args["proposal"], json.dumps(new_conf), str(args["apply"])])
 socket.send("['{}']".format(msg).encode())
diff --git a/webservice/webservice.py b/webservice/webservice.py
index 51e60a32a..76597931f 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -633,7 +633,11 @@ async def server_runner(config, mode):
 
                     for key, value in dconfig.items():
                         if not isinstance(value, bool):
-                            cmd += ["--{}".format(key), str(value)]
+                            if " " in str(value):
+                                cmd += ["--{}".format(key)]
+                                cmd += [str(v) for v in value.split(" ")]
+                            else:
+                                cmd += ["--{}".format(key), str(value)]
                         else:
                             cmd += ["--{}".format(key)]
                     ret = await run_correction(job_db, cmd, mode, proposal,
@@ -683,9 +687,14 @@ async def server_runner(config, mode):
                            detector, "CORRECT"]
                     for key, value in dconfig.items():
                         if not isinstance(value, bool):
-                            cmd += ["--{}".format(key), str(value)]
+                            if " " in str(value):
+                                cmd += ["--{}".format(key)]
+                                cmd += [str(v) for v in value.split(" ")]
+                            else:
+                                cmd += ["--{}".format(key), str(value)]
                         else:
-                            cmd += ["--{}".format(key)]
+                            if value:
+                                cmd += ["--{}".format(key)]
                     if priority:
                         cmd += ["--priority", str(priority)]
                     ret = await run_correction(job_db, cmd, mode, proposal,
-- 
GitLab


From 329d50cc5c974e89c94cb96dbea8644da0679b6f Mon Sep 17 00:00:00 2001
From: Karim Ahmed <karim.ahmed@xfel.eu>
Date: Mon, 14 Oct 2019 17:44:41 +0200
Subject: [PATCH 2/6] remove some changes

---
 webservice/update_config.py |  2 +-
 webservice/webservice.py    | 12 ++----------
 2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/webservice/update_config.py b/webservice/update_config.py
index 33fdc1347..9157e8d6c 100644
--- a/webservice/update_config.py
+++ b/webservice/update_config.py
@@ -86,7 +86,7 @@ print(f"Sending the following update: \n {pyaml}")
 print("-" * 80)
 con = zmq.Context()
 socket = con.socket(zmq.REQ)
-con = socket.connect("tcp://max-exfl001:5555")
+con = socket.connect("tcp://max-exfl016:5555")
 msg = "','".join(["update_conf", "SASEX", args["instrument"], args["cycle"],
                   args["proposal"], json.dumps(new_conf), str(args["apply"])])
 socket.send("['{}']".format(msg).encode())
diff --git a/webservice/webservice.py b/webservice/webservice.py
index 76597931f..e53b4d0ae 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -633,11 +633,7 @@ async def server_runner(config, mode):
 
                     for key, value in dconfig.items():
                         if not isinstance(value, bool):
-                            if " " in str(value):
-                                cmd += ["--{}".format(key)]
-                                cmd += [str(v) for v in value.split(" ")]
-                            else:
-                                cmd += ["--{}".format(key), str(value)]
+                            cmd += ["--{}".format(key), str(value)]
                         else:
                             cmd += ["--{}".format(key)]
                     ret = await run_correction(job_db, cmd, mode, proposal,
@@ -687,11 +683,7 @@ async def server_runner(config, mode):
                            detector, "CORRECT"]
                     for key, value in dconfig.items():
                         if not isinstance(value, bool):
-                            if " " in str(value):
-                                cmd += ["--{}".format(key)]
-                                cmd += [str(v) for v in value.split(" ")]
-                            else:
-                                cmd += ["--{}".format(key), str(value)]
+                            cmd += ["--{}".format(key), str(value)]
                         else:
                             if value:
                                 cmd += ["--{}".format(key)]
-- 
GitLab


From 94fa652b73d95b89c83211b56f5708a61807b1a9 Mon Sep 17 00:00:00 2001
From: Karim Ahmed <karim.ahmed@xfel.eu>
Date: Tue, 15 Oct 2019 09:16:25 +0200
Subject: [PATCH 3/6] avoid saving no-arg

---
 webservice/update_config.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/webservice/update_config.py b/webservice/update_config.py
index 9157e8d6c..779b1e420 100644
--- a/webservice/update_config.py
+++ b/webservice/update_config.py
@@ -71,6 +71,8 @@ for key, value in args.items():
     if key in available_options[detector] and value is not None:
         if 'no-' in  key and isinstance(value, bool):
             new_conf[task][instrument][detector][key.replace('no-','')] = False
+            # avoid saving the "no-"key into the updated config
+            continue
         new_conf[task][instrument][detector][key] = value
 
 pyaml = yaml.dump(new_conf, default_flow_style=False)
-- 
GitLab


From d209427b9fa7f8c390eeb4da385d3acda92902c9 Mon Sep 17 00:00:00 2001
From: Karim Ahmed <karim.ahmed@xfel.eu>
Date: Fri, 25 Oct 2019 15:19:27 +0200
Subject: [PATCH 4/6] adding the -no option to calibrate.py

---
 xfel_calibrate/calibrate.py | 85 +++++++++++++++++++++----------------
 1 file changed, 49 insertions(+), 36 deletions(-)

diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index 8e73594a2..717c205b9 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -23,6 +23,7 @@ import textwrap
 
 from cal_tools.tools import tex_escape
 
+
 # Add a class combining raw description formatting with
 # Metavariable default outputs
 class RawTypeFormatter(argparse.RawDescriptionHelpFormatter,
@@ -43,7 +44,7 @@ def make_initial_parser():
                         help='The detector to calibrate')
 
     parser.add_argument('type', metavar='TYPE', type=str,
-                        help='Type of calibration: '+",".join(notebooks.keys()))
+                        help='Type of calibration: ' + ",".join(notebooks.keys()))
 
     parser.add_argument('--no-cluster-job',
                         action="store_true",
@@ -68,6 +69,7 @@ def make_initial_parser():
 
 parser = make_initial_parser()
 
+
 # Helper functions for parser extensions
 
 def make_intelli_list(ltype):
@@ -86,17 +88,20 @@ def make_intelli_list(ltype):
 
             parsed_values = []
             values = ",".join(values)
-            if isinstance(values, str):
-                for rcomp in values.split(","):
-                    if "-" in rcomp:
-                        start, end = rcomp.split("-")
-                        parsed_values += list(range(int(start), int(end)))
-                    else:
-                        parsed_values += [int(rcomp)]
-            elif isinstance(values, (list, tuple)):
-                parsed_values = values
-            else:
-                parsed_values = [values, ]
+            try:
+                if isinstance(values, str):
+                    for rcomp in values.split(","):
+                        if "-" in rcomp:
+                            start, end = rcomp.split("-")
+                            parsed_values += list(range(int(start), int(end)))
+                        else:
+                            parsed_values += [int(rcomp)]
+                elif isinstance(values, (list, tuple)):
+                    parsed_values = values
+                else:
+                    parsed_values = [values, ]
+            except Exception as e:
+                print('ERROR:', e)
             parsed_values = [self.ltype(p) for p in parsed_values]
             print("Parsed input {} to {}".format(values, parsed_values))
             setattr(namespace, self.dest, parsed_values)
@@ -193,7 +198,7 @@ def make_epilog(nb, caltype=None):
         plines = pp.pformat(lines[1:])[1:-1].split("\n")
         for line in plines:
             sline = line.replace("'", "", 1)
-            sline = sline.replace("', '", " "*(17 if caltype else 0), 1)
+            sline = sline.replace("', '", " " * (17 if caltype else 0), 1)
             sline = sline[::-1].replace("'", "", 1)[::-1]
             sline = sline.replace(" ,", " ")
             if len(sline) > 1 and sline[0] == ",":
@@ -243,7 +248,7 @@ if len(sys.argv) == 3 and "-h" in sys.argv[2]:
         exit()
 
     msg = "Options for detector {}\n".format(detector)
-    msg += "*"*len(msg)+"\n\n"
+    msg += "*" * len(msg) + "\n\n"
 
     # basically, this creates help in the form of
     #
@@ -285,6 +290,7 @@ elif len(sys.argv) >= 3:
 
         ext_func = notebooks[detector][caltype].get("extend parms", None)
 
+
         def do_parse(nb, parser, overwrite_reqs=False):
             parser.description = make_epilog(nb)
             parms = extract_parameters(nb)
@@ -326,12 +332,23 @@ elif len(sys.argv) >= 3:
                                             required=required,
                                             action=make_intelli_list(ltype) if range_allowed else None)
                 elif p.type == bool:
-                    pars_group.add_argument("--{}".format(consolize_name(p.name)),
-                                            action="store_true",
-                                            default=default,
-                                            help=helpstr,
-                                            required=required)
-
+                    # check if an input arg is given with an extra "-no" for
+                    # forcing to convert a bool to False.
+                    # Otherwise leave the default value from the notebook
+                    # or convert to true if the bool arg is given.
+                    if consolize_name("--no-{}".format(p.name)) in sys.argv:
+                        pars_group.add_argument("--{}".format(consolize_name(p.name)),
+                                                action="store_false",
+                                                default=False,
+                                                help=helpstr,
+                                                required=required)
+                        sys.argv.remove(consolize_name("--no-{}".format(p.name)))
+                    else:
+                        pars_group.add_argument("--{}".format(consolize_name(p.name)),
+                                                action="store_true",
+                                                default=default,
+                                                help=helpstr,
+                                                required=required)
                 else:
                     pars_group.add_argument("--{}".format(consolize_name(p.name)),
                                             type=p.type,
@@ -376,7 +393,7 @@ elif len(sys.argv) >= 3:
 
                 extention = f(*callargs)
                 fcc = first_code_cell(nb)
-                fcc["source"] += "\n"+extention
+                fcc["source"] += "\n" + extention
                 parser = make_initial_parser()
                 do_parse(nb, parser, False)
 
@@ -608,7 +625,6 @@ def run():
             if func is None:
                 warnings.warn("Didn't find concurrency function {} in notebook".format(ext_func),
                               RuntimeWarning)
-
             else:
                 # remove help calls as they will cause the argument parser to exit
                 known, remaining = parser.parse_known_args()
@@ -624,7 +640,7 @@ def run():
 
                 extention = f(*callargs)
                 fcc = first_code_cell(nb)
-                fcc["source"] += "\n"+extention
+                fcc["source"] += "\n" + extention
 
         parms = extract_parameters(nb)
 
@@ -642,14 +658,14 @@ def run():
         run_uuid = uuid4()
 
         # check that a modules field is present if we run concurrently
-        if not has_parm(parms,  concurrency["parameter"]) and concurrency["parameter"] is not None:
+        if not has_parm(parms, concurrency["parameter"]) and concurrency["parameter"] is not None:
             msg = "Notebook cannot be run concurrently: no {} parameter".format(
                 concurrency["parameter"])
             warnings.warn(msg, RuntimeWarning)
 
         if not has_parm(parms, "cluster_profile"):
             warnings.warn("Notebook has no cluster_profile parameter, " +
-                          "running on cluster will likeyl fail!", RuntimeWarning)
+                          "running on cluster will likely fail!", RuntimeWarning)
         elif "cluster_profile" not in args or args["cluster_profile"] == parser.get_default('cluster_profile'):
             args["cluster_profile"] = "slurm_prof_{}".format(run_uuid)
 
@@ -740,16 +756,14 @@ def run():
                 if func is None:
                     warnings.warn("Didn't find concurrency function {} in notebook".format(con_func),
                                   RuntimeWarning)
-
                 else:
                     df = {}
-                    exec(func, df)
+                    exec (func, df)
                     f = df[con_func]
                     import inspect
                     sig = inspect.signature(f)
                     callargs = []
                     if cvals:
-
                         # in case default needs to be used for function call
                         args[cvar] = cvals
                     for arg in sig.parameters:
@@ -769,7 +783,7 @@ def run():
                 # Job is not final if there are dependent notebooks
                 jobid = concurrent_run(run_tmp_path, nb, notebook, args,
                                        cvar, [cval, ] if not isinstance(
-                                           cval, list) and cvtype is list else cval,
+                        cval, list) and cvtype is list else cval,
                                        cnum == len(list(cvals)) -
                                        1 and len(dep_notebooks) == 0,
                                        joblist, fmtcmd,
@@ -777,7 +791,6 @@ def run():
                                        show_title=show_title, ureservation=ureservation)
                 joblist.append(jobid)
 
-
         # Run dependent notebooks
         for i, notebook in enumerate(dep_notebooks):
             notebook_path = os.path.abspath(
@@ -786,12 +799,12 @@ def run():
                 nb = nbformat.read(f, as_version=4)
                 final_job = i == len(dep_notebooks) - 1
                 jobid = concurrent_run(run_tmp_path, nb, os.path.basename(notebook),
-                                   args,
-                                   final_job=final_job,
-                                   job_list=joblist, fmtcmd=fmtcmd,
-                                   cluster_cores=cluster_cores,
-                                   sequential=sequential, priority=priority,
-                                   dependent=True, ureservation=ureservation)
+                                       args,
+                                       final_job=final_job,
+                                       job_list=joblist, fmtcmd=fmtcmd,
+                                       cluster_cores=cluster_cores,
+                                       sequential=sequential, priority=priority,
+                                       dependent=True, ureservation=ureservation)
                 joblist.append(jobid)
 
         if not all([j is None for j in joblist]):
-- 
GitLab


From b48534f3066b7a8c65aae8c4f7ea17122dffcd05 Mon Sep 17 00:00:00 2001
From: Karim Ahmed <karim.ahmed@xfel.eu>
Date: Fri, 25 Oct 2019 15:22:15 +0200
Subject: [PATCH 5/6] remove space

---
 xfel_calibrate/calibrate.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index 717c205b9..ec2aa7835 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -758,7 +758,7 @@ def run():
                                   RuntimeWarning)
                 else:
                     df = {}
-                    exec (func, df)
+                    exec(func, df)
                     f = df[con_func]
                     import inspect
                     sig = inspect.signature(f)
-- 
GitLab


From 44513be0fa976b42c4940e27553068a818ac918c Mon Sep 17 00:00:00 2001
From: Karim Ahmed <karim.ahmed@xfel.eu>
Date: Mon, 28 Oct 2019 08:32:12 +0100
Subject: [PATCH 6/6] revert max-pulses in available options change

---
 webservice/update_config.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/webservice/update_config.py b/webservice/update_config.py
index 779b1e420..be928ee31 100644
--- a/webservice/update_config.py
+++ b/webservice/update_config.py
@@ -10,8 +10,7 @@ available_options = {
               "blc-noise": bool,
               "dont-zero-nans": bool,
               "dont-zero-orange": bool,
-              "max-pulses": str,
-              "calfile": str},
+              "max-pulses": int},
 }
 
 parser = argparse.ArgumentParser(
-- 
GitLab