From 6f208d6c649733a360b055bb86b1760b2f3bf8df Mon Sep 17 00:00:00 2001 From: Karim Ahmed <karim.ahmed@xfel.eu> Date: Mon, 14 Oct 2019 17:40:30 +0200 Subject: [PATCH 1/6] enable turning a true bool to false --- webservice/update_config.py | 16 +++++++++++++--- webservice/webservice.py | 15 ++++++++++++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/webservice/update_config.py b/webservice/update_config.py index 77f6ffe25..33fdc1347 100644 --- a/webservice/update_config.py +++ b/webservice/update_config.py @@ -10,7 +10,8 @@ available_options = { "blc-noise": bool, "dont-zero-nans": bool, "dont-zero-orange": bool, - "max-pulses": int}, + "max-pulses": str, + "calfile": str}, } parser = argparse.ArgumentParser( @@ -25,7 +26,6 @@ required_args.add_argument('--instrument', type=str, "DETLAB"], help='The instrument') # noqa required_args.add_argument('--cycle', type=str, help='The facility cycle') parser.add_argument('--apply', action='store_true') - # remove help calls as they will cause the argument parser to exit add_help = False if "-h" in sys.argv: @@ -35,6 +35,14 @@ if "--help" in sys.argv: sys.argv.remove("--help") add_help = True +for det, val in available_options.items(): + bool_keys = [] + for k, v in val.items(): + if v == bool: + bool_keys.append(k) + for b in bool_keys: + available_options[det]['no-{}'.format(b)] = bool + known, remaining = parser.parse_known_args() args = vars(known) detector = args["detector"] @@ -61,6 +69,8 @@ new_conf = {task: {instrument: {detector: {}}}} for key, value in args.items(): key = key.replace("_", "-") if key in available_options[detector] and value is not None: + if 'no-' in key and isinstance(value, bool): + new_conf[task][instrument][detector][key.replace('no-','')] = False new_conf[task][instrument][detector][key] = value pyaml = yaml.dump(new_conf, default_flow_style=False) @@ -76,7 +86,7 @@ print(f"Sending the following update: \n {pyaml}") print("-" * 80) con = zmq.Context() socket = con.socket(zmq.REQ) -con = socket.connect("tcp://max-exfl016:5555") +con = socket.connect("tcp://max-exfl001:5555") msg = "','".join(["update_conf", "SASEX", args["instrument"], args["cycle"], args["proposal"], json.dumps(new_conf), str(args["apply"])]) socket.send("['{}']".format(msg).encode()) diff --git a/webservice/webservice.py b/webservice/webservice.py index 51e60a32a..76597931f 100644 --- a/webservice/webservice.py +++ b/webservice/webservice.py @@ -633,7 +633,11 @@ async def server_runner(config, mode): for key, value in dconfig.items(): if not isinstance(value, bool): - cmd += ["--{}".format(key), str(value)] + if " " in str(value): + cmd += ["--{}".format(key)] + cmd += [str(v) for v in value.split(" ")] + else: + cmd += ["--{}".format(key), str(value)] else: cmd += ["--{}".format(key)] ret = await run_correction(job_db, cmd, mode, proposal, @@ -683,9 +687,14 @@ async def server_runner(config, mode): detector, "CORRECT"] for key, value in dconfig.items(): if not isinstance(value, bool): - cmd += ["--{}".format(key), str(value)] + if " " in str(value): + cmd += ["--{}".format(key)] + cmd += [str(v) for v in value.split(" ")] + else: + cmd += ["--{}".format(key), str(value)] else: - cmd += ["--{}".format(key)] + if value: + cmd += ["--{}".format(key)] if priority: cmd += ["--priority", str(priority)] ret = await run_correction(job_db, cmd, mode, proposal, -- GitLab From 329d50cc5c974e89c94cb96dbea8644da0679b6f Mon Sep 17 00:00:00 2001 From: Karim Ahmed <karim.ahmed@xfel.eu> Date: Mon, 14 Oct 2019 17:44:41 +0200 Subject: [PATCH 2/6] remove some changes --- webservice/update_config.py | 2 +- webservice/webservice.py | 12 ++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/webservice/update_config.py b/webservice/update_config.py index 33fdc1347..9157e8d6c 100644 --- a/webservice/update_config.py +++ b/webservice/update_config.py @@ -86,7 +86,7 @@ print(f"Sending the following update: \n {pyaml}") print("-" * 80) con = zmq.Context() socket = con.socket(zmq.REQ) -con = socket.connect("tcp://max-exfl001:5555") +con = socket.connect("tcp://max-exfl016:5555") msg = "','".join(["update_conf", "SASEX", args["instrument"], args["cycle"], args["proposal"], json.dumps(new_conf), str(args["apply"])]) socket.send("['{}']".format(msg).encode()) diff --git a/webservice/webservice.py b/webservice/webservice.py index 76597931f..e53b4d0ae 100644 --- a/webservice/webservice.py +++ b/webservice/webservice.py @@ -633,11 +633,7 @@ async def server_runner(config, mode): for key, value in dconfig.items(): if not isinstance(value, bool): - if " " in str(value): - cmd += ["--{}".format(key)] - cmd += [str(v) for v in value.split(" ")] - else: - cmd += ["--{}".format(key), str(value)] + cmd += ["--{}".format(key), str(value)] else: cmd += ["--{}".format(key)] ret = await run_correction(job_db, cmd, mode, proposal, @@ -687,11 +683,7 @@ async def server_runner(config, mode): detector, "CORRECT"] for key, value in dconfig.items(): if not isinstance(value, bool): - if " " in str(value): - cmd += ["--{}".format(key)] - cmd += [str(v) for v in value.split(" ")] - else: - cmd += ["--{}".format(key), str(value)] + cmd += ["--{}".format(key), str(value)] else: if value: cmd += ["--{}".format(key)] -- GitLab From 94fa652b73d95b89c83211b56f5708a61807b1a9 Mon Sep 17 00:00:00 2001 From: Karim Ahmed <karim.ahmed@xfel.eu> Date: Tue, 15 Oct 2019 09:16:25 +0200 Subject: [PATCH 3/6] avoid saving no-arg --- webservice/update_config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/webservice/update_config.py b/webservice/update_config.py index 9157e8d6c..779b1e420 100644 --- a/webservice/update_config.py +++ b/webservice/update_config.py @@ -71,6 +71,8 @@ for key, value in args.items(): if key in available_options[detector] and value is not None: if 'no-' in key and isinstance(value, bool): new_conf[task][instrument][detector][key.replace('no-','')] = False + # avoid saving the "no-"key into the updated config + continue new_conf[task][instrument][detector][key] = value pyaml = yaml.dump(new_conf, default_flow_style=False) -- GitLab From d209427b9fa7f8c390eeb4da385d3acda92902c9 Mon Sep 17 00:00:00 2001 From: Karim Ahmed <karim.ahmed@xfel.eu> Date: Fri, 25 Oct 2019 15:19:27 +0200 Subject: [PATCH 4/6] adding the -no option to calibrate.py --- xfel_calibrate/calibrate.py | 85 +++++++++++++++++++++---------------- 1 file changed, 49 insertions(+), 36 deletions(-) diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py index 8e73594a2..717c205b9 100755 --- a/xfel_calibrate/calibrate.py +++ b/xfel_calibrate/calibrate.py @@ -23,6 +23,7 @@ import textwrap from cal_tools.tools import tex_escape + # Add a class combining raw description formatting with # Metavariable default outputs class RawTypeFormatter(argparse.RawDescriptionHelpFormatter, @@ -43,7 +44,7 @@ def make_initial_parser(): help='The detector to calibrate') parser.add_argument('type', metavar='TYPE', type=str, - help='Type of calibration: '+",".join(notebooks.keys())) + help='Type of calibration: ' + ",".join(notebooks.keys())) parser.add_argument('--no-cluster-job', action="store_true", @@ -68,6 +69,7 @@ def make_initial_parser(): parser = make_initial_parser() + # Helper functions for parser extensions def make_intelli_list(ltype): @@ -86,17 +88,20 @@ def make_intelli_list(ltype): parsed_values = [] values = ",".join(values) - if isinstance(values, str): - for rcomp in values.split(","): - if "-" in rcomp: - start, end = rcomp.split("-") - parsed_values += list(range(int(start), int(end))) - else: - parsed_values += [int(rcomp)] - elif isinstance(values, (list, tuple)): - parsed_values = values - else: - parsed_values = [values, ] + try: + if isinstance(values, str): + for rcomp in values.split(","): + if "-" in rcomp: + start, end = rcomp.split("-") + parsed_values += list(range(int(start), int(end))) + else: + parsed_values += [int(rcomp)] + elif isinstance(values, (list, tuple)): + parsed_values = values + else: + parsed_values = [values, ] + except Exception as e: + print('ERROR:', e) parsed_values = [self.ltype(p) for p in parsed_values] print("Parsed input {} to {}".format(values, parsed_values)) setattr(namespace, self.dest, parsed_values) @@ -193,7 +198,7 @@ def make_epilog(nb, caltype=None): plines = pp.pformat(lines[1:])[1:-1].split("\n") for line in plines: sline = line.replace("'", "", 1) - sline = sline.replace("', '", " "*(17 if caltype else 0), 1) + sline = sline.replace("', '", " " * (17 if caltype else 0), 1) sline = sline[::-1].replace("'", "", 1)[::-1] sline = sline.replace(" ,", " ") if len(sline) > 1 and sline[0] == ",": @@ -243,7 +248,7 @@ if len(sys.argv) == 3 and "-h" in sys.argv[2]: exit() msg = "Options for detector {}\n".format(detector) - msg += "*"*len(msg)+"\n\n" + msg += "*" * len(msg) + "\n\n" # basically, this creates help in the form of # @@ -285,6 +290,7 @@ elif len(sys.argv) >= 3: ext_func = notebooks[detector][caltype].get("extend parms", None) + def do_parse(nb, parser, overwrite_reqs=False): parser.description = make_epilog(nb) parms = extract_parameters(nb) @@ -326,12 +332,23 @@ elif len(sys.argv) >= 3: required=required, action=make_intelli_list(ltype) if range_allowed else None) elif p.type == bool: - pars_group.add_argument("--{}".format(consolize_name(p.name)), - action="store_true", - default=default, - help=helpstr, - required=required) - + # check if an input arg is given with an extra "-no" for + # forcing to convert a bool to False. + # Otherwise leave the default value from the notebook + # or convert to true if the bool arg is given. + if consolize_name("--no-{}".format(p.name)) in sys.argv: + pars_group.add_argument("--{}".format(consolize_name(p.name)), + action="store_false", + default=False, + help=helpstr, + required=required) + sys.argv.remove(consolize_name("--no-{}".format(p.name))) + else: + pars_group.add_argument("--{}".format(consolize_name(p.name)), + action="store_true", + default=default, + help=helpstr, + required=required) else: pars_group.add_argument("--{}".format(consolize_name(p.name)), type=p.type, @@ -376,7 +393,7 @@ elif len(sys.argv) >= 3: extention = f(*callargs) fcc = first_code_cell(nb) - fcc["source"] += "\n"+extention + fcc["source"] += "\n" + extention parser = make_initial_parser() do_parse(nb, parser, False) @@ -608,7 +625,6 @@ def run(): if func is None: warnings.warn("Didn't find concurrency function {} in notebook".format(ext_func), RuntimeWarning) - else: # remove help calls as they will cause the argument parser to exit known, remaining = parser.parse_known_args() @@ -624,7 +640,7 @@ def run(): extention = f(*callargs) fcc = first_code_cell(nb) - fcc["source"] += "\n"+extention + fcc["source"] += "\n" + extention parms = extract_parameters(nb) @@ -642,14 +658,14 @@ def run(): run_uuid = uuid4() # check that a modules field is present if we run concurrently - if not has_parm(parms, concurrency["parameter"]) and concurrency["parameter"] is not None: + if not has_parm(parms, concurrency["parameter"]) and concurrency["parameter"] is not None: msg = "Notebook cannot be run concurrently: no {} parameter".format( concurrency["parameter"]) warnings.warn(msg, RuntimeWarning) if not has_parm(parms, "cluster_profile"): warnings.warn("Notebook has no cluster_profile parameter, " + - "running on cluster will likeyl fail!", RuntimeWarning) + "running on cluster will likely fail!", RuntimeWarning) elif "cluster_profile" not in args or args["cluster_profile"] == parser.get_default('cluster_profile'): args["cluster_profile"] = "slurm_prof_{}".format(run_uuid) @@ -740,16 +756,14 @@ def run(): if func is None: warnings.warn("Didn't find concurrency function {} in notebook".format(con_func), RuntimeWarning) - else: df = {} - exec(func, df) + exec (func, df) f = df[con_func] import inspect sig = inspect.signature(f) callargs = [] if cvals: - # in case default needs to be used for function call args[cvar] = cvals for arg in sig.parameters: @@ -769,7 +783,7 @@ def run(): # Job is not final if there are dependent notebooks jobid = concurrent_run(run_tmp_path, nb, notebook, args, cvar, [cval, ] if not isinstance( - cval, list) and cvtype is list else cval, + cval, list) and cvtype is list else cval, cnum == len(list(cvals)) - 1 and len(dep_notebooks) == 0, joblist, fmtcmd, @@ -777,7 +791,6 @@ def run(): show_title=show_title, ureservation=ureservation) joblist.append(jobid) - # Run dependent notebooks for i, notebook in enumerate(dep_notebooks): notebook_path = os.path.abspath( @@ -786,12 +799,12 @@ def run(): nb = nbformat.read(f, as_version=4) final_job = i == len(dep_notebooks) - 1 jobid = concurrent_run(run_tmp_path, nb, os.path.basename(notebook), - args, - final_job=final_job, - job_list=joblist, fmtcmd=fmtcmd, - cluster_cores=cluster_cores, - sequential=sequential, priority=priority, - dependent=True, ureservation=ureservation) + args, + final_job=final_job, + job_list=joblist, fmtcmd=fmtcmd, + cluster_cores=cluster_cores, + sequential=sequential, priority=priority, + dependent=True, ureservation=ureservation) joblist.append(jobid) if not all([j is None for j in joblist]): -- GitLab From b48534f3066b7a8c65aae8c4f7ea17122dffcd05 Mon Sep 17 00:00:00 2001 From: Karim Ahmed <karim.ahmed@xfel.eu> Date: Fri, 25 Oct 2019 15:22:15 +0200 Subject: [PATCH 5/6] remove space --- xfel_calibrate/calibrate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py index 717c205b9..ec2aa7835 100755 --- a/xfel_calibrate/calibrate.py +++ b/xfel_calibrate/calibrate.py @@ -758,7 +758,7 @@ def run(): RuntimeWarning) else: df = {} - exec (func, df) + exec(func, df) f = df[con_func] import inspect sig = inspect.signature(f) -- GitLab From 44513be0fa976b42c4940e27553068a818ac918c Mon Sep 17 00:00:00 2001 From: Karim Ahmed <karim.ahmed@xfel.eu> Date: Mon, 28 Oct 2019 08:32:12 +0100 Subject: [PATCH 6/6] revert max-pulses in available options change --- webservice/update_config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/webservice/update_config.py b/webservice/update_config.py index 779b1e420..be928ee31 100644 --- a/webservice/update_config.py +++ b/webservice/update_config.py @@ -10,8 +10,7 @@ available_options = { "blc-noise": bool, "dont-zero-nans": bool, "dont-zero-orange": bool, - "max-pulses": str, - "calfile": str}, + "max-pulses": int}, } parser = argparse.ArgumentParser( -- GitLab