diff --git a/webservice/update_config.py b/webservice/update_config.py
index 917f252f5e3ddbbea89c6880d235bd34f288f970..0d3cf58f1f45be6e51be83eaf31b6046aac43805 100644
--- a/webservice/update_config.py
+++ b/webservice/update_config.py
@@ -5,31 +5,48 @@ import sys
import yaml
import zmq
+
+# Defining available options
+agipd_options = {
+ "force-hg-if-below": {'typ': int},
+ "rel-gain": {'typ': bool},
+ "xray-gain": {'typ': bool},
+ "blc-noise": {'typ': bool},
+ "blc-set-min": {'typ': bool},
+ "dont-zero-nans": {'typ': bool},
+ "dont-zero-orange": {'typ': bool},
+ "max-pulses": {'typ': list,
+ 'msg': "Range list of maximum pulse indices "
+ "(--max-pulses start end step). "
+ "3 max input elements. "},
+ "calfile": {'typ': str},
+ }
+
+data_mapping = {
+ "karabo-da": {'typ': list,
+ 'choices': ["AGIPD{:02d}".format(i) for i in range(16)],
+ 'msg': "Choices: [AGIPD00 ... AGIPD15]. "}
+}
+
available_options = {
- "AGIPD": {"force-hg-if-below": float,
- "rel_gain": bool,
- "xray_gain": bool,
- "blc-noise": bool,
- "blc-stripes": bool,
- "blc-set-min": bool,
- "dont-zero-nans": bool,
- "dont-zero-orange": bool,
- "max-pulses": list,
- "calfile": str},
+ "SPB_DET_AGIPD1M-1": [agipd_options, data_mapping],
+ "MID_DET_AGIPD1M-1": [agipd_options, data_mapping]
}
+formatter = lambda prog: argparse.HelpFormatter(prog, max_help_position=52)
parser = argparse.ArgumentParser(
- description='Request update of configuration')
+ description='Request update of configuration', formatter_class=formatter)
required_args = parser.add_argument_group('required arguments')
-required_args.add_argument('--detector', type=str, choices=['AGIPD'])
-required_args.add_argument('--task', type=str, choices=['correct', 'dark'])
+required_args.add_argument('--karabo-id', type=str,
+ choices=['SPB_DET_AGIPD1M-1',
+ 'MID_DET_AGIPD1M-1'])
required_args.add_argument('--proposal', type=str,
- help='The proposal number, without leading p, but with leading zeros') # noqa
-required_args.add_argument('--instrument', type=str,
- choices=["SPB", "MID", "FXE", "SCS", "SQS", "HED",
- "DETLAB"], help='The instrument') # noqa
-required_args.add_argument('--cycle', type=str, help='The facility cycle')
-parser.add_argument('--apply', action='store_true')
+ help='The proposal number, without leading p, '
+ 'but with leading zeros.')
+required_args.add_argument('--cycle', type=str, help='The facility cycle.')
+parser.add_argument('--apply', action='store_true',
+ help='Apply and push the requested '
+ 'configuration update to the git.')
# remove help calls as they will cause the argument parser to exit
add_help = False
if "-h" in sys.argv:
@@ -39,60 +56,97 @@ if "--help" in sys.argv:
sys.argv.remove("--help")
add_help = True
-for det, val in available_options.items():
- bool_keys = []
- for k, v in val.items():
- if v == bool:
- bool_keys.append(k)
- for b in bool_keys:
- available_options[det]['no-{}'.format(b)] = bool
-
known, remaining = parser.parse_known_args()
args = vars(known)
-detector = args["detector"]
+karabo_id = args["karabo_id"]
+bool_keys = []
+
-if detector is not None:
- for option, typ in available_options[detector].items():
- if typ == list:
- nargs = '+'
- else:
- nargs = None
- parser.add_argument(f"--{option}", type=typ, nargs=nargs)
+# Avoid erros when karabo_id not yet given through the command line.
+if karabo_id is not None:
+ # adding "no" bools to available options
+ for det, val in available_options[karabo_id][0].items():
+ if val['typ'] == bool:
+ bool_keys.append(det)
+
+ for b in bool_keys:
+ available_options[karabo_id][0]['no-{}'.format(det)] = {'typ': bool}
+
+ for exposed_options in available_options[karabo_id]:
+ for option, info in exposed_options.items():
+ metavar = None
+ if info['typ'] == list:
+ nargs = '+'
+ typ = str
+ action = 'append'
+ # Avoid having a big line of choices in the help message.
+ if 'choices' in info.keys():
+ metavar = option.upper()
+ else:
+ action = None
+ nargs = None
+ typ = info['typ']
+ # Add help messages
+ help_msg = ""
+ if 'msg' in info.keys():
+ help_msg += info['msg']
+ if 'choices' in info.keys():
+ choices = info['choices']
+ else:
+ choices = None
+ help_msg += f"Type: {info['typ'].__name__} ".upper()
+ parser.add_argument(f"--{option}", type=typ, action=action,
+ metavar=metavar, nargs=nargs, choices=choices,
+ help=help_msg)
+
+parser.add_argument('--instrument', type=str, choices=["CALLAB"],
+ help='This is only used for testing purposes.')
if add_help:
sys.argv.append("--help")
args = vars(parser.parse_args())
-task = args['task']
-instrument = args['instrument']
+task = "correct"
+# check if instrument is not given from CLI (e.g. CALLAB)
+if args['instrument'] is None:
+ # extract instrument from karabo_id
+ instrument = karabo_id.split("_")[0]
+else:
+ instrument = args['instrument']
+
proposal = args['proposal']
cycle = args['cycle']
-if task is None or instrument is None or proposal is None or cycle is None:
+if instrument is None or proposal is None or cycle is None:
print("Need to define all fields")
exit()
-new_conf = {task: {instrument: {detector: {}}}}
+new_conf = {task: {instrument: {karabo_id: {}}}}
for key, value in args.items():
key = key.replace("_", "-")
- if key in available_options[detector] and value is not None:
-
- if isinstance(value, list):
- for v in value:
- value[value.index(v)] = ''.join(v)
-
- if 'no-' in key and isinstance(value, bool):
- if key not in bool_keys:
- new_conf[task][instrument][detector][key.replace('no-','')] = False #noqa
- # avoid saving the "no-"key into the updated config
- continue
- # Assure adding an empty string for new empty
- # str. updates (e.g. calfile)
- if isinstance(key, str) and (value == '' or value == ' '):
- value = '""'
-
- new_conf[task][instrument][detector][key] = value
+ for exposed_options in available_options[karabo_id]:
+ if key in exposed_options and value is not None:
+
+ if isinstance(value, list):
+ value = value[0]
+ # convert no arguments to bool false
+ if 'no-' in key and isinstance(value, bool):
+ if key not in bool_keys:
+ new_conf[task][instrument][karabo_id][key.replace('no-', '')] = False # noqa
+ # avoid saving the "no-"key into the updated config
+ continue
+ # Assure adding an empty string for new empty
+ # str. updates (e.g. calfile)
+ if isinstance(key, str) and (value == '' or value == ' '):
+ value = '""'
+ # checking if data-mapping was updated.
+ if key in data_mapping.keys():
+ if 'data-mapping' not in new_conf.keys():
+ new_conf['data-mapping'] = {karabo_id: {key: {}}}
+ new_conf['data-mapping'][karabo_id][key] = value
+ else:
+ new_conf[task][instrument][karabo_id][key] = value
pyaml = yaml.dump(new_conf, default_flow_style=False)
@@ -108,8 +162,9 @@ print("-" * 80)
con = zmq.Context()
socket = con.socket(zmq.REQ)
con = socket.connect("tcp://max-exfl016:5555")
-msg = "','".join(["update_conf", "SASEX", args["instrument"], args["cycle"],
- args["proposal"], json.dumps(new_conf), str(args["apply"])])
+msg = "','".join(["update_conf", "SASEX", args["karabo_id"],
+ instrument, args["cycle"], args["proposal"],
+ json.dumps(new_conf), str(args["apply"])])
socket.send("['{}']".format(msg).encode())
resp = socket.recv_multipart()[0]
print("Configuration now in place is:")
diff --git a/webservice/webservice.py b/webservice/webservice.py
index b4eb0841c7eb1c1b4e2752138f4324dfcca0cb5d..faa9d8b0ab58fdc9fe5aa09431f5d52b4fe7076a 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -10,6 +10,7 @@ import sqlite3
import subprocess
import urllib.parse
from datetime import datetime
+import traceback
import yaml
import zmq
@@ -128,6 +129,9 @@ def merge(source, destination):
"""
for key, value in source.items():
if isinstance(value, dict):
+ # check if destination (e.g. karabo-id) has none value
+ if key in destination.keys() and destination[key] is None:
+ destination[key] = {}
# get node or create one
node = destination.setdefault(key, {})
merge(value, node)
@@ -137,7 +141,7 @@ def merge(source, destination):
return destination
-async def change_config(socket, config, updated_config, instrument, cycle,
+async def change_config(socket, config, updated_config, karabo_id, instrument, cycle,
proposal, apply=False):
"""
Change the configuration of a proposal
@@ -169,7 +173,10 @@ async def change_config(socket, config, updated_config, instrument, cycle,
subconf = {}
for action, instruments in defconf.items():
subconf[action]= {}
- subconf[action][instrument] = instruments[instrument]
+ if action != "data-mapping":
+ subconf[action][instrument] = instruments[instrument]
+ else:
+ subconf[action][instrument] = instruments[karabo_id]
with open(fpath, "w") as wf:
wf.write(yaml.dump(subconf, default_flow_style=False))
new_conf = None
@@ -571,16 +578,17 @@ async def server_runner(config, mode):
if action in ['update_conf']:
updated_config = None
try:
- sase, instrument, cycle, proposal, config_yaml, apply = payload # noqa
+ sase, karabo_id, instrument, cycle, proposal, config_yaml, apply = payload # noqa
updated_config = json.loads(config_yaml)
await change_config(socket, config['config-repo'],
- updated_config, instrument, cycle,
+ updated_config, karabo_id, instrument, cycle,
proposal, apply.upper()=="TRUE")
except Exception as e:
e = str(e)
err_msg = f"Failure applying config for {proposal}:" + \
f" {e}: {updated_config}"
logging.error(err_msg)
+ logging.error(f"Unexpected error: {traceback.format_exc()}")
socket.send(yaml.dump(err_msg, default_flow_style=False).encode())
if action in ['dark', 'correct']:
@@ -777,7 +785,7 @@ async def server_runner(config, mode):
logging.error(Errors.MDC_RESPONSE.format(response))
return
except Exception as corr_e:
- logging.error(f"Correct Error: {corr_e}")
+ logging.error(f"Error during correction: {str(corr_e)}")
response = mdc.update_run_api(rid, {
'flg_cal_data_status': 'NA',
'cal_pipeline_reply': Errors.REQUEST_FAILED})