diff --git a/notebooks/generic/DB_Constants_to_HDF5_NBC.ipynb b/notebooks/generic/DB_Constants_to_HDF5_NBC.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..671479d9cdef689c10e7abe18c0e27bd29beb2ed
--- /dev/null
+++ b/notebooks/generic/DB_Constants_to_HDF5_NBC.ipynb
@@ -0,0 +1,271 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Constants from DB to HDF5 #\n",
+ "\n",
+ "Version 0.1, Author: S. Hauf"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "detector_instance = \"LPD1M1\" # the detector instance to get constants for, required\n",
+ "out_file = \"/gpfs/exfel/data/scratch/haufs/test/test.h5\" # HDF5 file to output constants into, required\n",
+ "valid_at = \"\" # ISO formatted date for which constants shoudl be valid. Leave empty to get most current ones\n",
+ "cal_db_interface = \"tcp://max-exfl015:5005\"\n",
+ "modules = [-1] # modules to get data from, in terms of numerical quadrant indices, range allowed"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "dtype = \"LPD\" if \"LPD\" in detector_instance.upper() else \"AGIPD\"\n",
+ "darkconst = [\"Offset\", \"Noise\", \"SlopesPC\", \"SlopesCI\", \"BadPixelsDark\", \"BadPixelsPC\", \"BadPixelsCI\"]\n",
+ "skip = [\"BadPixels\"]\n",
+ "\n",
+ "overwrites = {\"LPD\": {\"SlopesFF\": {\"memory_cells\": 1},\n",
+ " \"BadPixelsFF\": {\"memory_cells\": 1}}}\n",
+ "\n",
+ "if modules[0] == -1:\n",
+ " modules = list(range(16))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'pixels_y = 0. # required\\ncapacitor = \"\" # required\\nmemory_cells = 0. # required\\npixels_x = 0. # required\\nphoton_energy = 0. # required\\nbias_voltage = 0. # required\\nbeam_energy = \"\" # required'"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "import copy\n",
+ "import datetime\n",
+ "import h5py\n",
+ "import inspect\n",
+ "\n",
+ "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
+ "\n",
+ "def extend_parms(detector_instance):\n",
+ " from iCalibrationDB import Conditions\n",
+ " import inspect\n",
+ " \n",
+ " def extract_parms(cls):\n",
+ " args, varargs, varkw, defaults = inspect.getargspec(cls.__init__)\n",
+ " pList = []\n",
+ " for i, arg in enumerate(args[1:][::-1]):\n",
+ " if i < len(defaults):\n",
+ " default = defaults[::-1][i]\n",
+ " if str(default).isdigit():\n",
+ " pList.append(\"{} = 0. # required\".format(arg))\n",
+ " elif default is None or default == \"None\":\n",
+ " pList.append(\"{} = \\\"\\\" # required\".format(arg))\n",
+ " else:\n",
+ " pList.append(\"{} = \\\"\\\" # required\".format(arg))\n",
+ " else:\n",
+ " pList.append(\"{} = 0. # required\".format(arg))\n",
+ " return set(pList[::-1]) # mandatories first\n",
+ " dtype = \"LPD\" if \"LPD\" in detector_instance.upper() else \"AGIPD\"\n",
+ " all_conditions = set()\n",
+ " for c in dir(Conditions):\n",
+ " if c[:2] != \"__\":\n",
+ " condition = getattr(Conditions, c)\n",
+ " parms = extract_parms(getattr(condition, dtype))\n",
+ " [all_conditions.add(p) for p in parms]\n",
+ " return \"\\n\".join(all_conditions)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "det = getattr(Detectors, detector_instance)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "ip = get_ipython()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "first_cell = next(ip.history_manager.get_range(ip.history_manager.get_last_session_id(), 1, 2, raw=True))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "_, _, code = first_cell\n",
+ "code = code.split(\"\\n\")\n",
+ "parms = {}\n",
+ "for c in code:\n",
+ " n, v = c.split(\"=\")\n",
+ " n = n.strip()\n",
+ " v = v.strip()\n",
+ " try:\n",
+ " parms[n] = float(v)\n",
+ " except:\n",
+ " parms[n] = str(v) if not isinstance(v, str) else v\n",
+ " if parms[n] == \"None\" or parms[n] == \"'None'\":\n",
+ " parms[n] = None"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Failed for const RelativeGain of Q2M1: Error sending to database: {'reason': '\\'NoneType\\' object has no attribute \\'get\\': File \"/gpfs/exfel/data/scratch/haufs/karabo-2.2.1/karabo/devices/calibrationDbRemote/src/calibrationDBRemote/calibration_db_remote.py\", line 373, in zmq_server_runner\\n krb_ccv)\\n File \"/gpfs/exfel/data/scratch/haufs/karabo-2.2.1/karabo/devices/calibrationDbRemote/src/calibrationDBRemote/calibration_karabo.py\", line 53, in get_calib_const_version_file\\n file_name_abs_url = \\'{0}{1}\\'.format(ccv_krb_h.get(\\'hdf5path\\'),\\n', 'success': False}\n"
+ ]
+ }
+ ],
+ "source": [
+ "ofile = h5py.File(out_file, \"w\")\n",
+ "\n",
+ "detector = getattr(Detectors, detector_instance)\n",
+ "for i in modules:\n",
+ " qm = \"Q{}M{}\".format(i//4+1, i%4+1)\n",
+ " module = getattr(detector, qm)\n",
+ " dconstants = getattr(Constants, dtype)\n",
+ " for const in dir(dconstants):\n",
+ " if const[:2] != \"__\":\n",
+ " \n",
+ " if const in skip:\n",
+ " continue\n",
+ " \n",
+ " cparms = copy.copy(parms)\n",
+ " if dtype in overwrites:\n",
+ " do = overwrites[dtype]\n",
+ " if const in do:\n",
+ " for arg, v in do[const].items():\n",
+ " cparms[arg] = v\n",
+ " \n",
+ " try:\n",
+ " metadata = ConstantMetaData()\n",
+ " cons = getattr(dconstants, const)()\n",
+ " metadata.calibration_constant = cons\n",
+ "\n",
+ " # set the operating condition\n",
+ "\n",
+ " cond = Conditions.Dark if const in darkconst else Conditions.Illuminated\n",
+ "\n",
+ " condition = getattr(cond, dtype)\n",
+ "\n",
+ " args, varargs, varkw, defaults = inspect.getargspec(condition.__init__)\n",
+ " alist = []\n",
+ " plist = {}\n",
+ " for i, arg in enumerate(args[1:][::-1]):\n",
+ " #if i < len(defaults):\n",
+ " # plist[arg] = parms[arg]\n",
+ " #else:\n",
+ " # alist.append(parms[arg])\n",
+ " plist[arg] = cparms[arg]\n",
+ "\n",
+ "\n",
+ " condition = condition(**plist)\n",
+ "\n",
+ " metadata.detector_condition = condition\n",
+ "\n",
+ " # specify the a version for this constant\n",
+ " if valid_at is None or valid_at == \"\":\n",
+ " creation_time = datetime.datetime.now()\n",
+ " metadata.calibration_constant_version = Versions.Now(\n",
+ " device=module)\n",
+ " else:\n",
+ " metadata.calibration_constant_version = Versions.Timespan(\n",
+ " device=module,\n",
+ " start=valid_at)\n",
+ " creation_time = valid_at\n",
+ "\n",
+ " ctime = creation_time.isoformat() if not isinstance(creation_time, str) else creation_time\n",
+ "\n",
+ " metadata.retrieve(cal_db_interface, when=ctime)\n",
+ " \n",
+ " ofile[\"{}/{}/data\".format(qm, const)] = metadata.calibration_constant.data\n",
+ " except Exception as e:\n",
+ " print(\"Failed for const {} of {}: {}\".format(const, qm, e))\n",
+ "ofile.close() "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.4.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index 67e79a995a1c52f1dafb1ae5cbee95ca0df2d58d..63d5e61866251767201e16fd5e3506c5025a9042 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python
import argparse
+import copy
from datetime import datetime
import nbconvert
import nbformat
@@ -27,21 +28,25 @@ class RawTypeFormatter(argparse.RawDescriptionHelpFormatter,
# The argument parser for calibrate.py, will be extended depending
# on the options given.
-parser = argparse.ArgumentParser(description="Main entry point "
- "for offline calibration",
- formatter_class=RawTypeFormatter)
-parser.add_argument('detector', metavar='DETECTOR', type=str,
- help='The detector to calibrate')
+def make_initial_parser():
+ parser = argparse.ArgumentParser(description="Main entry point "
+ "for offline calibration",
+ formatter_class=RawTypeFormatter)
-parser.add_argument('type', metavar='TYPE', type=str,
- help='Type of calibration: '+",".join(notebooks.keys()))
+ parser.add_argument('detector', metavar='DETECTOR', type=str,
+ help='The detector to calibrate')
-parser.add_argument('--no-cluster-job',
- action="store_true",
- default=False,
- help="Do not run as a cluster job")
+ parser.add_argument('type', metavar='TYPE', type=str,
+ help='Type of calibration: '+",".join(notebooks.keys()))
+ parser.add_argument('--no-cluster-job',
+ action="store_true",
+ default=False,
+ help="Do not run as a cluster job")
+ return parser
+
+parser = make_initial_parser()
# helper functions for parser extensions
def make_intelli_list(ltype):
@@ -52,7 +57,7 @@ def make_intelli_list(ltype):
"""
class IntelliListAction(argparse.Action):
- ltype = ltype
+
def __init__(self, *args, **kwargs):
super(IntelliListAction, self).__init__(*args, **kwargs)
@@ -74,6 +79,7 @@ def make_intelli_list(ltype):
parsed_values = [self.ltype(p) for p in parsed_values]
print("Parsed input {} to {}".format(values, parsed_values))
setattr(namespace, self.dest, parsed_values)
+ IntelliListAction.ltype = ltype
return IntelliListAction
@@ -109,6 +115,12 @@ def extract_title_author_version(nb):
version = version[0] if len(version) else None
return title, author, version
+
+def first_code_cell(nb):
+ """ Return the first code cell of a notebook """
+ for cell in nb.cells:
+ if cell.cell_type == 'code':
+ return cell
def first_markdown_cell(nb):
""" Return the first markdown cell of a notebook """
@@ -139,6 +151,35 @@ def make_epilog(nb, caltype=None):
msg += sline + "\n"
msg += "\n"
return msg
+
+def get_notebook_function(nb, fname):
+ import re
+ flines = []
+ def_found = False
+ indent = None
+ for cell in nb.cells:
+ if cell.cell_type == 'code':
+ lines = cell.source.split("\n")
+ for line in lines:
+
+ if def_found:
+ lin = len(line) - len(line.lstrip())
+ if indent is None:
+ if lin != 0:
+ indent = lin
+ flines.append(line)
+ elif lin >= indent:
+ flines.append(line)
+ else:
+ return "\n".join(flines)
+
+ if re.search(r"def\s+{}\(.*\):\s*".format(fname), line) and not def_found:
+ # print("Found {} in line {}".format(fname, line))
+ # set this to indent level
+ def_found = True
+ flines.append(line)
+ return None
+
# extend the parser according to user input
# the first case is if a detector was given, but no calibration type
@@ -187,38 +228,85 @@ elif len(sys.argv) >= 3:
exit()
with open(notebook, "r") as f:
nb = nbformat.read(f, as_version=4)
- parser.description = make_epilog(nb)
- parms = extract_parameters(nb)
- for p in parms:
- helpstr = ("Default: %(default)s" if not p.comment
- else "{}. Default: %(default)s".format(p.comment.replace("#", " ").strip()))
- required = p.comment is not None and "required" in p.comment
+
+ ext_func = notebooks[detector][caltype].get("extend parms", None)
+
+ def do_parse(nb, parser, overwrite_reqs=False):
+ parser.description = make_epilog(nb)
+ parms = extract_parameters(nb)
- if p.type == list:
- if len(p.value):
- ltype = type(p.value[0])
+ for p in parms:
+
+ helpstr = ("Default: %(default)s" if not p.comment
+ else "{}. Default: %(default)s".format(p.comment.replace("#", " ").strip()))
+ required = (p.comment is not None and "required" in p.comment) and not overwrite_reqs
+
+ if p.type == list:
+ if len(p.value):
+ ltype = type(p.value[0])
+ else:
+ ltype = str
+ range_allowed = "RANGE ALLOWED" in p.comment.upper() if p.comment else False
+ parser.add_argument("--{}".format(consolize_name(p.name)),
+ nargs='+',
+ type=ltype if not range_allowed else str,
+ default=p.value if (not required) and p.name != cvar else None,
+ help=helpstr,
+ required=required and p.name != cvar,
+ action=make_intelli_list(ltype) if range_allowed else None)
+ elif p.type == bool:
+ parser.add_argument("--{}".format(consolize_name(p.name)),
+ action="store_true",
+ default=p.value if not required else None,
+ help=helpstr,
+ required=required)
else:
- ltype = str
- range_allowed = "RANGE ALLOWED" in p.comment.upper() if p.comment else False
- parser.add_argument("--{}".format(consolize_name(p.name)),
- nargs='+',
- type=ltype if not range_allowed else str,
- default=p.value if (not required) and p.name != cvar else None,
- help=helpstr,
- required=required and p.name != cvar,
- action=make_intelli_list(ltype) if range_allowed else None)
- elif p.type == bool:
- parser.add_argument("--{}".format(consolize_name(p.name)),
- action="store_true",
- default=p.value if not required else None,
- help=helpstr,
- required=required)
+ parser.add_argument("--{}".format(consolize_name(p.name)),
+ type=p.type,
+ default=p.value if not required else None,
+ help=helpstr,
+ required=required)
+
+ do_parse(nb, parser, True)
+
+ # extend parameters if needed
+ ext_func = notebooks[detector][caltype].get("extend parms", None)
+ if ext_func is not None:
+ func = get_notebook_function(nb, ext_func)
+
+ if func is None:
+ warnings.warn("Didn't find concurrency function {} in notebook".format(ext_func),
+ RuntimeWarning)
+
else:
- parser.add_argument("--{}".format(consolize_name(p.name)),
- type=p.type,
- default=p.value if not required else None,
- help=helpstr,
- required=required)
+ # remove help calls as they will cause the argument parser to exit
+ add_help = False
+ if "-h" in sys.argv:
+ sys.argv.remove("-h")
+ add_help = True
+ if "--help" in sys.argv:
+ sys.argv.remove("--help")
+ add_help = True
+ known, remaining = parser.parse_known_args()
+ if add_help:
+ sys.argv.append("--help")
+ args = deconsolize_args(vars(known))
+
+ df = {}
+
+ exec(func, df)
+ f = df[ext_func]
+ import inspect
+ sig = inspect.signature(f)
+ callargs = []
+ for arg in sig.parameters:
+ callargs.append(args[arg])
+
+ extention = f(*callargs)
+ fcc = first_code_cell(nb)
+ fcc["source"] += "\n"+extention
+ parser = make_initial_parser()
+ do_parse(nb, parser, False)
def has_parm(parms, name):
@@ -247,6 +335,7 @@ def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
args["cluster_profile"] = "{}_{}".format(args["cluster_profile"], suffix)
# first convert the notebook
+ parms = extract_parameters(nb)
params = parameter_values(parms, **args)
new_nb = replace_definitions(nb, params, execute=False)
base_name = nbname.replace(".ipynb", "")
@@ -299,35 +388,6 @@ def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
return jobid
-def get_concurrency_function(nb, fname):
- import re
- flines = []
- def_found = False
- indent = None
- for cell in nb.cells:
- if cell.cell_type == 'code':
- lines = cell.source.split("\n")
- for line in lines:
-
- if def_found:
- lin = len(line) - len(line.lstrip())
- if indent is None:
- if lin != 0:
- indent = lin
- flines.append(line)
- elif lin >= indent:
- flines.append(line)
- else:
- return "\n".join(flines)
-
- if re.search(r"def\s+{}\(.*\):\s*".format(fname), line):
- print("Found {} in line {}".format(fname, line))
- # set this to indent level
- def_found = True
- flines.append(line)
- return None
-
-
def run():
""" Run a calibration task with parser arguments """
@@ -351,7 +411,35 @@ def run():
return
with open(notebook, "r") as f:
nb = nbformat.read(f, as_version=4)
+
+ # extend parameters if needed
+ ext_func = notebooks[detector][caltype].get("extend parms", None)
+ if ext_func is not None:
+ func = get_notebook_function(nb, ext_func)
+
+ if func is None:
+ warnings.warn("Didn't find concurrency function {} in notebook".format(ext_func),
+ RuntimeWarning)
+
+ else:
+ # remove help calls as they will cause the argument parser to exit
+ known, remaining = parser.parse_known_args()
+ args = deconsolize_args(vars(known))
+ df = {}
+ exec(func, df)
+ f = df[ext_func]
+ import inspect
+ sig = inspect.signature(f)
+ callargs = []
+ for arg in sig.parameters:
+ callargs.append(args[arg])
+
+ extention = f(*callargs)
+ fcc = first_code_cell(nb)
+ fcc["source"] += "\n"+extention
+
parms = extract_parameters(nb)
+
title, author, version = extract_title_author_version(nb)
if not title:
@@ -423,7 +511,7 @@ def run():
cvals = defcval
if con_func:
- func = get_concurrency_function(nb, con_func)
+ func = get_notebook_function(nb, con_func)
if func is None:
warnings.warn("Didn't find concurrency function {} in notebook".format(con_func),
RuntimeWarning)
diff --git a/xfel_calibrate/notebooks.py b/xfel_calibrate/notebooks.py
index 972d6df66269b4596e5821b227a99492b8eb8298..182fbffb6d5effc8f5b68285621838bf32896c32 100644
--- a/xfel_calibrate/notebooks.py
+++ b/xfel_calibrate/notebooks.py
@@ -12,7 +12,7 @@ notebooks = {
"notebook": "notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb",
"concurrency": {"parameter": "modules",
"default concurrency": 16,
- "cluster cores": 32},
+ "cluster cores": 16},
},
"FF": {
"notebook": "notebooks/AGIPD/Characterize_AGIPD_Gain_FlatFields_NBC.ipynb",
@@ -74,6 +74,15 @@ notebooks = {
"default concurrency": None,
"cluster cores": 32},
},
+ },
+ "GENERIC": {
+ "DBTOH5": {
+ "notebook": "notebooks/generic/DB_Constants_to_HDF5_NBC.ipynb",
+ "concurrency": {"parameter": None,
+ "default concurrency": None,
+ "cluster cores": 32},
+ "extend parms": "extend_parms",
+ },
}
}
\ No newline at end of file