From defc38efa57be8b1906fa45715fcf0c2319a8d36 Mon Sep 17 00:00:00 2001
From: Steffen Hauf <haufs@max-exfl015.desy.de>
Date: Tue, 14 Aug 2018 12:48:31 +0200
Subject: [PATCH] Remove non-related updates
---
.../generic/DB_Constants_to_HDF5_NBC.ipynb | 291 ------------------
xfel_calibrate/calibrate.py | 259 ++++------------
2 files changed, 54 insertions(+), 496 deletions(-)
delete mode 100644 notebooks/generic/DB_Constants_to_HDF5_NBC.ipynb
diff --git a/notebooks/generic/DB_Constants_to_HDF5_NBC.ipynb b/notebooks/generic/DB_Constants_to_HDF5_NBC.ipynb
deleted file mode 100644
index 3aa953a03..000000000
--- a/notebooks/generic/DB_Constants_to_HDF5_NBC.ipynb
+++ /dev/null
@@ -1,291 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Constants from DB to HDF5 #\n",
- "\n",
- "Version 0.1, Author: S. Hauf\n",
- "\n",
- "Currently available instances are LPD1M1 and AGIPD1M1"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "detector_instance = \"LPD1M1\" # the detector instance to get constants for e.g. LPD1M1, required\n",
- "out_file = \"/gpfs/exfel/data/scratch/haufs/test/test.h5\" # HDF5 file to output constants into, required\n",
- "valid_at = \"\" # ISO formatted date for which constants shoudl be valid. Leave empty to get most current ones\n",
- "cal_db_interface = \"tcp://max-exfl015:5005\"\n",
- "modules = [-1] # modules to get data from, in terms of numerical quadrant indices, range allowed"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 12,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "dtype = \"LPD\" if \"LPD\" in detector_instance.upper() else \"AGIPD\"\n",
- "darkconst = [\"Offset\", \"Noise\", \"SlopesPC\", \"SlopesCI\", \"BadPixelsDark\", \"BadPixelsPC\", \"BadPixelsCI\"]\n",
- "skip = [\"BadPixels\"]\n",
- "\n",
- "overwrites = {\"LPD\": {\"SlopesFF\": {\"memory_cells\": 1},\n",
- " \"BadPixelsFF\": {\"memory_cells\": 1}}}\n",
- "\n",
- "if modules[0] == -1:\n",
- " modules = list(range(16))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 20,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Skipping capacitor\n",
- "Skipping pixels_y\n",
- "Skipping pixels_x\n",
- "Skipping bias_voltage\n",
- "Skipping memory_cells\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "'memory_cells = 0. # required\\npixels_y = 256 # required\\ncapacitor = \"5pF\" # required\\nbias_voltage = 0. # required\\nphoton_energy = 0. # required\\nbeam_energy = \"\" # required\\npixels_x = 256 # required'"
- ]
- },
- "execution_count": 20,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "import copy\n",
- "import datetime\n",
- "import h5py\n",
- "import inspect\n",
- "\n",
- "from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
- "\n",
- "def extend_parms(detector_instance):\n",
- " from iCalibrationDB import Conditions\n",
- " import inspect\n",
- " existing = set()\n",
- " def extract_parms(cls):\n",
- " args, varargs, varkw, defaults = inspect.getargspec(cls.__init__)\n",
- " pList = [] \n",
- " for i, arg in enumerate(args[1:][::-1]):\n",
- " if arg in existing:\n",
- " continue\n",
- " \n",
- " existing.add(arg)\n",
- " \n",
- " if i < len(defaults):\n",
- " default = defaults[::-1][i] \n",
- " if str(default).isdigit():\n",
- " pList.append(\"{} = {}\".format(arg, default))\n",
- " elif default is None or default == \"None\":\n",
- " pList.append(\"{} = \\\"None\\\"\".format(arg))\n",
- " else:\n",
- " pList.append(\"{} = \\\"{}\\\"\".format(arg, default))\n",
- " else:\n",
- " pList.append(\"{} = 0. # required\".format(arg))\n",
- " return set(pList[::-1]) # mandatories first\n",
- " dtype = \"LPD\" if \"LPD\" in detector_instance.upper() else \"AGIPD\"\n",
- " all_conditions = set()\n",
- " for c in dir(Conditions):\n",
- " if c[:2] != \"__\":\n",
- " condition = getattr(Conditions, c)\n",
- " parms = extract_parms(getattr(condition, dtype))\n",
- " [all_conditions.add(p) for p in parms]\n",
- " return \"\\n\".join(all_conditions)\n",
- "\n",
- "extend_parms(\"LPD1M1\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": [
- "det = getattr(Detectors, detector_instance)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "ip = get_ipython()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "first_cell = next(ip.history_manager.get_range(ip.history_manager.get_last_session_id(), 1, 2, raw=True))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "_, _, code = first_cell\n",
- "code = code.split(\"\\n\")\n",
- "parms = {}\n",
- "for c in code:\n",
- " n, v = c.split(\"=\")\n",
- " n = n.strip()\n",
- " v = v.strip()\n",
- " try:\n",
- " parms[n] = float(v)\n",
- " except:\n",
- " parms[n] = str(v) if not isinstance(v, str) else v\n",
- " if parms[n] == \"None\" or parms[n] == \"'None'\":\n",
- " parms[n] = None"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 10,
- "metadata": {
- "collapsed": false
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Failed for const RelativeGain of Q2M1: Error sending to database: {'reason': '\\'NoneType\\' object has no attribute \\'get\\': File \"/gpfs/exfel/data/scratch/haufs/karabo-2.2.1/karabo/devices/calibrationDbRemote/src/calibrationDBRemote/calibration_db_remote.py\", line 373, in zmq_server_runner\\n krb_ccv)\\n File \"/gpfs/exfel/data/scratch/haufs/karabo-2.2.1/karabo/devices/calibrationDbRemote/src/calibrationDBRemote/calibration_karabo.py\", line 53, in get_calib_const_version_file\\n file_name_abs_url = \\'{0}{1}\\'.format(ccv_krb_h.get(\\'hdf5path\\'),\\n', 'success': False}\n"
- ]
- }
- ],
- "source": [
- "ofile = h5py.File(out_file, \"w\")\n",
- "\n",
- "detector = getattr(Detectors, detector_instance)\n",
- "for i in modules:\n",
- " qm = \"Q{}M{}\".format(i//4+1, i%4+1)\n",
- " module = getattr(detector, qm)\n",
- " dconstants = getattr(Constants, dtype)\n",
- " for const in dir(dconstants):\n",
- " if const[:2] != \"__\":\n",
- " \n",
- " if const in skip:\n",
- " continue\n",
- " \n",
- " cparms = copy.copy(parms)\n",
- " if dtype in overwrites:\n",
- " do = overwrites[dtype]\n",
- " if const in do:\n",
- " for arg, v in do[const].items():\n",
- " cparms[arg] = v\n",
- " \n",
- " try:\n",
- " metadata = ConstantMetaData()\n",
- " cons = getattr(dconstants, const)()\n",
- " metadata.calibration_constant = cons\n",
- "\n",
- " # set the operating condition\n",
- "\n",
- " cond = Conditions.Dark if const in darkconst else Conditions.Illuminated\n",
- "\n",
- " condition = getattr(cond, dtype)\n",
- "\n",
- " args, varargs, varkw, defaults = inspect.getargspec(condition.__init__)\n",
- " alist = []\n",
- " plist = {}\n",
- " for i, arg in enumerate(args[1:][::-1]):\n",
- " #if i < len(defaults):\n",
- " # plist[arg] = parms[arg]\n",
- " #else:\n",
- " # alist.append(parms[arg])\n",
- " plist[arg] = cparms[arg]\n",
- "\n",
- "\n",
- " condition = condition(**plist)\n",
- "\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if valid_at is None or valid_at == \"\":\n",
- " creation_time = datetime.datetime.now()\n",
- " metadata.calibration_constant_version = Versions.Now(\n",
- " device=module)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(\n",
- " device=module,\n",
- " start=valid_at)\n",
- " creation_time = valid_at\n",
- "\n",
- " ctime = creation_time.isoformat() if not isinstance(creation_time, str) else creation_time\n",
- "\n",
- " metadata.retrieve(cal_db_interface, when=ctime)\n",
- " \n",
- " ofile[\"{}/{}/data\".format(qm, const)] = metadata.calibration_constant.data\n",
- " except Exception as e:\n",
- " print(\"Failed for const {} of {}: {}\".format(const, qm, e))\n",
- "ofile.close() "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": []
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.4.3"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index 63d5e6186..baa4cdff5 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -1,7 +1,6 @@
#!/usr/bin/env python
import argparse
-import copy
from datetime import datetime
import nbconvert
import nbformat
@@ -28,25 +27,21 @@ class RawTypeFormatter(argparse.RawDescriptionHelpFormatter,
# The argument parser for calibrate.py, will be extended depending
# on the options given.
+parser = argparse.ArgumentParser(description="Main entry point "
+ "for offline calibration",
+ formatter_class=RawTypeFormatter)
-def make_initial_parser():
- parser = argparse.ArgumentParser(description="Main entry point "
- "for offline calibration",
- formatter_class=RawTypeFormatter)
+parser.add_argument('detector', metavar='DETECTOR', type=str,
+ help='The detector to calibrate')
- parser.add_argument('detector', metavar='DETECTOR', type=str,
- help='The detector to calibrate')
+parser.add_argument('type', metavar='TYPE', type=str,
+ help='Type of calibration: '+",".join(notebooks.keys()))
- parser.add_argument('type', metavar='TYPE', type=str,
- help='Type of calibration: '+",".join(notebooks.keys()))
+parser.add_argument('--no-cluster-job',
+ action="store_true",
+ default=False,
+ help="Do not run as a cluster job")
- parser.add_argument('--no-cluster-job',
- action="store_true",
- default=False,
- help="Do not run as a cluster job")
- return parser
-
-parser = make_initial_parser()
# helper functions for parser extensions
def make_intelli_list(ltype):
@@ -57,7 +52,7 @@ def make_intelli_list(ltype):
"""
class IntelliListAction(argparse.Action):
-
+ ltype = ltype
def __init__(self, *args, **kwargs):
super(IntelliListAction, self).__init__(*args, **kwargs)
@@ -79,7 +74,6 @@ def make_intelli_list(ltype):
parsed_values = [self.ltype(p) for p in parsed_values]
print("Parsed input {} to {}".format(values, parsed_values))
setattr(namespace, self.dest, parsed_values)
- IntelliListAction.ltype = ltype
return IntelliListAction
@@ -115,12 +109,6 @@ def extract_title_author_version(nb):
version = version[0] if len(version) else None
return title, author, version
-
-def first_code_cell(nb):
- """ Return the first code cell of a notebook """
- for cell in nb.cells:
- if cell.cell_type == 'code':
- return cell
def first_markdown_cell(nb):
""" Return the first markdown cell of a notebook """
@@ -151,35 +139,6 @@ def make_epilog(nb, caltype=None):
msg += sline + "\n"
msg += "\n"
return msg
-
-def get_notebook_function(nb, fname):
- import re
- flines = []
- def_found = False
- indent = None
- for cell in nb.cells:
- if cell.cell_type == 'code':
- lines = cell.source.split("\n")
- for line in lines:
-
- if def_found:
- lin = len(line) - len(line.lstrip())
- if indent is None:
- if lin != 0:
- indent = lin
- flines.append(line)
- elif lin >= indent:
- flines.append(line)
- else:
- return "\n".join(flines)
-
- if re.search(r"def\s+{}\(.*\):\s*".format(fname), line) and not def_found:
- # print("Found {} in line {}".format(fname, line))
- # set this to indent level
- def_found = True
- flines.append(line)
- return None
-
# extend the parser according to user input
# the first case is if a detector was given, but no calibration type
@@ -228,85 +187,38 @@ elif len(sys.argv) >= 3:
exit()
with open(notebook, "r") as f:
nb = nbformat.read(f, as_version=4)
-
- ext_func = notebooks[detector][caltype].get("extend parms", None)
-
- def do_parse(nb, parser, overwrite_reqs=False):
- parser.description = make_epilog(nb)
- parms = extract_parameters(nb)
+ parser.description = make_epilog(nb)
+ parms = extract_parameters(nb)
+ for p in parms:
+ helpstr = ("Default: %(default)s" if not p.comment
+ else "{}. Default: %(default)s".format(p.comment.replace("#", " ").strip()))
+ required = p.comment is not None and "required" in p.comment
- for p in parms:
-
- helpstr = ("Default: %(default)s" if not p.comment
- else "{}. Default: %(default)s".format(p.comment.replace("#", " ").strip()))
- required = (p.comment is not None and "required" in p.comment) and not overwrite_reqs
-
- if p.type == list:
- if len(p.value):
- ltype = type(p.value[0])
- else:
- ltype = str
- range_allowed = "RANGE ALLOWED" in p.comment.upper() if p.comment else False
- parser.add_argument("--{}".format(consolize_name(p.name)),
- nargs='+',
- type=ltype if not range_allowed else str,
- default=p.value if (not required) and p.name != cvar else None,
- help=helpstr,
- required=required and p.name != cvar,
- action=make_intelli_list(ltype) if range_allowed else None)
- elif p.type == bool:
- parser.add_argument("--{}".format(consolize_name(p.name)),
- action="store_true",
- default=p.value if not required else None,
- help=helpstr,
- required=required)
+ if p.type == list:
+ if len(p.value):
+ ltype = type(p.value[0])
else:
- parser.add_argument("--{}".format(consolize_name(p.name)),
- type=p.type,
- default=p.value if not required else None,
- help=helpstr,
- required=required)
-
- do_parse(nb, parser, True)
-
- # extend parameters if needed
- ext_func = notebooks[detector][caltype].get("extend parms", None)
- if ext_func is not None:
- func = get_notebook_function(nb, ext_func)
-
- if func is None:
- warnings.warn("Didn't find concurrency function {} in notebook".format(ext_func),
- RuntimeWarning)
-
+ ltype = str
+ range_allowed = "RANGE ALLOWED" in p.comment.upper() if p.comment else False
+ parser.add_argument("--{}".format(consolize_name(p.name)),
+ nargs='+',
+ type=ltype if not range_allowed else str,
+ default=p.value if (not required) and p.name != cvar else None,
+ help=helpstr,
+ required=required and p.name != cvar,
+ action=make_intelli_list(ltype) if range_allowed else None)
+ elif p.type == bool:
+ parser.add_argument("--{}".format(consolize_name(p.name)),
+ action="store_true",
+ default=p.value if not required else None,
+ help=helpstr,
+ required=required)
else:
- # remove help calls as they will cause the argument parser to exit
- add_help = False
- if "-h" in sys.argv:
- sys.argv.remove("-h")
- add_help = True
- if "--help" in sys.argv:
- sys.argv.remove("--help")
- add_help = True
- known, remaining = parser.parse_known_args()
- if add_help:
- sys.argv.append("--help")
- args = deconsolize_args(vars(known))
-
- df = {}
-
- exec(func, df)
- f = df[ext_func]
- import inspect
- sig = inspect.signature(f)
- callargs = []
- for arg in sig.parameters:
- callargs.append(args[arg])
-
- extention = f(*callargs)
- fcc = first_code_cell(nb)
- fcc["source"] += "\n"+extention
- parser = make_initial_parser()
- do_parse(nb, parser, False)
+ parser.add_argument("--{}".format(consolize_name(p.name)),
+ type=p.type,
+ default=p.value if not required else None,
+ help=helpstr,
+ required=required)
def has_parm(parms, name):
@@ -317,10 +229,6 @@ def has_parm(parms, name):
return False
-def flatten_list(l):
- return "_".join([str(flatten_list(v)) for v in l]) if isinstance(l, list) else l
-
-
def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
final_job=False, job_list=[], fmtcmd="", cluster_cores=8,
sequential=False):
@@ -330,12 +238,11 @@ def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
if cparm is not None:
args[cparm] = cval
- suffix = flatten_list(cval)
+ suffix = "_".join([str(v) for v in cval]) if isinstance(cval, list) else cval
if "cluster_profile" in args:
args["cluster_profile"] = "{}_{}".format(args["cluster_profile"], suffix)
# first convert the notebook
- parms = extract_parameters(nb)
params = parameter_values(parms, **args)
new_nb = replace_definitions(nb, params, execute=False)
base_name = nbname.replace(".ipynb", "")
@@ -358,7 +265,9 @@ def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
os.chmod("{}/finalize.sh".format(temp_path), all_stats)
# then run an sbatch job
if not sequential:
- srun_base = launcher_command.format(temp_path=temp_path).split()
+ srun_base = ["sbatch", "-p", slurm_partion, "-t", "24:00:00",
+ "--mem", "500G", "--mail-type", "END", "--requeue",
+ "--output", "{}/slurm-%j.out".format(temp_path)]
else:
srun_base = []
@@ -380,13 +289,12 @@ def concurrent_run(temp_path, nb, nbname, args, cparm=None, cval=None,
output = check_output(srun_base).decode('utf8')
jobid = None
- if not sequential:
- for line in output.split("\n"):
- if "Submitted batch job " in line:
- jobid = line.split(" ")[3]
- print("Submitted job: {}".format(jobid))
+ for line in output.split("\n"):
+ if "Submitted batch job " in line:
+ jobid = line.split(" ")[3]
+ print("Submitted job: {}".format(jobid))
return jobid
-
+
def run():
""" Run a calibration task with parser arguments """
@@ -401,7 +309,6 @@ def run():
try:
notebook = notebooks[detector][caltype]["notebook"]
- notebook = os.path.abspath("{}/{}".format(os.path.dirname(__file__), notebook))
concurrency = notebooks[detector][caltype].get("concurrency", None)
version = notebooks[detector][caltype].get("version", "NA")
author = notebooks[detector][caltype].get("author", "anonymous")
@@ -411,35 +318,7 @@ def run():
return
with open(notebook, "r") as f:
nb = nbformat.read(f, as_version=4)
-
- # extend parameters if needed
- ext_func = notebooks[detector][caltype].get("extend parms", None)
- if ext_func is not None:
- func = get_notebook_function(nb, ext_func)
-
- if func is None:
- warnings.warn("Didn't find concurrency function {} in notebook".format(ext_func),
- RuntimeWarning)
-
- else:
- # remove help calls as they will cause the argument parser to exit
- known, remaining = parser.parse_known_args()
- args = deconsolize_args(vars(known))
- df = {}
- exec(func, df)
- f = df[ext_func]
- import inspect
- sig = inspect.signature(f)
- callargs = []
- for arg in sig.parameters:
- callargs.append(args[arg])
-
- extention = f(*callargs)
- fcc = first_code_cell(nb)
- fcc["source"] += "\n"+extention
-
parms = extract_parameters(nb)
-
title, author, version = extract_title_author_version(nb)
if not title:
@@ -501,46 +380,16 @@ def run():
cvals = args.get(cvar, None)
cluster_cores = concurrency.get("cluster cores", 8)
- con_func = concurrency.get("use function", None)
-
if cvals is None:
- defcval = concurrency.get("default concurrency", None)
- if not isinstance(defcval, (list, tuple)):
- cvals = range(defcval)
- else:
- cvals = defcval
-
- if con_func:
- func = get_notebook_function(nb, con_func)
- if func is None:
- warnings.warn("Didn't find concurrency function {} in notebook".format(con_func),
- RuntimeWarning)
-
- else:
- df = {}
- exec(func, df)
- f = df[con_func]
- import inspect
- sig = inspect.signature(f)
- callargs = []
- if cvals:
-
- args[cvar] = cvals # in case default needs to be used for function call
- for arg in sig.parameters:
- callargs.append(args[arg])
- cvals = f(*callargs)
- print("Split concurrency into {}".format(cvals))
+ cvals = range(concurrency["default concurrency"])
for cnum, cval in enumerate(cvals):
jobid = concurrent_run(run_tmp_path, nb, notebook, args,
- cvar, [cval,] if not isinstance(cval, list) else cval,
- cnum==len(list(cvals))-1, joblist, fmtcmd,
+ cvar, [cval,], cnum==len(list(cvals))-1, joblist, fmtcmd,
cluster_cores=cluster_cores, sequential=sequential)
joblist.append(jobid)
-
-
- if not all([j is None for j in joblist]):
- print("Submitted the following SLURM jobs: {}".format(",".join(joblist)))
+
+ print("Submitted the following SLURM jobs: {}".format(",".join(joblist)))
if __name__ == "__main__":
--
GitLab