Skip to content
Snippets Groups Projects
Commit 68e54e3b authored by Steffen Hauf's avatar Steffen Hauf
Browse files

Commit generic notebook and docs

parent 4db2ca0a
No related branches found
No related tags found
1 merge request!19Commit generic notebook and docs
Hide whitespace changes
Inline Side-by-side
docs/source/advanced.rst
+ 94
− 0
View file @ 68e54e3b
...@@ -108,4 +108,98 @@ This can be useful to add user requests while running. For this: ...@@ -108,4 +108,98 @@ This can be useful to add user requests while running. For this:
manually, assuming the correction notebook allows run and overwrite paramters:: manually, assuming the correction notebook allows run and overwrite paramters::
xfel-calibrate ...... --run XYZ,ZXY-YYS --overwrite xfel-calibrate ...... --run XYZ,ZXY-YYS --overwrite
Using a Parameter Generator Function
------------------------------------
By default, the parameters to be exposed to the command line are deduced from the
first code cell of the notebook, after resolving the notebook itself from the
detector and characterization type. For some applications it might be beneficial
to define a context-specific parameter range within the same notebook, based on
additional user input. This can be done via a parameter generation function which
is defined in one of the code cell::
def extend_parms(detector_instance):
from iCalibrationDB import Conditions
import inspect
existing = set()
def extract_parms(cls):
args, varargs, varkw, defaults = inspect.getargspec(cls.__init__)
pList = []
for i, arg in enumerate(args[1:][::-1]):
if arg in existing:
continue
existing.add(arg)
if i < len(defaults):
default = defaults[::-1][i]
if str(default).isdigit():
pList.append("{} = {}".format(arg, default))
elif default is None or default == "None":
pList.append("{} = \"None\"".format(arg))
else:
pList.append("{} = \"{}\"".format(arg, default))
else:
pList.append("{} = 0. # required".format(arg))
return set(pList[::-1]) # mandatories first
dtype = "LPD" if "LPD" in detector_instance.upper() else "AGIPD"
all_conditions = set()
for c in dir(Conditions):
if c[:2] != "__":
condition = getattr(Conditions, c)
parms = extract_parms(getattr(condition, dtype))
[all_conditions.add(p) for p in parms]
return "\n".join(all_conditions)
.. note::
Note how all imports are inlined, as the function is executed outside the
notebook context.
In the example, the function generates a list of additional parameters depending
on the `detector_instance` given. Here, `detector_instance` is defined in the first
code cell the usual way. Any other parameters defined such, that have names matching
those of the generator function signature are passed to this function. The function
should then return a string containing additional code to be appended to the first
code cell.
To make use of this functionality, the parameter generator function needs to be
configured in `notebooks.py`, e.g. ::
...
"GENERIC": {
"DBTOH5": {
"notebook": "notebooks/generic/DB_Constants_to_HDF5_NBC.ipynb",
"concurrency": {"parameter": None,
"default concurrency": None,
"cluster cores": 32},
"extend parms": "extend_parms",
},
}
...
To generically query which parameters are defined in the first code cell, the
code execution history feature of iPython can be used::
ip = get_ipython()
session = ip.history_manager.get_last_session_id()
first_cell = next(ip.history_manager.get_range(session, 1, 2, raw=True))
_, _, code = first_cell
code = code.split("\n")
parms = {}
for c in code:
n, v = c.split("=")
n = n.strip()
v = v.strip()
try:
parms[n] = float(v)
except:
parms[n] = str(v) if not isinstance(v, str) else v
if parms[n] == "None" or parms[n] == "'None'":
parms[n] = None
This will create a dictionary `parms` which contains all parameters either
as `float` or `str` values.
\ No newline at end of file
notebooks/generic/DB_Constants_to_HDF5_NBC.ipynb 0 → 100644
+ 267
− 0
View file @ 68e54e3b
%% Cell type:markdown id: tags:
# Constants from DB to HDF5 #
Version 0.1, Author: S. Hauf
Currently available instances are LPD1M1 and AGIPD1M1
%% Cell type:code id: tags:
``` python
detector_instance = "LPD1M1" # the detector instance to get constants for e.g. LPD1M1, required
out_file = "/gpfs/exfel/data/scratch/haufs/test/test.h5" # HDF5 file to output constants into, required
valid_at = "" # ISO formatted date for which constants shoudl be valid. Leave empty to get most current ones
cal_db_interface = "tcp://max-exfl015:5005"
modules = [-1] # modules to get data from, in terms of numerical quadrant indices, range allowed
```
%% Cell type:code id: tags:
``` python
dtype = "LPD" if "LPD" in detector_instance.upper() else "AGIPD"
darkconst = ["Offset", "Noise", "SlopesPC", "SlopesCI", "BadPixelsDark", "BadPixelsPC", "BadPixelsCI"]
skip = ["BadPixels"]
overwrites = {"LPD": {"SlopesFF": {"memory_cells": 1},
"BadPixelsFF": {"memory_cells": 1}}}
if modules[0] == -1:
modules = list(range(16))
```
%% Cell type:code id: tags:
``` python
import copy
import datetime
import h5py
import inspect
from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions
def extend_parms(detector_instance):
from iCalibrationDB import Conditions
import inspect
existing = set()
def extract_parms(cls):
args, varargs, varkw, defaults = inspect.getargspec(cls.__init__)
pList = []
for i, arg in enumerate(args[1:][::-1]):
if arg in existing:
continue
existing.add(arg)
if i < len(defaults):
default = defaults[::-1][i]
if str(default).isdigit():
pList.append("{} = {}".format(arg, default))
elif default is None or default == "None":
pList.append("{} = \"None\"".format(arg))
else:
pList.append("{} = \"{}\"".format(arg, default))
else:
pList.append("{} = 0. # required".format(arg))
return set(pList[::-1]) # mandatories first
dtype = "LPD" if "LPD" in detector_instance.upper() else "AGIPD"
all_conditions = set()
for c in dir(Conditions):
if c[:2] != "__":
condition = getattr(Conditions, c)
parms = extract_parms(getattr(condition, dtype))
[all_conditions.add(p) for p in parms]
return "\n".join(all_conditions)
```
%% Cell type:code id: tags:
``` python
det = getattr(Detectors, detector_instance)
```
%% Cell type:code id: tags:
``` python
ip = get_ipython()
```
%% Cell type:code id: tags:
``` python
first_cell = next(ip.history_manager.get_range(ip.history_manager.get_last_session_id(), 1, 2, raw=True))
```
%% Cell type:code id: tags:
``` python
_, _, code = first_cell
code = code.split("\n")
parms = {}
for c in code:
n, v = c.split("=")
n = n.strip()
v = v.strip()
try:
parms[n] = float(v)
except:
parms[n] = str(v) if not isinstance(v, str) else v
if parms[n] == "None" or parms[n] == "'None'":
parms[n] = None
```
%% Cell type:code id: tags:
``` python
ofile = h5py.File(out_file, "w")
detector = getattr(Detectors, detector_instance)
for i in modules:
qm = "Q{}M{}".format(i//4+1, i%4+1)
module = getattr(detector, qm)
dconstants = getattr(Constants, dtype)
for const in dir(dconstants):
if const[:2] != "__":
if const in skip:
continue
cparms = copy.copy(parms)
if dtype in overwrites:
do = overwrites[dtype]
if const in do:
for arg, v in do[const].items():
cparms[arg] = v
try:
metadata = ConstantMetaData()
cons = getattr(dconstants, const)()
metadata.calibration_constant = cons
# set the operating condition
cond = Conditions.Dark if const in darkconst else Conditions.Illuminated
condition = getattr(cond, dtype)
args, varargs, varkw, defaults = inspect.getargspec(condition.__init__)
alist = []
plist = {}
for i, arg in enumerate(args[1:][::-1]):
#if i < len(defaults):
# plist[arg] = parms[arg]
#else:
# alist.append(parms[arg])
plist[arg] = cparms[arg]
condition = condition(**plist)
metadata.detector_condition = condition
# specify the a version for this constant
if valid_at is None or valid_at == "":
creation_time = datetime.datetime.now()
metadata.calibration_constant_version = Versions.Now(
device=module)
else:
metadata.calibration_constant_version = Versions.Timespan(
device=module,
start=valid_at)
creation_time = valid_at
ctime = creation_time.isoformat() if not isinstance(creation_time, str) else creation_time
metadata.retrieve(cal_db_interface, when=ctime)
ofile["{}/{}/data".format(qm, const)] = metadata.calibration_constant.data
except Exception as e:
print("Failed for const {} of {}: {}".format(const, qm, e))
ofile.close()
```
%% Output
Failed for const RelativeGain of Q2M1: Error sending to database: {'reason': '\'NoneType\' object has no attribute \'get\': File "/gpfs/exfel/data/scratch/haufs/karabo-2.2.1/karabo/devices/calibrationDbRemote/src/calibrationDBRemote/calibration_db_remote.py", line 373, in zmq_server_runner\n krb_ccv)\n File "/gpfs/exfel/data/scratch/haufs/karabo-2.2.1/karabo/devices/calibrationDbRemote/src/calibrationDBRemote/calibration_karabo.py", line 53, in get_calib_const_version_file\n file_name_abs_url = \'{0}{1}\'.format(ccv_krb_h.get(\'hdf5path\'),\n', 'success': False}
%% Cell type:code id: tags:
``` python
```
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts