From 8adaa32ef40db4d6b92c31fe02988f62e7fc2cb7 Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Tue, 26 Nov 2019 10:24:58 +0100
Subject: [PATCH] Reduce changes
---
cal_tools/cal_tools/tools.py | 83 +-----------
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 2 +-
.../ePix/Characterize_Darks_ePix10K_NBC.ipynb | 2 +-
.../ePix/Characterize_Darks_ePix_NBC.ipynb | 2 +-
.../pnCCD/Characterize_pnCCD_Dark_NBC.ipynb | 4 +-
webservice/serve_overview.py | 127 ++----------------
webservice/templates/dark_overview.html | 81 +++--------
webservice/webservice.py | 3 +-
xfel_calibrate/calibrate.py | 27 +---
9 files changed, 51 insertions(+), 280 deletions(-)
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index c9ea07079..d1912490b 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -2,7 +2,6 @@ from collections import OrderedDict
import datetime
from glob import glob
from importlib.machinery import SourceFileLoader
-import inspect
import json
from os import chdir, environ, listdir, makedirs, path, remove, stat
from os.path import isdir, isfile, splitext
@@ -15,10 +14,7 @@ from time import sleep
from urllib.parse import urljoin
import dateutil.parser
-import h5py
-from iCalibrationDB import (ConstantMetaData, Constants,
- Conditions, Detectors, Versions,
- DetectorCondition, OperatingCondition)
+
import ipykernel
from metadata_client.metadata_client import MetadataClient
from notebook.notebookapp import list_running_servers
@@ -249,8 +245,9 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
if isinstance(v, str):
tmpl = '{} = "{}"\n'
+ # Set name of the latex document
if var == 'latex_documents' and len(v[0]) > 1:
- v[0] = v[0][:1] + ('{}.tex'.format(report_name),) + v[0][2:]
+ v[0] = v[0][:1] + ('{}.tex'.format(report_name), ) + v[0][2:]
mf.write(tmpl.format(var, v))
@@ -504,13 +501,11 @@ def get_dir_creation_date(directory, run, tsdir=False, verbosity=0):
def save_const_to_h5(metadata, out_folder):
"""
- Save constant to h5 file
+ Save constant in h5 file with it's metadata
+ (e.g. db_module, condition, creation_time)
:param metadata: Metadata
:param out_folder: path to output folder
- :param constant_name: constant name, e.g. Offset
- :param db_module: module name
- :return:
"""
dpar = {}
@@ -534,6 +529,8 @@ def save_const_to_h5(metadata, out_folder):
data_to_store['file_loc'] = raw_data
ofile = "{}/const_{}_{}.h5".format(out_folder, constant_name, db_module)
+ if isfile(ofile):
+ print('File {} already exists and will be overwritten'.format(ofile))
save_dict_to_hdf5(data_to_store, ofile)
@@ -628,72 +625,6 @@ def get_from_db(device, constant, condition, empty_constant,
return empty_constant, None
-def send_from_file(filename, cal_db_interface, verbosity=1, timeout=30000,
- ntries=120, doraise=False):
- """
-
- :param filename: Path to file
- :param cal_db_interface: Interface string, e.g. "tcp://max-exfl016:8015"
- :param verbosity: Level of verbosity (0 - silent)
- :param timeout: Timeout for zmq request
- :param ntries: number of tries to contact the database
- :param doraise: if True raise errors during communication with DB
- """
- with h5py.File(filename, 'r') as f:
-
- conds = {}
- for key in f['condition']:
- conds[key] = \
- {'lower_deviation':
- f['condition/{}/lower_deviation_value'.format(key)][()],
- 'upper_deviation':
- f['condition/{}/upper_deviation_value'.format(key)][()],
- 'value': f['condition/{}/value'.format(key)][()]
- }
-
- db_module = f['db_module'][()]
- const_name = f['constant'][()]
- file_loc = f['file_loc'][()]
- const_data = f['data'][()]
- creation_time = dateutil.parser.parse(f['creation_time'][()])
-
- det = Detectors().get_instance_by_name(db_module)
- dclass = det.detector_type.name
-
- if 'CCD' in dclass:
- dclass = 'CCD'
-
- # Get getector conditions
- if dclass == 'CCD':
- dconstants = getattr(Constants, dclass)(det.detector_type)
- else:
- dconstants = getattr(Constants, dclass)
-
- constant = getattr(dconstants, const_name)()
- constant.data = const_data
-
- # initialize detector conditions with value=1
- dcond = Conditions.Dark
- p = inspect.signature(getattr(dcond, dclass)).parameters
- pars = {key: 1 for key in p.keys()}
- # string should be used for LPD capacitor
- if 'capacitor' in pars:
- pars['capacitor'] = '5'
- mcond = getattr(dcond, dclass)(**pars)
-
- # set actual condition values taken from file
- for i, parm in enumerate(mcond.parameters):
- if parm.name in conds:
- parm.lower_deviation = conds[parm.name]['lower_deviation']
- parm.upper_deviation = conds[parm.name]['upper_deviation']
- parm.value = conds[parm.name]['value']
- else:
- del mcond.parameters[i]
-
- send_to_db(det, constant, mcond, file_loc, cal_db_interface, creation_time,
- verbosity, timeout, ntries, doraise)
-
-
def send_to_db(device, constant, condition, file_loc,
cal_db_interface, creation_time=None,
verbosity=1, timeout=30000, ntries=120, doraise=False):
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 1d315af88..e2f81fe25 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -34,7 +34,7 @@
"run_low = 266 # run number in which low gain data was recorded, required\n",
"\n",
"mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
- "local_output = True # output constants locally\n",
+ "local_output = False # output constants locally\n",
"db_output = True # output constants to database\n",
"bias_voltage = 300 # detector bias voltage\n",
"cal_db_interface = \"tcp://max-exfl016:8020\" # the database interface to use\n",
diff --git a/notebooks/ePix/Characterize_Darks_ePix10K_NBC.ipynb b/notebooks/ePix/Characterize_Darks_ePix10K_NBC.ipynb
index 678497609..c9e213439 100644
--- a/notebooks/ePix/Characterize_Darks_ePix10K_NBC.ipynb
+++ b/notebooks/ePix/Characterize_Darks_ePix10K_NBC.ipynb
@@ -39,7 +39,7 @@
"path_inset = \"EPIX03\" # file inset for image data\n",
"fix_temperature = 290. # fix temperature to this value\n",
"db_output = True # Output constants to the calibration database\n",
- "local_output = True # output constants locally"
+ "local_output = False # output constants locally"
]
},
{
diff --git a/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb b/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb
index 21c82e006..831a1c70a 100644
--- a/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb
+++ b/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb
@@ -39,7 +39,7 @@
"path_inset = \"DA01\" # file inset for image data\n",
"fix_temperature = 290. # fix temperature to this value\n",
"db_output = True # Output constants to the calibration database\n",
- "local_output = True # output constants locally"
+ "local_output = False # output constants locally"
]
},
{
diff --git a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
index 9a9d68b10..1543ff4b5 100644
--- a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
+++ b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
@@ -44,8 +44,8 @@
"gain = 0 # the detector gain setting, only 0 is currently implemented\n",
"bias_voltage = 300 # detector bias voltage\n",
"integration_time = 70 # detector integration time\n",
- "db_output = False # Output constants to the calibration database\n",
- "local_output = True # output constants locally"
+ "db_output = True # Output constants to the calibration database\n",
+ "local_output = False # output constants locally"
]
},
{
diff --git a/webservice/serve_overview.py b/webservice/serve_overview.py
index 067ce421d..a2b417aea 100644
--- a/webservice/serve_overview.py
+++ b/webservice/serve_overview.py
@@ -10,7 +10,6 @@ from uuid import uuid4
import yaml
from jinja2 import Template
-from cal_tools.tools import send_from_file
from xfel_calibrate.settings import (free_nodes_cmd, preempt_nodes_cmd,
reservation)
@@ -63,36 +62,6 @@ class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
- def check_const_files(pars):
- """
- Check if calibration constants already exists
- :param pars: Dictionary of parameters
- :return: True if constants already exist
- """
-
- path = '/gpfs/exfel/exp/{}/{}/p{:06d}/usr/dark/{}'
- if 'run' in pars:
- runs = 'runs_{}'.format(pars['run'])
- else:
- runs = 'runs_{}_{}_{}'.format(pars['run_high'],
- pars['run_med'],
- pars['run_low'])
-
- path = path.format(pars['instrument'],
- pars['cycle'],
- int(pars['proposal']),
- runs)
-
- dets = pars['detectors'].split(',')
- for detector in dets:
- opath = '{}/{}/const*h5'.format(path,
- detector.replace('-', '_'))
- files = glob.glob(opath)
- if len(files)>0:
- return True
- return False
-
-
if not self.conf_was_init:
self.init_config()
# Send response status code
@@ -115,48 +84,6 @@ class RequestHandler(BaseHTTPRequestHandler):
self.wfile.write(f.read())
return
- if "process_constants?" in self.path:
- self.send_header('Content-type', 'text/html')
- self.end_headers()
- pars = self.path.split("?")[1].split("&")
- pars = {x.split("=")[0]: x.split("=")[1] for x in pars}
-
- if 'action' not in pars or 'path' not in pars:
- return
-
- if pars['action'] == 'delete':
- try:
- files = glob.glob('{}/const*h5'.format(pars['path']))
- for file_name in files:
- os.remove(file_name)
- msg = "{} Constants were deleted".format(len(files))
- except Exception as e:
- msg = 'Error: {}'.format(e)
-
- self.wfile.write(bytes(msg, "utf8"))
- return
-
- if pars['action'] == 'inject':
- try:
- files = glob.glob('{}/const*h5'.format(pars['path']))
- for file_name in files:
- cal_db_interface = config["server-config"][
- "cal_db_interface"] # noqa
- cal_db_timeout = config["server-config"][
- "cal_db_timeout"] # noqa
- send_from_file(file_name, cal_db_interface,
- verbosity=0,
- timeout=int(cal_db_timeout), ntries=2,
- doraise=True)
- msg = "{} Constants were injected".format(len(files))
- except Exception as e:
- msg = 'Error: {}'.format(e)
-
- self.wfile.write(bytes(msg, "utf8"))
- return
-
- return
-
if "process_dark?" in self.path:
self.send_header('Content-type', 'text/html')
self.end_headers()
@@ -181,16 +108,11 @@ class RequestHandler(BaseHTTPRequestHandler):
par_list.append('{}'.format(v))
par_list = list(filter(None, par_list))
+ print('REQUEST DARK: ', par_list)
timeout = config["server-config"]["dark-timeout"]
try:
- # check if h5 files exist in folder
- if check_const_files(pars):
- msg = 'Constant exists, remove them before processing'
- msg = '<div style = "color:red;font-weight:bold;font-size:160%;" > {} </div> '.format(msg) # noqa
- else:
- print('REQUEST DARK: ', par_list)
- msg = check_output(par_list, shell=False,
- timeout=timeout).decode('utf8')
+ msg = check_output(par_list, shell=False,
+ timeout=timeout).decode('utf8')
except Exception as e:
msg = str(e)
@@ -241,44 +163,24 @@ class RequestHandler(BaseHTTPRequestHandler):
reports[instrument] = {}
for detector in detectors:
det_inset = detector.replace('-', '_')
- tmpl = '/gpfs/exfel/exp/{}/*/*/usr/dark/*/{}/'.format(
+ tmpl = '/gpfs/exfel/exp/{}/*/*/usr/dark/*/{}'.format(
instrument, det_inset)
- files = glob.glob(tmpl)
+ files = glob.glob(tmpl + '/*pdf')
+ files += glob.glob(tmpl + '/*/*pdf')
files.sort(key=os.path.getmtime, reverse=True)
file_info = []
for i, file in enumerate(files):
- reps = glob.glob(file + '*pdf')
- const = glob.glob(file + 'const*h5')
-
- reps.sort(key=os.path.getmtime, reverse=False)
- const.sort(key=os.path.getmtime, reverse=True)
-
- for rep in reps:
- time = os.stat(rep).st_mtime
- d_time = datetime.fromtimestamp(time).replace(
- tzinfo=timezone.utc)
- s_time = d_time.strftime('%y-%m-%d %H:%M')
- file_info.append({
- 'report': rep,
- 'path': file,
- 'report_time': s_time
- })
-
- if len(const) > 0:
- time = os.stat(const[-1]).st_mtime
- d_time = datetime.fromtimestamp(time).replace(
- tzinfo=timezone.utc)
- s_time = d_time.strftime('%y-%m-%d %H:%M')
-
- file_info[-1]['const'] = True
- file_info[-1]['const_time'] = s_time
-
- for i in range(len(file_info)):
- if (i % 2) == 0:
+ if 'xfel.pdf' in file:
+ continue
+ if (len(file_info) % 2) == 0:
bgcolor = 'EEEEEE'
else:
bgcolor = 'FFFFFF'
- file_info[i]['bgcolor'] = bgcolor
+ time = os.stat(file).st_mtime
+ d_time = datetime.fromtimestamp(time).replace(
+ tzinfo=timezone.utc)
+ s_time = d_time.strftime('%y-%m-%d %H:%M')
+ file_info.append([file, s_time, bgcolor])
reports[instrument][detector] = file_info
@@ -289,7 +191,6 @@ class RequestHandler(BaseHTTPRequestHandler):
return
if "/gpfs" in self.path:
-
sendReply = False
if self.path.endswith(".html"):
mimetype = 'text/html'
diff --git a/webservice/templates/dark_overview.html b/webservice/templates/dark_overview.html
index f7ab166af..200c63b3b 100644
--- a/webservice/templates/dark_overview.html
+++ b/webservice/templates/dark_overview.html
@@ -6,76 +6,29 @@
</head>
<body>
-
-<script type="text/javascript">
-
-function processConstants(file, action) {
-
- var xhttp = new XMLHttpRequest();
- xhttp.onreadystatechange = function() {
- if (this.readyState == 4 && this.status == 200) {
- var result_style = document.getElementById(file).style;
- result_style.display = '';
- document.getElementById("info"+file).innerHTML = this.responseText;
- }
- };
-
- msg = 'action=' + action;
- msg += '&path=' + file;
-
- var result_style = document.getElementById(file).style;
- result_style.display = '';
- document.getElementById("info"+file).innerHTML = 'Perform action: ' + action;
- xhttp.open("GET", "/process_constants?"+msg, true);
- xhttp.send();
-
-}
-</script>
-
-
<div class="block">
- <h1>Dark runs</h1>
- {% for instrument, detectors in reports.items() %}
- <h2>{{ instrument }}</h2>
+ <h1>Dark runs</h1>
+ {% for instrument, detectors in reports.items() %}
+ <h2>{{ instrument }}</h2>
- {% for detector, files in detectors.items() %}
- <h3>{{ detector }}</h3>
+ {% for detector, files in detectors.items() %}
+ <h3>{{ detector }}</h3>
<table>
- {% for file in files %}
-
- <tr bgcolor="{{ file['bgcolor'] }}">
- <td><a href=http://{{host}}:{{port}}/{{ file['report'] }} target="_blank">{{ file['report'] }}</a> </td>
- <td> {{ file['report_time'] }}</td>
-
- <td>
- {% if file['const'] %}
- {{ file['const_time'] }}
- <input type="button" value="inject" onclick="javascript:processConstants('{{ file['path'] }}', 'inject');">
- <input type="button" value="delete" onclick="javascript:processConstants('{{ file['path'] }}', 'delete');">
- {% endif %}
- </td>
- </tr>
-
- {% if file['const'] %}
- <tr id="{{ file['path'] }}" style="display: none;">
- <td colspan="3">
- <div id="info{{ file['path'] }}"></div>
- </td>
- </tr>
- {% endif %}
-
-
- {% endfor %}
- </table>
- <br>
- {% endfor %}
- <br>
+ {% for file in files %}
+
+ <tr bgcolor="{{ file[2] }}">
+ <td> <a href=http://{{host}}:{{port}}/file?{{ file[0] }} target="_blank">{{ file[0] }}</a> </td>
+ <td> {{ file[1] }} </td>
+ </tr>
+
+ {% endfor %}
+ </table>
+ <br>
+ {% endfor %}
+ <br>
{% endfor %}
</div>
-<div id="demo">
- </div>
-
</body>
</html>
diff --git a/webservice/webservice.py b/webservice/webservice.py
index 241a5ae36..be231a0a2 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -612,6 +612,7 @@ async def server_runner(config, mode):
return
print("Now doing: {}".format(action))
+ ts = datetime.now().strftime('%y%m%d_%H%M%S')
if action == 'dark':
print("Running dark cal")
status = []
@@ -636,7 +637,6 @@ async def server_runner(config, mode):
thisconf["out-folder"] = '/'.join((out_folder,
detector.replace('-', '_')))
- ts = datetime.now().strftime('%y%m%d_%H%M')
report_to = 'Dark_' + \
detector.replace('-', '_') + '_' + ts
thisconf["report-to"] = report_to.lower()
@@ -704,7 +704,6 @@ async def server_runner(config, mode):
thisconf = copy.copy(dconfig)
thisconf["in-folder"] = in_folder
thisconf["out-folder"] = out_folder
- ts = datetime.now().strftime('%y%m%d_%H%M')
report_to = 'Correct_' + \
detector.replace('-', '_') + '_' + ts
thisconf["report-to"] = report_to.lower()
diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index b03a1b48b..f1e7badbe 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -455,31 +455,18 @@ def has_parm(parms, name):
return False
-def get_par_value(parms, key, default=None):
- """
- Return the value of parameter with name key
- :param parms: List of parameters
- :param key: Name of the parameter to be considered
- :param default: Value to be returned if interested name is not found
- :return: The value of the parameter
- """
- for p in parms:
- if p.name == key:
- return p.value
- return default
-
-
-def get_par_type(parms, key, default=None):
+def get_par_attr(parms, key, attr, default=None):
"""
Return the type of parameter with name key
:param parms: List of parameters
:param key: Name of the parameter to be considered
+ :param attr: Name of the parameter attribute (e.g. value, type)
:param default: Type to be returned if interested name is not found
- :return: The type of the parameter
+ :return: The attribute of the parameter
"""
for p in parms:
if p.name == key:
- return p.type
+ return getattr(p, attr, default)
return default
@@ -797,7 +784,7 @@ def run():
if args["report_to"] is not None:
report_to = args["report_to"]
- folder = get_par_value(parms, 'in_folder', '')
+ folder = get_par_attr(parms, 'in_folder', 'value', '')
fmtcmd = cmd.format(run_path=run_tmp_path, out_path=out_path,
project=title, calibration=title,
@@ -833,7 +820,7 @@ def run():
cvals = defcval
if cvals is None:
- defcval = get_par_value(parms, cvar)
+ defcval = get_par_attr(parms, cvar, 'value')
if defcval is not None:
print("Concurrency parameter '{}' is taken from '{}'".format(
cvar, notebook))
@@ -863,7 +850,7 @@ def run():
print("Split concurrency into {}".format(cvals))
# get expected type
- cvtype = get_par_type(parms, cvar, list)
+ cvtype = get_par_attr(parms, cvar, 'type', list)
for cnum, cval in enumerate(cvals):
show_title = cnum == 0
--
GitLab