From bc1bb97125fac0d44763d676b750984cf9e1e168 Mon Sep 17 00:00:00 2001
From: karnem <mikhail.karnevskiy@desy.de>
Date: Tue, 12 Nov 2019 12:45:47 +0100
Subject: [PATCH] runs in input folder and report names
---
cal_tools/cal_tools/plotting.py | 4 +-
cal_tools/cal_tools/tools.py | 179 +++++++++++++-
.../Characterize_AGIPD_Gain_Darks_NBC.ipynb | 91 +++-----
...s_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb | 26 ++-
notebooks/LPD/LPDChar_Darks_NBC.ipynb | 66 +++---
notebooks/LPD/LPD_Correct_and_Verify.ipynb | 3 -
.../ePix/Characterize_Darks_ePix10K_NBC.ipynb | 35 +--
.../ePix/Characterize_Darks_ePix_NBC.ipynb | 27 ++-
notebooks/ePix/Correction_ePix10K_NBC.ipynb | 32 +--
notebooks/ePix/Correction_ePix_NBC.ipynb | 2 -
.../pnCCD/Characterize_pnCCD_Dark_NBC.ipynb | 219 +++++++-----------
notebooks/pnCCD/Correct_pnCCD_NBC.ipynb | 41 ++--
webservice/serve_overview.py | 169 ++++++++++++--
webservice/serve_overview.yaml | 2 +
webservice/templates/dark_overview.html | 81 +++++--
webservice/templates/last_correction.html | 2 +-
webservice/templates/request_dark.html | 2 +-
webservice/webservice.py | 14 +-
webservice/webservice.yaml | 2 +-
xfel_calibrate/calibrate.py | 78 ++++---
20 files changed, 669 insertions(+), 406 deletions(-)
diff --git a/cal_tools/cal_tools/plotting.py b/cal_tools/cal_tools/plotting.py
index fc38cc93d..ca8580fcd 100644
--- a/cal_tools/cal_tools/plotting.py
+++ b/cal_tools/cal_tools/plotting.py
@@ -21,7 +21,7 @@ def show_overview(d, cell_to_preview, gain_to_preview, out_folder=None, infix=No
i = 0
for key, item in data.items():
cf = 0
- if "Threshold" in key:
+ if "ThresholdsDark" in key:
cf = -1
if len(item.shape) == 4:
med = np.nanmedian(item[...,cell_to_preview, gain_to_preview + cf])
@@ -33,7 +33,7 @@ def show_overview(d, cell_to_preview, gain_to_preview, out_folder=None, infix=No
(item[...,cell_to_preview, gain_to_preview + cf] > med+np.abs(bound*med)))/item[...,cell_to_preview, gain_to_preview + cf].size > 0.01):
bound *=2
- if "BadPixels" in key:
+ if "BadPixelsDark" in key:
im = grid[i].imshow(np.log2(item[...,cell_to_preview, gain_to_preview + cf]), interpolation="nearest",
vmin=0, vmax=8, aspect='auto')
else:
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index eb791019f..c9ea07079 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -2,6 +2,7 @@ from collections import OrderedDict
import datetime
from glob import glob
from importlib.machinery import SourceFileLoader
+import inspect
import json
from os import chdir, environ, listdir, makedirs, path, remove, stat
from os.path import isdir, isfile, splitext
@@ -14,6 +15,10 @@ from time import sleep
from urllib.parse import urljoin
import dateutil.parser
+import h5py
+from iCalibrationDB import (ConstantMetaData, Constants,
+ Conditions, Detectors, Versions,
+ DetectorCondition, OperatingCondition)
import ipykernel
from metadata_client.metadata_client import MetadataClient
from notebook.notebookapp import list_running_servers
@@ -23,6 +28,7 @@ import tabulate
from jinja2 import Template
from .mdc_config import MDC_config
+from .ana_tools import save_dict_to_hdf5
from xfel_calibrate import settings
@@ -173,7 +179,7 @@ def make_timing_summary(run_path, joblist):
with open("{}/timing_summary.rst".format(run_path), "w+") as gfile:
- if len(pars_vals)>0:
+ if len(pars_vals) > 0:
table = tabulate.tabulate(pars_vals, tablefmt='latex',
headers=pars_name)
gfile.write(dedent(tmpl.render(table=table.split('\n'))))
@@ -243,6 +249,9 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
if isinstance(v, str):
tmpl = '{} = "{}"\n'
+ if var == 'latex_documents' and len(v[0]) > 1:
+ v[0] = v[0][:1] + ('{}.tex'.format(report_name),) + v[0][2:]
+
mf.write(tmpl.format(var, v))
remove("{}/conf.py".format(run_path))
@@ -281,8 +290,7 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
"can be inspected at: {}".format(run_path))
return
print("Moving report to final location: {}".format(out_path))
- for f in glob(r'{}/_build/latex/*.pdf'.format(run_path)):
- copy(f, out_path)
+ copy('{}/_build/latex/{}.pdf'.format(run_path, report_name), out_path)
print("Removing temporary files at: {}".format(tmp_path))
rmtree(tmp_path)
@@ -494,6 +502,41 @@ def get_dir_creation_date(directory, run, tsdir=False, verbosity=0):
ntries -= 1
+def save_const_to_h5(metadata, out_folder):
+ """
+ Save constant to h5 file
+
+ :param metadata: Metadata
+ :param out_folder: path to output folder
+ :param constant_name: constant name, e.g. Offset
+ :param db_module: module name
+ :return:
+ """
+
+ dpar = {}
+ for parm in metadata.detector_condition.parameters:
+ dpar[parm.name] = {'lower_deviation_value': parm.lower_deviation,
+ 'upper_deviation_value': parm.upper_deviation,
+ 'value': parm.value,
+ 'flg_logarithmic': parm.logarithmic}
+
+ creation_time = metadata.calibration_constant_version.begin_at
+ raw_data = metadata.calibration_constant_version.raw_data_location
+ db_module = metadata.calibration_constant_version.device_name
+ constant_name = metadata.calibration_constant.__class__.__name__
+
+ data_to_store = {}
+ data_to_store['condition'] = dpar
+ data_to_store['db_module'] = db_module
+ data_to_store['constant'] = constant_name
+ data_to_store['data'] = metadata.calibration_constant.data
+ data_to_store['creation_time'] = creation_time
+ data_to_store['file_loc'] = raw_data
+
+ ofile = "{}/const_{}_{}.h5".format(out_folder, constant_name, db_module)
+ save_dict_to_hdf5(data_to_store, ofile)
+
+
def get_random_db_interface(cal_db_interface):
"""
Return interface to calibration DB with random (with given range) port
@@ -585,6 +628,136 @@ def get_from_db(device, constant, condition, empty_constant,
return empty_constant, None
+def send_from_file(filename, cal_db_interface, verbosity=1, timeout=30000,
+ ntries=120, doraise=False):
+ """
+
+ :param filename: Path to file
+ :param cal_db_interface: Interface string, e.g. "tcp://max-exfl016:8015"
+ :param verbosity: Level of verbosity (0 - silent)
+ :param timeout: Timeout for zmq request
+ :param ntries: number of tries to contact the database
+ :param doraise: if True raise errors during communication with DB
+ """
+ with h5py.File(filename, 'r') as f:
+
+ conds = {}
+ for key in f['condition']:
+ conds[key] = \
+ {'lower_deviation':
+ f['condition/{}/lower_deviation_value'.format(key)][()],
+ 'upper_deviation':
+ f['condition/{}/upper_deviation_value'.format(key)][()],
+ 'value': f['condition/{}/value'.format(key)][()]
+ }
+
+ db_module = f['db_module'][()]
+ const_name = f['constant'][()]
+ file_loc = f['file_loc'][()]
+ const_data = f['data'][()]
+ creation_time = dateutil.parser.parse(f['creation_time'][()])
+
+ det = Detectors().get_instance_by_name(db_module)
+ dclass = det.detector_type.name
+
+ if 'CCD' in dclass:
+ dclass = 'CCD'
+
+ # Get getector conditions
+ if dclass == 'CCD':
+ dconstants = getattr(Constants, dclass)(det.detector_type)
+ else:
+ dconstants = getattr(Constants, dclass)
+
+ constant = getattr(dconstants, const_name)()
+ constant.data = const_data
+
+ # initialize detector conditions with value=1
+ dcond = Conditions.Dark
+ p = inspect.signature(getattr(dcond, dclass)).parameters
+ pars = {key: 1 for key in p.keys()}
+ # string should be used for LPD capacitor
+ if 'capacitor' in pars:
+ pars['capacitor'] = '5'
+ mcond = getattr(dcond, dclass)(**pars)
+
+ # set actual condition values taken from file
+ for i, parm in enumerate(mcond.parameters):
+ if parm.name in conds:
+ parm.lower_deviation = conds[parm.name]['lower_deviation']
+ parm.upper_deviation = conds[parm.name]['upper_deviation']
+ parm.value = conds[parm.name]['value']
+ else:
+ del mcond.parameters[i]
+
+ send_to_db(det, constant, mcond, file_loc, cal_db_interface, creation_time,
+ verbosity, timeout, ntries, doraise)
+
+
+def send_to_db(device, constant, condition, file_loc,
+ cal_db_interface, creation_time=None,
+ verbosity=1, timeout=30000, ntries=120, doraise=False):
+ """
+ Return calibration constants and metadata requested from CalDB
+
+ :param device: Instance of detector
+ :param constant: Calibration constant known for given detector
+ :param condition: Calibration condition
+ :param file_loc: Location of raw data.
+ :param cal_db_interface: Interface string, e.g. "tcp://max-exfl016:8015"
+ :param creation_time: Latest time for constant to be created
+ :param verbosity: Level of verbosity (0 - silent)
+ :param timeout: Timeout for zmq request
+ :param ntries: number of tries to contact the database
+ :param doraise: if True raise errors during communication with DB
+ """
+ from iCalibrationDB import ConstantMetaData, Versions
+ import zmq
+
+ if device:
+ metadata = ConstantMetaData()
+ metadata.calibration_constant = constant
+ metadata.detector_condition = condition
+ if creation_time is None:
+ metadata.calibration_constant_version = Versions.Now(
+ device=device)
+ else:
+ metadata.calibration_constant_version = Versions.Timespan(
+ device=device,
+ start=creation_time)
+
+ metadata.calibration_constant_version.raw_data_location = file_loc
+
+ while ntries > 0:
+
+ this_interface = get_random_db_interface(cal_db_interface)
+ try:
+ r = metadata.send(this_interface, timeout=timeout)
+ break
+ except zmq.error.Again:
+ ntries -= 1
+ sleep(np.random.randint(30))
+ if ntries == 0 and doraise:
+ raise
+ except Exception as e:
+ if verbosity > 0:
+ print(e)
+ if 'missing_token' in str(e):
+ ntries -= 1
+ else:
+ ntries = 0
+ if ntries == 0 and doraise:
+ raise
+
+ if ntries > 0:
+ if verbosity > 0:
+ if constant.name not in already_printed or verbosity > 1:
+ already_printed[constant.name] = True
+ begin_at = metadata.calibration_constant_version.begin_at
+ print("{} was sent on: {}".format(constant.name,
+ begin_at))
+
+
def get_constant_from_db(device, constant, condition, empty_constant,
cal_db_interface, creation_time=None,
print_once=True, timeout=30000, ntries=120,
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 91e853812..1d315af88 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -34,10 +34,11 @@
"run_low = 266 # run number in which low gain data was recorded, required\n",
"\n",
"mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
- "local_output = False # output constants locally\n",
+ "local_output = True # output constants locally\n",
"db_output = True # output constants to database\n",
"bias_voltage = 300 # detector bias voltage\n",
"cal_db_interface = \"tcp://max-exfl016:8020\" # the database interface to use\n",
+ "cal_db_timeout = 3000000 # timeout on caldb requests\"\n",
"interlaced = False # assume interlaced data format, for data prior to Dec. 2017\n",
"rawversion = 2 # RAW file format version\n",
"dont_use_dir_date = False # don't use the dir creation date for determining the creation time\n",
@@ -78,7 +79,9 @@
"import matplotlib.pyplot as plt\n",
"%matplotlib inline\n",
"\n",
- "from cal_tools.tools import gain_map_files, parse_runs, run_prop_seq_from_path, get_notebook_name, get_dir_creation_date\n",
+ "from cal_tools.tools import (gain_map_files, parse_runs, \n",
+ " run_prop_seq_from_path, get_notebook_name, \n",
+ " get_dir_creation_date, save_const_to_h5)\n",
"from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import show_overview, plot_badpix_3d, create_constant_overview\n",
@@ -389,8 +392,8 @@
" qm = \"Q{}M{}\".format(i//4+1, i%4+1)\n",
" res[qm] = {'Offset': offset_g[qm],\n",
" 'Noise': noise_g[qm],\n",
- " 'Threshold': thresholds_g[qm],\n",
- " 'BadPixels': badpix_g[qm] \n",
+ " 'ThresholdsDark': thresholds_g[qm],\n",
+ " 'BadPixelsDark': badpix_g[qm] \n",
" }\n",
" \n",
"if local_output:\n",
@@ -425,79 +428,43 @@
},
"outputs": [],
"source": [
- "if db_output:\n",
- " for qm in offset_g.keys():\n",
+ "for qm in res:\n",
+ " for const in res[qm]:\n",
" metadata = ConstantMetaData()\n",
- " offset = Constants.AGIPD.Offset()\n",
- " offset.data = offset_g[qm]\n",
- " metadata.calibration_constant = offset\n",
+ " dconst = getattr(Constants.AGIPD, const)()\n",
+ " dconst.data = res[qm][const]\n",
+ " metadata.calibration_constant = dconst\n",
"\n",
" # set the operating condition\n",
- " condition = Conditions.Dark.AGIPD(memory_cells=max_cells, bias_voltage=bias_voltage, acquisition_rate=acq_rate)\n",
+ " condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
+ " bias_voltage=bias_voltage,\n",
+ " acquisition_rate=acq_rate)\n",
" detinst = getattr(Detectors, dinstance)\n",
" device = getattr(detinst, qm)\n",
" \n",
" metadata.detector_condition = condition\n",
- " \n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
- " metadata.send(cal_db_interface, timeout=3000000)\n",
- " \n",
- " \n",
- " metadata = ConstantMetaData()\n",
- " noise = Constants.AGIPD.Noise()\n",
- " noise.data = noise_g[qm]\n",
- " metadata.calibration_constant = noise\n",
- "\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.AGIPD(memory_cells=max_cells, bias_voltage=bias_voltage, acquisition_rate=acq_rate)\n",
- " metadata.detector_condition = condition\n",
"\n",
" # specify the a version for this constant\n",
" if creation_time is None:\n",
" metadata.calibration_constant_version = Versions.Now(device=device)\n",
" else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
- " metadata.send(cal_db_interface, timeout=3000000)\n",
- " \n",
- " metadata = ConstantMetaData()\n",
- " thresholds = Constants.AGIPD.ThresholdsDark()\n",
- " thresholds.data = thresholds_g[qm]\n",
- " metadata.calibration_constant = thresholds\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
+ " start=creation_time)\n",
"\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.AGIPD(memory_cells=max_cells, bias_voltage=bias_voltage, acquisition_rate=acq_rate)\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
" metadata.calibration_constant_version.raw_data_location = file_loc\n",
- " metadata.send(cal_db_interface, timeout=3000000)\n",
- " \n",
- " metadata = ConstantMetaData()\n",
- " badpixels = Constants.AGIPD.BadPixelsDark()\n",
- " badpixels.data = badpix_g[qm]\n",
- " metadata.calibration_constant = badpixels\n",
"\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.AGIPD(memory_cells=max_cells, bias_voltage=bias_voltage, acquisition_rate=acq_rate)\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
- " metadata.send(cal_db_interface, timeout=3000000)"
+ " if db_output:\n",
+ " try:\n",
+ " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
+ " msg = 'Const {} for module {} was injected to the calibration DB. Begin at: {}'\n",
+ " print(msg.format(const, qm,\n",
+ " metadata.calibration_constant_version.begin_at))\n",
+ " except Exception as e:\n",
+ " print(e)\n",
+ "\n",
+ " if local_output:\n",
+ " save_const_to_h5(metadata, out_folder)\n",
+ " print(\"Calibration constant {} is stored locally.\".format(const))\n"
]
},
{
diff --git a/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb b/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
index 0b56642e1..e27e4a3dc 100644
--- a/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
+++ b/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
@@ -50,7 +50,8 @@
" # high gain: conversion gain is 1 ADU = 6.1e-\n",
"ADU_to_electron_lower_hg = 6.2 # and for lower hemisphere and high gain: conversion gain is 1 ADU = 6.2e- \n",
"run_parallel = True # For parallel computation \n",
- "db_output = True # Output constants to the calibration database"
+ "db_output = True # Output constants to the calibration database\n",
+ "local_output = True # output constants locally"
]
},
{
@@ -82,7 +83,7 @@
"\n",
"from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
"from iCalibrationDB.detectors import DetectorTypes\n",
- "from cal_tools.tools import get_dir_creation_date\n",
+ "from cal_tools.tools import get_dir_creation_date, save_const_to_h5\n",
"from cal_tools.enums import BadPixels\n",
"from XFELDetAna import xfelpyanatools as xana\n",
"from XFELDetAna import xfelpycaltools as xcal\n",
@@ -1110,21 +1111,22 @@
" else:\n",
" metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
" \n",
+ " metadata.calibration_constant_version.raw_data_location = file_loc\n",
+ " \n",
" if db_output:\n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
- " metadata.send(cal_db_interface, timeout=cal_db_timeout) \n",
+ " try:\n",
+ " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
+ " print(\"Calibration constant {} is sent to the database.\".format(const))\n",
+ " except Exception as e:\n",
+ " print(e)\n",
+ " \n",
+ " if local_output:\n",
+ " save_const_to_h5(metadata, out_folder)\n",
+ " print(\"Calibration constant {} is stored locally.\".format(const))\n",
" \n",
- "print(\"Calibration constants (offsetMap, noiseMapCM_2nd and bad_pixels) are sent to the calibration database.\")\n",
"print(\"Creation time is: {}\".format(creation_time))\n",
"print(\"Raw data location is: {}\".format(file_loc))"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
diff --git a/notebooks/LPD/LPDChar_Darks_NBC.ipynb b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
index 521c9573b..09826cea8 100644
--- a/notebooks/LPD/LPDChar_Darks_NBC.ipynb
+++ b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
@@ -120,7 +120,8 @@
" run_prop_seq_from_path, \n",
" get_notebook_name, \n",
" get_dir_creation_date, get_from_db,\n",
- " get_random_db_interface)\n",
+ " get_random_db_interface,\n",
+ " save_const_to_h5)\n",
"from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import (show_overview, plot_badpix_3d, \n",
@@ -452,37 +453,44 @@
"outputs": [],
"source": [
"# Save constants in the calibration DB\n",
- "if db_output:\n",
- " for cap in capacitor_settings:\n",
- " for qm in res[cap]:\n",
- " for const in res[cap][qm]:\n",
- "\n",
- " metadata = ConstantMetaData()\n",
- " dconst = getattr(Constants.LPD, const)()\n",
- " dconst.data = res[cap][qm][const]\n",
- " metadata.calibration_constant = dconst\n",
+ "for cap in capacitor_settings:\n",
+ " for qm in res[cap]:\n",
+ " for const in res[cap][qm]:\n",
"\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.LPD(memory_cells=max_cells,\n",
- " bias_voltage=bias_voltage,\n",
- " capacitor=cap)\n",
- " device = getattr(Detectors.LPD1M1, qm)\n",
- " if device:\n",
+ " metadata = ConstantMetaData()\n",
+ " dconst = getattr(Constants.LPD, const)()\n",
+ " dconst.data = res[cap][qm][const]\n",
+ " metadata.calibration_constant = dconst\n",
"\n",
- " metadata.detector_condition = condition\n",
+ " # set the operating condition\n",
+ " condition = Conditions.Dark.LPD(memory_cells=max_cells,\n",
+ " bias_voltage=bias_voltage,\n",
+ " capacitor=cap)\n",
+ " device = getattr(Detectors.LPD1M1, qm)\n",
+ " if device:\n",
+ " metadata.detector_condition = condition\n",
"\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
- " start=creation_time)\n",
- " \n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
- " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
- " msg = 'Const {} for module {} was injected to the calibration DB. Begin at: {}'\n",
- " print(msg.format(const, qm,\n",
- " metadata.calibration_constant_version.begin_at))"
+ " # specify the a version for this constant\n",
+ " if creation_time is None:\n",
+ " metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ " else:\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
+ " start=creation_time)\n",
+ "\n",
+ " metadata.calibration_constant_version.raw_data_location = file_loc\n",
+ "\n",
+ " if db_output:\n",
+ " try:\n",
+ " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
+ " msg = 'Const {} for module {} was injected to the calibration DB. Begin at: {}'\n",
+ " print(msg.format(const, qm,\n",
+ " metadata.calibration_constant_version.begin_at))\n",
+ " except Exception as e:\n",
+ " print(e)\n",
+ "\n",
+ " if local_output:\n",
+ " save_const_to_h5(metadata, out_folder)\n",
+ " print(\"Calibration constant {} is stored locally.\".format(const))"
]
},
{
diff --git a/notebooks/LPD/LPD_Correct_and_Verify.ipynb b/notebooks/LPD/LPD_Correct_and_Verify.ipynb
index 81a17d23a..883892ccb 100644
--- a/notebooks/LPD/LPD_Correct_and_Verify.ipynb
+++ b/notebooks/LPD/LPD_Correct_and_Verify.ipynb
@@ -137,9 +137,6 @@
"DET_FILE_INSET = \"LPD\"\n",
"CHUNK_SIZE = 512\n",
"MAX_PAR = 32\n",
- " \n",
- "out_folder = \"{}/r{:04d}\".format(out_folder, run)\n",
- "print(\"Outputting to {}\".format(out_folder))\n",
"\n",
"if not os.path.exists(out_folder):\n",
" os.makedirs(out_folder)\n",
diff --git a/notebooks/ePix/Characterize_Darks_ePix10K_NBC.ipynb b/notebooks/ePix/Characterize_Darks_ePix10K_NBC.ipynb
index 84e92b3f2..678497609 100644
--- a/notebooks/ePix/Characterize_Darks_ePix10K_NBC.ipynb
+++ b/notebooks/ePix/Characterize_Darks_ePix10K_NBC.ipynb
@@ -29,7 +29,6 @@
"h5path_t = '/INSTRUMENT/{}/DET/RECEIVER:daqOutput/data/backTemp' # path to find temperature at\n",
"h5path_cntrl = '/CONTROL/{}/DET' # path to control data\n",
"cal_db_interface = \"tcp://max-exfl016:8020\" # calibration DB interface to use\n",
- "local_output = False # output also in as H5 files\n",
"temp_limits = 5 # limit for parameter Operational temperature\n",
"sequence = 0 # sequence file to use\n",
"use_dir_creation_date = True\n",
@@ -38,7 +37,9 @@
"in_vacuum = False # detector operated in vacuum\n",
"instance = \"HED_IA1_EPIX10K-1\" # karabo instance\n",
"path_inset = \"EPIX03\" # file inset for image data\n",
- "fix_temperature = 290. # fix temperature to this value"
+ "fix_temperature = 290. # fix temperature to this value\n",
+ "db_output = True # Output constants to the calibration database\n",
+ "local_output = True # output constants locally"
]
},
{
@@ -64,7 +65,7 @@
"prettyPlotting = True\n",
"from XFELDetAna.xfelreaders import ChunkReader\n",
"from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
- "from cal_tools.tools import get_dir_creation_date\n",
+ "from cal_tools.tools import get_dir_creation_date, save_const_to_h5\n",
"\n",
"from iCalibrationDB import (ConstantMetaData, Constants, Conditions, Detectors,\n",
" Versions)\n",
@@ -261,9 +262,7 @@
"source": [
"# Save constants to DB\n",
"dclass=\"ePix10K\"\n",
- "consts = [\"Offset\", \"Noise\"]\n",
- "\n",
- "for const_name in consts:\n",
+ "for const_name in constant_maps.keys():\n",
" metadata = ConstantMetaData()\n",
" det = getattr(Constants, dclass)\n",
" const = getattr(det, const_name)()\n",
@@ -291,22 +290,24 @@
" # specify the a version for this constant\n",
" if creation_time is None:\n",
" metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " \n",
" else:\n",
" metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
" start=creation_time)\n",
+ " \n",
" metadata.calibration_constant_version.raw_data_location = file_loc\n",
- " metadata.send(cal_db_interface)\n",
- " print(\"Inject {} constants from {}\".format(const_name, \n",
- " metadata.calibration_constant_version.begin_at))"
+ " \n",
+ " if db_output:\n",
+ " try:\n",
+ " metadata.send(cal_db_interface)\n",
+ " print(\"Inject {} constants from {}\".format(const_name, \n",
+ " metadata.calibration_constant_version.begin_at))\n",
+ " except Exception as e:\n",
+ " print(e)\n",
+ " \n",
+ " if local_output:\n",
+ " save_const_to_h5(metadata, out_folder)\n",
+ " print(\"Calibration constant {} is stored locally.\".format(const))"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
diff --git a/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb b/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb
index 45dc2bd97..21c82e006 100644
--- a/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb
+++ b/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb
@@ -29,7 +29,6 @@
"h5path_t = '/INSTRUMENT/{}/DET/RECEIVER:daqOutput/data/backTemp' # path to find temperature at\n",
"h5path_cntrl = '/CONTROL/{}/DET' # path to control data\n",
"cal_db_interface = \"tcp://max-exfl016:8020\" # calibration DB interface to use\n",
- "local_output = False # output also in as H5 files\n",
"temp_limits = 5 # limit for parameter Operational temperature\n",
"sequence = 0 # sequence file to use\n",
"use_dir_creation_date = True\n",
@@ -38,7 +37,9 @@
"in_vacuum = False # detector operated in vacuum\n",
"instance = \"MID_EXP_EPIX-1\" # karabo instance\n",
"path_inset = \"DA01\" # file inset for image data\n",
- "fix_temperature = 290. # fix temperature to this value"
+ "fix_temperature = 290. # fix temperature to this value\n",
+ "db_output = True # Output constants to the calibration database\n",
+ "local_output = True # output constants locally"
]
},
{
@@ -64,7 +65,7 @@
"prettyPlotting = True\n",
"from XFELDetAna.xfelreaders import ChunkReader\n",
"from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
- "from cal_tools.tools import get_dir_creation_date\n",
+ "from cal_tools.tools import get_dir_creation_date, save_const_to_h5\n",
"\n",
"from iCalibrationDB import (ConstantMetaData, Constants, Conditions, Detectors,\n",
" Versions)\n",
@@ -254,9 +255,7 @@
"source": [
"# Save constants to DB\n",
"dclass=\"ePix100\"\n",
- "consts = [\"Offset\", \"Noise\"]\n",
- "\n",
- "for const_name in consts:\n",
+ "for const_name in constant_maps.keys():\n",
" metadata = ConstantMetaData()\n",
" det = getattr(Constants, dclass)\n",
" const = getattr(det, const_name)()\n",
@@ -277,22 +276,28 @@
" parm.upper_deviation = temp_limits\n",
"\n",
" device = getattr(Detectors, db_module)\n",
- "\n",
" metadata.detector_condition = condition\n",
"\n",
" # specify the a version for this constant\n",
" if creation_time is None:\n",
" metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " \n",
" else:\n",
" metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
" start=creation_time)\n",
" \n",
" metadata.calibration_constant_version.raw_data_location = file_loc\n",
" \n",
- " metadata.send(cal_db_interface)\n",
- " print(\"Inject {} constants from {}\".format(const_name, \n",
- " metadata.calibration_constant_version.begin_at))"
+ " if db_output:\n",
+ " try:\n",
+ " metadata.send(cal_db_interface)\n",
+ " print(\"Inject {} constants from {}\".format(const_name, \n",
+ " metadata.calibration_constant_version.begin_at))\n",
+ " except Exception as e:\n",
+ " print(e)\n",
+ " \n",
+ " if local_output:\n",
+ " save_const_to_h5(metadata, out_folder)\n",
+ " print(\"Calibration constant {} is stored locally.\".format(const))"
]
}
],
diff --git a/notebooks/ePix/Correction_ePix10K_NBC.ipynb b/notebooks/ePix/Correction_ePix10K_NBC.ipynb
index dc5c75d04..1dbefa31b 100644
--- a/notebooks/ePix/Correction_ePix10K_NBC.ipynb
+++ b/notebooks/ePix/Correction_ePix10K_NBC.ipynb
@@ -18,8 +18,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:23.218849Z",
"start_time": "2018-12-06T15:54:23.166497Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -82,8 +81,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:23.455376Z",
"start_time": "2018-12-06T15:54:23.413579Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -139,8 +137,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:23.679069Z",
"start_time": "2018-12-06T15:54:23.662821Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -148,7 +145,6 @@
"y = 384 # columns of the ePix10K\n",
" \n",
"ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
- "out_folder = \"{}/r{:04d}\".format(out_folder, run)\n",
"fp_name = path_template.format(run, path_inset)\n",
"fp_path = '{}/{}'.format(ped_dir, fp_name)\n",
"\n",
@@ -173,7 +169,6 @@
"end_time": "2018-12-06T15:54:23.913269Z",
"start_time": "2018-12-06T15:54:23.868910Z"
},
- "collapsed": false,
"scrolled": true
},
"outputs": [],
@@ -203,7 +198,6 @@
"\n",
"if not os.path.exists(out_folder):\n",
" os.makedirs(out_folder)\n",
- " \n",
"elif not overwrite:\n",
" raise AttributeError(\"Output path exists! Exiting\") \n"
]
@@ -254,8 +248,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T18:43:39.776018Z",
"start_time": "2018-12-06T18:43:39.759185Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -285,8 +278,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:28.254544Z",
"start_time": "2018-12-06T15:54:24.709521Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -353,8 +345,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:28.771629Z",
"start_time": "2018-12-06T15:54:28.346051Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -593,8 +584,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:13:12.889583Z",
"start_time": "2018-12-06T16:13:11.122653Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -652,8 +642,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:11:08.317130Z",
"start_time": "2018-12-06T16:11:05.788655Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -680,8 +669,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:11:10.908912Z",
"start_time": "2018-12-06T16:11:08.318486Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -718,7 +706,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.4.3"
+ "version": "3.6.7"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
diff --git a/notebooks/ePix/Correction_ePix_NBC.ipynb b/notebooks/ePix/Correction_ePix_NBC.ipynb
index fbf437b79..ec4a20eb7 100644
--- a/notebooks/ePix/Correction_ePix_NBC.ipynb
+++ b/notebooks/ePix/Correction_ePix_NBC.ipynb
@@ -154,7 +154,6 @@
"y = 768 # columns of the ePix100\n",
" \n",
"ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
- "out_folder = \"{}/r{:04d}\".format(out_folder, run)\n",
"fp_name = path_template.format(run, path_inset)\n",
"fp_path = '{}/{}'.format(ped_dir, fp_name)\n",
"\n",
@@ -206,7 +205,6 @@
"\n",
"if not os.path.exists(out_folder):\n",
" os.makedirs(out_folder)\n",
- " \n",
"elif not overwrite:\n",
" raise AttributeError(\"Output path exists! Exiting\") \n"
]
diff --git a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
index 4cf0a0a8c..9a9d68b10 100644
--- a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
+++ b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
@@ -34,7 +34,6 @@
"sigma_noise = 10. # Pixel exceeding 'sigmaNoise' * noise value in that pixel will be masked\n",
"h5path = '/INSTRUMENT/SQS_NQS_PNCCD1MP/CAL/PNCCD_FMT-0:output/data/image/' # path in the HDF5 file the data is at\n",
"cal_db_interface = \"tcp://max-exfl016:8021\" # calibration DB interface to use\n",
- "local_output = False # output also in as H5 files\n",
"temp_limits = 5 # temperature limits in which to consider calibration parameters equal\n",
"sequence = 0 # sequence file to use\n",
"multi_iteration = False # use multiple iterations\n",
@@ -44,7 +43,9 @@
"fix_temperature = 233. # fix temperature to this value in K, set to -1 to use value from slow data\n",
"gain = 0 # the detector gain setting, only 0 is currently implemented\n",
"bias_voltage = 300 # detector bias voltage\n",
- "integration_time = 70 # detector integration time"
+ "integration_time = 70 # detector integration time\n",
+ "db_output = False # Output constants to the calibration database\n",
+ "local_output = True # output constants locally"
]
},
{
@@ -70,8 +71,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T10:54:39.467334Z",
"start_time": "2018-12-06T10:54:39.427784Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -91,7 +91,7 @@
"prettyPlotting=True\n",
"from XFELDetAna.xfelreaders import ChunkReader\n",
"from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
- "from cal_tools.tools import get_dir_creation_date\n",
+ "from cal_tools.tools import get_dir_creation_date, save_const_to_h5\n",
"\n",
"import numpy as np\n",
"import h5py\n",
@@ -115,6 +115,16 @@
"temperature_k = fix_temperature"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
+ "file_loc = 'proposal:{} runs:{}'.format(proposal, run)"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -122,8 +132,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T10:54:40.058101Z",
"start_time": "2018-12-06T10:54:40.042615Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -156,8 +165,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T10:54:40.555804Z",
"start_time": "2018-12-06T10:54:40.452978Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -241,8 +249,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T10:55:21.238009Z",
"start_time": "2018-12-06T10:54:54.586435Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -267,8 +274,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T10:56:20.686534Z",
"start_time": "2018-12-06T10:56:11.721829Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -331,8 +337,9 @@
" [512, 512], \n",
" 'row',\n",
" nCells = memoryCells, \n",
- " noiseMap = noiseMap,\n",
- " runParallel=False,\n",
+ " dType=np.float32,\n",
+ " noiseMap = noiseMap.astype(np.float32),\n",
+ " runParallel=run_parallel,\n",
" stats=True)\n",
"cmCorrection.debug()\n",
"noiseCalCM = xcal.NoiseCalculator(sensorSize, memoryCells, \n",
@@ -352,7 +359,8 @@
" data = data.astype(np.float32)\n",
" dx = np.count_nonzero(data, axis=(0, 1))\n",
" data = data[:,:,dx != 0]\n",
- " data = cmCorrection.correct(data)\n",
+ " cellTable=np.zeros(data.shape[2], np.int32) # Common mode correction\n",
+ " data = cmCorrection.correct(data.astype(np.float32), cellTable=cellTable) \n",
" #Filling calculators with data\n",
" noiseCalCM.fill(data)\n",
" \n",
@@ -364,9 +372,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"#*****NOISE MAP HISTOGRAM FROM THE OFFSET CORRECTED DATA*******#\n",
@@ -396,81 +402,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2018-12-06T10:56:22.741284Z",
- "start_time": "2018-12-06T10:56:20.688393Z"
- },
- "collapsed": true
- },
- "outputs": [],
- "source": [
- "\n",
- "## offset\n",
- "\n",
- "metadata = ConstantMetaData()\n",
- "offset = Constants.CCD(DetectorTypes.pnCCD).Offset()\n",
- "offset.data = offsetMapCM.data\n",
- "metadata.calibration_constant = offset\n",
- "\n",
- "# set the operating condition\n",
- "condition = Conditions.Dark.CCD(bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " gain_setting=gain,\n",
- " temperature=temperature_k,\n",
- " pixels_x=1024,\n",
- " pixels_y=1024)\n",
- "\n",
- "device = Detectors.PnCCD1\n",
- "\n",
- "\n",
- "\n",
- "metadata.detector_condition = condition\n",
- "\n",
- "# specify the a version for this constant\n",
- "if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- "else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- "metadata.send(cal_db_interface, timeout=300000)\n",
- "\n",
- "## noise\n",
- "\n",
- "metadata = ConstantMetaData()\n",
- "noise = Constants.CCD(DetectorTypes.pnCCD).Noise()\n",
- "noise.data = noiseMapCM.data\n",
- "metadata.calibration_constant = noise\n",
- "\n",
- "# set the operating condition\n",
- "condition = Conditions.Dark.CCD(bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " gain_setting=gain,\n",
- " temperature=temperature_k,\n",
- " pixels_x=1024,\n",
- " pixels_y=1024)\n",
- "\n",
- "\n",
- "\n",
- "device = Detectors.PnCCD1\n",
- "\n",
- "\n",
- "metadata.detector_condition = condition\n",
- "\n",
- "# specify the a version for this constant\n",
- "if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- "else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- "metadata.send(cal_db_interface, timeout=300000)\n",
- "\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"from cal_tools.enums import BadPixels\n",
@@ -496,35 +428,60 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
+ "ExecuteTime": {
+ "end_time": "2018-12-06T10:56:22.741284Z",
+ "start_time": "2018-12-06T10:56:20.688393Z"
+ },
"collapsed": true
},
"outputs": [],
"source": [
- "metadata = ConstantMetaData()\n",
- "badpix = Constants.CCD(DetectorTypes.pnCCD).BadPixelsDark()\n",
- "badpix.data = bad_pixels.data\n",
- "metadata.calibration_constant = badpix\n",
- "\n",
- "# set the operating condition\n",
- "condition = Conditions.Dark.CCD(bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " gain_setting=gain,\n",
- " temperature=temperature_k,\n",
- " pixels_x=1024,\n",
- " pixels_y=1024)\n",
- "\n",
- "\n",
- "device = Detectors.PnCCD1\n",
- "\n",
- "\n",
- "metadata.detector_condition = condition\n",
- "\n",
- "# specify the a version for this constant\n",
- "if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- "else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- "metadata.send(cal_db_interface, timeout=300000)"
+ "constant_maps = {\n",
+ " 'Offset': offsetMapCM,\n",
+ " 'Noise': noiseMapCM,\n",
+ " 'BadPixelsDark': bad_pixels\n",
+ "}\n",
+ "\n",
+ "for const_name in constant_maps.keys():\n",
+ " metadata = ConstantMetaData()\n",
+ " det = Constants.CCD(DetectorTypes.pnCCD)\n",
+ " const = getattr(det, const_name)()\n",
+ " const.data = constant_maps[const_name].data\n",
+ "\n",
+ " metadata.calibration_constant = const\n",
+ "\n",
+ " # set the operating condition\n",
+ " condition = Conditions.Dark.CCD(bias_voltage=bias_voltage,\n",
+ " integration_time=integration_time,\n",
+ " gain_setting=gain,\n",
+ " temperature=temperature_k,\n",
+ " pixels_x=1024,\n",
+ " pixels_y=1024)\n",
+ "\n",
+ "\n",
+ " device = Detectors.PnCCD1\n",
+ " metadata.detector_condition = condition\n",
+ "\n",
+ " # specify the a version for this constant\n",
+ " if creation_time is None:\n",
+ " metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ " else:\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
+ " start=creation_time)\n",
+ " \n",
+ " metadata.calibration_constant_version.raw_data_location = file_loc\n",
+ " \n",
+ " if db_output:\n",
+ " try:\n",
+ " metadata.send(cal_db_interface)\n",
+ " print(\"Inject {} constants from {}\".format(const_name, \n",
+ " metadata.calibration_constant_version.begin_at))\n",
+ " except Exception as e:\n",
+ " print(e)\n",
+ " \n",
+ " if local_output:\n",
+ " save_const_to_h5(metadata, out_folder)\n",
+ " print(\"Calibration constant {} is stored locally.\".format(const)) \n"
]
},
{
@@ -550,9 +507,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"ho,eo,co,so = histCalCorr.get()\n",
@@ -575,24 +530,6 @@
" y_log=True, x_range=(-50,500),\n",
" legend='top-center-frame-2col')"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": []
}
],
"metadata": {
@@ -611,7 +548,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.6"
+ "version": "3.6.7"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
diff --git a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
index 1af1d0ca1..5ffe35f6e 100644
--- a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
+++ b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
@@ -18,8 +18,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:23.218849Z",
"start_time": "2018-12-06T15:54:23.166497Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -84,8 +83,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:23.455376Z",
"start_time": "2018-12-06T15:54:23.413579Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -143,8 +141,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:23.679069Z",
"start_time": "2018-12-06T15:54:23.662821Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -167,9 +164,7 @@
"print(\"Run is: {}\".format(run))\n",
"print(\"HDF5 path: {}\".format(h5path))\n",
"if creation_time:\n",
- " print(\"Using {} as creation time\".format(creation_time.isoformat()))\n",
- " \n",
- "out_folder = \"{}/r{:04d}\".format(out_folder, run)"
+ " print(\"Using {} as creation time\".format(creation_time.isoformat()))\n"
]
},
{
@@ -250,8 +245,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T18:43:39.776018Z",
"start_time": "2018-12-06T18:43:39.759185Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -282,8 +276,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:28.254544Z",
"start_time": "2018-12-06T15:54:24.709521Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -753,7 +746,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
"scrolled": false
},
"outputs": [],
@@ -773,8 +765,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:10:56.126409Z",
"start_time": "2018-12-06T16:10:56.096242Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -830,8 +821,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:10:56.176160Z",
"start_time": "2018-12-06T16:10:56.127853Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -887,8 +877,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:10:56.190150Z",
"start_time": "2018-12-06T16:10:56.177570Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -955,8 +944,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:10:56.212586Z",
"start_time": "2018-12-06T16:10:56.204731Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -989,8 +977,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:11:08.317130Z",
"start_time": "2018-12-06T16:11:05.788655Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -1024,8 +1011,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:11:10.908912Z",
"start_time": "2018-12-06T16:11:08.318486Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -1083,7 +1069,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
"scrolled": false
},
"outputs": [],
@@ -1151,7 +1136,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.6"
+ "version": "3.6.7"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
diff --git a/webservice/serve_overview.py b/webservice/serve_overview.py
index d06fcd0e8..067ce421d 100644
--- a/webservice/serve_overview.py
+++ b/webservice/serve_overview.py
@@ -10,6 +10,7 @@ from uuid import uuid4
import yaml
from jinja2 import Template
+from cal_tools.tools import send_from_file
from xfel_calibrate.settings import (free_nodes_cmd, preempt_nodes_cmd,
reservation)
@@ -62,6 +63,36 @@ class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
+ def check_const_files(pars):
+ """
+ Check if calibration constants already exists
+ :param pars: Dictionary of parameters
+ :return: True if constants already exist
+ """
+
+ path = '/gpfs/exfel/exp/{}/{}/p{:06d}/usr/dark/{}'
+ if 'run' in pars:
+ runs = 'runs_{}'.format(pars['run'])
+ else:
+ runs = 'runs_{}_{}_{}'.format(pars['run_high'],
+ pars['run_med'],
+ pars['run_low'])
+
+ path = path.format(pars['instrument'],
+ pars['cycle'],
+ int(pars['proposal']),
+ runs)
+
+ dets = pars['detectors'].split(',')
+ for detector in dets:
+ opath = '{}/{}/const*h5'.format(path,
+ detector.replace('-', '_'))
+ files = glob.glob(opath)
+ if len(files)>0:
+ return True
+ return False
+
+
if not self.conf_was_init:
self.init_config()
# Send response status code
@@ -84,17 +115,51 @@ class RequestHandler(BaseHTTPRequestHandler):
self.wfile.write(f.read())
return
- if "file?" in self.path:
- fpath = self.path.split("?")[1]
- if fpath is None:
+ if "process_constants?" in self.path:
+ self.send_header('Content-type', 'text/html')
+ self.end_headers()
+ pars = self.path.split("?")[1].split("&")
+ pars = {x.split("=")[0]: x.split("=")[1] for x in pars}
+
+ if 'action' not in pars or 'path' not in pars:
return
- if os.path.isfile(fpath) and os.path.splitext(fpath)[1] == '.pdf':
- self.send_header('Content-type', 'application/pdf')
- self.end_headers()
- with open(fpath, "rb") as f:
- self.wfile.write(f.read())
+
+ if pars['action'] == 'delete':
+ try:
+ files = glob.glob('{}/const*h5'.format(pars['path']))
+ for file_name in files:
+ os.remove(file_name)
+ msg = "{} Constants were deleted".format(len(files))
+ except Exception as e:
+ msg = 'Error: {}'.format(e)
+
+ self.wfile.write(bytes(msg, "utf8"))
+ return
+
+ if pars['action'] == 'inject':
+ try:
+ files = glob.glob('{}/const*h5'.format(pars['path']))
+ for file_name in files:
+ cal_db_interface = config["server-config"][
+ "cal_db_interface"] # noqa
+ cal_db_timeout = config["server-config"][
+ "cal_db_timeout"] # noqa
+ send_from_file(file_name, cal_db_interface,
+ verbosity=0,
+ timeout=int(cal_db_timeout), ntries=2,
+ doraise=True)
+ msg = "{} Constants were injected".format(len(files))
+ except Exception as e:
+ msg = 'Error: {}'.format(e)
+
+ self.wfile.write(bytes(msg, "utf8"))
+ return
+
return
- if "update_form?" in self.path:
+
+ if "process_dark?" in self.path:
+ self.send_header('Content-type', 'text/html')
+ self.end_headers()
pars = self.path.split("?")[1].split("&")
pars = {x.split("=")[0]: x.split("=")[1] for x in pars}
if pars['instrument'] in ['Nothing', 'none']:
@@ -116,11 +181,16 @@ class RequestHandler(BaseHTTPRequestHandler):
par_list.append('{}'.format(v))
par_list = list(filter(None, par_list))
- print('REQUEST DARK: ', par_list)
timeout = config["server-config"]["dark-timeout"]
try:
- msg = check_output(par_list, shell=False,
- timeout=timeout).decode('utf8')
+ # check if h5 files exist in folder
+ if check_const_files(pars):
+ msg = 'Constant exists, remove them before processing'
+ msg = '<div style = "color:red;font-weight:bold;font-size:160%;" > {} </div> '.format(msg) # noqa
+ else:
+ print('REQUEST DARK: ', par_list)
+ msg = check_output(par_list, shell=False,
+ timeout=timeout).decode('utf8')
except Exception as e:
msg = str(e)
@@ -171,24 +241,44 @@ class RequestHandler(BaseHTTPRequestHandler):
reports[instrument] = {}
for detector in detectors:
det_inset = detector.replace('-', '_')
- tmpl = '/gpfs/exfel/exp/{}/*/*/usr/dark/*/{}'.format(
+ tmpl = '/gpfs/exfel/exp/{}/*/*/usr/dark/*/{}/'.format(
instrument, det_inset)
- files = glob.glob(tmpl + '/*pdf')
- files += glob.glob(tmpl + '/*/*pdf')
+ files = glob.glob(tmpl)
files.sort(key=os.path.getmtime, reverse=True)
file_info = []
for i, file in enumerate(files):
- if 'xfel.pdf' in file:
- continue
- if (len(file_info) % 2) == 0:
+ reps = glob.glob(file + '*pdf')
+ const = glob.glob(file + 'const*h5')
+
+ reps.sort(key=os.path.getmtime, reverse=False)
+ const.sort(key=os.path.getmtime, reverse=True)
+
+ for rep in reps:
+ time = os.stat(rep).st_mtime
+ d_time = datetime.fromtimestamp(time).replace(
+ tzinfo=timezone.utc)
+ s_time = d_time.strftime('%y-%m-%d %H:%M')
+ file_info.append({
+ 'report': rep,
+ 'path': file,
+ 'report_time': s_time
+ })
+
+ if len(const) > 0:
+ time = os.stat(const[-1]).st_mtime
+ d_time = datetime.fromtimestamp(time).replace(
+ tzinfo=timezone.utc)
+ s_time = d_time.strftime('%y-%m-%d %H:%M')
+
+ file_info[-1]['const'] = True
+ file_info[-1]['const_time'] = s_time
+
+ for i in range(len(file_info)):
+ if (i % 2) == 0:
bgcolor = 'EEEEEE'
else:
bgcolor = 'FFFFFF'
- time = os.stat(file).st_mtime
- d_time = datetime.fromtimestamp(time).replace(
- tzinfo=timezone.utc)
- s_time = d_time.strftime('%y-%m-%d %H:%M')
- file_info.append([file, s_time, bgcolor])
+ file_info[i]['bgcolor'] = bgcolor
reports[instrument][detector] = file_info
@@ -198,6 +288,39 @@ class RequestHandler(BaseHTTPRequestHandler):
self.wfile.write(bytes(message, "utf8"))
return
+ if "/gpfs" in self.path:
+
+ sendReply = False
+ if self.path.endswith(".html"):
+ mimetype = 'text/html'
+ sendReply = True
+ if self.path.endswith(".jpg"):
+ mimetype = 'image/jpg'
+ sendReply = True
+ if self.path.endswith(".gif"):
+ mimetype = 'image/gif'
+ sendReply = True
+ if self.path.endswith(".png"):
+ mimetype = 'image/png'
+ sendReply = True
+ if self.path.endswith(".pdf"):
+ mimetype = 'application/pdf'
+ sendReply = True
+ if self.path.endswith(".js"):
+ mimetype = 'application/javascript'
+ sendReply = True
+ if self.path.endswith(".css"):
+ mimetype = 'text/css'
+ sendReply = True
+
+ if sendReply == True and os.path.isfile(self.path):
+ with open(self.path, "rb") as f:
+ self.send_header('Content-type', mimetype)
+ self.end_headers()
+ self.wfile.write(f.read())
+ return
+
+
# Send headers
self.send_header('Content-type', 'text/html')
self.end_headers()
diff --git a/webservice/serve_overview.yaml b/webservice/serve_overview.yaml
index ed80c5182..12984eb21 100644
--- a/webservice/serve_overview.yaml
+++ b/webservice/serve_overview.yaml
@@ -50,6 +50,8 @@ server-config:
host: max-exfl016
dark-timeout: 30
n-calib: 10
+ cal_db_interface: tcp://max-exfl016:8022
+ cal_db_timeout: 30000
web-service:
job-db: ./webservice_jobs.sqlite
diff --git a/webservice/templates/dark_overview.html b/webservice/templates/dark_overview.html
index 200c63b3b..f7ab166af 100644
--- a/webservice/templates/dark_overview.html
+++ b/webservice/templates/dark_overview.html
@@ -6,29 +6,76 @@
</head>
<body>
+
+<script type="text/javascript">
+
+function processConstants(file, action) {
+
+ var xhttp = new XMLHttpRequest();
+ xhttp.onreadystatechange = function() {
+ if (this.readyState == 4 && this.status == 200) {
+ var result_style = document.getElementById(file).style;
+ result_style.display = '';
+ document.getElementById("info"+file).innerHTML = this.responseText;
+ }
+ };
+
+ msg = 'action=' + action;
+ msg += '&path=' + file;
+
+ var result_style = document.getElementById(file).style;
+ result_style.display = '';
+ document.getElementById("info"+file).innerHTML = 'Perform action: ' + action;
+ xhttp.open("GET", "/process_constants?"+msg, true);
+ xhttp.send();
+
+}
+</script>
+
+
<div class="block">
- <h1>Dark runs</h1>
- {% for instrument, detectors in reports.items() %}
- <h2>{{ instrument }}</h2>
+ <h1>Dark runs</h1>
+ {% for instrument, detectors in reports.items() %}
+ <h2>{{ instrument }}</h2>
- {% for detector, files in detectors.items() %}
- <h3>{{ detector }}</h3>
+ {% for detector, files in detectors.items() %}
+ <h3>{{ detector }}</h3>
<table>
- {% for file in files %}
-
- <tr bgcolor="{{ file[2] }}">
- <td> <a href=http://{{host}}:{{port}}/file?{{ file[0] }} target="_blank">{{ file[0] }}</a> </td>
- <td> {{ file[1] }} </td>
- </tr>
-
- {% endfor %}
- </table>
- <br>
- {% endfor %}
- <br>
+ {% for file in files %}
+
+ <tr bgcolor="{{ file['bgcolor'] }}">
+ <td><a href=http://{{host}}:{{port}}/{{ file['report'] }} target="_blank">{{ file['report'] }}</a> </td>
+ <td> {{ file['report_time'] }}</td>
+
+ <td>
+ {% if file['const'] %}
+ {{ file['const_time'] }}
+ <input type="button" value="inject" onclick="javascript:processConstants('{{ file['path'] }}', 'inject');">
+ <input type="button" value="delete" onclick="javascript:processConstants('{{ file['path'] }}', 'delete');">
+ {% endif %}
+ </td>
+ </tr>
+
+ {% if file['const'] %}
+ <tr id="{{ file['path'] }}" style="display: none;">
+ <td colspan="3">
+ <div id="info{{ file['path'] }}"></div>
+ </td>
+ </tr>
+ {% endif %}
+
+
+ {% endfor %}
+ </table>
+ <br>
+ {% endfor %}
+ <br>
{% endfor %}
</div>
+<div id="demo">
+ </div>
+
</body>
</html>
diff --git a/webservice/templates/last_correction.html b/webservice/templates/last_correction.html
index 893266cb2..5a799066e 100644
--- a/webservice/templates/last_correction.html
+++ b/webservice/templates/last_correction.html
@@ -18,7 +18,7 @@
<td> run: {{ item[2] }} </td>
<td>
{% for file, path in item[3].items() %}
- <a href="http://{{host}}:{{port}}/file?{{ path }}" target="_blank">{{ file }}</a>
+ <a href="http://{{host}}:{{port}}/{{ path }}" target="_blank">{{ file }}</a>
{% endfor %}
</td>
</tr>
diff --git a/webservice/templates/request_dark.html b/webservice/templates/request_dark.html
index c7dfefbae..efd129e37 100644
--- a/webservice/templates/request_dark.html
+++ b/webservice/templates/request_dark.html
@@ -75,7 +75,7 @@ function loadCheckbox(command) {
document.getElementById("demo").innerHTML = 'Requesting dark...'
}
- xhttp.open("GET", "{{host}}:{{port}}/update_form?"+msg, true);
+ xhttp.open("GET", "{{host}}:{{port}}/process_dark?"+msg, true);
xhttp.send();
}
diff --git a/webservice/webservice.py b/webservice/webservice.py
index f2df2c70b..241a5ae36 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -636,6 +636,11 @@ async def server_runner(config, mode):
thisconf["out-folder"] = '/'.join((out_folder,
detector.replace('-', '_')))
+ ts = datetime.now().strftime('%y%m%d_%H%M')
+ report_to = 'Dark_' + \
+ detector.replace('-', '_') + '_' + ts
+ thisconf["report-to"] = report_to.lower()
+
# don't need this for xfel-calibrate
del thisconf["inset"]
detectors[detector] = thisconf
@@ -684,7 +689,9 @@ async def server_runner(config, mode):
rpath = "{}/r{:04d}/".format(in_folder, int(runnr))
out_folder = config[action]['out-folder'].format(
- instrument=instrument, cycle=cycle, proposal=proposal)
+ instrument=instrument, cycle=cycle, proposal=proposal,
+ run='r{:04d}'.format(int(runnr)))
+
corr_file_list = set()
copy_file_list = set(glob.glob("{}/*.h5".format(rpath)))
detectors = {}
@@ -697,6 +704,11 @@ async def server_runner(config, mode):
thisconf = copy.copy(dconfig)
thisconf["in-folder"] = in_folder
thisconf["out-folder"] = out_folder
+ ts = datetime.now().strftime('%y%m%d_%H%M')
+ report_to = 'Correct_' + \
+ detector.replace('-', '_') + '_' + ts
+ thisconf["report-to"] = report_to.lower()
+
thisconf["run"] = runnr
del thisconf[
"inset"] # don't need this for xfel-calibrate
diff --git a/webservice/webservice.yaml b/webservice/webservice.yaml
index 6e8eef82a..4ed2b966a 100644
--- a/webservice/webservice.yaml
+++ b/webservice/webservice.yaml
@@ -24,7 +24,7 @@ metadata-client:
correct:
in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw
- out-folder: /gpfs/exfel/d/proc/{instrument}/{cycle}/p{proposal}
+ out-folder: /gpfs/exfel/d/proc/{instrument}/{cycle}/p{proposal}/{run}
dark:
in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw
diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index 09bc6978e..b03a1b48b 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -455,6 +455,34 @@ def has_parm(parms, name):
return False
+def get_par_value(parms, key, default=None):
+ """
+ Return the value of parameter with name key
+ :param parms: List of parameters
+ :param key: Name of the parameter to be considered
+ :param default: Value to be returned if interested name is not found
+ :return: The value of the parameter
+ """
+ for p in parms:
+ if p.name == key:
+ return p.value
+ return default
+
+
+def get_par_type(parms, key, default=None):
+ """
+ Return the type of parameter with name key
+ :param parms: List of parameters
+ :param key: Name of the parameter to be considered
+ :param default: Type to be returned if interested name is not found
+ :return: The type of the parameter
+ """
+ for p in parms:
+ if p.name == key:
+ return p.type
+ return default
+
+
def flatten_list(l):
return "_".join([str(flatten_list(v)) for v in l]) if isinstance(l, list) else l
@@ -755,12 +783,6 @@ def run():
if try_report_to_output:
if "out_folder" in args:
out_path = os.path.abspath(args["out_folder"])
- if "run" in args:
- rr = args["run"]
- if isinstance(rr, int):
- out_path = "{}/r{:04d}/".format(out_path, rr)
- else:
- out_path = "{}/{}/".format(out_path, rr)
else:
print("No 'out_folder' defined as argument, outputting to '{}' instead.".format(
out_path))
@@ -775,10 +797,7 @@ def run():
if args["report_to"] is not None:
report_to = args["report_to"]
- folder = ''
- for p in parms:
- if p.name == 'in_folder':
- folder = p.value
+ folder = get_par_value(parms, 'in_folder', '')
fmtcmd = cmd.format(run_path=run_tmp_path, out_path=out_path,
project=title, calibration=title,
@@ -788,9 +807,14 @@ def run():
joblist = []
if concurrency.get("parameter", None) is None:
cluster_cores = concurrency.get("cluster cores", 8)
- jobid = concurrent_run(run_tmp_path, nb, os.path.basename(notebook), args,
- final_job=True, job_list=joblist, fmtcmd=fmtcmd,
- cluster_cores=cluster_cores, sequential=sequential)
+
+ jobid = concurrent_run(run_tmp_path, nb,
+ os.path.basename(notebook), args,
+ final_job=True, job_list=joblist,
+ fmtcmd=fmtcmd,
+ cluster_cores=cluster_cores,
+ sequential=sequential)
+
joblist.append(jobid)
else:
cvar = concurrency["parameter"]
@@ -802,18 +826,14 @@ def run():
if cvals is None:
defcval = concurrency.get("default concurrency", None)
if defcval is not None:
- print(
- "Concurrency parameter '{}' is taken from notebooks.py".format(cvar))
+ print("Concurrency parameter '{}' is taken from notebooks.py".format(cvar))
if not isinstance(defcval, (list, tuple)):
cvals = range(defcval)
else:
cvals = defcval
if cvals is None:
- print(parms)
- for p in parms:
- if p.name == cvar:
- defcval = p.value
+ defcval = get_par_value(parms, cvar)
if defcval is not None:
print("Concurrency parameter '{}' is taken from '{}'".format(
cvar, notebook))
@@ -843,22 +863,19 @@ def run():
print("Split concurrency into {}".format(cvals))
# get expected type
- cvtype = list
- for p in parms:
- if p.name == cvar:
- cvtype = p.type
- break
+ cvtype = get_par_type(parms, cvar, list)
for cnum, cval in enumerate(cvals):
show_title = cnum == 0
# Job is not final if there are dependent notebooks
+ final_job = (cnum == len(list(cvals)) - 1 and len(dep_notebooks) == 0)
+ cval = [cval, ] if not isinstance(cval, list) and cvtype is list else cval
+
jobid = concurrent_run(run_tmp_path, nb, notebook, args,
- cvar, [cval, ] if not isinstance(
- cval, list) and cvtype is list else cval,
- cnum == len(list(cvals)) -
- 1 and len(dep_notebooks) == 0,
+ cvar, cval, final_job,
joblist, fmtcmd,
- cluster_cores=cluster_cores, sequential=sequential,
+ cluster_cores=cluster_cores,
+ sequential=sequential,
show_title=show_title)
joblist.append(jobid)
@@ -869,7 +886,8 @@ def run():
with open(notebook_path, "r") as f:
nb = nbformat.read(f, as_version=4)
final_job = i == len(dep_notebooks) - 1
- jobid = concurrent_run(run_tmp_path, nb, os.path.basename(notebook),
+ jobid = concurrent_run(run_tmp_path, nb,
+ os.path.basename(notebook),
args,
final_job=final_job,
job_list=joblist, fmtcmd=fmtcmd,
--
GitLab