diff --git a/cal_tools/cal_tools/plotting.py b/cal_tools/cal_tools/plotting.py
index fc38cc93d0c796d5815d3aa2d7a6102261c0c5d6..ca8580fcd218a5287d997eb0844f3399b2fa459f 100644
--- a/cal_tools/cal_tools/plotting.py
+++ b/cal_tools/cal_tools/plotting.py
@@ -21,7 +21,7 @@ def show_overview(d, cell_to_preview, gain_to_preview, out_folder=None, infix=No
i = 0
for key, item in data.items():
cf = 0
- if "Threshold" in key:
+ if "ThresholdsDark" in key:
cf = -1
if len(item.shape) == 4:
med = np.nanmedian(item[...,cell_to_preview, gain_to_preview + cf])
@@ -33,7 +33,7 @@ def show_overview(d, cell_to_preview, gain_to_preview, out_folder=None, infix=No
(item[...,cell_to_preview, gain_to_preview + cf] > med+np.abs(bound*med)))/item[...,cell_to_preview, gain_to_preview + cf].size > 0.01):
bound *=2
- if "BadPixels" in key:
+ if "BadPixelsDark" in key:
im = grid[i].imshow(np.log2(item[...,cell_to_preview, gain_to_preview + cf]), interpolation="nearest",
vmin=0, vmax=8, aspect='auto')
else:
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index eb791019f6f999153743651fda93e2c57688d7cc..d1912490bdc4bedf9a2a67a90b50d636ad65e203 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -14,6 +14,7 @@ from time import sleep
from urllib.parse import urljoin
import dateutil.parser
+
import ipykernel
from metadata_client.metadata_client import MetadataClient
from notebook.notebookapp import list_running_servers
@@ -23,6 +24,7 @@ import tabulate
from jinja2 import Template
from .mdc_config import MDC_config
+from .ana_tools import save_dict_to_hdf5
from xfel_calibrate import settings
@@ -173,7 +175,7 @@ def make_timing_summary(run_path, joblist):
with open("{}/timing_summary.rst".format(run_path), "w+") as gfile:
- if len(pars_vals)>0:
+ if len(pars_vals) > 0:
table = tabulate.tabulate(pars_vals, tablefmt='latex',
headers=pars_name)
gfile.write(dedent(tmpl.render(table=table.split('\n'))))
@@ -243,6 +245,10 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
if isinstance(v, str):
tmpl = '{} = "{}"\n'
+ # Set name of the latex document
+ if var == 'latex_documents' and len(v[0]) > 1:
+ v[0] = v[0][:1] + ('{}.tex'.format(report_name), ) + v[0][2:]
+
mf.write(tmpl.format(var, v))
remove("{}/conf.py".format(run_path))
@@ -281,8 +287,7 @@ def make_report(run_path, tmp_path, out_path, project, author, version,
"can be inspected at: {}".format(run_path))
return
print("Moving report to final location: {}".format(out_path))
- for f in glob(r'{}/_build/latex/*.pdf'.format(run_path)):
- copy(f, out_path)
+ copy('{}/_build/latex/{}.pdf'.format(run_path, report_name), out_path)
print("Removing temporary files at: {}".format(tmp_path))
rmtree(tmp_path)
@@ -494,6 +499,41 @@ def get_dir_creation_date(directory, run, tsdir=False, verbosity=0):
ntries -= 1
+def save_const_to_h5(metadata, out_folder):
+ """
+ Save constant in h5 file with it's metadata
+ (e.g. db_module, condition, creation_time)
+
+ :param metadata: Metadata
+ :param out_folder: path to output folder
+ """
+
+ dpar = {}
+ for parm in metadata.detector_condition.parameters:
+ dpar[parm.name] = {'lower_deviation_value': parm.lower_deviation,
+ 'upper_deviation_value': parm.upper_deviation,
+ 'value': parm.value,
+ 'flg_logarithmic': parm.logarithmic}
+
+ creation_time = metadata.calibration_constant_version.begin_at
+ raw_data = metadata.calibration_constant_version.raw_data_location
+ db_module = metadata.calibration_constant_version.device_name
+ constant_name = metadata.calibration_constant.__class__.__name__
+
+ data_to_store = {}
+ data_to_store['condition'] = dpar
+ data_to_store['db_module'] = db_module
+ data_to_store['constant'] = constant_name
+ data_to_store['data'] = metadata.calibration_constant.data
+ data_to_store['creation_time'] = creation_time
+ data_to_store['file_loc'] = raw_data
+
+ ofile = "{}/const_{}_{}.h5".format(out_folder, constant_name, db_module)
+ if isfile(ofile):
+ print('File {} already exists and will be overwritten'.format(ofile))
+ save_dict_to_hdf5(data_to_store, ofile)
+
+
def get_random_db_interface(cal_db_interface):
"""
Return interface to calibration DB with random (with given range) port
@@ -585,6 +625,70 @@ def get_from_db(device, constant, condition, empty_constant,
return empty_constant, None
+def send_to_db(device, constant, condition, file_loc,
+ cal_db_interface, creation_time=None,
+ verbosity=1, timeout=30000, ntries=120, doraise=False):
+ """
+ Return calibration constants and metadata requested from CalDB
+
+ :param device: Instance of detector
+ :param constant: Calibration constant known for given detector
+ :param condition: Calibration condition
+ :param file_loc: Location of raw data.
+ :param cal_db_interface: Interface string, e.g. "tcp://max-exfl016:8015"
+ :param creation_time: Latest time for constant to be created
+ :param verbosity: Level of verbosity (0 - silent)
+ :param timeout: Timeout for zmq request
+ :param ntries: number of tries to contact the database
+ :param doraise: if True raise errors during communication with DB
+ """
+ from iCalibrationDB import ConstantMetaData, Versions
+ import zmq
+
+ if device:
+ metadata = ConstantMetaData()
+ metadata.calibration_constant = constant
+ metadata.detector_condition = condition
+ if creation_time is None:
+ metadata.calibration_constant_version = Versions.Now(
+ device=device)
+ else:
+ metadata.calibration_constant_version = Versions.Timespan(
+ device=device,
+ start=creation_time)
+
+ metadata.calibration_constant_version.raw_data_location = file_loc
+
+ while ntries > 0:
+
+ this_interface = get_random_db_interface(cal_db_interface)
+ try:
+ r = metadata.send(this_interface, timeout=timeout)
+ break
+ except zmq.error.Again:
+ ntries -= 1
+ sleep(np.random.randint(30))
+ if ntries == 0 and doraise:
+ raise
+ except Exception as e:
+ if verbosity > 0:
+ print(e)
+ if 'missing_token' in str(e):
+ ntries -= 1
+ else:
+ ntries = 0
+ if ntries == 0 and doraise:
+ raise
+
+ if ntries > 0:
+ if verbosity > 0:
+ if constant.name not in already_printed or verbosity > 1:
+ already_printed[constant.name] = True
+ begin_at = metadata.calibration_constant_version.begin_at
+ print("{} was sent on: {}".format(constant.name,
+ begin_at))
+
+
def get_constant_from_db(device, constant, condition, empty_constant,
cal_db_interface, creation_time=None,
print_once=True, timeout=30000, ntries=120,
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 91e853812ce551bef66bb714e89ef67a8d0d9ea7..e2f81fe255fd731721a177049083ff020104b0e4 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -38,6 +38,7 @@
"db_output = True # output constants to database\n",
"bias_voltage = 300 # detector bias voltage\n",
"cal_db_interface = \"tcp://max-exfl016:8020\" # the database interface to use\n",
+ "cal_db_timeout = 3000000 # timeout on caldb requests\"\n",
"interlaced = False # assume interlaced data format, for data prior to Dec. 2017\n",
"rawversion = 2 # RAW file format version\n",
"dont_use_dir_date = False # don't use the dir creation date for determining the creation time\n",
@@ -78,7 +79,9 @@
"import matplotlib.pyplot as plt\n",
"%matplotlib inline\n",
"\n",
- "from cal_tools.tools import gain_map_files, parse_runs, run_prop_seq_from_path, get_notebook_name, get_dir_creation_date\n",
+ "from cal_tools.tools import (gain_map_files, parse_runs, \n",
+ " run_prop_seq_from_path, get_notebook_name, \n",
+ " get_dir_creation_date, save_const_to_h5)\n",
"from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import show_overview, plot_badpix_3d, create_constant_overview\n",
@@ -389,8 +392,8 @@
" qm = \"Q{}M{}\".format(i//4+1, i%4+1)\n",
" res[qm] = {'Offset': offset_g[qm],\n",
" 'Noise': noise_g[qm],\n",
- " 'Threshold': thresholds_g[qm],\n",
- " 'BadPixels': badpix_g[qm] \n",
+ " 'ThresholdsDark': thresholds_g[qm],\n",
+ " 'BadPixelsDark': badpix_g[qm] \n",
" }\n",
" \n",
"if local_output:\n",
@@ -425,79 +428,43 @@
},
"outputs": [],
"source": [
- "if db_output:\n",
- " for qm in offset_g.keys():\n",
+ "for qm in res:\n",
+ " for const in res[qm]:\n",
" metadata = ConstantMetaData()\n",
- " offset = Constants.AGIPD.Offset()\n",
- " offset.data = offset_g[qm]\n",
- " metadata.calibration_constant = offset\n",
+ " dconst = getattr(Constants.AGIPD, const)()\n",
+ " dconst.data = res[qm][const]\n",
+ " metadata.calibration_constant = dconst\n",
"\n",
" # set the operating condition\n",
- " condition = Conditions.Dark.AGIPD(memory_cells=max_cells, bias_voltage=bias_voltage, acquisition_rate=acq_rate)\n",
+ " condition = Conditions.Dark.AGIPD(memory_cells=max_cells,\n",
+ " bias_voltage=bias_voltage,\n",
+ " acquisition_rate=acq_rate)\n",
" detinst = getattr(Detectors, dinstance)\n",
" device = getattr(detinst, qm)\n",
" \n",
" metadata.detector_condition = condition\n",
- " \n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
- " metadata.send(cal_db_interface, timeout=3000000)\n",
- " \n",
- " \n",
- " metadata = ConstantMetaData()\n",
- " noise = Constants.AGIPD.Noise()\n",
- " noise.data = noise_g[qm]\n",
- " metadata.calibration_constant = noise\n",
- "\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.AGIPD(memory_cells=max_cells, bias_voltage=bias_voltage, acquisition_rate=acq_rate)\n",
- " metadata.detector_condition = condition\n",
"\n",
" # specify the a version for this constant\n",
" if creation_time is None:\n",
" metadata.calibration_constant_version = Versions.Now(device=device)\n",
" else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
- " metadata.send(cal_db_interface, timeout=3000000)\n",
- " \n",
- " metadata = ConstantMetaData()\n",
- " thresholds = Constants.AGIPD.ThresholdsDark()\n",
- " thresholds.data = thresholds_g[qm]\n",
- " metadata.calibration_constant = thresholds\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
+ " start=creation_time)\n",
"\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.AGIPD(memory_cells=max_cells, bias_voltage=bias_voltage, acquisition_rate=acq_rate)\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
" metadata.calibration_constant_version.raw_data_location = file_loc\n",
- " metadata.send(cal_db_interface, timeout=3000000)\n",
- " \n",
- " metadata = ConstantMetaData()\n",
- " badpixels = Constants.AGIPD.BadPixelsDark()\n",
- " badpixels.data = badpix_g[qm]\n",
- " metadata.calibration_constant = badpixels\n",
"\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.AGIPD(memory_cells=max_cells, bias_voltage=bias_voltage, acquisition_rate=acq_rate)\n",
- " metadata.detector_condition = condition\n",
- "\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
- " metadata.send(cal_db_interface, timeout=3000000)"
+ " if db_output:\n",
+ " try:\n",
+ " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
+ " msg = 'Const {} for module {} was injected to the calibration DB. Begin at: {}'\n",
+ " print(msg.format(const, qm,\n",
+ " metadata.calibration_constant_version.begin_at))\n",
+ " except Exception as e:\n",
+ " print(e)\n",
+ "\n",
+ " if local_output:\n",
+ " save_const_to_h5(metadata, out_folder)\n",
+ " print(\"Calibration constant {} is stored locally.\".format(const))\n"
]
},
{
diff --git a/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb b/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
index 0b56642e1a91ff2f68c87245995eac562c4677e6..e27e4a3dc0c3869e8c8efc4cf82d026fba3b8494 100644
--- a/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
+++ b/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
@@ -50,7 +50,8 @@
" # high gain: conversion gain is 1 ADU = 6.1e-\n",
"ADU_to_electron_lower_hg = 6.2 # and for lower hemisphere and high gain: conversion gain is 1 ADU = 6.2e- \n",
"run_parallel = True # For parallel computation \n",
- "db_output = True # Output constants to the calibration database"
+ "db_output = True # Output constants to the calibration database\n",
+ "local_output = True # output constants locally"
]
},
{
@@ -82,7 +83,7 @@
"\n",
"from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
"from iCalibrationDB.detectors import DetectorTypes\n",
- "from cal_tools.tools import get_dir_creation_date\n",
+ "from cal_tools.tools import get_dir_creation_date, save_const_to_h5\n",
"from cal_tools.enums import BadPixels\n",
"from XFELDetAna import xfelpyanatools as xana\n",
"from XFELDetAna import xfelpycaltools as xcal\n",
@@ -1110,21 +1111,22 @@
" else:\n",
" metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
" \n",
+ " metadata.calibration_constant_version.raw_data_location = file_loc\n",
+ " \n",
" if db_output:\n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
- " metadata.send(cal_db_interface, timeout=cal_db_timeout) \n",
+ " try:\n",
+ " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
+ " print(\"Calibration constant {} is sent to the database.\".format(const))\n",
+ " except Exception as e:\n",
+ " print(e)\n",
+ " \n",
+ " if local_output:\n",
+ " save_const_to_h5(metadata, out_folder)\n",
+ " print(\"Calibration constant {} is stored locally.\".format(const))\n",
" \n",
- "print(\"Calibration constants (offsetMap, noiseMapCM_2nd and bad_pixels) are sent to the calibration database.\")\n",
"print(\"Creation time is: {}\".format(creation_time))\n",
"print(\"Raw data location is: {}\".format(file_loc))"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
diff --git a/notebooks/LPD/LPDChar_Darks_NBC.ipynb b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
index 521c9573b46edd894a989f17b8bf26882bb4e0ad..09826cea884ffff7117d8da4bf1ec86f41df7d63 100644
--- a/notebooks/LPD/LPDChar_Darks_NBC.ipynb
+++ b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
@@ -120,7 +120,8 @@
" run_prop_seq_from_path, \n",
" get_notebook_name, \n",
" get_dir_creation_date, get_from_db,\n",
- " get_random_db_interface)\n",
+ " get_random_db_interface,\n",
+ " save_const_to_h5)\n",
"from cal_tools.influx import InfluxLogger\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import (show_overview, plot_badpix_3d, \n",
@@ -452,37 +453,44 @@
"outputs": [],
"source": [
"# Save constants in the calibration DB\n",
- "if db_output:\n",
- " for cap in capacitor_settings:\n",
- " for qm in res[cap]:\n",
- " for const in res[cap][qm]:\n",
- "\n",
- " metadata = ConstantMetaData()\n",
- " dconst = getattr(Constants.LPD, const)()\n",
- " dconst.data = res[cap][qm][const]\n",
- " metadata.calibration_constant = dconst\n",
+ "for cap in capacitor_settings:\n",
+ " for qm in res[cap]:\n",
+ " for const in res[cap][qm]:\n",
"\n",
- " # set the operating condition\n",
- " condition = Conditions.Dark.LPD(memory_cells=max_cells,\n",
- " bias_voltage=bias_voltage,\n",
- " capacitor=cap)\n",
- " device = getattr(Detectors.LPD1M1, qm)\n",
- " if device:\n",
+ " metadata = ConstantMetaData()\n",
+ " dconst = getattr(Constants.LPD, const)()\n",
+ " dconst.data = res[cap][qm][const]\n",
+ " metadata.calibration_constant = dconst\n",
"\n",
- " metadata.detector_condition = condition\n",
+ " # set the operating condition\n",
+ " condition = Conditions.Dark.LPD(memory_cells=max_cells,\n",
+ " bias_voltage=bias_voltage,\n",
+ " capacitor=cap)\n",
+ " device = getattr(Detectors.LPD1M1, qm)\n",
+ " if device:\n",
+ " metadata.detector_condition = condition\n",
"\n",
- " # specify the a version for this constant\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
- " start=creation_time)\n",
- " \n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
- " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
- " msg = 'Const {} for module {} was injected to the calibration DB. Begin at: {}'\n",
- " print(msg.format(const, qm,\n",
- " metadata.calibration_constant_version.begin_at))"
+ " # specify the a version for this constant\n",
+ " if creation_time is None:\n",
+ " metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ " else:\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
+ " start=creation_time)\n",
+ "\n",
+ " metadata.calibration_constant_version.raw_data_location = file_loc\n",
+ "\n",
+ " if db_output:\n",
+ " try:\n",
+ " metadata.send(cal_db_interface, timeout=cal_db_timeout)\n",
+ " msg = 'Const {} for module {} was injected to the calibration DB. Begin at: {}'\n",
+ " print(msg.format(const, qm,\n",
+ " metadata.calibration_constant_version.begin_at))\n",
+ " except Exception as e:\n",
+ " print(e)\n",
+ "\n",
+ " if local_output:\n",
+ " save_const_to_h5(metadata, out_folder)\n",
+ " print(\"Calibration constant {} is stored locally.\".format(const))"
]
},
{
diff --git a/notebooks/LPD/LPD_Correct_and_Verify.ipynb b/notebooks/LPD/LPD_Correct_and_Verify.ipynb
index 81a17d23aa92d2e11e7471295c05cbb8b6766675..883892ccb833fb43a4b54f0e2bfdee73d4824100 100644
--- a/notebooks/LPD/LPD_Correct_and_Verify.ipynb
+++ b/notebooks/LPD/LPD_Correct_and_Verify.ipynb
@@ -137,9 +137,6 @@
"DET_FILE_INSET = \"LPD\"\n",
"CHUNK_SIZE = 512\n",
"MAX_PAR = 32\n",
- " \n",
- "out_folder = \"{}/r{:04d}\".format(out_folder, run)\n",
- "print(\"Outputting to {}\".format(out_folder))\n",
"\n",
"if not os.path.exists(out_folder):\n",
" os.makedirs(out_folder)\n",
diff --git a/notebooks/ePix/Characterize_Darks_ePix10K_NBC.ipynb b/notebooks/ePix/Characterize_Darks_ePix10K_NBC.ipynb
index 84e92b3f25d54fe1b2c1d910acfb98722b653cbe..c9e2134392de07938f485dd217b9d0372d1e7c9e 100644
--- a/notebooks/ePix/Characterize_Darks_ePix10K_NBC.ipynb
+++ b/notebooks/ePix/Characterize_Darks_ePix10K_NBC.ipynb
@@ -29,7 +29,6 @@
"h5path_t = '/INSTRUMENT/{}/DET/RECEIVER:daqOutput/data/backTemp' # path to find temperature at\n",
"h5path_cntrl = '/CONTROL/{}/DET' # path to control data\n",
"cal_db_interface = \"tcp://max-exfl016:8020\" # calibration DB interface to use\n",
- "local_output = False # output also in as H5 files\n",
"temp_limits = 5 # limit for parameter Operational temperature\n",
"sequence = 0 # sequence file to use\n",
"use_dir_creation_date = True\n",
@@ -38,7 +37,9 @@
"in_vacuum = False # detector operated in vacuum\n",
"instance = \"HED_IA1_EPIX10K-1\" # karabo instance\n",
"path_inset = \"EPIX03\" # file inset for image data\n",
- "fix_temperature = 290. # fix temperature to this value"
+ "fix_temperature = 290. # fix temperature to this value\n",
+ "db_output = True # Output constants to the calibration database\n",
+ "local_output = False # output constants locally"
]
},
{
@@ -64,7 +65,7 @@
"prettyPlotting = True\n",
"from XFELDetAna.xfelreaders import ChunkReader\n",
"from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
- "from cal_tools.tools import get_dir_creation_date\n",
+ "from cal_tools.tools import get_dir_creation_date, save_const_to_h5\n",
"\n",
"from iCalibrationDB import (ConstantMetaData, Constants, Conditions, Detectors,\n",
" Versions)\n",
@@ -261,9 +262,7 @@
"source": [
"# Save constants to DB\n",
"dclass=\"ePix10K\"\n",
- "consts = [\"Offset\", \"Noise\"]\n",
- "\n",
- "for const_name in consts:\n",
+ "for const_name in constant_maps.keys():\n",
" metadata = ConstantMetaData()\n",
" det = getattr(Constants, dclass)\n",
" const = getattr(det, const_name)()\n",
@@ -291,22 +290,24 @@
" # specify the a version for this constant\n",
" if creation_time is None:\n",
" metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " \n",
" else:\n",
" metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
" start=creation_time)\n",
+ " \n",
" metadata.calibration_constant_version.raw_data_location = file_loc\n",
- " metadata.send(cal_db_interface)\n",
- " print(\"Inject {} constants from {}\".format(const_name, \n",
- " metadata.calibration_constant_version.begin_at))"
+ " \n",
+ " if db_output:\n",
+ " try:\n",
+ " metadata.send(cal_db_interface)\n",
+ " print(\"Inject {} constants from {}\".format(const_name, \n",
+ " metadata.calibration_constant_version.begin_at))\n",
+ " except Exception as e:\n",
+ " print(e)\n",
+ " \n",
+ " if local_output:\n",
+ " save_const_to_h5(metadata, out_folder)\n",
+ " print(\"Calibration constant {} is stored locally.\".format(const))"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
diff --git a/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb b/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb
index 45dc2bd9730cf0d4bcaba0c9fb8467f541d89b30..831a1c70a4ae4e150e70fdb0788732098391cb5b 100644
--- a/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb
+++ b/notebooks/ePix/Characterize_Darks_ePix_NBC.ipynb
@@ -29,7 +29,6 @@
"h5path_t = '/INSTRUMENT/{}/DET/RECEIVER:daqOutput/data/backTemp' # path to find temperature at\n",
"h5path_cntrl = '/CONTROL/{}/DET' # path to control data\n",
"cal_db_interface = \"tcp://max-exfl016:8020\" # calibration DB interface to use\n",
- "local_output = False # output also in as H5 files\n",
"temp_limits = 5 # limit for parameter Operational temperature\n",
"sequence = 0 # sequence file to use\n",
"use_dir_creation_date = True\n",
@@ -38,7 +37,9 @@
"in_vacuum = False # detector operated in vacuum\n",
"instance = \"MID_EXP_EPIX-1\" # karabo instance\n",
"path_inset = \"DA01\" # file inset for image data\n",
- "fix_temperature = 290. # fix temperature to this value"
+ "fix_temperature = 290. # fix temperature to this value\n",
+ "db_output = True # Output constants to the calibration database\n",
+ "local_output = False # output constants locally"
]
},
{
@@ -64,7 +65,7 @@
"prettyPlotting = True\n",
"from XFELDetAna.xfelreaders import ChunkReader\n",
"from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
- "from cal_tools.tools import get_dir_creation_date\n",
+ "from cal_tools.tools import get_dir_creation_date, save_const_to_h5\n",
"\n",
"from iCalibrationDB import (ConstantMetaData, Constants, Conditions, Detectors,\n",
" Versions)\n",
@@ -254,9 +255,7 @@
"source": [
"# Save constants to DB\n",
"dclass=\"ePix100\"\n",
- "consts = [\"Offset\", \"Noise\"]\n",
- "\n",
- "for const_name in consts:\n",
+ "for const_name in constant_maps.keys():\n",
" metadata = ConstantMetaData()\n",
" det = getattr(Constants, dclass)\n",
" const = getattr(det, const_name)()\n",
@@ -277,22 +276,28 @@
" parm.upper_deviation = temp_limits\n",
"\n",
" device = getattr(Detectors, db_module)\n",
- "\n",
" metadata.detector_condition = condition\n",
"\n",
" # specify the a version for this constant\n",
" if creation_time is None:\n",
" metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " \n",
" else:\n",
" metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
" start=creation_time)\n",
" \n",
" metadata.calibration_constant_version.raw_data_location = file_loc\n",
" \n",
- " metadata.send(cal_db_interface)\n",
- " print(\"Inject {} constants from {}\".format(const_name, \n",
- " metadata.calibration_constant_version.begin_at))"
+ " if db_output:\n",
+ " try:\n",
+ " metadata.send(cal_db_interface)\n",
+ " print(\"Inject {} constants from {}\".format(const_name, \n",
+ " metadata.calibration_constant_version.begin_at))\n",
+ " except Exception as e:\n",
+ " print(e)\n",
+ " \n",
+ " if local_output:\n",
+ " save_const_to_h5(metadata, out_folder)\n",
+ " print(\"Calibration constant {} is stored locally.\".format(const))"
]
}
],
diff --git a/notebooks/ePix/Correction_ePix10K_NBC.ipynb b/notebooks/ePix/Correction_ePix10K_NBC.ipynb
index dc5c75d040810f136d869756d1a542b2b4748d98..1dbefa31bffb08879d303a1626fe6a928fa15814 100644
--- a/notebooks/ePix/Correction_ePix10K_NBC.ipynb
+++ b/notebooks/ePix/Correction_ePix10K_NBC.ipynb
@@ -18,8 +18,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:23.218849Z",
"start_time": "2018-12-06T15:54:23.166497Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -82,8 +81,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:23.455376Z",
"start_time": "2018-12-06T15:54:23.413579Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -139,8 +137,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:23.679069Z",
"start_time": "2018-12-06T15:54:23.662821Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -148,7 +145,6 @@
"y = 384 # columns of the ePix10K\n",
" \n",
"ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
- "out_folder = \"{}/r{:04d}\".format(out_folder, run)\n",
"fp_name = path_template.format(run, path_inset)\n",
"fp_path = '{}/{}'.format(ped_dir, fp_name)\n",
"\n",
@@ -173,7 +169,6 @@
"end_time": "2018-12-06T15:54:23.913269Z",
"start_time": "2018-12-06T15:54:23.868910Z"
},
- "collapsed": false,
"scrolled": true
},
"outputs": [],
@@ -203,7 +198,6 @@
"\n",
"if not os.path.exists(out_folder):\n",
" os.makedirs(out_folder)\n",
- " \n",
"elif not overwrite:\n",
" raise AttributeError(\"Output path exists! Exiting\") \n"
]
@@ -254,8 +248,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T18:43:39.776018Z",
"start_time": "2018-12-06T18:43:39.759185Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -285,8 +278,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:28.254544Z",
"start_time": "2018-12-06T15:54:24.709521Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -353,8 +345,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:28.771629Z",
"start_time": "2018-12-06T15:54:28.346051Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -593,8 +584,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:13:12.889583Z",
"start_time": "2018-12-06T16:13:11.122653Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -652,8 +642,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:11:08.317130Z",
"start_time": "2018-12-06T16:11:05.788655Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -680,8 +669,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:11:10.908912Z",
"start_time": "2018-12-06T16:11:08.318486Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -718,7 +706,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.4.3"
+ "version": "3.6.7"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
diff --git a/notebooks/ePix/Correction_ePix_NBC.ipynb b/notebooks/ePix/Correction_ePix_NBC.ipynb
index fbf437b79267e09f108e48f2265ec70d0a09c354..ec4a20eb7bc1d21c23acbe4994c3dbca5fab0e05 100644
--- a/notebooks/ePix/Correction_ePix_NBC.ipynb
+++ b/notebooks/ePix/Correction_ePix_NBC.ipynb
@@ -154,7 +154,6 @@
"y = 768 # columns of the ePix100\n",
" \n",
"ped_dir = \"{}/r{:04d}\".format(in_folder, run)\n",
- "out_folder = \"{}/r{:04d}\".format(out_folder, run)\n",
"fp_name = path_template.format(run, path_inset)\n",
"fp_path = '{}/{}'.format(ped_dir, fp_name)\n",
"\n",
@@ -206,7 +205,6 @@
"\n",
"if not os.path.exists(out_folder):\n",
" os.makedirs(out_folder)\n",
- " \n",
"elif not overwrite:\n",
" raise AttributeError(\"Output path exists! Exiting\") \n"
]
diff --git a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
index 4cf0a0a8cc2753ad4dc0492bbab3dd35cddfa387..1543ff4b5aaf9597d8f1a40c1a44b5302f62661a 100644
--- a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
+++ b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
@@ -34,7 +34,6 @@
"sigma_noise = 10. # Pixel exceeding 'sigmaNoise' * noise value in that pixel will be masked\n",
"h5path = '/INSTRUMENT/SQS_NQS_PNCCD1MP/CAL/PNCCD_FMT-0:output/data/image/' # path in the HDF5 file the data is at\n",
"cal_db_interface = \"tcp://max-exfl016:8021\" # calibration DB interface to use\n",
- "local_output = False # output also in as H5 files\n",
"temp_limits = 5 # temperature limits in which to consider calibration parameters equal\n",
"sequence = 0 # sequence file to use\n",
"multi_iteration = False # use multiple iterations\n",
@@ -44,7 +43,9 @@
"fix_temperature = 233. # fix temperature to this value in K, set to -1 to use value from slow data\n",
"gain = 0 # the detector gain setting, only 0 is currently implemented\n",
"bias_voltage = 300 # detector bias voltage\n",
- "integration_time = 70 # detector integration time"
+ "integration_time = 70 # detector integration time\n",
+ "db_output = True # Output constants to the calibration database\n",
+ "local_output = False # output constants locally"
]
},
{
@@ -70,8 +71,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T10:54:39.467334Z",
"start_time": "2018-12-06T10:54:39.427784Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -91,7 +91,7 @@
"prettyPlotting=True\n",
"from XFELDetAna.xfelreaders import ChunkReader\n",
"from XFELDetAna.detectors.fastccd import readerh5 as fastccdreaderh5\n",
- "from cal_tools.tools import get_dir_creation_date\n",
+ "from cal_tools.tools import get_dir_creation_date, save_const_to_h5\n",
"\n",
"import numpy as np\n",
"import h5py\n",
@@ -115,6 +115,16 @@
"temperature_k = fix_temperature"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
+ "file_loc = 'proposal:{} runs:{}'.format(proposal, run)"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -122,8 +132,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T10:54:40.058101Z",
"start_time": "2018-12-06T10:54:40.042615Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -156,8 +165,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T10:54:40.555804Z",
"start_time": "2018-12-06T10:54:40.452978Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -241,8 +249,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T10:55:21.238009Z",
"start_time": "2018-12-06T10:54:54.586435Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -267,8 +274,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T10:56:20.686534Z",
"start_time": "2018-12-06T10:56:11.721829Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -331,8 +337,9 @@
" [512, 512], \n",
" 'row',\n",
" nCells = memoryCells, \n",
- " noiseMap = noiseMap,\n",
- " runParallel=False,\n",
+ " dType=np.float32,\n",
+ " noiseMap = noiseMap.astype(np.float32),\n",
+ " runParallel=run_parallel,\n",
" stats=True)\n",
"cmCorrection.debug()\n",
"noiseCalCM = xcal.NoiseCalculator(sensorSize, memoryCells, \n",
@@ -352,7 +359,8 @@
" data = data.astype(np.float32)\n",
" dx = np.count_nonzero(data, axis=(0, 1))\n",
" data = data[:,:,dx != 0]\n",
- " data = cmCorrection.correct(data)\n",
+ " cellTable=np.zeros(data.shape[2], np.int32) # Common mode correction\n",
+ " data = cmCorrection.correct(data.astype(np.float32), cellTable=cellTable) \n",
" #Filling calculators with data\n",
" noiseCalCM.fill(data)\n",
" \n",
@@ -364,9 +372,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"#*****NOISE MAP HISTOGRAM FROM THE OFFSET CORRECTED DATA*******#\n",
@@ -396,81 +402,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "ExecuteTime": {
- "end_time": "2018-12-06T10:56:22.741284Z",
- "start_time": "2018-12-06T10:56:20.688393Z"
- },
- "collapsed": true
- },
- "outputs": [],
- "source": [
- "\n",
- "## offset\n",
- "\n",
- "metadata = ConstantMetaData()\n",
- "offset = Constants.CCD(DetectorTypes.pnCCD).Offset()\n",
- "offset.data = offsetMapCM.data\n",
- "metadata.calibration_constant = offset\n",
- "\n",
- "# set the operating condition\n",
- "condition = Conditions.Dark.CCD(bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " gain_setting=gain,\n",
- " temperature=temperature_k,\n",
- " pixels_x=1024,\n",
- " pixels_y=1024)\n",
- "\n",
- "device = Detectors.PnCCD1\n",
- "\n",
- "\n",
- "\n",
- "metadata.detector_condition = condition\n",
- "\n",
- "# specify the a version for this constant\n",
- "if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- "else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- "metadata.send(cal_db_interface, timeout=300000)\n",
- "\n",
- "## noise\n",
- "\n",
- "metadata = ConstantMetaData()\n",
- "noise = Constants.CCD(DetectorTypes.pnCCD).Noise()\n",
- "noise.data = noiseMapCM.data\n",
- "metadata.calibration_constant = noise\n",
- "\n",
- "# set the operating condition\n",
- "condition = Conditions.Dark.CCD(bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " gain_setting=gain,\n",
- " temperature=temperature_k,\n",
- " pixels_x=1024,\n",
- " pixels_y=1024)\n",
- "\n",
- "\n",
- "\n",
- "device = Detectors.PnCCD1\n",
- "\n",
- "\n",
- "metadata.detector_condition = condition\n",
- "\n",
- "# specify the a version for this constant\n",
- "if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- "else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- "metadata.send(cal_db_interface, timeout=300000)\n",
- "\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"from cal_tools.enums import BadPixels\n",
@@ -496,35 +428,60 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
+ "ExecuteTime": {
+ "end_time": "2018-12-06T10:56:22.741284Z",
+ "start_time": "2018-12-06T10:56:20.688393Z"
+ },
"collapsed": true
},
"outputs": [],
"source": [
- "metadata = ConstantMetaData()\n",
- "badpix = Constants.CCD(DetectorTypes.pnCCD).BadPixelsDark()\n",
- "badpix.data = bad_pixels.data\n",
- "metadata.calibration_constant = badpix\n",
- "\n",
- "# set the operating condition\n",
- "condition = Conditions.Dark.CCD(bias_voltage=bias_voltage,\n",
- " integration_time=integration_time,\n",
- " gain_setting=gain,\n",
- " temperature=temperature_k,\n",
- " pixels_x=1024,\n",
- " pixels_y=1024)\n",
- "\n",
- "\n",
- "device = Detectors.PnCCD1\n",
- "\n",
- "\n",
- "metadata.detector_condition = condition\n",
- "\n",
- "# specify the a version for this constant\n",
- "if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- "else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- "metadata.send(cal_db_interface, timeout=300000)"
+ "constant_maps = {\n",
+ " 'Offset': offsetMapCM,\n",
+ " 'Noise': noiseMapCM,\n",
+ " 'BadPixelsDark': bad_pixels\n",
+ "}\n",
+ "\n",
+ "for const_name in constant_maps.keys():\n",
+ " metadata = ConstantMetaData()\n",
+ " det = Constants.CCD(DetectorTypes.pnCCD)\n",
+ " const = getattr(det, const_name)()\n",
+ " const.data = constant_maps[const_name].data\n",
+ "\n",
+ " metadata.calibration_constant = const\n",
+ "\n",
+ " # set the operating condition\n",
+ " condition = Conditions.Dark.CCD(bias_voltage=bias_voltage,\n",
+ " integration_time=integration_time,\n",
+ " gain_setting=gain,\n",
+ " temperature=temperature_k,\n",
+ " pixels_x=1024,\n",
+ " pixels_y=1024)\n",
+ "\n",
+ "\n",
+ " device = Detectors.PnCCD1\n",
+ " metadata.detector_condition = condition\n",
+ "\n",
+ " # specify the a version for this constant\n",
+ " if creation_time is None:\n",
+ " metadata.calibration_constant_version = Versions.Now(device=device)\n",
+ " else:\n",
+ " metadata.calibration_constant_version = Versions.Timespan(device=device,\n",
+ " start=creation_time)\n",
+ " \n",
+ " metadata.calibration_constant_version.raw_data_location = file_loc\n",
+ " \n",
+ " if db_output:\n",
+ " try:\n",
+ " metadata.send(cal_db_interface)\n",
+ " print(\"Inject {} constants from {}\".format(const_name, \n",
+ " metadata.calibration_constant_version.begin_at))\n",
+ " except Exception as e:\n",
+ " print(e)\n",
+ " \n",
+ " if local_output:\n",
+ " save_const_to_h5(metadata, out_folder)\n",
+ " print(\"Calibration constant {} is stored locally.\".format(const)) \n"
]
},
{
@@ -550,9 +507,7 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "collapsed": false
- },
+ "metadata": {},
"outputs": [],
"source": [
"ho,eo,co,so = histCalCorr.get()\n",
@@ -575,24 +530,6 @@
" y_log=True, x_range=(-50,500),\n",
" legend='top-center-frame-2col')"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": []
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": []
}
],
"metadata": {
@@ -611,7 +548,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.6"
+ "version": "3.6.7"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
diff --git a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
index 1af1d0ca1743f1a6c1d5fee347bcd2c30104712b..5ffe35f6e0ab747019c131517f07d5fecf52526d 100644
--- a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
+++ b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
@@ -18,8 +18,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:23.218849Z",
"start_time": "2018-12-06T15:54:23.166497Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -84,8 +83,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:23.455376Z",
"start_time": "2018-12-06T15:54:23.413579Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -143,8 +141,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:23.679069Z",
"start_time": "2018-12-06T15:54:23.662821Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -167,9 +164,7 @@
"print(\"Run is: {}\".format(run))\n",
"print(\"HDF5 path: {}\".format(h5path))\n",
"if creation_time:\n",
- " print(\"Using {} as creation time\".format(creation_time.isoformat()))\n",
- " \n",
- "out_folder = \"{}/r{:04d}\".format(out_folder, run)"
+ " print(\"Using {} as creation time\".format(creation_time.isoformat()))\n"
]
},
{
@@ -250,8 +245,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T18:43:39.776018Z",
"start_time": "2018-12-06T18:43:39.759185Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -282,8 +276,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T15:54:28.254544Z",
"start_time": "2018-12-06T15:54:24.709521Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -753,7 +746,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
"scrolled": false
},
"outputs": [],
@@ -773,8 +765,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:10:56.126409Z",
"start_time": "2018-12-06T16:10:56.096242Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -830,8 +821,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:10:56.176160Z",
"start_time": "2018-12-06T16:10:56.127853Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -887,8 +877,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:10:56.190150Z",
"start_time": "2018-12-06T16:10:56.177570Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -955,8 +944,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:10:56.212586Z",
"start_time": "2018-12-06T16:10:56.204731Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -989,8 +977,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:11:08.317130Z",
"start_time": "2018-12-06T16:11:05.788655Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -1024,8 +1011,7 @@
"ExecuteTime": {
"end_time": "2018-12-06T16:11:10.908912Z",
"start_time": "2018-12-06T16:11:08.318486Z"
- },
- "collapsed": false
+ }
},
"outputs": [],
"source": [
@@ -1083,7 +1069,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false,
"scrolled": false
},
"outputs": [],
@@ -1151,7 +1136,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.6.6"
+ "version": "3.6.7"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
diff --git a/webservice/serve_overview.py b/webservice/serve_overview.py
index d06fcd0e88ae030727fef10a471f56a5108dc1f2..a2b417aea7aa3285dcaa28940dac42dc5e2b0797 100644
--- a/webservice/serve_overview.py
+++ b/webservice/serve_overview.py
@@ -84,17 +84,9 @@ class RequestHandler(BaseHTTPRequestHandler):
self.wfile.write(f.read())
return
- if "file?" in self.path:
- fpath = self.path.split("?")[1]
- if fpath is None:
- return
- if os.path.isfile(fpath) and os.path.splitext(fpath)[1] == '.pdf':
- self.send_header('Content-type', 'application/pdf')
- self.end_headers()
- with open(fpath, "rb") as f:
- self.wfile.write(f.read())
- return
- if "update_form?" in self.path:
+ if "process_dark?" in self.path:
+ self.send_header('Content-type', 'text/html')
+ self.end_headers()
pars = self.path.split("?")[1].split("&")
pars = {x.split("=")[0]: x.split("=")[1] for x in pars}
if pars['instrument'] in ['Nothing', 'none']:
@@ -198,6 +190,38 @@ class RequestHandler(BaseHTTPRequestHandler):
self.wfile.write(bytes(message, "utf8"))
return
+ if "/gpfs" in self.path:
+ sendReply = False
+ if self.path.endswith(".html"):
+ mimetype = 'text/html'
+ sendReply = True
+ if self.path.endswith(".jpg"):
+ mimetype = 'image/jpg'
+ sendReply = True
+ if self.path.endswith(".gif"):
+ mimetype = 'image/gif'
+ sendReply = True
+ if self.path.endswith(".png"):
+ mimetype = 'image/png'
+ sendReply = True
+ if self.path.endswith(".pdf"):
+ mimetype = 'application/pdf'
+ sendReply = True
+ if self.path.endswith(".js"):
+ mimetype = 'application/javascript'
+ sendReply = True
+ if self.path.endswith(".css"):
+ mimetype = 'text/css'
+ sendReply = True
+
+ if sendReply == True and os.path.isfile(self.path):
+ with open(self.path, "rb") as f:
+ self.send_header('Content-type', mimetype)
+ self.end_headers()
+ self.wfile.write(f.read())
+ return
+
+
# Send headers
self.send_header('Content-type', 'text/html')
self.end_headers()
diff --git a/webservice/serve_overview.yaml b/webservice/serve_overview.yaml
index ed80c518255cf9ccf5c6e503867875c1ca257375..12984eb21cac3077aaac598e894385204c249e76 100644
--- a/webservice/serve_overview.yaml
+++ b/webservice/serve_overview.yaml
@@ -50,6 +50,8 @@ server-config:
host: max-exfl016
dark-timeout: 30
n-calib: 10
+ cal_db_interface: tcp://max-exfl016:8022
+ cal_db_timeout: 30000
web-service:
job-db: ./webservice_jobs.sqlite
diff --git a/webservice/templates/last_correction.html b/webservice/templates/last_correction.html
index 893266cb2ec311c144f1a8f73f0744609ec842ff..5a799066ea3ae1b5d8be8f1ab571605cd1926a18 100644
--- a/webservice/templates/last_correction.html
+++ b/webservice/templates/last_correction.html
@@ -18,7 +18,7 @@
<td> run: {{ item[2] }} </td>
<td>
{% for file, path in item[3].items() %}
- <a href="http://{{host}}:{{port}}/file?{{ path }}" target="_blank">{{ file }}</a>
+ <a href="http://{{host}}:{{port}}/{{ path }}" target="_blank">{{ file }}</a>
{% endfor %}
</td>
</tr>
diff --git a/webservice/templates/request_dark.html b/webservice/templates/request_dark.html
index c7dfefbaeca5f0e17c5561d7be3353bb739b7f18..efd129e37a7a273a0011d2a4a72e7f5716b617b0 100644
--- a/webservice/templates/request_dark.html
+++ b/webservice/templates/request_dark.html
@@ -75,7 +75,7 @@ function loadCheckbox(command) {
document.getElementById("demo").innerHTML = 'Requesting dark...'
}
- xhttp.open("GET", "{{host}}:{{port}}/update_form?"+msg, true);
+ xhttp.open("GET", "{{host}}:{{port}}/process_dark?"+msg, true);
xhttp.send();
}
diff --git a/webservice/webservice.py b/webservice/webservice.py
index f2df2c70b98da64550261707183e0dffbfd9b0b0..be231a0a29d34b3e5438b42675a3712d42b15040 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -612,6 +612,7 @@ async def server_runner(config, mode):
return
print("Now doing: {}".format(action))
+ ts = datetime.now().strftime('%y%m%d_%H%M%S')
if action == 'dark':
print("Running dark cal")
status = []
@@ -636,6 +637,10 @@ async def server_runner(config, mode):
thisconf["out-folder"] = '/'.join((out_folder,
detector.replace('-', '_')))
+ report_to = 'Dark_' + \
+ detector.replace('-', '_') + '_' + ts
+ thisconf["report-to"] = report_to.lower()
+
# don't need this for xfel-calibrate
del thisconf["inset"]
detectors[detector] = thisconf
@@ -684,7 +689,9 @@ async def server_runner(config, mode):
rpath = "{}/r{:04d}/".format(in_folder, int(runnr))
out_folder = config[action]['out-folder'].format(
- instrument=instrument, cycle=cycle, proposal=proposal)
+ instrument=instrument, cycle=cycle, proposal=proposal,
+ run='r{:04d}'.format(int(runnr)))
+
corr_file_list = set()
copy_file_list = set(glob.glob("{}/*.h5".format(rpath)))
detectors = {}
@@ -697,6 +704,10 @@ async def server_runner(config, mode):
thisconf = copy.copy(dconfig)
thisconf["in-folder"] = in_folder
thisconf["out-folder"] = out_folder
+ report_to = 'Correct_' + \
+ detector.replace('-', '_') + '_' + ts
+ thisconf["report-to"] = report_to.lower()
+
thisconf["run"] = runnr
del thisconf[
"inset"] # don't need this for xfel-calibrate
diff --git a/webservice/webservice.yaml b/webservice/webservice.yaml
index 6e8eef82a41321fb28d1246833a5c784fec3ec7b..4ed2b966ab36cac4a1a7db6d1245eb61ce63ca9a 100644
--- a/webservice/webservice.yaml
+++ b/webservice/webservice.yaml
@@ -24,7 +24,7 @@ metadata-client:
correct:
in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw
- out-folder: /gpfs/exfel/d/proc/{instrument}/{cycle}/p{proposal}
+ out-folder: /gpfs/exfel/d/proc/{instrument}/{cycle}/p{proposal}/{run}
dark:
in-folder: /gpfs/exfel/exp/{instrument}/{cycle}/p{proposal}/raw
diff --git a/xfel_calibrate/calibrate.py b/xfel_calibrate/calibrate.py
index 09bc6978ec39be88d822faffd2c186af53ca5e2c..f1e7badbe68455a16658bf1b4ab7b68e75b6bcf0 100755
--- a/xfel_calibrate/calibrate.py
+++ b/xfel_calibrate/calibrate.py
@@ -455,6 +455,21 @@ def has_parm(parms, name):
return False
+def get_par_attr(parms, key, attr, default=None):
+ """
+ Return the type of parameter with name key
+ :param parms: List of parameters
+ :param key: Name of the parameter to be considered
+ :param attr: Name of the parameter attribute (e.g. value, type)
+ :param default: Type to be returned if interested name is not found
+ :return: The attribute of the parameter
+ """
+ for p in parms:
+ if p.name == key:
+ return getattr(p, attr, default)
+ return default
+
+
def flatten_list(l):
return "_".join([str(flatten_list(v)) for v in l]) if isinstance(l, list) else l
@@ -755,12 +770,6 @@ def run():
if try_report_to_output:
if "out_folder" in args:
out_path = os.path.abspath(args["out_folder"])
- if "run" in args:
- rr = args["run"]
- if isinstance(rr, int):
- out_path = "{}/r{:04d}/".format(out_path, rr)
- else:
- out_path = "{}/{}/".format(out_path, rr)
else:
print("No 'out_folder' defined as argument, outputting to '{}' instead.".format(
out_path))
@@ -775,10 +784,7 @@ def run():
if args["report_to"] is not None:
report_to = args["report_to"]
- folder = ''
- for p in parms:
- if p.name == 'in_folder':
- folder = p.value
+ folder = get_par_attr(parms, 'in_folder', 'value', '')
fmtcmd = cmd.format(run_path=run_tmp_path, out_path=out_path,
project=title, calibration=title,
@@ -788,9 +794,14 @@ def run():
joblist = []
if concurrency.get("parameter", None) is None:
cluster_cores = concurrency.get("cluster cores", 8)
- jobid = concurrent_run(run_tmp_path, nb, os.path.basename(notebook), args,
- final_job=True, job_list=joblist, fmtcmd=fmtcmd,
- cluster_cores=cluster_cores, sequential=sequential)
+
+ jobid = concurrent_run(run_tmp_path, nb,
+ os.path.basename(notebook), args,
+ final_job=True, job_list=joblist,
+ fmtcmd=fmtcmd,
+ cluster_cores=cluster_cores,
+ sequential=sequential)
+
joblist.append(jobid)
else:
cvar = concurrency["parameter"]
@@ -802,18 +813,14 @@ def run():
if cvals is None:
defcval = concurrency.get("default concurrency", None)
if defcval is not None:
- print(
- "Concurrency parameter '{}' is taken from notebooks.py".format(cvar))
+ print("Concurrency parameter '{}' is taken from notebooks.py".format(cvar))
if not isinstance(defcval, (list, tuple)):
cvals = range(defcval)
else:
cvals = defcval
if cvals is None:
- print(parms)
- for p in parms:
- if p.name == cvar:
- defcval = p.value
+ defcval = get_par_attr(parms, cvar, 'value')
if defcval is not None:
print("Concurrency parameter '{}' is taken from '{}'".format(
cvar, notebook))
@@ -843,22 +850,19 @@ def run():
print("Split concurrency into {}".format(cvals))
# get expected type
- cvtype = list
- for p in parms:
- if p.name == cvar:
- cvtype = p.type
- break
+ cvtype = get_par_attr(parms, cvar, 'type', list)
for cnum, cval in enumerate(cvals):
show_title = cnum == 0
# Job is not final if there are dependent notebooks
+ final_job = (cnum == len(list(cvals)) - 1 and len(dep_notebooks) == 0)
+ cval = [cval, ] if not isinstance(cval, list) and cvtype is list else cval
+
jobid = concurrent_run(run_tmp_path, nb, notebook, args,
- cvar, [cval, ] if not isinstance(
- cval, list) and cvtype is list else cval,
- cnum == len(list(cvals)) -
- 1 and len(dep_notebooks) == 0,
+ cvar, cval, final_job,
joblist, fmtcmd,
- cluster_cores=cluster_cores, sequential=sequential,
+ cluster_cores=cluster_cores,
+ sequential=sequential,
show_title=show_title)
joblist.append(jobid)
@@ -869,7 +873,8 @@ def run():
with open(notebook_path, "r") as f:
nb = nbformat.read(f, as_version=4)
final_job = i == len(dep_notebooks) - 1
- jobid = concurrent_run(run_tmp_path, nb, os.path.basename(notebook),
+ jobid = concurrent_run(run_tmp_path, nb,
+ os.path.basename(notebook),
args,
final_job=final_job,
job_list=joblist, fmtcmd=fmtcmd,