diff --git a/notebooks/DynamicFF/Characterize_DynamicFF_NBC.ipynb b/notebooks/DynamicFF/Characterize_DynamicFF_NBC.ipynb
index e9595656c8fe6624f0e3d91ff8e27a49af416df6..9339eae9c535260072335592a462bbcff1d162e6 100644
--- a/notebooks/DynamicFF/Characterize_DynamicFF_NBC.ipynb
+++ b/notebooks/DynamicFF/Characterize_DynamicFF_NBC.ipynb
@@ -64,10 +64,12 @@
"from cal_tools.step_timing import StepTimer\n",
"from cal_tools.tools import (\n",
" get_dir_creation_date,\n",
+ " run_prop_seq_from_path,\n",
" save_dict_to_hdf5\n",
")\n",
"from cal_tools.restful_config import calibration_client, extra_calibration_client\n",
"from cal_tools.shimadzu import ShimadzuHPVX2\n",
+ "from cal_tools.constants import write_ccv, inject_ccv\n",
"\n",
"import dynflatfield as dffc\n",
"from dynflatfield.draw import plot_images, plot_camera_image"
@@ -168,7 +170,7 @@
" module_constants = constants.setdefault(meta[\"db_module\"], {})\n",
" module_constants[\"Offset\"] = dict(\n",
" conditions=conditions, data=dark, pdu_no=meta[\"pdu_no\"],\n",
- " creation_time=dark_creation_time\n",
+ " creation_time=dark_creation_time, dims=['ss', 'fs']\n",
" )\n",
" step_timer.done_step(\"Process dark images\")\n",
" display()\n",
@@ -243,7 +245,7 @@
" module_constants = constants.setdefault(meta[\"db_module\"], {})\n",
" module_constants[\"DynamicFF\"] = dict(\n",
" conditions=conditions, data=flat_data, pdu_no=meta[\"pdu_no\"],\n",
- " creation_time=flat_creation_time\n",
+ " creation_time=flat_creation_time, dims=['component', 'ss', 'fs']\n",
" )\n",
" step_timer.done_step(\"Process flat-field images\")\n",
"\n",
@@ -283,39 +285,27 @@
"source": [
"step_timer.start()\n",
"\n",
+ "_, proposal, _ = run_prop_seq_from_path(in_folder)\n",
+ "\n",
"# Output Folder Creation:\n",
"if local_output:\n",
" os.makedirs(out_folder, exist_ok=True)\n",
"\n",
- "def inject_ccv(in_folder, metadata_folder, runs, calibration, cond, pdu, const_input, begin_at):\n",
- " print(\"* Send to db:\", const_input)\n",
- " print(\" - in folder:\", in_folder)\n",
- " print(\" - metadata folder:\", metadata_folder)\n",
- " print(\" - runs:\", runs)\n",
- " print(\" -\", calibration)\n",
- " print(\" -\", cond)\n",
- " print(\" -\", begin_at)\n",
- "\n",
"for db_module, module_constants in constants.items():\n",
" for constant_name, constant in module_constants.items():\n",
" conditions = constant[\"conditions\"]\n",
- " conditions_dict = conditions.make_dict(\n",
- " conditions.calibration_types[constant_name])\n",
- "\n",
- " data_to_store = {db_module: {constant_name: {'0': {\n",
- " 'conditions': conditions_dict,\n",
- " 'data': constant[\"data\"],\n",
- " }}}}\n",
+ " pdu = pdus[\"data\"][constant[\"pdu_no\"]]\n",
"\n",
" with NamedTemporaryFile() as tempf:\n",
- " save_dict_to_hdf5(data_to_store, tempf)\n",
+ " ccv_root = write_ccv(\n",
+ " tempf.name,\n",
+ " pdu['physical_name'], pdu['uuid'], pdu['detector_type']['name'],\n",
+ " constant_name, conditions, constant['creation_time'],\n",
+ " proposal, [dark_run, flat_run],\n",
+ " constant[\"data\"], constant['dims'])\n",
" \n",
" if db_output:\n",
- " inject_ccv(\n",
- " in_folder, metadata_folder, [dark_run, flat_run],\n",
- " constant_name, conditions, pdus[\"data\"][constant[\"pdu_no\"]],\n",
- " ofile, constant[\"creation_time\"]\n",
- " )\n",
+ " inject_ccv(tempf.name, ccv_root, metadata_folder)\n",
" \n",
" if local_output:\n",
" ofile = f\"{out_folder}/const_{constant_name}_{db_module}.h5\"\n",
diff --git a/notebooks/DynamicFF/Correct_DynamicFF_NBC.ipynb b/notebooks/DynamicFF/Correct_DynamicFF_NBC.ipynb
index 258c3712daea0e707b6346deff7f404a982d193c..3e6a1875c027ef1796d7f5687588becf8539fdf1 100644
--- a/notebooks/DynamicFF/Correct_DynamicFF_NBC.ipynb
+++ b/notebooks/DynamicFF/Correct_DynamicFF_NBC.ipynb
@@ -146,9 +146,8 @@
"corrections = {}\n",
"for da in modules:\n",
" try:\n",
- " # !!! REMOVE caldb_root for production\n",
- " dark = caldata[\"Offset\", da].ndarray(caldb_root=caldb_root)\n",
- " flat = caldata[\"DynamicFF\", da].ndarray(caldb_root=caldb_root)\n",
+ " dark = caldata[\"Offset\", da].ndarray()\n",
+ " flat = caldata[\"DynamicFF\", da].ndarray()\n",
" \n",
" components = flat[1:][:n_components]\n",
" flat = flat[0]\n",
diff --git a/src/cal_tools/constants.py b/src/cal_tools/constants.py
new file mode 100644
index 0000000000000000000000000000000000000000..cd4abb00f819ef75f2da7b3315adaf4f863bd3c6
--- /dev/null
+++ b/src/cal_tools/constants.py
@@ -0,0 +1,199 @@
+
+from datetime import datetime, timezone
+from struct import pack, unpack
+from pathlib import Path
+from shutil import copyfile
+from hashlib import md5
+import binascii
+import time
+
+import numpy as np
+import h5py
+
+from calibration_client import CalibrationClient
+from cal_tools.calcat_interface2 import _get_default_caldb_root, get_client
+from cal_tools.tools import run_prop_seq_from_path
+from cal_tools.restful_config import calibration_client
+
+
+def write_ccv(
+ const_path,
+ pdu_name, pdu_uuid, detector_type,
+ calibration, conditions, created_at, proposal, runs,
+ data, dims, key='0'
+):
+ """Write CCV data file.
+
+ Args:
+ const_path (os.PathLike): Path to CCV file to write
+ pdu_name (str): Physical detector unit name
+ pdu_uuid (int): Physical detector unit UUID
+ detector_type (str): Detector type name
+ calibration (str): Calibration name
+ conditions (ConditionsBase): Detector operating conditions
+ created_at (datetime): Validity start for calibration
+ proposal (int): Raw data proposal the calibration data is
+ generated from
+ runs (Iterable of int): Raw data runs the calibration data is
+ generated from
+ data (ndarray): Calibration constant data
+ dims (Iterable of str):
+ key (str, optional):
+
+ Returns:
+ (str) CCV HDF group name.
+
+ """
+
+ if data.ndim != len(dims):
+ raise ValueError('data.ndims != len(dims)')
+
+ with h5py.File(const_path, 'a') as const_file:
+ const_file.attrs['version'] = 0
+
+ pdu_group = const_file.require_group(pdu_name)
+ pdu_group.attrs['uuid'] = pdu_uuid
+ pdu_group.attrs['detector_type'] = detector_type
+
+ calibration_group = pdu_group.require_group(calibration)
+
+ if key is None:
+ key = str(len(calibration_group))
+
+ ccv_group = calibration_group.create_group(key)
+ ccv_group.attrs['begin_at'] = created_at.isoformat()
+ ccv_group.attrs['proposal'] = proposal
+ ccv_group.attrs['runs'] = np.array(runs, dtype=np.int32)
+ ccv_group_name = ccv_group.name
+
+ opcond_group = ccv_group.create_group('operating_condition')
+ opcond_dict = conditions.make_dict(
+ conditions.calibration_types[calibration])
+ for db_name, value in opcond_dict.items():
+ key = db_name.lower().replace(' ', '_')
+ dset = opcond_group.create_dataset(key, data=value,
+ dtype=np.float64)
+ dset.attrs['lower_deviation'] = 0.0
+ dset.attrs['upper_deviation'] = 0.0
+ dset.attrs['database_name'] = db_name
+
+ dset = ccv_group.create_dataset('data', data=data)
+ dset.attrs['dims'] = dims
+
+ return ccv_group_name
+
+
+def inject_ccv(const_src, ccv_root, report_to=None):
+ """Inject new CCV into CalCat.
+
+ Args:
+ const_path (str or Path): Path to CCV data file.
+ ccv_root (str): CCV HDF group name.
+ report_to (str): Metadata location.
+
+ Returns:
+ None
+
+ Raises:
+ RuntimeError: If CalCat POST request fails.
+ """
+
+ pdu_name, calibration, key = ccv_root.lstrip('/').split('/')
+
+ with h5py.File(const_src, 'r') as const_file:
+ pdu_group = const_file[pdu_name]
+ pdu_uuid = pdu_group.attrs['uuid']
+ detector_type = pdu_group.attrs['detector_type']
+
+ ccv_group = const_file[ccv_root]
+ proposal, runs = ccv_group.attrs['proposal'], ccv_group.attrs['runs']
+ begin_at_str = ccv_group.attrs['begin_at']
+
+ condition_group = ccv_group['operating_condition']
+
+ cond_params = []
+
+ # It's really not ideal we're mixing conditionS and condition now.
+ for parameter in condition_group:
+ param_dset = condition_group[parameter]
+ cond_params.append({
+ 'parameter_name': param_dset.attrs['database_name'],
+ 'value': float(param_dset[()]),
+ 'lower_deviation_value': param_dset.attrs['lower_deviation'],
+ 'upper_deviation_value': param_dset.attrs['upper_deviation'],
+ 'flg_available': True
+ })
+
+ const_rel_path = f'xfel/cal/{detector_type.lower()}/{pdu_name.lower()}'
+ const_filename = f'cal.{time.time()}.h5'
+
+ if proposal and len(runs) > 0:
+ raw_data_location = 'proposal:{} runs: {}'.format(
+ proposal, ' '.join([str(x) for x in runs]))
+ else:
+ pass # Fallback for non-run based constants
+
+ # Generate condition name.
+ unique_name = detector_type[:detector_type.index('-Type')] + ' Def'
+ cond_hash = md5(pdu_name.encode())
+ cond_hash.update(int(pdu_uuid).to_bytes(
+ length=8, byteorder='little', signed=False))
+
+ for param_dict in cond_params:
+ cond_hash.update(str(param_dict['parameter_name']).encode())
+ cond_hash.update(str(param_dict['value']).encode())
+
+ unique_name += binascii.b2a_base64(cond_hash.digest()).decode()
+ unique_name = unique_name[:60]
+
+ # Add PDU "UUID" to parameters.
+ cond_params.append({
+ 'parameter_name': 'Detector UUID',
+ 'value': unpack('d', pack('q', pdu_uuid))[0],
+ 'lower_deviation_value': 0.0,
+ 'upper_deviation_value': 0.0,
+ 'flg_available': True
+ })
+
+ inject_h = {
+ 'detector_condition': {
+ 'name': unique_name,
+ 'parameters': cond_params
+ },
+ 'calibration_constant': {
+ 'calibration_name': calibration,
+ 'detector_type_name': detector_type,
+ 'flg_auto_approve': True
+ },
+ 'calibration_constant_version': {
+ 'raw_data_location': raw_data_location,
+ 'file_name': const_filename,
+ 'path_to_file': const_rel_path,
+ 'data_set_name': f'{pdu_name}/{calibration}/0',
+ 'start_idx': '0',
+ 'end_idx': '0',
+ 'begin_validity_at': begin_at_str,
+ 'end_validity_at': '',
+ 'begin_at': begin_at_str,
+ 'pdu_physical_name': pdu_name,
+ 'flg_good_quality': True
+ }
+ }
+
+ if report_to:
+ report_path = Path(report_to).absolute().with_suffix('.pdf')
+ inject_h['report'] = {
+ 'name': report_path.stem,
+ 'file_path': str(report_path)
+ }
+
+ const_dest = _get_default_caldb_root() / const_rel_path / const_filename
+ const_dest.parent.mkdir(parents=True, exist_ok=True)
+ copyfile(const_src, const_dest)
+
+ resp = CalibrationClient.inject_new_calibration_constant_version(
+ calibration_client(), inject_h)
+
+ if not resp['success']:
+ const_dest.unlink() # Delete already copied CCV file.
+ raise RuntimeError(resp)