diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index f89f643fdd7c3d71ecd4c74e9825776bf1380b1f..9b40845e024ba0655bb835fac0b1e31be02016d1 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -1,6 +1,6 @@
import traceback
from pathlib import Path
-from typing import Optional, Tuple
+from typing import Any, Dict, Optional, Tuple
import h5py
import numpy as np
@@ -178,8 +178,8 @@ class AgipdCorrections:
for mod in modules:
qm = f"Q{mod // 4 + 1}M{mod % 4 + 1}"
- device = getattr(getattr(Detectors, dinstance), qm)
- agipd_corr.initialize_from_yaml(const_yaml, mod, device)
+ agipd_corr.initialize_from_yaml(karabo_da,
+ const_yaml, mod)
data_shape = (n_images_max, 512, 128)
agipd_corr.allocate_images(data_shape, n_cores_files)
@@ -989,19 +989,22 @@ class AgipdCorrections:
data=cntsv,
fletcher32=True)
- def retrieve_constant_and_time(self, const_dict, device,
- cal_db_interface, creation_time):
- """
- A wrapper around get_constant_from_db_and_time to get
+ def retrieve_constant_and_time(self, karabo_id: str, karabo_da: str,
+ const_dict: Dict[str, Any],
+ cal_db_interface: str,
+ creation_time: str
+ ) -> Tuple[Dict[str, Any], Dict[str, Any]]:
+ """A wrapper around get_constant_from_db_and_time to get
constant image and creation time using a dictionary of
the names of the required constant from the data base
and their parameter operating conditions.
+ :param karabo_id: karabo identifier
+ :param karabo_da: karabo data aggregator
:param const_dict: (Dict) A dictionary with constants to retrieve
and their operating conditions.
e.g.{ "Offset": ["zeros", (128, 512, memory_cells, 3), "Dark"]
"Dark-cond": {Condition-name}: condition-value }
- :param device: (iCalibrationDB obj) Object for AGIPD's device
:param cal_db_interface: (Str) The port interface for
calibrationDBRemote
:param creation_time: (Str) Raw data creation time
@@ -1016,7 +1019,7 @@ class AgipdCorrections:
condition = \
getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])
cons_data[cname], when[cname] = \
- get_constant_from_db_and_time(device,
+ get_constant_from_db_and_time(karabo_id, karabo_da,
getattr(Constants.AGIPD, cname)(), # noqa
condition,
getattr(np, cval[0])(cval[1]),
@@ -1214,24 +1217,27 @@ class AgipdCorrections:
return
- def initialize_from_yaml(self, const_yaml, module_idx, device):
- """
- Initialize calibration constants from a yaml file
+ def initialize_from_yaml(self, karabo_da: str,
+ const_yaml: Dict[str, Any],
+ module_idx: int
+ ) -> Dict[str, Any]:
+ """Initialize calibration constants from a yaml file
+ :param karabo-da: karabo data aggerator
:param const_yaml: (Dict) from the "retrieved-constants" part of a yaml
file in pre-notebook, which consists of metadata of either the constant
file path or the empty constant shape, and the creation-time of the
retrieved constants
- :param module_idx: Index of module
- :return:
+ :param module_idx: Index of module.
+ :return when: Dictionary of retrieved constants with
+ their creation-time.
"""
# string of the device name.
- dname = device.device_name
cons_data = dict()
when = dict()
-
- for cname, mdata in const_yaml[dname].items():
+ db_module = const_yaml[karabo_da]["physical-detector-unit"]
+ for cname, mdata in const_yaml[karabo_da]["constants"].items():
when[cname] = mdata["creation-time"]
if when[cname]:
# This path is only used when testing new flat fields from
@@ -1239,33 +1245,33 @@ class AgipdCorrections:
# cells. Consequently, the shape needs to be fixed when less
# cells are used.
with h5py.File(mdata["file-path"], "r") as cf:
- cons_data[cname] = np.copy(cf[f"{dname}/{cname}/0/data"])
+ cons_data[cname] = np.copy(cf[f"{db_module}/{cname}/0/data"]) # noqa
else:
# Create empty constant using the list elements
cons_data[cname] = getattr(np, mdata["file-path"][0])(mdata["file-path"][1]) # noqa
-
+
self.init_constants(cons_data, when, module_idx)
return when
- def initialize_from_db(self, cal_db_interface, creation_time, memory_cells,
- bias_voltage, photon_energy, gain_setting,
- acquisition_rate, module_idx, device, only_dark=False):
+ def initialize_from_db(self, karabo_id: str, karabo_da: str,
+ cal_db_interface: str, creation_time: str,
+ memory_cells: float, bias_voltage: int,
+ photon_energy: float, gain_setting: float,
+ acquisition_rate: float, module_idx: int,
+ only_dark: bool = False):
""" Initialize calibration constants from the calibration database
- :param dbparms: a tuple containing relevant database parameters,
- can be either:
- * cal_db_interface, creation_time, max_cells_db, bias_voltage,
- photon_energy in which case the db timeout is set to 300 seconds,
- the cells to query dark image derived constants from the
- database is set to the global value
-
- * cal_db_interface, creation_time, max_cells_db, bias_voltage,
- photon_energy, max_cells_db_dark
- additionally a timeout is given
-
- :param qm: quadrant and module of the constants to load in Q1M1
- notation
+ :param karabo_id: karabo identifier
+ :param karabo_da: karabo data aggregator
+ :param cal_db_interface: database interaface port
+ :param creation_time: time for desired calibration constant version
+ :param memory_cells: number of memory cells used for CCV conditions
+ :param bias_voltage: bias voltage used for CCV conditions
+ :param photon_energy: photon energy used for CCV conditions
+ :param gain_setting: gain setting used for CCV conditions
+ :param acquisition_rate: acquistion rate used for CCV conditions
+ :param module_idx: module index to save retrieved CCV in sharedmem
:param only_dark: load only dark image derived constants. This
implies that a `calfile` is used to load the remaining
constants. Useful to reduce DB traffic and interactions
@@ -1315,9 +1321,8 @@ class AgipdCorrections:
beam_energy=None, only_dark=only_dark)
cons_data, when = \
- self.retrieve_constant_and_time(const_dict, device,
- cal_db_interface,
- creation_time)
+ self.retrieve_constant_and_time(karabo_id, karabo_da, const_dict,
+ cal_db_interface, creation_time)
self.init_constants(cons_data, when, module_idx)
@@ -1383,4 +1388,4 @@ class AgipdCorrections:
self.shared_dict[i]['rel_corr'] = sharedmem.empty(shape,
dtype='f4')
self.shared_dict[i]['t0_rgain'] = sharedmem.empty(shape,
- dtype='u2')
\ No newline at end of file
+ dtype='u2')
diff --git a/cal_tools/cal_tools/lpdlib.py b/cal_tools/cal_tools/lpdlib.py
index dbf86d33f3e8a9a3ec30ff973295df32d964dc5b..ea78384d4736f866cbed7043aa78669659d6d843 100644
--- a/cal_tools/cal_tools/lpdlib.py
+++ b/cal_tools/cal_tools/lpdlib.py
@@ -1,9 +1,12 @@
import copy
+from typing import Optional, Tuple
import h5py
import numpy as np
+
from cal_tools.enums import BadPixels
-from cal_tools.tools import get_constant_from_db, get_constant_from_db_and_time
+from cal_tools.tools import (get_constant_from_db,
+ get_constant_from_db_and_time)
from iCalibrationDB import Conditions, Constants, Detectors
@@ -556,7 +559,9 @@ class LpdCorrections:
self.hists_gain_vs_signal),
(self.low_edges, self.high_edges, self.signal_edges))
- def initialize_from_db(self, dbparms, qm, only_dark=False):
+ def initialize_from_db(self, dbparms: Tuple['DBParms', 'DBParms_timeout'],
+ karabo_id: str, karabo_da: str,
+ only_dark: Optional[bool] = False):
""" Initialize calibration constants from the calibration database
:param dbparms: a tuple containing relevant database parameters,
@@ -571,8 +576,8 @@ class LpdCorrections:
bias_voltage, photon_energy, timeout
additionally a timeout is given
- :param qm: quadrant and module of the constants to load in Q1M1
- notation
+ :param karabo_id: karabo identifier
+ :param karabo_da: karabo data aggregator
:param only_dark: load only dark image derived constants. This
implies that a `calfile` is used to load the remaining
constants. Useful to reduce DB traffic and interactions
@@ -625,9 +630,9 @@ class LpdCorrections:
else:
(cal_db_interface, creation_time, max_cells_db, capacitor,
bias_voltage, photon_energy, timeout) = dbparms
-
+ #TODO: remove sw duplication during LPD correction modifications
offset, when = get_constant_from_db_and_time(
- getattr(Detectors.LPD1M1, qm),
+ karabo_id, karabo_da,
Constants.LPD.Offset(),
Conditions.Dark.LPD(
memory_cells=max_cells_db,
@@ -638,7 +643,7 @@ class LpdCorrections:
creation_time=creation_time,
timeout=timeout)
- noise = get_constant_from_db(getattr(Detectors.LPD1M1, qm),
+ noise = get_constant_from_db(karabo_id, karabo_da,
Constants.LPD.Noise(),
Conditions.Dark.LPD(
memory_cells=max_cells_db,
@@ -649,7 +654,7 @@ class LpdCorrections:
creation_time=creation_time,
timeout=timeout)
- bpixels = get_constant_from_db(getattr(Detectors.LPD1M1, qm),
+ bpixels = get_constant_from_db(karabo_id, karabo_da,
Constants.LPD.BadPixelsDark(),
Conditions.Dark.LPD(
memory_cells=max_cells_db,
@@ -665,7 +670,7 @@ class LpdCorrections:
self.initialize(offset, None, None, bpixels, noise, None)
return when
- slopesCI = get_constant_from_db(getattr(Detectors.LPD1M1, qm),
+ slopesCI = get_constant_from_db(karabo_id, karabo_da,
Constants.LPD.SlopesCI(),
Conditions.Dark.LPD(
memory_cells=max_cells_db,
@@ -680,7 +685,7 @@ class LpdCorrections:
rel_gain_offset = slopesCI[..., 1]
flat_fields = np.squeeze(
- get_constant_from_db(getattr(Detectors.LPD1M1, qm),
+ get_constant_from_db(karabo_id, karabo_da,
Constants.LPD.SlopesFF(),
Conditions.Illuminated.LPD(max_cells_db,
bias_voltage,
@@ -694,7 +699,7 @@ class LpdCorrections:
creation_time=creation_time,
timeout=timeout))
- bpixels |= get_constant_from_db(getattr(Detectors.LPD1M1, qm),
+ bpixels |= get_constant_from_db(karabo_id, karabo_da,
Constants.LPD.BadPixelsCI(),
Conditions.Dark.LPD(
memory_cells=max_cells_db,
@@ -705,7 +710,7 @@ class LpdCorrections:
creation_time=creation_time,
timeout=timeout).astype(np.uint32)
- bpix = get_constant_from_db(getattr(Detectors.LPD1M1, qm),
+ bpix = get_constant_from_db(karabo_id, karabo_da,
Constants.LPD.BadPixelsFF(),
Conditions.Illuminated.LPD(
max_cells_db, bias_voltage,
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index 5f190d55780946bbf880842c10ba407c41009f40..f112c3cf089b62279ee18692ca719e37fd37ed7a 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -8,7 +8,7 @@ from os.path import isfile
from pathlib import Path
from queue import Queue
from time import sleep
-from typing import Union
+from typing import List, Optional, Tuple, Union
from urllib.parse import urljoin
import dateutil.parser
@@ -64,7 +64,8 @@ def map_modules_from_folder(in_folder, run, path_template, karabo_da,
:param in_folder: Input folder with raw data
:param run: Run number
- :param path_template: Template for file name e.g. `RAW-R{:04d}-{}-S{:05d}.h5`
+ :param path_template: Template for file name
+ e.g. `RAW-R{:04d}-{}-S{:05d}.h5`
:param karabo_da: List of data aggregators e.g. [AGIPD00, AGIPD01]
:param sequences: List of sequences to be considered
:return: Dictionary of queues of files, dictionary of module indexes,
@@ -113,7 +114,8 @@ def map_gain_stages(in_folder, runs, path_template, karabo_da, sequences=None):
and gain name as a keys
:param in_folder: Input folder with raw data
:param runs: Dictionary of runs with key naming the gain stages
- :param path_template: Template for file name e.g. `RAW-R{:04d}-{}-S{:05d}.h5`
+ :param path_template: Template for file name
+ e.g. `RAW-R{:04d}-{}-S{:05d}.h5`
:param karabo_da: List of data aggregators e.g. [AGIPD00, AGIPD01]
:param sequences: List of sequences to be considered
:return: Dictionary of queues of files,
@@ -240,8 +242,8 @@ def get_dir_creation_date(directory: Union[str, Path], run: int,
in [directory]/[run] or, if the dataset is older than 2020, from the oldest
file's modified time.
- If the data is not available from either source, this function will raise a
- ValueError.
+ If the data is not available from either source,
+ this function will raise a ValueError.
:param directory: path to directory which contains runs
:param run: run number
@@ -284,37 +286,48 @@ def get_dir_creation_date(directory: Union[str, Path], run: int,
raise ValueError(msg, directory)
-def save_const_to_h5(device, constant, condition, data,
- file_loc, creation_time, out_folder):
- """
- Save constant in h5 file with its metadata
- (e.g. db_module, condition, creation_time)
+def _init_metadata(constant: 'iCalibrationDB.calibration_constant',
+ condition: 'iCalibrationDB.detector_conditions',
+ creation_time: Optional[str] = None
+ ) -> 'ConstantMetaData':
- :param device: Instance of detector
- :type device: iCalibrationDB.detectors object
- :param constant: Calibration constant known for given detector
- :type constant: iCalibrationDB.know_constants object
- :param condition: Calibration condition
- :type condition: iCalibrationDB.know_detector_conditions object
- :param data: Constant data to save
- :type data: ndarray
- :param file_loc: Location of raw data "proposal:{} runs:{} {} {}"
- :type file_loc: str
- :param creation_time: creation_time for the saved constant
- :type creation_time: datetime object
- :param out_folder: path to output folder
- :type out_folder: str
+ """Initializing a ConstantMetaData class instance and
+ add the correct creation time of the constant metadata.
"""
metadata = ConstantMetaData()
metadata.calibration_constant = constant
metadata.detector_condition = condition
if creation_time is None:
- metadata.calibration_constant_version = Versions.Now(
- device=device)
+ metadata.calibration_constant_version = Versions.Now()
else:
metadata.calibration_constant_version = Versions.Timespan(
- device=device,
start=creation_time)
+ return metadata
+
+
+def save_const_to_h5(db_module: str, karabo_id: str,
+ constant: 'iCalibrationDB.calibration_constant',
+ condition: 'iCalibrationDB.detector_conditions',
+ data: np.array, file_loc: str,
+ report: str,
+ creation_time: datetime.datetime,
+ out_folder: str) -> 'ConstantMetaData':
+
+ """ Save constant in h5 file with its metadata
+ (e.g. db_module, condition, creation_time)
+
+ :param db_module: database module (PDU/Physical Detector Unit).
+ :param karabo_id: karabo identifier.
+ :param constant: Calibration constant known for given detector.
+ :param condition: Calibration condition.
+ :param data: Constant data to save.
+ :param file_loc: Location of raw data "proposal:{} runs:{} {} {}".
+ :param creation_time: creation_time for the saved constant.
+ :param out_folder: path to output folder.
+ :return: metadata of the saved constant.
+ """
+
+ metadata = _init_metadata(constant, condition, creation_time)
metadata.calibration_constant_version.raw_data_location = file_loc
@@ -327,16 +340,17 @@ def save_const_to_h5(device, constant, condition, data,
creation_time = metadata.calibration_constant_version.begin_at
raw_data = metadata.calibration_constant_version.raw_data_location
- db_module = metadata.calibration_constant_version.device_name
constant_name = metadata.calibration_constant.__class__.__name__
data_to_store = {}
data_to_store['condition'] = dpar
data_to_store['db_module'] = db_module
+ data_to_store['karabo_id'] = karabo_id
data_to_store['constant'] = constant_name
data_to_store['data'] = data
data_to_store['creation_time'] = creation_time
data_to_store['file_loc'] = raw_data
+ data_to_store['report'] = report
ofile = f"{out_folder}/const_{constant_name}_{db_module}.h5"
if isfile(ofile):
@@ -345,9 +359,10 @@ def save_const_to_h5(device, constant, condition, data,
return metadata
+
def get_random_db_interface(cal_db_interface):
"""
- Return interface to calibration DB with random (with given range) port
+ Return interface to calibration DB with random (with given range) port.
"""
if "#" in cal_db_interface:
prot, serv, ran = cal_db_interface.split(":")
@@ -356,59 +371,150 @@ def get_random_db_interface(cal_db_interface):
return cal_db_interface
-already_printed = {}
-
+def get_report(out_folder: str):
+ """Get the report path from calibration_metadata.yml
+ stored in the out_folder.
+ """
-def get_from_db(device, constant, condition, empty_constant,
- cal_db_interface, creation_time=None,
- verbosity=1, timeout=30000, ntries=7, meta_only=True,
- version_info=False, doraise=False):
+ metadata = CalibrationMetadata(out_folder)
+ report_path = metadata.get("report-path", "")
+ if not report_path:
+ print("WARNING: No report path will be injected "
+ "with the constants.\n")
+ return report_path
+
+
+def get_pdu_from_db(karabo_id: str, karabo_da: Union[str, list],
+ constant: 'iCalibrationDB.calibration_constant',
+ condition: 'iCalibrationDB.detector_conditions',
+ cal_db_interface: str,
+ snapshot_at: Optional[datetime.datetime] = None,
+ timeout: int = 30000) -> List[str]:
+
+ """Return all physical detector units for a
+ karabo_id and list of karabo_da
+
+ :param karabo_id: Karabo identifier.
+ :param karabo_da: Karabo data aggregator.
+ :param cal_db_interface: Interface string, e.g. "tcp://max-exfl016:8015".
+ :param snapshot_at: Database snapshot.
+ :param timeout: Calibration Database timeout.
+ :return: List of physical detector units (db_modules)
"""
- Return calibration constants and metadata requested from CalDB
+ if not isinstance(karabo_da, (str, list)):
+ raise TypeError("karabo_da should either be a list of multiple "
+ "karabo_da or a string of one karabo_da or 'all'")
+
+ metadata = _init_metadata(constant, condition, None)
+
+ # CalibrationDBRemote expects a string.
+ if snapshot_at is not None and hasattr(snapshot_at, 'isoformat'):
+ snapshot_at = snapshot_at.isoformat()
+
+ pdu_dicts = metadata.retrieve_pdus_for_detector(receiver=cal_db_interface,
+ karabo_id=karabo_id,
+ snapshot_at=snapshot_at,
+ timeout=timeout)
+ # Get a list of pdus based on requested karabo_das
+ if karabo_da == 'all':
+ db_modules = [d["pdu_physical_name"] for d in pdu_dicts]
+ else:
+ k_indices = []
+ if isinstance(karabo_da, str):
+ karabo_da = [karabo_da]
+ # Get indices of dict with the right karabo_da,
+ # else use None.
+ for k in karabo_da:
+ pdu_found = False
+ for i, d in enumerate(pdu_dicts):
+ if d["karabo_da"] == k:
+ k_indices.append(i)
+ pdu_found = True
+ break
+ if not pdu_found:
+ k_indices.append(None)
+
+ db_modules = []
+ for i in k_indices:
+ if i is None:
+ db_modules.append(None)
+ else:
+ db_modules.append(pdu_dicts[i]["pdu_physical_name"])
- :param device: Instance of detector
- :param constant: Calibration constant known for given detector
- :param condition: Calibration condition
+ return db_modules
+
+
+already_printed = {}
+
+
+def get_from_db(karabo_id: str, karabo_da: str,
+ constant: 'iCalibrationDB.calibration_constant',
+ condition: 'iCalibrationDB.detector_conditions',
+ empty_constant: np.array,
+ cal_db_interface: str,
+ creation_time: Optional[datetime.datetime] = None,
+ verbosity: int = 1,
+ timeout: int = 30000,
+ ntries: int = 7,
+ meta_only: bool = True,
+ load_data: bool = True,
+ version_info: bool = False,
+ doraise: bool = False,
+ strategy: str = "pdu_closest_by_time"
+ ) -> Tuple[np.array, 'ConstantMetaData']:
+
+ """Return calibration constants and metadata requested from CalDB
+
+ This feature uses the karabo-id and karabo-da to retrieve the
+ desired CCV
+
+ :param karabo_id: karabo identifier (detector identifier).
+ :param karabo_da: karabo data aggregator.
+ :param constant: Calibration constant known for given detector.
+ :param condition: Calibration condition.
:param empty_constant: Constant to be returned in case of failure.
:param cal_db_interface: Interface string, e.g. "tcp://max-exfl016:8015"
- :param creation_time: Latest time for constant to be created
+ :param creation_time: Latest time for constant to be created.
:param verbosity: Level of verbosity (0 - silent)
:param timeout: Timeout for zmq request
ntries is set to 7 so that if the timeout started at 30s last timeout
will be ~ 1h.
- :param ntries: number of tries to contact the database
- :param meta_only: (bool) Retrieve only metadata via ZMQ. Constants are
+ :param ntries: number of tries to contact the database.
+ :param meta_only: Retrieve only metadata via ZMQ. Constants are
taken directly from the h5 file on maxwell.
- :param version_info: (bool) to show the info for the retrieved Constant
- :param doraise: (bool) if True raise errors during communication with DB.
- :return: Calibration constant, metadata
+ :param version_info: Flag to show the info for the retrieved Constant.
+ :param doraise: if True raise errors during communication with DB.
+ :param strategy: Retrieving strategy for calibrationDBRemote.
+ :return: Calibration constant, metadata.
"""
if version_info:
meta_only = False
- if device:
- metadata = ConstantMetaData()
- metadata.calibration_constant = constant
- metadata.detector_condition = condition
+ metadata = _init_metadata(constant, condition, creation_time)
+
+ if karabo_id and karabo_da:
when = None
- if creation_time is None:
- metadata.calibration_constant_version = Versions.Now(
- device=device)
- else:
- if hasattr(creation_time, 'isoformat'):
- when = creation_time.isoformat()
- metadata.calibration_constant_version = Versions.Timespan(
- device=device,
- start=creation_time)
- while ntries > 0:
+ if creation_time is not None and hasattr(creation_time, 'isoformat'):
+ when = creation_time.isoformat()
+
+ metadata.calibration_constant_version.karabo_id = karabo_id
+ metadata.calibration_constant_version.karabo_da = karabo_da
+ # make sure to remove device name from metadata dict before
+ # retrieving to keep using karabo_id and karabo_da only
+ # during retrieval. As device_name could have been set after
+ # retrieval from iCalibrationDB
+ metadata.calibration_constant_version.device_name = None
+
+ while ntries > 0:
this_interface = get_random_db_interface(cal_db_interface)
try:
r = metadata.retrieve(this_interface, timeout=timeout,
when=when, meta_only=meta_only,
- version_info=version_info)
+ version_info=version_info,
+ strategy=strategy)
if version_info:
return r
break
@@ -429,10 +535,18 @@ def get_from_db(device, constant, condition, empty_constant,
raise RuntimeError(f'{e}')
if ntries > 0:
+ if load_data and meta_only:
+ mdata_const = metadata.calibration_constant_version
+ fpath = Path(mdata_const.hdf5path,
+ mdata_const.filename)
+ with h5py.File(fpath, "r") as f:
+ arr = f[f"{mdata_const.h5path}/data"][()]
+ metadata.calibration_constant.data = arr
+
if verbosity > 0:
if constant.name not in already_printed or verbosity > 1:
already_printed[constant.name] = True
- begin_at = metadata.calibration_constant_version.begin_at
+ begin_at = mdata_const.begin_at
print(f"Retrieved {constant.name} "
f"with creation time: {begin_at}")
return constant.data, metadata
@@ -442,54 +556,54 @@ def get_from_db(device, constant, condition, empty_constant,
return empty_constant, None
-def send_to_db(device, constant, condition, file_loc,
- cal_db_interface, creation_time=None,
- timeout=30000, ntries=7, doraise=False):
- """
- Return calibration constants and metadata requested from CalDB
+def send_to_db(db_module: str, karabo_id: str, constant, condition,
+ file_loc: str, report_path: str, cal_db_interface: str,
+ creation_time: Optional[datetime.datetime] = None,
+ timeout: int = 30000,
+ ntries: int = 7,
+ doraise: bool = False):
+ """Return calibration constants and metadata requested from CalDB
- :param device: Instance of detector
- :type device: iCalibrationDB.detectors object
+ :param db_module: database module (PDU/Physical Detector Unit)
+ :param karabo_id: karabo identifier
:param constant: Calibration constant known for given detector
- :type constant: iCalibrationDB.know_constants object
:param condition: Calibration condition
- :type condition: iCalibrationDB.know_detector_conditions object
:param file_loc: Location of raw data.
- :type file_loc: str
:param cal_db_interface: Interface string, e.g. "tcp://max-exfl016:8015"
- :type cal_db_interface: str
:param creation_time: Latest time for constant to be created
- :type creation_time: datetime object
:param timeout: Timeout for zmq request
- :type timeout: int
:param ntries: number of tries to contact the database,
- ntries is set to 7 so that if the timeout started at 30s last timeout
- will be ~ 1h.
- :type ntries: int
+ ntries is set to 7 so that if the timeout started
+ at 30s last timeout will be ~ 1h.
:param doraise: if True raise errors during communication with DB
- :type doraise: bool
"""
success = False
- if device:
- metadata = ConstantMetaData()
- metadata.calibration_constant = constant
- metadata.detector_condition = condition
- if creation_time is None:
- metadata.calibration_constant_version = Versions.Now(
- device=device)
- else:
- metadata.calibration_constant_version = Versions.Timespan(
- device=device,
- start=creation_time)
+ metadata = _init_metadata(constant, condition, creation_time)
+
+ if db_module:
+
+ # Add injected constant's file source info as a file location
+ metadata.calibration_constant_version.raw_data_location = file_loc
+
+ if report_path:
+ # calibration_client expects a dict of injected report path
+ # of at least 2 characters for each key.
+ report = {"name": path.basename(report_path),
+ "file_path": report_path}
+ metadata.calibration_constant_version.report_path = report
+
+ metadata.calibration_constant_version.karabo_id = karabo_id
+ metadata.calibration_constant_version.device_name = db_module
+ metadata.calibration_constant_version.karabo_da = None
metadata.calibration_constant_version.raw_data_location = file_loc
while ntries > 0:
this_interface = get_random_db_interface(cal_db_interface)
try:
- r = metadata.send(this_interface, timeout=timeout)
+ metadata.send(this_interface, timeout=timeout)
success = True
break
except zmq.error.Again:
@@ -499,12 +613,13 @@ def send_to_db(device, constant, condition, file_loc,
if ntries == 0 and doraise:
raise
except Exception as e:
+ # TODO: refactor to use custom exception class
if "has already been taken" in str(e):
print(f"WARNING: {constant.name} has already been "
"injected with the same parameter "
"conditions\n")
else:
- print(f"{e}\n")
+ print(f"{e}\n")
if 'missing_token' in str(e):
ntries -= 1
@@ -518,27 +633,30 @@ def send_to_db(device, constant, condition, file_loc,
f"{metadata.calibration_constant_version.begin_at}\n")
return metadata
-def get_constant_from_db(device, constant, condition, empty_constant,
- cal_db_interface, creation_time=None,
+
+def get_constant_from_db(karabo_id: str, karabo_da: str,
+ constant, condition, empty_constant,
+ cal_db_interface: str, creation_time=None,
print_once=True, timeout=30000, ntries=120,
meta_only=True):
+ """Return calibration constants requested from CalDB
"""
- Return calibration constants requested from CalDB
- """
- data, _ = get_from_db(device, constant, condition, empty_constant,
+ data, _ = get_from_db(karabo_id, karabo_da, constant,
+ condition, empty_constant,
cal_db_interface, creation_time,
int(print_once), timeout, ntries, meta_only)
return data
-def get_constant_from_db_and_time(device, constant, condition, empty_constant,
- cal_db_interface, creation_time=None,
+def get_constant_from_db_and_time(karabo_id: str, karabo_da: str,
+ constant, condition, empty_constant,
+ cal_db_interface: str, creation_time=None,
print_once=True, timeout=30000, ntries=120):
+ """Return calibration constants requested from CalDB,
+ alongside injection time
"""
- Return calibration constants requested from CalDB, alongside injection
- time
- """
- data, m = get_from_db(device, constant, condition, empty_constant,
+ data, m = get_from_db(karabo_id, karabo_da, constant,
+ condition, empty_constant,
cal_db_interface, creation_time,
int(print_once), timeout, ntries)
if m:
@@ -569,14 +687,13 @@ class CalibrationMetadata(dict):
if isinstance(data, dict):
self.update(data)
else:
- print(f"Warning: existing {self._yaml_fn} is malformed, will be overwritten")
-
+ print(f"Warning: existing {self._yaml_fn} is malformed, "
+ "will be overwritten")
def save(self):
with self._yaml_fn.open("w") as fd:
yaml.safe_dump(dict(self), fd)
-
def save_copy(self, copy_dir: Path):
with (copy_dir / self._yaml_fn.name).open("w") as fd:
yaml.safe_dump(dict(self), fd)
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 12326abe6dc0583d5a54d15da341f4678291ee7d..926498bd0c239437fbe7df7592337c1e3748af8d 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -433,20 +433,21 @@
" \n",
" Metadata for constants is taken from yml file or retrieved from the DB\n",
" \"\"\"\n",
- " device = getattr(getattr(Detectors, dinstance), mod_name(mod))\n",
" err = ''\n",
+ " # TODO: parallelize over modules\n",
+ " k_da = karabo_da[mod]\n",
" try:\n",
" # check if there is a yaml file in out_folder that has the device constants.\n",
- " if device.device_name in const_yaml:\n",
- " when = agipd_corr.initialize_from_yaml(const_yaml, mod, device)\n",
+ " if k_da in const_yaml:\n",
+ " when = agipd_corr.initialize_from_yaml(k_da, const_yaml, mod)\n",
" else:\n",
" # TODO: should we save what is found here in metadata?\n",
- " when = agipd_corr.initialize_from_db(cal_db_interface, creation_time, mem_cells_db, bias_voltage,\n",
- " photon_energy, gain_setting, acq_rate, mod, device, False)\n",
+ " when = agipd_corr.initialize_from_db(karabo_id, k_da, cal_db_interface, creation_time, mem_cells_db, bias_voltage,\n",
+ " photon_energy, gain_setting, acq_rate, mod, False)\n",
" except Exception as e:\n",
" err = f\"Error: {e}\\nError traceback: {traceback.format_exc()}\"\n",
" when = None\n",
- " return err, mod, when, device.device_name\n",
+ " return err, mod, when, k_da\n",
"\n",
"\n",
"ts = perf_counter()\n",
@@ -589,13 +590,13 @@
"fst_print = True\n",
"timestamps = {}\n",
"\n",
- "for i, (error, modno, when, mod_dev) in enumerate(const_out):\n",
+ "for i, (error, modno, when, k_da) in enumerate(const_out):\n",
" qm = mod_name(modno)\n",
" # expose errors while applying correction\n",
" if error:\n",
" print(\"Error: {}\".format(error) )\n",
"\n",
- " if mod_dev not in const_yaml:\n",
+ " if k_da not in const_yaml:\n",
" if fst_print:\n",
" print(\"Constants are retrieved with creation time: \")\n",
" fst_print = False\n",
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index e33f11761de1207d6dffb25133e3af0b15dc6d2c..6ce5f20bfc96f50caf3a6f6dd72a89d38c7bf467 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -93,6 +93,8 @@
"source": [
"import sys\n",
"from collections import OrderedDict\n",
+ "from functools import partial\n",
+ "from typing import List, Tuple\n",
"\n",
"import h5py\n",
"import matplotlib\n",
@@ -236,13 +238,13 @@
"metadata": {},
"outputs": [],
"source": [
- "from functools import partial\n",
- "\n",
- "\n",
- "def retrieve_constants(karabo_id, bias_voltage, max_cells, acq_rate, \n",
- " gain_setting, photon_energy, only_dark, nodb_with_dark, \n",
- " cal_db_interface, creation_time, \n",
- " corr_bools, pc_bools, inp):\n",
+ "def retrieve_constants(karabo_id: str, bias_voltage: int, max_cells: float,\n",
+ " acq_rate: float, gain_setting: float, photon_energy: float,\n",
+ " only_dark: bool, nodb_with_dark: bool, \n",
+ " cal_db_interface: str, creation_time: str, \n",
+ " corr_bools: dict, pc_bools: List[bool],\n",
+ " inp: Tuple[str, str, str, int]\n",
+ " ) -> Tuple[str, str, float, float, str, dict]:\n",
" \"\"\"\n",
" Retreive constant for each module in parallel and produce a dictionary\n",
" with the creation-time and constant file path.\n",
@@ -261,8 +263,12 @@
" :param pc_bools: (LIST) list of bools to retrieve pulse capacitor constants\n",
" :param inp: (LIST) input for the parallel cluster of the partial function\n",
" :return:\n",
- " mdata_dict: (DICT) dictionary with the metadata for the retrieved constants\n",
- " dev.device_name: (STR) device name\n",
+ " qm: module virtual name i.e. Q1M1.\n",
+ " karabo_da: karabo data aggregator.\n",
+ " acq_rate: acquisition rate parameter.\n",
+ " max_cells: number of memory cells.\n",
+ " err: string of faced errors.\n",
+ " mdata_dict: (DICT) dictionary with the metadata for the retrieved constants.\n",
" \"\"\"\n",
"\n",
" import sys\n",
@@ -276,7 +282,7 @@
"\n",
" err = None\n",
"\n",
- " qm_files, qm, dev, idx = inp\n",
+ " qm_files, qm, karabo_da, idx = inp\n",
" # get number of memory cells from a sequence file with image data\n",
" for f in qm_files:\n",
" if not max_cells:\n",
@@ -304,35 +310,39 @@
" # Retrieve multiple constants through an input dictionary\n",
" # to return a dict of useful metadata.\n",
" mdata_dict = dict()\n",
+ " mdata_dict['constants'] = dict()\n",
+ " mdata_dict['physical-detector-unit'] = None # initialization\n",
+ "\n",
" for cname, cval in const_dict.items():\n",
" # saving metadata in a dict\n",
- " mdata_dict[cname] = dict()\n",
- " \n",
+ " mdata_dict['constants'][cname] = dict()\n",
+ "\n",
" if slopes_ff_from_files and cname in [\"SlopesFF\", \"BadPixelsFF\"]:\n",
- " mdata_dict[cname][\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
- " mdata_dict[cname][\"creation-time\"] = \"00:00:00\"\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = f\"{slopes_ff_from_files}/slopesff_bpmask_module_{qm}.h5\"\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = \"00:00:00\"\n",
" else:\n",
" try:\n",
" condition = getattr(Conditions, cval[2][0]).AGIPD(**cval[2][1])\n",
" co, mdata = \\\n",
- " get_from_db(dev, getattr(Constants.AGIPD, cname)(),\n",
+ " get_from_db(karabo_id, karabo_da, getattr(Constants.AGIPD, cname)(),\n",
" condition, getattr(np, cval[0])(cval[1]),\n",
" cal_db_interface, creation_time, meta_only=True, verbosity=0)\n",
" mdata_const = mdata.calibration_constant_version\n",
- " \n",
+ " device_name = mdata.calibration_constant_version.device_name\n",
" # check if constant was sucessfully retrieved.\n",
- " if mdata.comm_db_success: \n",
- " mdata_dict[cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
+ " if mdata.comm_db_success:\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = f\"{mdata_const.hdf5path}\" \\\n",
" f\"{mdata_const.filename}\"\n",
- " mdata_dict[cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = f\"{mdata_const.begin_at}\"\n",
+ " mdata_dict['physical-detector-unit'] = mdata_const.device_name\n",
" else:\n",
- " mdata_dict[cname][\"file-path\"] = const_dict[cname][:2]\n",
- " mdata_dict[cname][\"creation-time\"] = None\n",
+ " mdata_dict['constants'][cname][\"file-path\"] = const_dict[cname][:2]\n",
+ " mdata_dict['constants'][cname][\"creation-time\"] = None\n",
" except Exception as e:\n",
" err = f\"Error: {e}, Traceback: {traceback.format_exc()}\"\n",
" print(err)\n",
"\n",
- " return qm, mdata_dict, dev.device_name, acq_rate, max_cells, err\n",
+ " return qm, mdata_dict, karabo_da, acq_rate, max_cells, err\n",
"\n",
"pc_bools = [corr_bools.get(\"rel_gain\"),\n",
" corr_bools.get(\"adjust_mg_baseline\"),\n",
@@ -351,7 +361,7 @@
"\n",
"# A dict to connect virtual device\n",
"# to actual device name.\n",
- "for i in modules:\n",
+ "for i, k_da in zip(modules, karabo_da):\n",
" qm = f\"Q{i//4+1}M{i%4+1}\"\n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
" device = getattr(getattr(Detectors, dinstance), qm)\n",
@@ -361,26 +371,25 @@
" print(f\"Skipping {qm}\")\n",
" continue\n",
"\n",
- " inp.append((qm_files, qm, device, i))\n",
+ " inp.append((qm_files, qm, k_da, i))\n",
"\n",
"p = partial(retrieve_constants, karabo_id, bias_voltage, max_cells, \n",
" acq_rate, gain_setting, photon_energy, only_dark, nodb_with_dark, \n",
" cal_db_interface, creation_time, \n",
" corr_bools, pc_bools)\n",
"\n",
- "with mp.Pool(processes=16) as pool:\n",
+ "with mp.Pool(processes=nmods) as pool:\n",
" results = pool.map(p, inp)\n",
"\n",
"mod_dev = dict()\n",
"mdata_dict = dict()\n",
"for r in results:\n",
" if r:\n",
- " qm, md_dict, dname, acq_rate, max_cells, err = r\n",
- " mod_dev[dname] = {\"mod\": qm, \"err\": err}\n",
+ " qm, md_dict, karabo_da, acq_rate, max_cells, err = r\n",
+ " mod_dev[karabo_da] = {\"mod\": qm, \"err\": err}\n",
" if err:\n",
" print(f\"Error for module {qm}: {err}\")\n",
- " mdata_dict[dname] = md_dict\n",
- "\n",
+ " mdata_dict[karabo_da] = md_dict\n",
"# check if it is requested not to retrieve any constants from the database\n",
"if not nodb_with_dark:\n",
" metadata.update({\"retrieved-constants\": mdata_dict})\n",
@@ -389,7 +398,7 @@
" f\"{[', '.join([f'Q{x//4+1}M{x%4+1}' for x in modules])]}\")\n",
" print(f\"Operating conditions are:\\n• Bias voltage: {bias_voltage}\\n• Memory cells: {max_cells}\\n\"\n",
" f\"• Acquisition rate: {acq_rate}\\n• Gain setting: {gain_setting}\\n• Photon Energy: {photon_energy}\\n\")\n",
- " print(f\"Constant metadata is saved under \\\"retrieved-constants\\\" in metadata.yml\\n\")\n",
+ " print(f\"Constant metadata is saved under \\\"retrieved-constants\\\" in calibration_metadata.yml\\n\")\n",
"else:\n",
" print(\"No constants were retrieved as calibrated files will be used.\")"
]
@@ -404,24 +413,24 @@
"i = 0\n",
"timestamps = {}\n",
"\n",
- "for dname, dinfo in mod_dev.items():\n",
+ "for k_da, dinfo in mod_dev.items():\n",
" print(dinfo[\"mod\"], \":\")\n",
" module_timestamps = {}\n",
" module_name = dinfo[\"mod\"]\n",
- " if dname in mdata_dict:\n",
- " for cname, mdata in mdata_dict[dname].items():\n",
+ " if k_da in mdata_dict:\n",
+ " for cname, mdata in mdata_dict[k_da][\"constants\"].items():\n",
" if hasattr(mdata[\"creation-time\"], 'strftime'):\n",
" mdata[\"creation-time\"] = mdata[\"creation-time\"].strftime('%y-%m-%d %H:%M')\n",
" print(f'{cname:.<12s}', mdata[\"creation-time\"])\n",
" # Store few time stamps if exists\n",
" # Add NA to keep array structure\n",
" for cname in ['Offset', 'SlopesPC', 'SlopesFF']:\n",
- " if not dname in mdata_dict or dinfo[\"err\"]:\n",
+ " if not k_da in mdata_dict or dinfo[\"err\"]:\n",
" module_timestamps[cname] = \"Err\"\n",
" else:\n",
- " if cname in mdata_dict[dname]:\n",
- " if mdata_dict[dname][cname][\"creation-time\"]:\n",
- " module_timestamps[cname] = mdata_dict[dname][cname][\"creation-time\"]\n",
+ " if cname in mdata_dict[k_da]:\n",
+ " if mdata_dict[k_da][cname][\"creation-time\"]:\n",
+ " module_timestamps[cname] = mdata_dict[k_da][cname][\"creation-time\"]\n",
" else:\n",
" module_timestamps[cname] = \"NA\"\n",
" else:\n",
diff --git a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
index 90643887cbf5becf4ee4d4346475dbea9d99aaaa..18d532b1639687b7f0b043d03178befa8748deb0 100644
--- a/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
+++ b/notebooks/AGIPD/Characterize_AGIPD_Gain_Darks_NBC.ipynb
@@ -103,8 +103,9 @@
"from IPython.display import display, Markdown, Latex\n",
"%matplotlib inline\n",
"\n",
- "from cal_tools.tools import (get_from_db, get_dir_creation_date,\n",
- " get_notebook_name, get_random_db_interface,\n",
+ "from cal_tools.tools import (get_dir_creation_date, get_from_db,\n",
+ " get_notebook_name, get_pdu_from_db,\n",
+ " get_random_db_interface, get_report,\n",
" map_gain_stages, parse_runs,\n",
" run_prop_seq_from_path, save_const_to_h5,\n",
" send_to_db)\n",
@@ -531,8 +532,26 @@
"metadata": {},
"outputs": [],
"source": [
+ "# Read report path and create file location tuple to add with the injection\n",
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
- "file_loc = 'proposal:{} runs:{} {} {}'.format(proposal, run_low, run_med, run_high)"
+ "file_loc = 'proposal:{} runs:{} {} {}'.format(proposal, run_low, run_med, run_high)\n",
+ "\n",
+ "report = get_report(out_folder)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# TODO: add db_module when received from myMDC\n",
+ "# Create the modules dict of karabo_das and PDUs\n",
+ "qm_dict = OrderedDict()\n",
+ "for i, k_da in zip(modules, karabo_da):\n",
+ " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ " qm_dict[qm] = {\"karabo_da\": k_da,\n",
+ " \"db_module\": \"\"}"
]
},
{
@@ -552,6 +571,8 @@
"print('Retrieve pre-existing constants for comparison.')\n",
"\n",
"for qm in res:\n",
+ " qm_db = qm_dict[qm]\n",
+ " karabo_da = qm_db[\"karabo_da\"]\n",
" for const in res[qm]:\n",
" dconst = getattr(Constants.AGIPD, const)()\n",
" dconst.data = res[qm][const]\n",
@@ -561,12 +582,20 @@
" bias_voltage=bias_voltage,\n",
" acquisition_rate=acq_rate,\n",
" gain_setting=gain_setting)\n",
- " device = getattr(detinst, qm)\n",
- " data, mdata = get_from_db(device,\n",
- " getattr(Constants.AGIPD, const)(),\n",
- " condition,\n",
- " None,\n",
- " cal_db_interface, creation_time=creation_time,\n",
+ "\n",
+ " # This should be used in case of running notebook \n",
+ " # by a different method other than myMDC which already\n",
+ " # sends CalCat info.\n",
+ " # TODO: Set db_module to \"\" by default in the first cell\n",
+ " if not qm_db[\"db_module\"]:\n",
+ " qm_db[\"db_module\"] = get_pdu_from_db(karabo_id, karabo_da, dconst,\n",
+ " condition, cal_db_interface,\n",
+ " snapshot_at=creation_time)[0]\n",
+ "\n",
+ " data, mdata = get_from_db(karabo_id, karabo_da,\n",
+ " dconst, condition,\n",
+ " None, cal_db_interface,\n",
+ " creation_time=creation_time,\n",
" verbosity=2, timeout=cal_db_timeout)\n",
"\n",
" old_const[const] = data\n",
@@ -574,9 +603,9 @@
" time = mdata.calibration_constant_version.begin_at\n",
" old_mdata[const] = time.isoformat()\n",
" os.makedirs('{}/old/'.format(out_folder), exist_ok=True)\n",
- " save_const_to_h5(device,\n",
- " getattr(Constants.AGIPD, const)(),\n",
- " condition, data, file_loc, creation_time,\n",
+ " save_const_to_h5(qm_db[\"db_module\"], karabo_id,\n",
+ " dconst, condition, data,\n",
+ " file_loc, report, creation_time,\n",
" f'{out_folder}/old/')\n",
" else:\n",
" old_mdata[const] = \"Not found\""
@@ -594,8 +623,10 @@
"outputs": [],
"source": [
"md = None\n",
- "for qm in res:\n",
"\n",
+ "for qm in res:\n",
+ " karabo_da = qm_dict[qm][\"karabo_da\"]\n",
+ " db_module = qm_dict[qm][\"db_module\"]\n",
" for const in res[qm]:\n",
" dconst = getattr(Constants.AGIPD, const)()\n",
" dconst.data = res[qm][const]\n",
@@ -605,15 +636,14 @@
" bias_voltage=bias_voltage,\n",
" acquisition_rate=acq_rate,\n",
" gain_setting=gain_setting)\n",
- " detinst = getattr(Detectors, dinstance)\n",
- " device = getattr(detinst, qm)\n",
- "\n",
" if db_output:\n",
- " md = send_to_db(device, dconst, condition, file_loc, \n",
- " cal_db_interface, creation_time=creation_time, timeout=cal_db_timeout)\n",
+ " md = send_to_db(db_module, karabo_id, dconst, condition, file_loc, \n",
+ " report, cal_db_interface, creation_time=creation_time,\n",
+ " timeout=cal_db_timeout)\n",
"\n",
" if local_output:\n",
- " md = save_const_to_h5(device, dconst, condition, dconst.data, file_loc, creation_time, out_folder)\n",
+ " md = save_const_to_h5(db_module, karabo_id, dconst, condition, dconst.data, \n",
+ " file_loc, report, creation_time, out_folder)\n",
" print(f\"Calibration constant {const} is stored locally.\\n\")\n",
" \n",
" print(\"Constants parameter conditions are:\\n\")\n",
diff --git a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
index 47098ca1f4aece4c4f68300fc57eab468b83c3fe..d3d9dbfc349e4114ce4c09772b8f068f8e507940 100644
--- a/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
+++ b/notebooks/AGIPD/Chracterize_AGIPD_Gain_PC_NBC.ipynb
@@ -56,6 +56,7 @@
"use_dir_creation_date = True\n",
"creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC.ms e.g. 2019-07-04 11:02:41.00\n",
"gain_setting = 0.1 # gain setting can have value 0 or 1, Default=0.1 for no (None) gain-setting\n",
+ "karabo_da = [\"all\"]\n",
"karabo_da_control = \"AGIPD1MCTRL00\" # karabo DA for control infromation\n",
"h5path_ctrl = '/CONTROL/{}/MDL/FPGA_COMP_TEST' # path to control information"
]
@@ -92,7 +93,8 @@
"from cal_tools.enums import BadPixels\n",
"from cal_tools.plotting import plot_badpix_3d, show_overview\n",
"from cal_tools.tools import (gain_map_files, get_dir_creation_date, get_notebook_name,\n",
- " parse_runs, run_prop_seq_from_path, get_dir_creation_date, send_to_db)\n",
+ " parse_runs, run_prop_seq_from_path, get_dir_creation_date,\n",
+ " get_pdu_from_db, get_report, send_to_db)\n",
"from iCalibrationDB import ConstantMetaData, Constants, Conditions, Detectors, Versions\n",
"import XFELDetAna.xfelpyanatools as xana\n",
"\n",
@@ -100,6 +102,13 @@
"view = Client(profile=cluster_profile)[:]\n",
"view.use_dill()\n",
"\n",
+ "if karabo_da[0] == [\"all\"]:\n",
+ " if modules[0] == -1:\n",
+ " modules = list(range(nmods))\n",
+ " karabo_da = [\"AGIPD{:02d}\".format(i) for i in modules]\n",
+ "else:\n",
+ " modules = [int(x[-2:]) for x in karabo_da]\n",
+ "\n",
"IL_MODE = interlaced \n",
"maxcells = mem_cells if not interlaced else mem_cells*2\n",
"cells = mem_cells\n",
@@ -1412,8 +1421,11 @@
"metadata": {},
"outputs": [],
"source": [
+ "# Read report path and create file location tuple to add with the injection\n",
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
- "file_loc = proposal + ' ' + ' '.join(list(map(str,runs)))"
+ "file_loc = proposal + ' ' + ' '.join(list(map(str,runs)))\n",
+ "\n",
+ "report = get_report(out_folder)"
]
},
{
@@ -1427,6 +1439,14 @@
},
"outputs": [],
"source": [
+ "\n",
+ "# TODO: add db_module when received from myMDC\n",
+ "# Create the modules dict of karabo_das and PDUs\n",
+ "qm_dict = OrderedDict()\n",
+ "for i, k_da in zip(modules, karabo_da):\n",
+ " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ " qm_dict[qm] = {\"karabo_da\": k_da,\n",
+ " \"db_module\": \"\"}\n",
"md = None\n",
"\n",
"if db_output:\n",
@@ -1434,22 +1454,31 @@
" det = getattr(Detectors, dinstance)\n",
"\n",
" for qm, r in fres.items():\n",
- "\n",
+ " qm_db = qm_dict[qm]\n",
+ " karabo_da = qm_db[\"karabo_da\"]\n",
" for const in [\"SlopesPC\", \"BadPixelsPC\"]:\n",
"\n",
" dbconst = getattr(Constants.AGIPD, const)()\n",
- "\n",
+ " # This should be used in case of running notebook \n",
+ " # by a different method other than myMDC which already\n",
+ " # sends CalCat info.\n",
+ " # TODO: Set db_module to \"\" by default in the first cell\n",
+ " if not qm_db[\"db_module\"]:\n",
+ " qm_db[\"db_module\"] = get_pdu_from_db(karabo_id, karabo_da, dbconst,\n",
+ " condition, cal_db_interface,\n",
+ " snapshot_at=creation_time)[0]\n",
" # set the operating condition\n",
" condition = Conditions.Dark.AGIPD(memory_cells=maxcells, bias_voltage=bias_voltage,\n",
- " acquisition_rate=acq_rate, gain_setting=gain_setting)\n",
+ " acquisition_rate=acq_rate, gain_setting=gain_setting)\n",
"\n",
" if const == \"SlopesPC\":\n",
" dbconst.data = slope_dict_to_arr(r)\n",
" else:\n",
" dbconst.data = bad_pixels[qm]\n",
"\n",
- " md = send_to_db(getattr(det, qm), dbconst, condition, file_loc, \n",
- " cal_db_interface, creation_time=creation_time)\n",
+ " md = send_to_db(qm_db[\"db_module\"], karabo_id, dbconst, condition,\n",
+ " file_loc, report, cal_db_interface,\n",
+ " creation_time=creation_time)\n",
"\n",
"print(\"Constants parameter conditions are:\\n\")\n",
"print(f\"• memory_cells: {maxcells}\\n• bias_voltage: {bias_voltage}\\n\"\n",
diff --git a/notebooks/DSSC/DSSC_Correct_and_Verify.ipynb b/notebooks/DSSC/DSSC_Correct_and_Verify.ipynb
index 41f9f5555881109ce67b84b27d7e5b480e9d2773..75ead05e79776c5489d0d7907799369c6b74c3c6 100644
--- a/notebooks/DSSC/DSSC_Correct_and_Verify.ipynb
+++ b/notebooks/DSSC/DSSC_Correct_and_Verify.ipynb
@@ -232,7 +232,7 @@
" import struct\n",
" import binascii\n",
" \n",
- " filename, filename_out, channel, qm, conditions = inp\n",
+ " filename, filename_out, channel, karabo_da, qm, conditions = inp\n",
" \n",
" # DSSC correction requires path without the leading \"/\"\n",
" if h5path[0] == '/':\n",
@@ -278,14 +278,13 @@
" with h5py.File(filename, \"r\", driver=\"core\") as infile:\n",
" y = infile[f\"{h5path}/data\"].shape[2]\n",
" x = infile[f\"{h5path}/data\"].shape[3]\n",
- " offset, when = get_constant_from_db_and_time(device,\n",
- " Constants.DSSC.Offset(),\n",
- " condition,\n",
- " None,\n",
- " cal_db_interface,\n",
- " creation_time=creation_time,\n",
- " timeout=cal_db_timeout)\n",
- " \n",
+ " offset, when = get_constant_from_db_and_time(karabo_id, karabo_da,\n",
+ " Constants.DSSC.Offset(),\n",
+ " condition,\n",
+ " None,\n",
+ " cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " timeout=cal_db_timeout)\n",
" if offset is not None:\n",
" offset = np.moveaxis(np.moveaxis(offset[...], 2, 0), 2, 1)\n",
" else:\n",
@@ -412,8 +411,7 @@
"Errors = []\n",
"while not done:\n",
" dones = []\n",
- " # TODO: add adaptive number of modules\n",
- " for i in range(16):\n",
+ " for i, k_da in zip(modules, karabo_da):\n",
" qm = \"Q{}M{}\".format(i//4 +1, i % 4 + 1)\n",
"\n",
" if qm in mapped_files:\n",
@@ -433,7 +431,7 @@
" conditions['acquisition_rate'] = operatingFreq[qm]\n",
" conditions['target_gain'] = tGain[qm]\n",
" conditions['encoded_gain'] = encodedGain[qm]\n",
- " inp.append((fname_in, fout, i, qm, conditions))\n",
+ " inp.append((fname_in, fout, i, k_da, qm, conditions))\n",
" \n",
" if len(inp) >= min(MAX_PAR, left):\n",
" print(f\"Running {len(inp)} tasks parallel\")\n",
diff --git a/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb b/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
index 9ed3d1515431bf50ef715a7b35cb54c862b5fe08..33242e8d5c28f601d8d2069de5334c519f45f2c1 100644
--- a/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
+++ b/notebooks/FastCCD/Characterize_Darks_NewDAQ_FastCCD_NBC_New_Common_Mode.ipynb
@@ -91,6 +91,7 @@
"from prettytable import PrettyTable\n",
"\n",
"from cal_tools.tools import (get_dir_creation_date, get_random_db_interface,\n",
+ " get_pdu_from_db, get_report,\n",
" save_const_to_h5, send_to_db)\n",
"from cal_tools.enums import BadPixels\n",
"from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
@@ -148,7 +149,9 @@
"outputs": [],
"source": [
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
- "file_loc = 'proposal:{} runs:{}'.format(proposal, run)"
+ "file_loc = 'proposal:{} runs:{}'.format(proposal, run)\n",
+ "\n",
+ "report = get_report(out_folder)"
]
},
{
@@ -1111,14 +1114,22 @@
" parm.lower_deviation = temp_limits\n",
" parm.upper_deviation = temp_limits\n",
"\n",
- " device = Detectors.fastCCD1\n",
+ " # This should be used in case of running notebook \n",
+ " # by a different method other than myMDC which already\n",
+ " # sends CalCat info.\n",
+ " # TODO: Set db_module to \"\" by default in the first cell\n",
+ " if not db_module:\n",
+ " db_module = get_pdu_from_db(karabo_id, karabo_da, dconst,\n",
+ " condition, cal_db_interface,\n",
+ " snapshot_at=creation_time)[0]\n",
" \n",
" if db_output:\n",
- " md = send_to_db(device, dconst, condition, file_loc, \n",
+ " md = send_to_db(db_module, karabo_id, dconst, condition, file_loc, report,\n",
" cal_db_interface, creation_time=creation_time, timeout=cal_db_timeout)\n",
"\n",
" if local_output:\n",
- " md = save_const_to_h5(device, dconst, condition, dconst.data, file_loc, creation_time, out_folder)\n",
+ " md = save_const_to_h5(db_module, karabo_id, dconst, condition, dconst.data, \n",
+ " file_loc, report, creation_time, out_folder)\n",
" print(f\"Calibration constant {const} is stored locally.\")\n",
" \n",
"print(\"Constants parameter conditions are:\\n\")\n",
diff --git a/notebooks/FastCCD/CorrectionNotebook_NewDAQ_FastCCD_NBC.ipynb b/notebooks/FastCCD/CorrectionNotebook_NewDAQ_FastCCD_NBC.ipynb
index c03287057d6fd1f7279818895f161004507b20df..c879d49e8fb22ba19c94ffc59b1982443f33ddf1 100644
--- a/notebooks/FastCCD/CorrectionNotebook_NewDAQ_FastCCD_NBC.ipynb
+++ b/notebooks/FastCCD/CorrectionNotebook_NewDAQ_FastCCD_NBC.ipynb
@@ -362,7 +362,7 @@
" bpix = Constants.CCD(DetectorTypes.fastCCD).BadPixelsDark()\n",
" \n",
" ## retrieve_offset\n",
- " offset, offset_time = get_constant_from_db_and_time(device=Detectors.fastCCD1,\n",
+ " offset, offset_time = get_constant_from_db_and_time(karabo_id, karabo_da,\n",
" constant=offset,\n",
" condition=condition,\n",
" empty_constant=None,\n",
@@ -388,7 +388,7 @@
" .format(offset_temperature, offset_time))\n",
"\n",
" ## retrieve_noise\n",
- " noise, noise_time = get_constant_from_db_and_time(device=Detectors.fastCCD1,\n",
+ " noise, noise_time = get_constant_from_db_and_time(karabo_id, karabo_da,\n",
" constant=noise,\n",
" condition=condition,\n",
" empty_constant=None,\n",
@@ -406,7 +406,7 @@
" .format(g_name[g], noise_time))\n",
"\n",
" ## retrieve_bad pixels\n",
- " bpix, bpix_time = get_constant_from_db_and_time(device=Detectors.fastCCD1,\n",
+ " bpix, bpix_time = get_constant_from_db_and_time(karabo_id, karabo_da,\n",
" constant=bpix,\n",
" condition=condition,\n",
" empty_constant=None,\n",
@@ -456,7 +456,7 @@
" pixels_x=x, pixels_y=y,\n",
" photon_energy=photon_energy_gain_map)\n",
"\n",
- " relgain, relgain_time = get_constant_from_db_and_time(device=Detectors.fastCCD1,\n",
+ " relgain, relgain_time = get_constant_from_db_and_time(karabo_id, karabo_da,\n",
" constant=relgain,\n",
" condition=condition,\n",
" empty_constant=None,\n",
diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
index 80ee348612f97700881ea5b088ed7a3a61867faa..00da5997abba27a77fd5aa54890785ae082ef76c 100644
--- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
@@ -220,9 +220,8 @@
" bias_voltage=bias_voltage,\n",
" integration_time=integration_time)\n",
"\n",
- "\n",
- "def get_constant_for_module(condition, cal_db_interface, creation_time, cal_db_timeout,\n",
- " memoryCells, db_module):\n",
+ "def get_constant_for_module(karabo_id, condition, cal_db_interface, creation_time, cal_db_timeout,\n",
+ " memoryCells, karabo_da):\n",
" \"\"\" Get calibration constants for given module of Jungfrau\n",
" \n",
" Function contains all includes to be used with ipCluster\n",
@@ -244,36 +243,33 @@
" import numpy as np\n",
" \n",
" when = {}\n",
- " \n",
- " offset_map, time = \\\n",
- " get_constant_from_db_and_time(getattr(Detectors, db_module),\n",
+ "\n",
+ " #TODO: Remove condition + constant retrieval duplication from notebook \n",
+ "\n",
+ " offset_map, when['Offset'] = \\\n",
+ " get_constant_from_db_and_time(karabo_id, karabo_da,\n",
" Constants.jungfrau.Offset(),\n",
" condition,\n",
" np.zeros((1024, 512, 1, 3)),\n",
" cal_db_interface,\n",
" creation_time=creation_time,\n",
" timeout=cal_db_timeout)\n",
- " when['Offset'] = time\n",
- " \n",
- " mask, _ = \\\n",
- " get_constant_from_db_and_time(getattr(Detectors, db_module),\n",
+ " mask, when['BadPixels'] = \\\n",
+ " get_constant_from_db_and_time(karabo_id, karabo_da,\n",
" Constants.jungfrau.BadPixelsDark(),\n",
" condition,\n",
" np.zeros((1024, 512, 1, 3)),\n",
" cal_db_interface,\n",
" creation_time=creation_time,\n",
" timeout=cal_db_timeout)\n",
- " when['BadPixels'] = time\n",
- "\n",
- " gain_map, _ = \\\n",
- " get_constant_from_db_and_time(getattr(Detectors, db_module),\n",
+ " gain_map, when['Gain'] = \\\n",
+ " get_constant_from_db_and_time(karabo_id, karabo_da,\n",
" Constants.jungfrau.RelativeGain(),\n",
" condition,\n",
" None,\n",
" cal_db_interface,\n",
" creation_time=creation_time,\n",
" timeout=cal_db_timeout)\n",
- " when['Gain'] = time\n",
"\n",
" # move from x,y,cell,gain to cell,x,y,gain\n",
" offset_map = np.squeeze(offset_map)\n",
@@ -289,27 +285,28 @@
" gain_map = np.squeeze(gain_map)\n",
" gain_map = np.moveaxis(gain_map, 1, 0)\n",
"\n",
- " return offset_map, mask, gain_map, db_module, when\n",
+ " return offset_map, mask, gain_map, karabo_da, when\n",
"\n",
"\n",
"# Retrieve Offset, BadPixels and gain constants for a JungFrau module.\n",
"# Run ip Cluster parallelization over modules\n",
- "p = partial(get_constant_for_module, condition, cal_db_interface,\n",
+ "p = partial(get_constant_for_module, karabo_id, condition, cal_db_interface, \n",
" creation_time, cal_db_timeout, memoryCells)\n",
"\n",
- "r = view.map_sync(p, db_module)\n",
- "#r = list(map(p, db_module))\n",
+ "r = view.map_sync(p, karabo_da)\n",
+ "#r = list(map(p, karabo_da))\n",
"\n",
"constants = {}\n",
"for rr in r:\n",
- " offset_map, mask, gain_map, db_mod, when = rr\n",
- " print(f'Constants for module {db_mod}:')\n",
+ " offset_map, mask, gain_map, k_da, when = rr\n",
+ " print(f'Constants for module {k_da}:')\n",
" for const in when:\n",
" print(f'{const} injected at {when[const]}')\n",
" if gain_map is None:\n",
" print(\"No gain map found\")\n",
" no_relative_gain = True\n",
- " constants[db_mod] = (offset_map, mask, gain_map)"
+ " \n",
+ " constants[k_da] = (offset_map, mask, gain_map)"
]
},
{
@@ -418,7 +415,7 @@
" # Loop over sequences for given module\n",
" for k, f in enumerate(list(mapped_files[key].queue)):\n",
" \n",
- " offset_map, mask, gain_map = constants[db_module[i]]\n",
+ " offset_map, mask, gain_map = constants[karabo_da[i]]\n",
" h5path_f = h5path.format(int(karabo_da[i][-2:]))\n",
" \n",
" with h5py.File(f, 'r') as infile:\n",
@@ -477,11 +474,13 @@
" p = partial(correct_chunk, offset_map, mask, gain_map, memoryCells, no_relative_gain)\n",
"\n",
" r = view.map_sync(p, inp)\n",
+ " # Used for debugging correct chunk\n",
" #r = list(map(p, inp))\n",
+ " \n",
" if k==0:\n",
" (_,_,_,\n",
- " rim_data[db_module[i]], fim_data[db_module[i]],\n",
- " gim_data[db_module[i]], msk_data[db_module[i]], _) = r[0]\n",
+ " rim_data[karabo_da[i]], fim_data[karabo_da[i]],\n",
+ " gim_data[karabo_da[i]], msk_data[karabo_da[i]], _) = r[0]\n",
"\n",
" print('Correction time: ', time.time() - ts)\n",
" ts = time.time()\n",
diff --git a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
index 08ee2d799fe00614d3036d16fdec037080408f47..a0d0bfef4937f2d075385a66f2acb824924fac50 100644
--- a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
@@ -78,7 +78,8 @@
"import numpy as np\n",
"\n",
"from cal_tools.enums import BadPixels\n",
- "from cal_tools.tools import (get_dir_creation_date, get_random_db_interface,\n",
+ "from cal_tools.tools import (get_dir_creation_date, get_pdu_from_db,\n",
+ " get_random_db_interface, get_report,\n",
" save_const_to_h5, send_to_db)\n",
"from cal_tools.ana_tools import save_dict_to_hdf5\n",
"from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
@@ -104,6 +105,8 @@
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
"file_loc = 'proposal:{} runs:{} {} {}'.format(proposal, run_high, run_med, run_low)\n",
"\n",
+ "report = get_report(out_folder)\n",
+ "\n",
"os.makedirs(out_folder, exist_ok=True)\n",
"\n",
"# TODO \n",
@@ -253,11 +256,12 @@
" \n",
" if i > 0 and memoryCells == 16: ## throwing away all the SC entries except the first for lower gains\n",
" acelltable[1:] = 255\n",
- " \n",
- " \n",
- " images, gainmaps, acelltable = rollout_data([images, gainmaps, acelltable]) # makes 4-dim vecs into 3-dim\n",
- " # makes 2-dim into 1-dim\n",
- " # leaves 1-dim and 3-dim vecs\n",
+ "\n",
+ " # makes 4-dim vecs into 3-dim\n",
+ " # makes 2-dim into 1-dim\n",
+ " # leaves 1-dim and 3-dim vecs\n",
+ "\n",
+ " images, gainmaps, acelltable = rollout_data([images, gainmaps, acelltable]) \n",
" \n",
" images, gainmaps, acelltable = sanitize_data_cellid([images, gainmaps], acelltable) # removes entries with cellID 255\n",
" valid_data.append(images)\n",
@@ -445,6 +449,8 @@
"metadata": {},
"outputs": [],
"source": [
+ "# TODO: this cell in the notebook is not designed to run for more than one module\n",
+ "# Constants need to be able to have constant for each module as for big detectors\n",
"constants = {'Offset': np.moveaxis(offset_map, 0, 1),\n",
" 'Noise': np.moveaxis(noise_map, 0, 1), \n",
" 'BadPixelsDark': np.moveaxis(bad_pixels_map, 0, 1)}\n",
@@ -462,15 +468,25 @@
" if parm.name == \"Integration Time\":\n",
" parm.lower_deviation = time_limits\n",
" parm.upper_deviation = time_limits\n",
- " \n",
- " device = getattr(Detectors, db_module)\n",
"\n",
+ " # This should be used in case of running notebook \n",
+ " # by a different method other than myMDC which already\n",
+ " # sends CalCat info.\n",
+ " # TODO: Set db_module to \"\" by default in the first cell\n",
+ " if not db_module:\n",
+ " db_module = get_pdu_from_db(karabo_id, karabo_da, const,\n",
+ " condition, cal_db_interface,\n",
+ " snapshot_at=creation_time)[0]\n",
" if db_output:\n",
- " md = send_to_db(device, const, condition, file_loc, \n",
- " cal_db_interface, creation_time=creation_time, timeout=cal_db_timeout)\n",
- "\n",
+ " md = send_to_db(db_module, karabo_id, const, condition,\n",
+ " file_loc=file_loc, report_path=report,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " timeout=cal_db_timeout)\n",
" if local_output:\n",
- " md = save_const_to_h5(device, const, condition, const_data, file_loc, creation_time, out_folder)\n",
+ " md = save_const_to_h5(db_module, karabo_id, const, condition,\n",
+ " const.data, file_loc, report,\n",
+ " creation_time, out_folder)\n",
" print(f\"Calibration constant {key} is stored locally at {out_folder}.\\n\")\n",
" \n",
"print(\"Constants parameter conditions are:\\n\")\n",
diff --git a/notebooks/LPD/LPDChar_Darks_NBC.ipynb b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
index 5b947b0689ac75dc240873e3c14019de3b86d450..d9c9fa6dddbc646fb9202a3c4dcf4f2ccd651e70 100644
--- a/notebooks/LPD/LPDChar_Darks_NBC.ipynb
+++ b/notebooks/LPD/LPDChar_Darks_NBC.ipynb
@@ -91,8 +91,8 @@
" create_constant_overview,\n",
" show_processed_modules)\n",
"from cal_tools.tools import (get_dir_creation_date, get_from_db,\n",
- " get_notebook_name, \n",
- " get_random_db_interface,\n",
+ " get_notebook_name, get_pdu_from_db,\n",
+ " get_random_db_interface, get_report,\n",
" map_gain_stages, parse_runs, \n",
" run_prop_seq_from_path,\n",
" save_const_to_h5, send_to_db)\n",
@@ -334,8 +334,26 @@
"metadata": {},
"outputs": [],
"source": [
+ "# Read report path and create file location tuple to add with the injection\n",
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
- "file_loc = 'proposal:{} runs:{} {} {}'.format(proposal, run_low, run_med, run_high)"
+ "file_loc = 'proposal:{} runs:{} {} {}'.format(proposal, run_low, run_med, run_high)\n",
+ "\n",
+ "report = get_report(out_folder)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# TODO: add db_module when received from myMDC\n",
+ "# Create the modules dict of karabo_das and PDUs\n",
+ "qm_dict = OrderedDict()\n",
+ "for i, k_da in zip(modules, karabo_da):\n",
+ " qm = f\"Q{i//4+1}M{i%4+1}\"\n",
+ " qm_dict[qm] = {\"karabo_da\": k_da,\n",
+ " \"db_module\": \"\"}"
]
},
{
@@ -354,27 +372,41 @@
"print('Retrieve pre-existing constants for comparison.')\n",
"for cap in capacitor_settings:\n",
" for qm in offset_g[cap].keys():\n",
- " for const in clist:\n",
"\n",
- " condition = Conditions.Dark.LPD(memory_cells=max_cells, bias_voltage=bias_voltage,\n",
- " capacitor=cap)\n",
+ " qm_db = qm_dict[qm]\n",
+ " karabo_da = qm_db[\"karabo_da\"]\n",
"\n",
- " data, mdata = get_from_db(getattr(detinst, qm),\n",
- " getattr(Constants.LPD, const)(),\n",
- " condition,\n",
- " None,\n",
- " cal_db_interface, creation_time=creation_time,\n",
+ " condition = Conditions.Dark.LPD(memory_cells=max_cells,\n",
+ " bias_voltage=bias_voltage,\n",
+ " capacitor=cap)\n",
+ " for const in clist:\n",
+ " constant = getattr(Constants.LPD, const)()\n",
+ " if not qm_db[\"db_module\"]:\n",
+ " # This should be used in case of running notebook\n",
+ " # by a different method other than myMDC which already\n",
+ " # sends CalCat info.\n",
+ " qm_db[\"db_module\"] = get_pdu_from_db(karabo_id, [karabo_da], constant,\n",
+ " condition, cal_db_interface,\n",
+ " snapshot_at=creation_time)[0]\n",
+ "\n",
+ " data, mdata = get_from_db(karabo_id, karabo_da,\n",
+ " constant,\n",
+ " condition, None,\n",
+ " cal_db_interface,\n",
+ " creation_time=creation_time,\n",
" verbosity=2, timeout=cal_db_timeout)\n",
"\n",
" old_const[const] = data\n",
"\n",
+ " # TODO: Save old constant file paths (meta_only)\n",
+ " # instead of retrieving the whole data\n",
" if mdata is not None and data is not None:\n",
" time = mdata.calibration_constant_version.begin_at\n",
" old_mdata[const] = time.isoformat()\n",
" os.makedirs('{}/old/'.format(out_folder), exist_ok=True)\n",
- " save_const_to_h5(getattr(detinst, qm),\n",
- " getattr(Constants.LPD, const)(),\n",
- " condition, data, file_loc, creation_time,\n",
+ " save_const_to_h5(qm_db[\"db_module\"], karabo_id,\n",
+ " constant, condition, data,\n",
+ " file_loc, report, creation_time,\n",
" f'{out_folder}/old/')\n",
" else:\n",
" old_mdata[const] = \"Not found\""
@@ -408,6 +440,10 @@
"md = None\n",
"for cap in capacitor_settings:\n",
" for qm in res[cap]:\n",
+ "\n",
+ " karabo_da = qm_dict[qm][\"karabo_da\"]\n",
+ " db_module = qm_dict[qm][\"db_module\"]\n",
+ "\n",
" # Do not store empty constants\n",
" # In case of 0 trains data_g is initiated with nans and never refilled.\n",
" if np.count_nonzero(~np.isnan(data_g[cap][qm]))==0:\n",
@@ -421,15 +457,17 @@
" condition = Conditions.Dark.LPD(memory_cells=max_cells,\n",
" bias_voltage=bias_voltage,\n",
" capacitor=cap)\n",
- " device = getattr(Detectors.LPD1M1, qm)\n",
"\n",
" if db_output:\n",
- " md = send_to_db(device, dconst, condition, file_loc, \n",
- " cal_db_interface, creation_time=creation_time, timeout=cal_db_timeout)\n",
+ " md = send_to_db(db_module, karabo_id, dconst, condition,\n",
+ " file_loc, report_path=report,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " timeout=cal_db_timeout)\n",
"\n",
" if local_output:\n",
- " md = save_const_to_h5(device, dconst, condition,\n",
- " dconst.data, file_loc, creation_time, out_folder)\n",
+ " md = save_const_to_h5(db_module, karabo_id, dconst, condition,\n",
+ " dconst.data, file_loc, report, creation_time, out_folder)\n",
" print(f\"Calibration constant {const} is stored locally.\\n\")\n",
"\n",
" print(\"Constants parameter conditions are:\\n\")\n",
diff --git a/notebooks/LPD/LPD_Correct_and_Verify.ipynb b/notebooks/LPD/LPD_Correct_and_Verify.ipynb
index d2599fb38bdd95212092fe590b41944b0b19990e..07ba55d4a3d3396b3a1145f2cab11104e52ebfe9 100644
--- a/notebooks/LPD/LPD_Correct_and_Verify.ipynb
+++ b/notebooks/LPD/LPD_Correct_and_Verify.ipynb
@@ -231,7 +231,7 @@
"def correct_module(max_cells, do_ff, index_v, CHUNK_SIZE, total_sequences, sequences_qm, \n",
" bins_gain_vs_signal, bins_signal_low_range, bins_signal_high_range, max_pulses,\n",
" dbparms, fileparms, nodb, no_non_linear_corrections, mark_non_lin_region, linear_between,\n",
- " nlc_version, h5path, h5path_idx, inp):\n",
+ " nlc_version, h5path, h5path_idx, karabo_id, inp):\n",
" import numpy as np\n",
" import copy\n",
" import h5py\n",
@@ -255,7 +255,7 @@
" start = datetime.now()\n",
" success = True\n",
" reason = \"\"\n",
- " filename, filename_out, channel, qm = inp\n",
+ " filename, filename_out, channel, karabo_da, qm = inp\n",
"\n",
" infile = h5py.File(filename, \"r\", driver=\"core\")\n",
" outfile = h5py.File(filename_out, \"w\")\n",
@@ -281,7 +281,8 @@
" except IOError:\n",
" return\n",
" if not nodb:\n",
- " when = lpd_corr.initialize_from_db(dbparms, qm, only_dark=(fileparms != \"\"))\n",
+ " when = lpd_corr.initialize_from_db(dbparms, karabo_id, karabo_da, only_dark=(fileparms != \"\"))\n",
+ " print(when)\n",
" if fileparms != \"\":\n",
" lpd_corr.initialize_from_file(fileparms, qm, with_dark=nodb)\n",
" print(\"Initialized constants\")\n",
@@ -307,7 +308,8 @@
" reason = \"Error\"\n",
" err = e\n",
"\n",
- " return hists_signal_low, hists_signal_high, hists_gain_vs_signal, low_edges, high_edges, signal_edges, when, qm, err\n",
+ " return (hists_signal_low, hists_signal_high, hists_gain_vs_signal, low_edges,\n",
+ " high_edges, signal_edges, when, qm, err)\n",
" \n",
"done = False\n",
"first_files = []\n",
@@ -330,7 +332,7 @@
" \n",
" dones = []\n",
" first = True\n",
- " for i in range(16):\n",
+ " for i, k_da in zip(modules, karabo_da):\n",
" qm = \"Q{}M{}\".format(i//4 +1, i % 4 + 1)\n",
" if qm in mapped_files and not mapped_files[qm].empty():\n",
" fname_in = str(mapped_files[qm].get())\n",
@@ -342,14 +344,14 @@
" fout = os.path.abspath(\"{}/{}\".format(out_folder, (os.path.split(fname_in)[-1]).replace(\"RAW\", \"CORR\")))\n",
" if first:\n",
" first_files.append((fname_in, fout))\n",
- " inp.append((fname_in, fout, i, qm))\n",
+ " inp.append((fname_in, fout, i, k_da, qm))\n",
" first = False\n",
" if len(inp) >= min(MAX_PAR, left):\n",
" print(\"Running {} tasks parallel\".format(len(inp)))\n",
" p = partial(correct_module, max_cells, do_ff, index_v, CHUNK_SIZE, total_sequences, sequences_qm,\n",
" bins_gain_vs_signal, bins_signal_low_range, bins_signal_high_range, max_pulses, dbparms,\n",
" fileparms, nodb, no_non_linear_corrections, mark_non_lin_region, linear_between, nlc_version,\n",
- " h5path, h5path_idx)\n",
+ " h5path, h5path_idx, karabo_id)\n",
" \n",
" r = view.map_sync(p, inp)\n",
" #r = list(map(p, inp))\n",
diff --git a/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb b/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
index d1a9fa05e4108aa1a4440c5c0183c8cca435845e..8270ef2e5deb6c40bcc55ad394c2a9f0b28e0773 100644
--- a/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Characterize_Darks_ePix100_NBC.ipynb
@@ -64,7 +64,8 @@
"%matplotlib inline\n",
"import numpy as np\n",
"\n",
- "from cal_tools.tools import (get_dir_creation_date, get_random_db_interface,\n",
+ "from cal_tools.tools import (get_dir_creation_date, get_pdu_from_db,\n",
+ " get_random_db_interface, get_report,\n",
" save_const_to_h5, send_to_db)\n",
"from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
"from iCalibrationDB.detectors import DetectorTypes\n",
@@ -97,9 +98,12 @@
"metadata": {},
"outputs": [],
"source": [
+ "# Read report path and create file location tuple to add with the injection\n",
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
"file_loc = f'proposal:{proposal} runs:{run}'\n",
"\n",
+ "report = get_report(out_folder)\n",
+ "\n",
"x = 708 # rows of the xPix100\n",
"y = 768 # columns of the xPix100\n",
"\n",
@@ -260,6 +264,9 @@
"dclass=\"ePix100\"\n",
"md = None\n",
"\n",
+ "# If PDU(db_module) is not given with input parameters \n",
+ "# retrieve the connected physical detector unit\n",
+ "\n",
"for const_name in constant_maps.keys():\n",
" det = getattr(Constants, dclass)\n",
" const = getattr(det, const_name)()\n",
@@ -270,21 +277,30 @@
" integration_time=integration_time,\n",
" temperature=temperature_k,\n",
" in_vacuum=in_vacuum)\n",
- "\n",
" for parm in condition.parameters:\n",
" if parm.name == \"Sensor Temperature\":\n",
" parm.lower_deviation = temp_limits\n",
" parm.upper_deviation = temp_limits\n",
"\n",
- " device = getattr(Detectors, db_module)\n",
+ " # This should be used in case of running notebook \n",
+ " # by a different method other than myMDC which already\n",
+ " # sends CalCat info.\n",
+ " # TODO: Set db_module to \"\" by default in the first cell\n",
+ " if not db_module:\n",
+ " db_module = get_pdu_from_db(karabo_id, karabo_da, const,\n",
+ " condition, cal_db_interface,\n",
+ " snapshot_at=creation_time)[0]\n",
"\n",
" if db_output:\n",
- " md = send_to_db(device, const, condition, file_loc, \n",
- " cal_db_interface, creation_time=creation_time)\n",
- "\n",
+ " md = send_to_db(db_module, karabo_id, const, condition,\n",
+ " file_loc=file_loc, report_path=report,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " timeout=cal_db_timeout)\n",
" if local_output:\n",
- " md = save_const_to_h5(device, const, condition,\n",
- " const.data, file_loc, creation_time, out_folder)\n",
+ " md = save_const_to_h5(db_module, karabo_id, const, condition,\n",
+ " const.data, file_loc, report,\n",
+ " creation_time, out_folder)\n",
" print(f\"Calibration constant {const_name} is stored locally at {out_folder} \\n\")\n",
"\n",
"print(\"Constants parameter conditions are:\\n\")\n",
diff --git a/notebooks/ePix100/Correction_ePix100_NBC.ipynb b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
index 3be55a3b1c3e82798ef00b97dfb52695d7cde4c3..2c4fd8da983e9f9f7476a63995bba52221066a8b 100644
--- a/notebooks/ePix100/Correction_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
@@ -285,7 +285,7 @@
"\n",
"\n",
"\n",
- "offsetMap = get_constant_from_db(det,\n",
+ "offsetMap = get_constant_from_db(karabo_id, karabo_da,\n",
" getattr(dconstants, const_name)(),\n",
" condition,\n",
" None,\n",
@@ -301,7 +301,7 @@
" temperature=temperature_k,\n",
" in_vacuum=in_vacuum)\n",
"\n",
- "noiseMap = get_constant_from_db(det,\n",
+ "noiseMap = get_constant_from_db(karabo_id, karabo_da,\n",
" getattr(dconstants, const_name)(),\n",
" condition,\n",
" None,\n",
@@ -319,7 +319,7 @@
" temperature=temperature_k,\n",
" in_vacuum=in_vacuum)\n",
"\n",
- " gainMap = get_constant_from_db(det,\n",
+ " gainMap = get_constant_from_db(karabo_id, karabo_da,\n",
" getattr(dconstants, const_name)(),\n",
" condition,\n",
" None,\n",
@@ -329,7 +329,7 @@
" timeout=cal_db_timeout)\n",
" \n",
" if gainMap is None:\n",
- " print(\"Gain map requisted, but not found\")\n",
+ " print(\"Gain map requested, but not found\")\n",
" print(\"No gain correction will be applied\")\n",
" no_relative_gain = True\n",
" plot_unit = 'ADU'\n",
diff --git a/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb b/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb
index 4aeecc749b9af0854b4e40077c69cd9c11e15a61..9023752ac105dd3f9b985e7c638edf9f586aebbe 100644
--- a/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb
+++ b/notebooks/ePix10K/Characterize_Darks_ePix10K_NBC.ipynb
@@ -63,8 +63,9 @@
"%matplotlib inline\n",
"import numpy as np\n",
"\n",
- "from cal_tools.tools import (get_dir_creation_date, save_const_to_h5,\n",
- " get_random_db_interface, send_to_db)\n",
+ "from cal_tools.tools import (get_dir_creation_date, get_pdu_from_db,\n",
+ " get_random_db_interface, get_report,\n",
+ " save_const_to_h5, send_to_db)\n",
"from iCalibrationDB import Constants, Conditions, Detectors, Versions\n",
"from iCalibrationDB.detectors import DetectorTypes\n",
"from XFELDetAna import xfelpyanatools as xana\n",
@@ -96,9 +97,12 @@
"metadata": {},
"outputs": [],
"source": [
+ "# Read report path and create file location tuple to add with the injection\n",
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
"file_loc = 'proposal:{} runs:{}'.format(proposal, run)\n",
"\n",
+ "report = get_report(out_folder)\n",
+ "\n",
"x = 356 # rows of the ePix10K\n",
"y = 384 # columns of the ePix10K\n",
"\n",
@@ -282,15 +286,25 @@
" parm.lower_deviation = temp_limits\n",
" parm.upper_deviation = temp_limits\n",
"\n",
- " device = getattr(Detectors, db_module)\n",
- " \n",
- " if db_output:\n",
- " md = send_to_db(device, const, condition, file_loc, \n",
- " cal_db_interface, creation_time=creation_time, timeout=cal_db_timeout)\n",
+ " # This should be used in case of running notebook \n",
+ " # by a different method other than myMDC which already\n",
+ " # sends CalCat info.\n",
+ " # TODO: Set db_module to \"\" by default in the first cell\n",
+ " if not db_module:\n",
+ " db_module = get_pdu_from_db(karabo_id, karabo_da, const,\n",
+ " condition, cal_db_interface,\n",
+ " snapshot_at=creation_time)[0]\n",
"\n",
+ " if db_output:\n",
+ " md = send_to_db(db_module, karabo_id, const, condition,\n",
+ " file_loc=file_loc, report_path=report,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " timeout=cal_db_timeout)\n",
" if local_output:\n",
- " md = save_const_to_h5(device, const, condition,\n",
- " const.data, file_loc, creation_time, out_folder)\n",
+ " md = save_const_to_h5(db_module, karabo_id, const, condition,\n",
+ " const.data, file_loc, report,\n",
+ " creation_time, out_folder)\n",
" print(f\"Calibration constant {const_name} is stored locally.\")\n",
"\n",
"print(\"Constants parameter conditions are:\\n\")\n",
diff --git a/notebooks/ePix10K/Correction_ePix10K_NBC.ipynb b/notebooks/ePix10K/Correction_ePix10K_NBC.ipynb
index 8a49dd47ffe08bdd94a5a49f3e6a24539d9bea71..53646280d7b2bee429531cdff5be19a3ce6213fe 100644
--- a/notebooks/ePix10K/Correction_ePix10K_NBC.ipynb
+++ b/notebooks/ePix10K/Correction_ePix10K_NBC.ipynb
@@ -263,7 +263,6 @@
"temp_limits = 5.\n",
"\n",
"# offset\n",
- "det = getattr(Detectors, db_module)\n",
"dconstants = getattr(Constants, dclass)\n",
"\n",
"dcond = Conditions.Dark\n",
@@ -279,7 +278,7 @@
" parm.upper_deviation = temp_limits\n",
"\n",
"\n",
- "offsetMap = get_constant_from_db(det,\n",
+ "offsetMap = get_constant_from_db(karabo_id, karabo_da,\n",
" getattr(dconstants, const_name)(),\n",
" condition,\n",
" None,\n",
@@ -295,7 +294,7 @@
" gain_setting=gain_setting)\n",
"\n",
"const_name = \"Noise\"\n",
- "noiseMap = get_constant_from_db(det,\n",
+ "noiseMap = get_constant_from_db(karabo_id, karabo_da,\n",
" getattr(dconstants, const_name)(),\n",
" condition,\n",
" None,\n",
diff --git a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
index 87030c5496f00cc71dbd92c8a83635e601a246ef..d1d59f69a31ad596e68135872f2d1f96964701a0 100644
--- a/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
+++ b/notebooks/pnCCD/Characterize_pnCCD_Dark_NBC.ipynb
@@ -99,9 +99,10 @@
"\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.pnccdlib import extract_slow_data, VALID_GAINS\n",
- "from cal_tools.tools import (get_dir_creation_date, save_const_to_h5,\n",
- " get_random_db_interface, send_to_db)\n",
- "from iCalibrationDB import (Conditions, Constants, Detectors, Versions)\n",
+ "from cal_tools.tools import (get_dir_creation_date, get_pdu_from_db,\n",
+ " get_random_db_interface, get_report, \n",
+ " save_const_to_h5, send_to_db)\n",
+ "from iCalibrationDB import (Constants, Conditions, Detectors, Versions)\n",
"from iCalibrationDB.detectors import DetectorTypes\n",
"import XFELDetAna.xfelprofiler as xprof\n",
"profiler = xprof.Profiler()\n",
@@ -140,9 +141,13 @@
"metadata": {},
"outputs": [],
"source": [
+ "# Read report path and create file location tuple to add with the injection\n",
"proposal = list(filter(None, in_folder.strip('/').split('/')))[-2]\n",
"file_loc = f'Proposal: {proposal}, Run: {run}'\n",
- "print(\"File Location:\", file_loc)"
+ "print(\"File Location:\", file_loc)\n",
+ "\n",
+ "report = get_report(out_folder)\n",
+ "print(\"Report Path:\", report)"
]
},
{
@@ -1001,13 +1006,25 @@
" parm.lower_deviation = temp_limits\n",
" parm.upper_deviation = temp_limits\n",
"\n",
- " device = getattr(Detectors, db_module)\n",
+ " # This should be used in case of running notebook \n",
+ " # by a different method other than myMDC which already\n",
+ " # sends CalCat info.\n",
+ " # TODO: Set db_module to \"\" by default in the first cell\n",
+ " if not db_module:\n",
+ " db_module = get_pdu_from_db(karabo_id, karabo_da, const,\n",
+ " condition, cal_db_interface,\n",
+ " snapshot_at=creation_time)[0]\n",
"\n",
" if db_output:\n",
- " md = send_to_db(device, const, condition, file_loc, \n",
- " cal_db_interface, creation_time=creation_time, timeout=cal_db_timeout)\n",
+ " md = send_to_db(db_module, karabo_id, const, condition,\n",
+ " file_loc=file_loc, report_path=report,\n",
+ " cal_db_interface=cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " timeout=cal_db_timeout)\n",
" if local_output:\n",
- " md = save_const_to_h5(device, const, condition, const.data, file_loc, creation_time, out_folder)\n",
+ " md = save_const_to_h5(db_module, karabo_id, const, condition,\n",
+ " const.data, file_loc, report,\n",
+ " creation_time, out_folder)\n",
" print(f\"Calibration constant {const_name} is stored to {out_folder}.\\n\")\n",
"\n",
"print(\"Constants parameter conditions are:\\n\")\n",
diff --git a/notebooks/pnCCD/Characterize_pnCCD_Gain.ipynb b/notebooks/pnCCD/Characterize_pnCCD_Gain.ipynb
index aab6f938d8149cae36c43d8217c60967f73544dd..4c6a9340026e64e924a6cca47705b28b283a6c5a 100644
--- a/notebooks/pnCCD/Characterize_pnCCD_Gain.ipynb
+++ b/notebooks/pnCCD/Characterize_pnCCD_Gain.ipynb
@@ -101,7 +101,8 @@
"from prettytable import PrettyTable\n",
"\n",
"from cal_tools.pnccdlib import extract_slow_data, VALID_GAINS\n",
- "from cal_tools.tools import get_dir_creation_date, save_const_to_h5, send_to_db\n",
+ "from cal_tools.tools import (get_dir_creation_date, get_pdu_from_db,\n",
+ " get_report, save_const_to_h5, send_to_db)\n",
"from iCalibrationDB import (Conditions, ConstantMetaData,\n",
" Constants, Detectors, Versions)\n",
"from iCalibrationDB.detectors import DetectorTypes\n",
@@ -141,6 +142,9 @@
"file_loc =f'Proposal: {proposal}, Run: {run}'\n",
"print(file_loc)\n",
"\n",
+ "report = get_report(out_folder)\n",
+ "print(\"report_path:\", report)\n",
+ "\n",
"# Paths to the data:\n",
"ped_dir = \"{}/r{:04d}\".format(out_folder, run)\n",
"ped_dir_ctrl = \"{}/r{:04d}\".format(in_folder, run)\n",
@@ -1178,7 +1182,7 @@
"source": [
"constant_maps = {\n",
" 'RelativeGain' : gainmap,\n",
- " 'CTI' : ctemap} # Instead of CTI map, one should send the CTE map because this is the one that can correct data\n",
+ " 'CTE' : ctemap}\n",
"\n",
"md = None\n",
"for cname in constant_maps.keys():\n",
@@ -1196,32 +1200,27 @@
" temperature=fix_temperature_top,\n",
" pixels_x=pixels_x,\n",
" pixels_y=pixels_y)\n",
- "\n",
- " device = getattr(Detectors, db_module)\n",
- " metadata.detector_condition = condition\n",
- " \n",
- " # Specifying the a version for this constant:\n",
- " if creation_time is None:\n",
- " metadata.calibration_constant_version = Versions.Now(device=device)\n",
- " else:\n",
- " metadata.calibration_constant_version = Versions.Timespan(device=device, start=creation_time)\n",
- " \n",
- " metadata.calibration_constant_version.raw_data_location = file_loc\n",
- " \n",
+ " # This should be used in case of running notebook \n",
+ " # by a different method other than myMDC which already\n",
+ " # sends CalCat info.\n",
+ " # TODO: Set db_module to \"\" by default in the first cell\n",
+ " if not db_module:\n",
+ " db_module = get_pdu_from_db(karabo_id, karabo_da, const,\n",
+ " condition, cal_db_interface,\n",
+ " snapshot_at=creation_time)[0]\n",
" if db_output:\n",
" try:\n",
- " md = send_to_db(device, const, condition, file_loc, \n",
- " cal_db_interface, creation_time=creation_time, timeout=cal_db_timeout)\n",
- " print(f\"Inject {cname} constants from {metadata.calibration_constant_version.begin_at}\\n\")\n",
- " except Exception as e: \n",
- " if \"has already been take\" in str(e):\n",
- " print(f\"WARN: {cname} has already been injected with the same parameter conditions.\\n\")\n",
- " else:\n",
- " # To prevent having big error message out of the pdf report's page.\n",
- " print(\"\\n\".join(textwrap.wrap(str(e), 100)))\n",
+ " md = send_to_db(db_module, karabo_da, const, condition,\n",
+ " file_loc, report,\n",
+ " cal_db_interface,\n",
+ " creation_time=creation_time,\n",
+ " timeout=cal_db_timeout)\n",
" \n",
" if local_output:\n",
- " md = save_const_to_h5(device, const, condition, const.data, file_loc, creation_time, out_folder)\n",
+ " md = save_const_to_h5(db_module, karabo_id, \n",
+ " const, condition, const.data,\n",
+ " file_loc, report,\n",
+ " creation_time, out_folder)\n",
" print(f\"Calibration constant {cname} is stored to {out_folder}.\")\n",
" \n",
"print(\"\\nGenerated constants with conditions:\\n\")\n",
diff --git a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
index e7436b99dc9d6f7cfcd64cf9eefbdfc513af933b..1f206f73f61493e04264ed614faf9befa0d32d4e 100644
--- a/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
+++ b/notebooks/pnCCD/Correct_pnCCD_NBC.ipynb
@@ -420,7 +420,7 @@
" \n",
" for const in constants.keys():\n",
" constants[const], when[const] = \\\n",
- " get_constant_from_db_and_time(getattr(Detectors, db_module),\n",
+ " get_constant_from_db_and_time(karabo_id, karabo_da,\n",
" getattr(Constants.CCD(DetectorTypes.pnCCD), const)(),\n",
" condition,\n",
" np.zeros((pixels_x, pixels_y, 1)),\n",
@@ -493,7 +493,7 @@
" photon_energy=photon_energy)\n",
"\n",
" constants[\"RelativeGain\"], relgain_time = \\\n",
- " get_constant_from_db_and_time(getattr(Detectors, db_module),\n",
+ " get_constant_from_db_and_time(karabo_id, karabo_da,\n",
" Constants.CCD(DetectorTypes.pnCCD).RelativeGain(),\n",
" condition,\n",
" np.zeros((pixels_x, pixels_y)),\n",
diff --git a/requirements.txt b/requirements.txt
index ebfcbd4ad4105c42a3d480c875d11b27805c0c3b..199e14468d4b2977690218251f9dcb09aaa5c9dc 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,6 @@
-git+file:///gpfs/exfel/sw/calsoft/git/cal_db_interactive@1.6.4
+git+file:///gpfs/exfel/sw/calsoft/git/cal_db_interactive@2.0.0
git+file:///gpfs/exfel/sw/calsoft/git/nbparameterise@0.3
-git+file:///gpfs/exfel/sw/calsoft/git/pyDetLib@2.5.5-2.9.1#subdirectory=lib
+git+file:///gpfs/exfel/sw/calsoft/git/pyDetLib@2.5.6-2.10.0#subdirectory=lib
astcheck == 0.2.5
astsearch == 0.1.3
Cython == 0.29.21