diff --git a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
index 5b4a4640028a4bada1d6729196e24d3b47959121..14358ef6fd28396300539ce3e8a2d4a26a82bc28 100644
--- a/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_dark_analysis_all_gains_burst_mode_NBC.ipynb
@@ -33,7 +33,6 @@
"ctrl_source_template = '{}/DET/CONTROL' # template for control source name (filled with karabo_id_control)\n",
"\n",
"# Parameters for calibration database and storing constants.\n",
- "use_dir_creation_date = True # use dir creation date\n",
"cal_db_interface = 'tcp://max-exfl-cal001:8016#8045' # calibrate db interface to connect to\n",
"cal_db_timeout = 300000 # timeout on caldb requests\n",
"local_output = True # output constants locally\n",
@@ -47,6 +46,7 @@
"min_trains = 100 # Minimum number of trains to process dark constants. Raise a warning if the run has fewer trains.\n",
"manual_slow_data = False # if true, use manually entered bias_voltage and integration_time values\n",
"time_limits = 0.025 # to find calibration constants later on, the integration time is allowed to vary by 0.5 us\n",
+ "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
"\n",
"# Parameters to be used for injecting dark calibration constants.\n",
"integration_time = -1 # Integration time in us. Set to -1 to overwrite by value in file.\n",
@@ -73,12 +73,12 @@
"source": [
"import os\n",
"import warnings\n",
+ "from datetime import timedelta\n",
"from logging import warning\n",
"warnings.filterwarnings('ignore')\n",
"\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
- "import multiprocessing\n",
"import numpy as np\n",
"import pasha as psh\n",
"import yaml\n",
@@ -91,10 +91,14 @@
"from XFELDetAna.plotting.heatmap import heatmapPlot\n",
"from XFELDetAna.plotting.histogram import histPlot\n",
"from cal_tools import step_timing\n",
+ "from cal_tools.calcat_interface2 import (\n",
+ " CalibrationData,\n",
+ " JUNGFRAUConditions,\n",
+ ")\n",
"from cal_tools.jungfrau import jungfraulib\n",
"from cal_tools.enums import BadPixels, JungfrauGainMode\n",
"from cal_tools.tools import (\n",
- " get_dir_creation_date,\n",
+ " calcat_creation_time,\n",
" get_pdu_from_db,\n",
" get_random_db_interface,\n",
" get_report,\n",
@@ -121,10 +125,8 @@
" JungfrauGainMode.FORCE_SWITCH_HG1.value, JungfrauGainMode.FORCE_SWITCH_HG2.value]\n",
"old_fixed_settings = [\"fixgain1\", \"fixgain2\"]\n",
"\n",
- "creation_time = None\n",
- "if use_dir_creation_date:\n",
- " creation_time = get_dir_creation_date(in_folder, run_high)\n",
- " print(f\"Using {creation_time} as creation time\")\n",
+ "creation_time = calcat_creation_time(in_folder, run_high, creation_time)\n",
+ "print(f\"Using {creation_time} as creation time\")\n",
"os.makedirs(out_folder, exist_ok=True)\n",
"\n",
"cal_db_interface = get_random_db_interface(cal_db_interface)\n",
@@ -250,58 +252,6 @@
"step_timer.done_step('Set conditions and get PDU names from CalCat.')"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Start retrieving existing constants for comparison\n",
- "step_timer.start()\n",
- "mod_x_const = [(mod, const) for const in [\"Offset\", \"Noise\", \"BadPixelsDark\"] for mod in karabo_da]\n",
- "\n",
- "from cal_tools.tools import get_from_db\n",
- "from datetime import timedelta\n",
- "\n",
- "def retrieve_old_constant(mod, const):\n",
- " dconst = getattr(Constants.jungfrau, const)()\n",
- "\n",
- " data, mdata = get_from_db(\n",
- " karabo_id=karabo_id,\n",
- " karabo_da=mod,\n",
- " constant=dconst,\n",
- " condition=condition,\n",
- " empty_constant=None,\n",
- " cal_db_interface=cal_db_interface,\n",
- " creation_time=creation_time-timedelta(seconds=60) if creation_time else None,\n",
- " strategy=\"pdu_prior_in_time\",\n",
- " verbosity=1,\n",
- " timeout=cal_db_timeout\n",
- " )\n",
- "\n",
- " if mdata is None or data is None:\n",
- " timestamp = \"Not found\"\n",
- " filepath = None\n",
- " h5path = None\n",
- " else:\n",
- " timestamp = mdata.calibration_constant_version.begin_at.isoformat()\n",
- " filepath = os.path.join(\n",
- " mdata.calibration_constant_version.hdf5path,\n",
- " mdata.calibration_constant_version.filename\n",
- " )\n",
- " h5path = mdata.calibration_constant_version.h5path\n",
- "\n",
- " return data, timestamp, filepath, h5path\n",
- "\n",
- "\n",
- "old_retrieval_pool = multiprocessing.Pool()\n",
- "old_retrieval_res = old_retrieval_pool.starmap_async(\n",
- " retrieve_old_constant, mod_x_const\n",
- ")\n",
- "old_retrieval_pool.close()\n",
- "step_timer.done_step('Retrieved old dark constants for comparison.')"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -707,19 +657,55 @@
"metadata": {},
"outputs": [],
"source": [
- "# now we need the old constants\n",
+ "# Start retrieving existing constants for comparison\n",
+ "step_timer.start()\n",
+ "\n",
"old_const = {}\n",
"old_mdata = {}\n",
- "old_retrieval_res.wait()\n",
- "\n",
- "for (mod, const), (data, timestamp, filepath, h5path) in zip(\n",
- " mod_x_const, old_retrieval_res.get()):\n",
- " old_const.setdefault(mod, {})[const] = data\n",
- " old_mdata.setdefault(mod, {})[const] = {\n",
- " \"timestamp\": timestamp,\n",
- " \"filepath\": filepath,\n",
- " \"h5path\": h5path,\n",
- " }"
+ "\n",
+ "# set the operating condition\n",
+ "conditions = JUNGFRAUConditions(\n",
+ " sensor_bias_voltage=bias_voltage,\n",
+ " memory_cells=memory_cells,\n",
+ " integration_time=integration_time,\n",
+ " gain_setting=gain_setting,\n",
+ " gain_mode=gain_mode,\n",
+ ")\n",
+ "\n",
+ "jf_caldata = CalibrationData.from_condition(\n",
+ " conditions,\n",
+ " karabo_id,\n",
+ " event_at=creation_time-timedelta(seconds=60),\n",
+ " begin_at_strategy=\"prior\",\n",
+ ")\n",
+ "\n",
+ "# TODO: remove when injection part is updated.\n",
+ "_cnames_mapping = {\n",
+ " \"Offset\": \"Offset10Hz\",\n",
+ " \"Noise\": \"Noise10Hz\",\n",
+ " \"BadPixelsDark\": \"BadPixelsDark10Hz\",\n",
+ "}\n",
+ "for mod in karabo_da:\n",
+ " old_const[mod] = {}\n",
+ " old_mdata[mod] = {}\n",
+ " for cname in constants.keys():\n",
+ " cmdata = jf_caldata.get(_cnames_mapping[cname], None)\n",
+ " data_found = cmdata and mod in cmdata.aggregator_names\n",
+ " if data_found:\n",
+ " old_const[mod][cname] = cmdata[mod].ndarray()\n",
+ " old_mdata[mod][cname] = {\n",
+ " \"timestamp\": cmdata[mod].metadata(\"begin_validity_at\"),\n",
+ " \"filepath\": str(cmdata[mod].file_path()),\n",
+ " \"dataset\": cmdata[mod].dataset,\n",
+ " }\n",
+ " else:\n",
+ " old_const[mod][cname] = None\n",
+ " old_mdata[mod][cname] = {\n",
+ " \"timestamp\": \"Not found\",\n",
+ " \"filepath\": None,\n",
+ " \"dataset\": None,\n",
+ " }\n",
+ "step_timer.done_step('Retrieved old dark constants for comparison.')"
]
},
{
diff --git a/notebooks/Jungfrau/Jungfrau_darks_Summary_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_darks_Summary_NBC.ipynb
index 08494dc42112fba9e6135c7291795ff1d4c4e048..607f15be1f61fca8639abe9eb1efdd6ae121c6b5 100644
--- a/notebooks/Jungfrau/Jungfrau_darks_Summary_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_darks_Summary_NBC.ipynb
@@ -148,7 +148,7 @@
"\n",
" if cname in old_mod_mdata: # a module can be missing from detector dark processing.\n",
" filepath = old_mod_mdata[cname][\"filepath\"]\n",
- " h5path = old_mod_mdata[cname][\"h5path\"]\n",
+ " h5path = old_mod_mdata[cname][\"dataset\"]\n",
" if not filepath or not h5path:\n",
" excluded_modules.append(mod)\n",
" prev_constants[cname][i, ...].fill(np.nan)\n",
diff --git a/src/cal_tools/calcat_interface2.py b/src/cal_tools/calcat_interface2.py
index 0d87fd7f2a3fda716353e614d90d9e8b785b0aaa..a26af505e7520854a6c097ae6462c9f6f9940aea 100644
--- a/src/cal_tools/calcat_interface2.py
+++ b/src/cal_tools/calcat_interface2.py
@@ -258,13 +258,15 @@ class SingleConstant:
_have_calcat_metadata=True,
)
- def dataset_obj(self, caldb_root=None) -> h5py.Dataset:
+ def file_path(self, caldb_root=None) -> Path:
if caldb_root is not None:
caldb_root = Path(caldb_root)
else:
caldb_root = _get_default_caldb_root()
+ return caldb_root / self.path
- f = h5py.File(caldb_root / self.path, "r")
+ def dataset_obj(self, caldb_root=None) -> h5py.Dataset:
+ f = h5py.File(self.file_path(caldb_root), "r")
return f[self.dataset]["data"]
def ndarray(self, caldb_root=None):
@@ -495,7 +497,14 @@ class CalibrationData(Mapping):
client=None,
event_at=None,
pdu_snapshot_at=None,
+ begin_at_strategy="closest",
):
+ accepted_strategies = ["closest", "prior"]
+ if begin_at_strategy not in accepted_strategies:
+ raise ValueError(
+ "Invalid begin_at_strategy. "
+ f"Expected one of {accepted_strategies}")
+
if calibrations is None:
calibrations = set(condition.calibration_types)
if pdu_snapshot_at is None:
@@ -539,6 +548,7 @@ class CalibrationData(Mapping):
"karabo_da": "",
"event_at": client.format_time(event_at),
"pdu_snapshot_at": client.format_time(pdu_snapshot_at),
+ "begin_at_strategy": begin_at_strategy,
},
data=json.dumps(cls._format_cond(condition_dict)),
)
@@ -927,6 +937,46 @@ class DSSCConditions(ConditionsBase):
}
+@dataclass
+class JUNGFRAUConditions(ConditionsBase):
+ """Conditions for JUNGFRAU detectors"""
+ sensor_bias_voltage: float
+ memory_cells: int
+ integration_time: float
+ gain_setting: int
+ gain_mode: Optional[int] = None
+ sensor_temperature: float = 291
+ pixels_x: int = 1024
+ pixels_y: int = 512
+
+ _params = [
+ "Sensor Bias Voltage",
+ "Memory Cells",
+ "Pixels X",
+ "Pixels Y",
+ "Integration Time",
+ "Sensor temperature",
+ "Gain Setting",
+ "Gain mode",
+ ]
+ calibration_types = {
+ "Offset10Hz": _params,
+ "Noise10Hz": _params,
+ "BadPixelsDark10Hz": _params,
+ "RelativeGain10Hz": _params,
+ "BadPixelsFF10Hz": _params,
+ }
+
+ def make_dict(self, parameters):
+ cond = super().make_dict(parameters)
+
+ # Fix-up some database quirks.
+ if int(cond.get("Gain mode", -1)) == 0:
+ del cond["Gain mode"]
+
+ return cond
+
+
@dataclass
class ShimadzuHPVX2Conditions(ConditionsBase):
burst_frame_count: float