diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index aeafce49cdd1c00adea7ea7433fc4ef4ff5360c4..e38e3245e24c1af6486941f4c6120fcb6efa8dfd 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -36,19 +36,18 @@
"\n",
"slopes_ff_from_files = \"\" # Path to locally stored SlopesFF and BadPixelsFF constants, loaded in precorrection notebook\n",
"\n",
- "use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
+ "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
"cal_db_interface = \"tcp://max-exfl016:8015#8045\" # the database interface to use\n",
"cal_db_timeout = 30000 # in milliseconds\n",
"creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
"\n",
- "mem_cells = 0 # Number of memory cells used, set to 0 to automatically infer\n",
- "bias_voltage = 0 # bias voltage, set to 0 to use stored value in slow data.\n",
- "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
+ "mem_cells = -1 # Number of memory cells used, set to 0 to automatically infer\n",
+ "bias_voltage = -1 # bias voltage, set to 0 to use stored value in slow data.\n",
+ "acq_rate = -1. # the detector acquisition rate, use 0 to try to auto-determine\n",
"gain_setting = -1 # the gain setting, use -1 to use value stored in slow data.\n",
"gain_mode = -1 # gain mode (0: adaptive, 1-3 fixed high/med/low, -1: read from CONTROL data)\n",
- "overwrite = True # set to True if existing data should be overwritten\n",
"max_pulses = [0, 352, 1] # range list [st, end, step] of memory cell indices to be processed within a train. 3 allowed maximum list input elements.\n",
- "mem_cells_db = 0 # set to a value different than 0 to use this value for DB queries\n",
+ "mem_cells_db = -1 # set to a value different than 0 to use this value for DB queries\n",
"integration_time = -1 # integration time, negative values for auto-detection.\n",
"\n",
"# Correction parameters\n",
@@ -141,7 +140,7 @@
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"import yaml\n",
- "from extra_data import H5File, RunDirectory, stack_detector_data, by_id\n",
+ "from extra_data import RunDirectory, stack_detector_data\n",
"from extra_geom import AGIPD_1MGeometry, AGIPD_500K2GGeometry\n",
"from matplotlib import cm as colormap\n",
"from matplotlib.colors import LogNorm\n",
@@ -165,10 +164,12 @@
"from cal_tools.ana_tools import get_range\n",
"from cal_tools.enums import AgipdGainMode, BadPixels\n",
"from cal_tools.step_timing import StepTimer\n",
- "\n",
- "sns.set()\n",
- "sns.set_context(\"paper\", font_scale=1.4)\n",
- "sns.set_style(\"ticks\")"
+ "from cal_tools.tools import (\n",
+ " CalibrationMetadata,\n",
+ " calcat_creation_time,\n",
+ " map_modules_from_folder,\n",
+ " module_index_to_qm,\n",
+ ")"
]
},
{
@@ -282,7 +283,7 @@
"else:\n",
" modules = [int(x[-2:]) for x in karabo_da]\n",
"\n",
- "print(\"Process modules:\", ', '.join(cal_tools.tools.module_index_to_qm(x) for x in modules))\n",
+ "print(\"Process modules:\", ', '.join(module_index_to_qm(x) for x in modules))\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"print(f\"Instrument {instrument}\")\n",
"print(f\"Detector instance {dinstance}\")"
@@ -325,7 +326,7 @@
"outputs": [],
"source": [
"# set everything up filewise\n",
- "mapped_files, _, total_sequences, _, _ = cal_tools.tools.map_modules_from_folder(\n",
+ "mapped_files, _, total_sequences, _, _ = map_modules_from_folder(\n",
" str(in_folder), run, path_template, karabo_da, sequences\n",
")\n",
"file_list = []\n",
@@ -375,22 +376,21 @@
"metadata": {},
"outputs": [],
"source": [
- "# Evaluate creation time\n",
- "creation_time = None\n",
- "if use_dir_creation_date:\n",
- " creation_time = cal_tools.tools.get_dir_creation_date(str(in_folder), run)\n",
- " offset = parser.parse(creation_date_offset)\n",
- " delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
- " creation_time += delta\n",
- "\n",
- "if acq_rate == 0.:\n",
+ "# Run's creation time:\n",
+ "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
+ "offset = parser.parse(creation_date_offset)\n",
+ "delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
+ "creation_time += delta\n",
+ "print(f\"Creation time: {creation_time}\")\n",
+ "\n",
+ "if acq_rate == -1.:\n",
" acq_rate = agipd_cond.get_acq_rate()\n",
- "if mem_cells == 0.:\n",
+ "if mem_cells == -1:\n",
" mem_cells = agipd_cond.get_num_cells()\n",
"# TODO: look for alternative for passing creation_time\n",
"if gain_setting == -1:\n",
" gain_setting = agipd_cond.get_gain_setting(creation_time)\n",
- "if bias_voltage == 0.:\n",
+ "if bias_voltage == -1:\n",
" bias_voltage = agipd_cond.get_bias_voltage(karabo_id_control)\n",
"if integration_time == -1:\n",
" integration_time = agipd_cond.get_integration_time()\n",
@@ -407,9 +407,9 @@
"outputs": [],
"source": [
"if mem_cells is None:\n",
- " raise ValueError(f\"No raw images found in {filename}\")\n",
+ " raise ValueError(f\"No raw images found for {instrument_src_mod}\")\n",
"\n",
- "mem_cells_db = mem_cells if mem_cells_db == 0 else mem_cells_db\n",
+ "mem_cells_db = mem_cells if mem_cells_db == -1 else mem_cells_db\n",
"\n",
"print(f\"Maximum memory cells to calibrate: {mem_cells}\")"
]
@@ -570,7 +570,6 @@
"\n",
" Metadata for constants is taken from yml file or retrieved from the DB\n",
" \"\"\"\n",
- " err = \"\"\n",
" k_da = module_index_to_karabo_da[mod]\n",
" # check if there is a yaml file in out_folder that has the device constants.\n",
" if k_da in const_yaml:\n",
@@ -756,11 +755,8 @@
"fst_print = True\n",
"timestamps = {}\n",
"\n",
- "for i, (error, modno, when, k_da) in enumerate(const_out):\n",
- " qm = cal_tools.tools.module_index_to_qm(modno)\n",
- " # expose errors while applying correction\n",
- " if error:\n",
- " print(\"Error: {}\".format(error) )\n",
+ "for i, (modno, when, k_da) in enumerate(const_out):\n",
+ " qm = module_index_to_qm(modno)\n",
"\n",
" if k_da not in const_yaml:\n",
" if fst_print:\n",
@@ -769,14 +765,12 @@
"\n",
" module_timestamps = {}\n",
"\n",
- " # If correction is crashed\n",
- " if not error:\n",
- " print(f\"{qm}:\")\n",
- " for key, item in when.items():\n",
- " if hasattr(item, 'strftime'):\n",
- " item = item.strftime('%y-%m-%d %H:%M')\n",
- " when[key] = item\n",
- " print('{:.<12s}'.format(key), item)\n",
+ " print(f\"{qm}:\")\n",
+ " for key, item in when.items():\n",
+ " if hasattr(item, 'strftime'):\n",
+ " item = item.strftime('%y-%m-%d %H:%M')\n",
+ " when[key] = item\n",
+ " print('{:.<12s}'.format(key), item)\n",
"\n",
" # Store few time stamps if exists\n",
" # Add NA to keep array structure\n",
@@ -784,10 +778,7 @@
" if when and key in when and when[key]:\n",
" module_timestamps[key] = when[key]\n",
" else:\n",
- " if error is not None:\n",
- " module_timestamps[key] = \"Err\"\n",
- " else:\n",
- " module_timestamps[key] = \"NA\"\n",
+ " module_timestamps[key] = \"NA\"\n",
" timestamps[qm] = module_timestamps\n",
"\n",
"seq = sequences[0] if sequences else 0\n",
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 4f155d4a8c82d35192fe855734d580ad065ff00b..bbb32e928c2784a85ea2c1b28b58a1c39cb8388f 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -20,28 +20,25 @@
"in_folder = \"/gpfs/exfel/exp/SPB/202030/p900119/raw\" # the folder to read data from, required\n",
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/AGIPD_\" # the folder to output to, required\n",
"metadata_folder = \"\" # Directory containing calibration_metadata.yml when run by xfel-calibrate\n",
- "sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
"run = 80 # runs to process, required\n",
"\n",
"karabo_id = \"SPB_DET_AGIPD1M-1\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
- "path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # the template to use to access data\n",
"ctrl_source_template = '{}/MDL/FPGA_COMP_TEST' # path to control information\n",
"instrument_source_template = '{}/DET/{}:xtdf' # path in the HDF5 file to images\n",
"receiver_template = \"{}CH0\" # inset for receiver devices\n",
"karabo_id_control = \"SPB_IRU_AGIPD1M1\" # karabo-id for control device\n",
"\n",
- "use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
+ "# Parameters for calibration database.\n",
"cal_db_interface = \"tcp://max-exfl016:8015#8045\" # the database interface to use\n",
"creation_date_offset = \"00:00:00\" # add an offset to creation date, e.g. to get different constants\n",
+ "creation_time = \"\" # To overwrite the measured creation_time. Required Format: YYYY-MM-DD HR:MN:SC e.g. \"2022-06-28 13:00:00\"\n",
"\n",
"slopes_ff_from_files = \"\" # Path to locally stored SlopesFF and BadPixelsFF constants\n",
- "calfile = \"\" # path to calibration file. Leave empty if all data should come from DB\n",
- "nodb = False # if set only file-based constants will be used\n",
- "mem_cells = 0 # number of memory cells used, set to 0 to automatically infer\n",
- "bias_voltage = 0 # bias voltage, set to 0 to use stored value in slow data.\n",
- "acq_rate = 0. # the detector acquisition rate, use 0 to try to auto-determine\n",
+ "mem_cells = -1 # number of memory cells used, set to 0 to automatically infer\n",
+ "bias_voltage = -1 # bias voltage, set to 0 to use stored value in slow data.\n",
+ "acq_rate = -1. # the detector acquisition rate, use 0 to try to auto-determine\n",
"gain_setting = -1 # the gain setting, use -1 to use value stored in slow data.\n",
"gain_mode = -1 # gain mode (0: adaptive, 1-3 fixed high/med/low, -1: read from CONTROL data)\n",
"integration_time = -1 # integration time, negative values for auto-detection.\n",
@@ -53,7 +50,6 @@
"blc_noise = False # if set, baseline correction via noise peak location is attempted\n",
"blc_stripes = False # if set, baseline corrected via stripes\n",
"blc_hmatch = False # if set, base line correction via histogram matching is attempted\n",
- "match_asics = False # if set, inner ASIC borders are matched to the same signal level\n",
"adjust_mg_baseline = False # adjust medium gain baseline to match highest high gain value"
]
},
@@ -90,10 +86,7 @@
"from pathlib import Path\n",
"from typing import Tuple\n",
"\n",
- "import matplotlib\n",
- "import matplotlib.pyplot as plt\n",
"import multiprocessing\n",
- "import numpy as np\n",
"from datetime import timedelta\n",
"from dateutil import parser\n",
"from extra_data import RunDirectory\n",
@@ -122,7 +115,9 @@
"# slopes_ff_from_files left as str for now\n",
"in_folder = Path(in_folder)\n",
"out_folder = Path(out_folder)\n",
- "metadata = tools.CalibrationMetadata(metadata_folder or out_folder)"
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
+ "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})"
]
},
{
@@ -131,21 +126,17 @@
"metadata": {},
"outputs": [],
"source": [
- "creation_time = None\n",
- "if use_dir_creation_date:\n",
- " creation_time = tools.get_dir_creation_date(str(in_folder), run)\n",
- " offset = parser.parse(creation_date_offset)\n",
- " delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
- " creation_time += delta\n",
- " print(f\"Using {creation_time} as creation time\")\n",
- "\n",
- "if sequences[0] == -1:\n",
- " sequences = None\n",
+ "# Run's creation time:\n",
+ "creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
+ "offset = parser.parse(creation_date_offset)\n",
+ "delta = timedelta(hours=offset.hour, minutes=offset.minute, seconds=offset.second)\n",
+ "creation_time += delta\n",
+ "print(f\"Creation time: {creation_time}\")\n",
" \n",
"print(f\"Outputting to {out_folder}\")\n",
"out_folder.mkdir(parents=True, exist_ok=True)\n",
"\n",
- "melt_snow = False if corr_bools[\"only_offset\"] else agipdlib.SnowResolution.NONE"
+ "melt_snow = False if corr_bools[\"only_offset\"] else SnowResolution.NONE"
]
},
{
@@ -200,7 +191,7 @@
"metadata": {},
"outputs": [],
"source": [
- "agipd_cond = agipdlib.AgipdCtrl(\n",
+ "agipd_cond = AgipdCtrl(\n",
" run_dc=run_dc,\n",
" image_src=None, # Not neededed, as we wont read mem_cells or acq_rate.\n",
" ctrl_src=ctrl_src,\n",
@@ -208,7 +199,7 @@
"\n",
"if gain_setting == -1:\n",
" gain_setting = agipd_cond.get_gain_setting(creation_time)\n",
- "if bias_voltage == 0.:\n",
+ "if bias_voltage == -1:\n",
" bias_voltage = agipd_cond.get_bias_voltage(karabo_id_control)\n",
"if integration_time == -1:\n",
" integration_time = agipd_cond.get_integration_time()\n",
@@ -373,39 +364,8 @@
"metadata": {},
"outputs": [],
"source": [
- "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
- "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "pc_bools = [corr_bools.get(\"rel_gain\"),\n",
- " corr_bools.get(\"adjust_mg_baseline\"),\n",
- " corr_bools.get('blc_noise'),\n",
- " corr_bools.get('blc_hmatch'),\n",
- " corr_bools.get('blc_stripes'),\n",
- " melt_snow]\n",
- "inp = []\n",
- "only_dark = False\n",
- "nodb_with_dark = False\n",
- "if not nodb:\n",
- " only_dark = (calfile != \"\")\n",
- "if calfile != \"\" and not corr_bools[\"only_offset\"]:\n",
- " nodb_with_dark = nodb\n",
- "\n",
- "da_to_qm = dict()\n",
- "for module_index, k_da in zip(modules, karabo_da):\n",
- " da_to_qm[k_da] = tools.module_index_to_qm(module_index)\n",
- " if k_da in retrieved_constants:\n",
- " print(\n",
- " f\"Constant for {k_da} already in calibration_metadata.yml, won't query again.\")\n",
- " continue\n",
- "\n",
- " inp.append((k_da, module_index))"
+ "with multiprocessing.Pool(processes=nmods) as pool:\n",
+ " results = pool.starmap(retrieve_constants, inp)"
]
},
{
diff --git a/notebooks/LPD/LPD_Correct_Fast.ipynb b/notebooks/LPD/LPD_Correct_Fast.ipynb
index 1364bbbf8f4c31ddba3635f9113caf89b576b07c..4b8dfa4576a2ce551562609a818c18da4063ee6e 100644
--- a/notebooks/LPD/LPD_Correct_Fast.ipynb
+++ b/notebooks/LPD/LPD_Correct_Fast.ipynb
@@ -104,10 +104,6 @@
"\n",
"from extra_data.components import LPD1M\n",
"\n",
-<<<<<<< HEAD
- "from cal_tools.calcat_interface import CalCatError, LPD_CalibrationData\n",
-=======
->>>>>>> remove LPD changes
"from cal_tools.lpdalgs import correct_lpd_frames\n",
"from cal_tools.tools import CalibrationMetadata, calcat_creation_time\n",
"from cal_tools.files import DataFile\n",
@@ -223,8 +219,6 @@
"metadata": {},
"outputs": [],
"source": [
-<<<<<<< HEAD
-=======
"metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
"# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
"const_yaml = metadata.setdefault(\"retrieved-constants\", {})"
@@ -236,7 +230,6 @@
"metadata": {},
"outputs": [],
"source": [
->>>>>>> remove LPD changes
"const_data = {}\n",
"const_load_mp = psh.ProcessContext(num_workers=24)\n",
"\n",
@@ -248,15 +241,6 @@
"\n",
" dtype = np.uint32 if calibration_name.startswith('BadPixels') else np.float32\n",
"\n",
-<<<<<<< HEAD
-<<<<<<< HEAD
- " const_data[(da, calibration_name)] = dict(\n",
- " path=Path(ccv['path']),\n",
- " dataset=ccv['dataset'],\n",
- " data=const_load_mp.alloc(shape=(256, 256, mem_cells, 3), dtype=dtype)\n",
- " )\n",
- "else: # Retrieve constants from CALCAT.\n",
-=======
" const_data[(da, calibration_name)] = dict(\n",
" path=Path(ccv['file-path']),\n",
" dataset=ccv['dataset-name'],\n",
@@ -289,7 +273,6 @@
" dict(parameter_id=25, value=category) # category\n",
" ]\n",
"\n",
->>>>>>> remove LPD changes
" print('Querying calibration database', end='', flush=True)\n",
" start = perf_counter()\n",
" for calibrations, condition in [\n",
@@ -299,12 +282,8 @@
" resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(\n",
" client, karabo_id, list(calibrations.keys()),\n",
" {'parameters_conditions_attributes': condition},\n",
-<<<<<<< HEAD
" karabo_da='', event_at=creation_time.isoformat()\n",
" )\n",
-=======
- " karabo_da='', event_at=creation_time.isoformat(), snapshot_at=None)\n",
->>>>>>> remove LPD changes
"\n",
" if not resp['success']:\n",
" raise RuntimeError(resp)\n",
@@ -323,63 +302,7 @@
" )\n",
" print('.', end='', flush=True)\n",
" \n",
-<<<<<<< HEAD
-=======
- "else:\n",
- " lpd_cal = LPD_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " modules=karabo_da,\n",
- " sensor_bias_voltage=bias_voltage,\n",
- " memory_cells=mem_cells,\n",
- " feedback_capacitor=capacitor,\n",
- " source_energy=photon_energy,\n",
- " category=category,\n",
- " event_at=creation_time,\n",
- " snapshot_at=creation_time,\n",
- " client=client,\n",
- " )\n",
- " const_data = lpd_cal.ndarray_map()\n",
- "\n",
- "# Validate the constants availability and raise/warn correspondingly. \n",
- "for mod, calibrations in const_data.items():\n",
- " missing_dark_constants = set(\n",
- " c for c in [\"Offset\", \"BadPixelsDark\"] if c not in calibrations.keys())\n",
- " missing_gain_constants = set(\n",
- " c for c in [\"BadPixelsFF\", \"GainAmpMap\", \"FFMap\", \"RelativeGain\"] if c not in calibrations.keys()) # noqa\n",
- " if missing_dark_constants:\n",
- " raise KeyError(\n",
- " f\"Dark constants {missing_dark_constants} are not available for correction. Module: {mod}\") # noqa\n",
- " if missing_gain_constants:\n",
- " warning(\n",
- " f\"Gain constants {missing_gain_constants} were not retrieved. Module: {mod}\")\n",
- " if calibrations[\"BadPixelsDark\"].dtype != np.uint32: # Old LPD constants are stored as float32.\n",
- " calibrations[\"BadPixelsDark\"] = calibrations[\"BadPixelsDark\"].astype(np.uint32)\n",
->>>>>>> add missing client input args
- "total_time = perf_counter() - start\n",
- "print(f'{total_time:.1f}s')"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "def load_constant_dataset(wid, index, const_descr):\n",
- " ccv_entry = const_data[const_descr]\n",
- " \n",
- " with h5py.File(cal_db_root / ccv_entry['path'], 'r') as fp:\n",
- " fp[ccv_entry['dataset'] + '/data'].read_direct(ccv_entry['data'])\n",
- " \n",
- " print('.', end='', flush=True)\n",
- "\n",
- "print('Loading calibration data', end='', flush=True)\n",
- "start = perf_counter()\n",
- "const_load_mp.map(load_constant_dataset, list(const_data.keys()))\n",
-=======
->>>>>>> remove LPD changes
"total_time = perf_counter() - start\n",
- "\n",
"print(f'{total_time:.1f}s')"
]
},
@@ -722,9 +645,9 @@
],
"metadata": {
"kernelspec": {
- "display_name": ".cal2_venv",
+ "display_name": "pycal",
"language": "python",
- "name": "python3"
+ "name": "pycal"
},
"language_info": {
"codemirror_mode": {
@@ -736,12 +659,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.11 (default, Jul 2 2021, 14:23:46) \n[GCC 4.8.5 20150623 (Red Hat 4.8.5-44)]"
- },
- "vscode": {
- "interpreter": {
- "hash": "8817b9fbc45d4ec68a62bc54fa52c1c5ac00d5619f7fe87d01def1798ea4889e"
- }
+ "version": "3.8.11"
}
},
"nbformat": 4,
diff --git a/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
index 54b59af989660bf95979b0d60549a6705a9382a3..7dfde91a4fdec82803f954e386b617f347b23008 100644
--- a/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
+++ b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
@@ -122,6 +122,28 @@
" dict(parameter_id=14, value=256), # Pixels Y\n",
"]\n",
"\n",
+ "illuminated_calibrations = {\n",
+ " 20: 'BadPixelsFF',\n",
+ " 42: 'GainAmpMap',\n",
+ " 43: 'FFMap',\n",
+ " 44: 'RelativeGain',\n",
+ "}\n",
+ "\n",
+ "illuminated_condition = dark_condition.copy()\n",
+ "illuminated_condition += [\n",
+ " dict(parameter_id=3, value=photon_energy), # Source energy\n",
+ " dict(parameter_id=25, value=category) # category\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "const_data = {}\n",
+ "\n",
"print('Querying calibration database', end='', flush=True)\n",
"start = perf_counter()\n",
"for k_da in karabo_da:\n",
@@ -140,31 +162,27 @@
" {'parameters_conditions_attributes': condition},\n",
" karabo_da=k_da, event_at=creation_time.isoformat())\n",
"\n",
- "lpd_cal_metadata = lpd_cal.metadata()\n",
- "\n",
- "# Validate the constants availability and raise/warn correspondingly. \n",
- "for mod, ccv_dict in lpd_cal_metadata.items():\n",
- " missing_dark_constants = set(\n",
- " c for c in [\"Offset\", \"BadPixelsDark\"] if c not in ccv_dict.keys())\n",
- " missing_gain_constants = set(\n",
- " c for c in [\"BadPixelsFF\", \"GainAmpMap\", \"FFMap\", \"RelativeGain\"] if c not in ccv_dict.keys()) # noqa\n",
- " if missing_dark_constants:\n",
- " raise KeyError(\n",
- " f\"Dark constants {missing_dark_constants} are not available for correction. Module: {mod}\") # noqa\n",
- " if missing_gain_constants:\n",
- " warning(\n",
- " f\"Gain constants {missing_gain_constants} were not retrieved. Module: {mod}\")\n",
- "\n",
- "for mod, ccv_dict in lpd_cal_metadata.items():\n",
- " mdata_dict = {\"constants\": dict()}\n",
- " for cname, ccv_metadata in ccv_dict.items():\n",
- " mdata_dict[\"constants\"][cname] = {\n",
- " \"path\": str(lpd_cal.caldb_root / ccv_metadata[\"path\"]),\n",
- " \"dataset\": ccv_metadata[\"dataset\"],\n",
- " \"creation-time\": ccv_metadata[\"begin_validity_at\"],\n",
- " }\n",
- " mdata_dict[\"physical-name\"] = ccv_metadata[\"physical_name\"]\n",
- " retrieved_constants[mod] = mdata_dict\n",
+ " if not resp[\"success\"]:\n",
+ " print(f\"ERROR: Constants {list(calibrations.values())} \"\n",
+ " f\"were not retrieved, {resp['app_info']}\")\n",
+ " for cname in calibrations.values():\n",
+ " const_mdata[cname] = dict()\n",
+ " const_mdata[cname][\"file-path\"] = None\n",
+ " const_mdata[cname][\"dataset-name\"] = None\n",
+ " const_mdata[cname][\"creation-time\"] = None \n",
+ " continue\n",
+ "\n",
+ " for ccv in resp[\"data\"]:\n",
+ " cc = ccv['calibration_constant']\n",
+ " cname = calibrations[cc['calibration_id']]\n",
+ " const_mdata[cname] = dict()\n",
+ " const_mdata[cname][\"file-path\"] = str(Path(ccv['path_to_file']) / ccv['file_name'])\n",
+ " const_mdata[cname][\"dataset-name\"] = ccv['data_set_name']\n",
+ " const_mdata[cname][\"creation-time\"] = ccv['begin_at']\n",
+ " pdu = ccv['physical_detector_unit']['physical_name']\n",
+ "\n",
+ " print('.', end='', flush=True)\n",
+ " retrieved_constants[k_da][\"physical-detector-unit\"] = pdu\n",
"metadata.save()\n",
"\n",
"total_time = perf_counter() - start\n",
@@ -175,9 +193,9 @@
],
"metadata": {
"kernelspec": {
- "display_name": "cal4_venv",
+ "display_name": "Python 3.8.11 ('.cal4_venv')",
"language": "python",
- "name": "cal4_venv"
+ "name": "python3"
},
"language_info": {
"codemirror_mode": {
@@ -189,7 +207,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.11 (default, Jul 2 2021, 14:23:46) \n[GCC 4.8.5 20150623 (Red Hat 4.8.5-44)]"
+ "version": "3.8.11"
},
"orig_nbformat": 4,
"vscode": {
diff --git a/notebooks/ePix100/Correction_ePix100_NBC.ipynb b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
index b7ef354bff5d828dad118edfd7fde16ebf7cd12a..03b9bbded634fbf2c8ff1f18914ac062c8cf07af 100644
--- a/notebooks/ePix100/Correction_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
@@ -895,7 +895,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.11 (default, Jul 2 2021, 14:23:46) \n[GCC 4.8.5 20150623 (Red Hat 4.8.5-44)]"
+ "version": "3.8.11"
},
"latex_envs": {
"LaTeX_envs_menu_present": true,
@@ -913,11 +913,6 @@
"latex_user_defs": false,
"report_style_numbering": false,
"user_envs_cfg": false
- },
- "vscode": {
- "interpreter": {
- "hash": "8817b9fbc45d4ec68a62bc54fa52c1c5ac00d5619f7fe87d01def1798ea4889e"
- }
}
},
"nbformat": 4,
diff --git a/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb b/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
index 9c19e81d0e87cd690b1bf81250b83e7bf1d43995..591d3b84267c154a26b662327c34485b41172a74 100644
--- a/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
+++ b/notebooks/ePix100/ePix100_retrieve_constants_precorrection.ipynb
@@ -201,9 +201,9 @@
],
"metadata": {
"kernelspec": {
- "display_name": "cal2_venv",
+ "display_name": "Python 3.8.11 ('.cal4_venv')",
"language": "python",
- "name": "cal2_venv"
+ "name": "python3"
},
"language_info": {
"codemirror_mode": {
@@ -215,7 +215,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.11 (default, Jul 2 2021, 14:23:46) \n[GCC 4.8.5 20150623 (Red Hat 4.8.5-44)]"
+ "version": "3.8.11"
},
"orig_nbformat": 4,
"vscode": {