diff --git a/notebooks/LPD/LPD_Correct_Fast.ipynb b/notebooks/LPD/LPD_Correct_Fast.ipynb
index c7d23ccdd19bd1b46174894a9a5c16ac6f86b684..1364bbbf8f4c31ddba3635f9113caf89b576b07c 100644
--- a/notebooks/LPD/LPD_Correct_Fast.ipynb
+++ b/notebooks/LPD/LPD_Correct_Fast.ipynb
@@ -26,7 +26,7 @@
"metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate.\n",
"sequences = [-1] # Sequences to correct, use [-1] for all\n",
"modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty\n",
- "karabo_da = ['all'] # Data aggregators names to correct, use [''] for all\n",
+ "karabo_da = [''] # Data aggregators names to correct, use [''] for all\n",
"run = 10 # run to process, required\n",
"\n",
"# Source parameters\n",
@@ -104,7 +104,10 @@
"\n",
"from extra_data.components import LPD1M\n",
"\n",
+<<<<<<< HEAD
"from cal_tools.calcat_interface import CalCatError, LPD_CalibrationData\n",
+=======
+>>>>>>> remove LPD changes
"from cal_tools.lpdalgs import correct_lpd_frames\n",
"from cal_tools.tools import CalibrationMetadata, calcat_creation_time\n",
"from cal_tools.files import DataFile\n",
@@ -135,20 +138,16 @@
"cal_db_root = Path(cal_db_root)\n",
"\n",
"metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
- "retrieved_constants = metadata.setdefault(\"retrieved-constants\", {})\n",
"\n",
"creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
"print(f'Using {creation_time.isoformat()} as creation time')\n",
"\n",
"# Pick all modules/aggregators or those selected.\n",
- "if karabo_da == ['all']:\n",
- " if modules == [-1]:\n",
+ "if not karabo_da or karabo_da == ['']:\n",
+ " if not modules or modules == [-1]:\n",
" modules = list(range(16))\n",
"\n",
" karabo_da = [f'LPD{i:02d}' for i in modules]\n",
- "else:\n",
- " modules = [int(x[-2:]) for x in karabo_da]\n",
" \n",
"# Pick all sequences or those selected.\n",
"if not sequences or sequences == [-1]:\n",
@@ -198,17 +197,6 @@
"# Obtain and prepare calibration constants"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
- "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
- "const_yaml = metadata.setdefault(\"retrieved-constants\", {})"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -232,11 +220,23 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {
- "tags": []
- },
+ "metadata": {},
+ "outputs": [],
+ "source": [
+<<<<<<< HEAD
+=======
+ "metadata = CalibrationMetadata(metadata_folder or out_folder)\n",
+ "# Constant paths & timestamps are saved under retrieved-constants in calibration_metadata.yml\n",
+ "const_yaml = metadata.setdefault(\"retrieved-constants\", {})"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
"outputs": [],
"source": [
+>>>>>>> remove LPD changes
"const_data = {}\n",
"const_load_mp = psh.ProcessContext(num_workers=24)\n",
"\n",
@@ -248,6 +248,7 @@
"\n",
" dtype = np.uint32 if calibration_name.startswith('BadPixels') else np.float32\n",
"\n",
+<<<<<<< HEAD
<<<<<<< HEAD
" const_data[(da, calibration_name)] = dict(\n",
" path=Path(ccv['path']),\n",
@@ -255,6 +256,40 @@
" data=const_load_mp.alloc(shape=(256, 256, mem_cells, 3), dtype=dtype)\n",
" )\n",
"else: # Retrieve constants from CALCAT.\n",
+=======
+ " const_data[(da, calibration_name)] = dict(\n",
+ " path=Path(ccv['file-path']),\n",
+ " dataset=ccv['dataset-name'],\n",
+ " data=const_load_mp.alloc(shape=(256, 256, mem_cells, 3), dtype=dtype)\n",
+ " )\n",
+ "else: # Retrieve constants from CALCAT.\n",
+ " dark_calibrations = {\n",
+ " 1: 'Offset', # np.float32\n",
+ " 14: 'BadPixelsDark' # should be np.uint32, but is np.float64\n",
+ " }\n",
+ "\n",
+ " dark_condition = [\n",
+ " dict(parameter_id=1, value=bias_voltage), # Sensor bias voltage\n",
+ " dict(parameter_id=7, value=mem_cells), # Memory cells\n",
+ " dict(parameter_id=15, value=capacitor), # Feedback capacitor\n",
+ " dict(parameter_id=13, value=256), # Pixels X\n",
+ " dict(parameter_id=14, value=256), # Pixels Y\n",
+ " ]\n",
+ "\n",
+ " illuminated_calibrations = {\n",
+ " 20: 'BadPixelsFF', # np.uint32\n",
+ " 42: 'GainAmpMap', # np.float32\n",
+ " 43: 'FFMap', # np.float32\n",
+ " 44: 'RelativeGain' # np.float32\n",
+ " }\n",
+ "\n",
+ " illuminated_condition = dark_condition.copy()\n",
+ " illuminated_condition += [\n",
+ " dict(parameter_id=3, value=photon_energy), # Source energy\n",
+ " dict(parameter_id=25, value=category) # category\n",
+ " ]\n",
+ "\n",
+>>>>>>> remove LPD changes
" print('Querying calibration database', end='', flush=True)\n",
" start = perf_counter()\n",
" for calibrations, condition in [\n",
@@ -264,8 +299,12 @@
" resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(\n",
" client, karabo_id, list(calibrations.keys()),\n",
" {'parameters_conditions_attributes': condition},\n",
+<<<<<<< HEAD
" karabo_da='', event_at=creation_time.isoformat()\n",
" )\n",
+=======
+ " karabo_da='', event_at=creation_time.isoformat(), snapshot_at=None)\n",
+>>>>>>> remove LPD changes
"\n",
" if not resp['success']:\n",
" raise RuntimeError(resp)\n",
@@ -284,6 +323,7 @@
" )\n",
" print('.', end='', flush=True)\n",
" \n",
+<<<<<<< HEAD
=======
"else:\n",
" lpd_cal = LPD_CalibrationData(\n",
@@ -319,6 +359,30 @@
"print(f'{total_time:.1f}s')"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def load_constant_dataset(wid, index, const_descr):\n",
+ " ccv_entry = const_data[const_descr]\n",
+ " \n",
+ " with h5py.File(cal_db_root / ccv_entry['path'], 'r') as fp:\n",
+ " fp[ccv_entry['dataset'] + '/data'].read_direct(ccv_entry['data'])\n",
+ " \n",
+ " print('.', end='', flush=True)\n",
+ "\n",
+ "print('Loading calibration data', end='', flush=True)\n",
+ "start = perf_counter()\n",
+ "const_load_mp.map(load_constant_dataset, list(const_data.keys()))\n",
+=======
+>>>>>>> remove LPD changes
+ "total_time = perf_counter() - start\n",
+ "\n",
+ "print(f'{total_time:.1f}s')"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -588,7 +652,8 @@
"ExecuteTime": {
"end_time": "2018-11-13T18:24:57.547563Z",
"start_time": "2018-11-13T18:24:56.995005Z"
- }
+ },
+ "scrolled": false
},
"outputs": [],
"source": [
diff --git a/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
index 37468ba95bb853a8631f491c5d48cbf8968b0b4b..54b59af989660bf95979b0d60549a6705a9382a3 100644
--- a/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
+++ b/notebooks/LPD/LPD_retrieve_constants_precorrection.ipynb
@@ -22,7 +22,7 @@
"out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/remove/LPD_test\" # the folder to output to, required\n",
"metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate.\n",
"modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty\n",
- "karabo_da = ['all'] # Data aggregators names to correct, use [''] for all\n",
+ "karabo_da = [''] # Data aggregators names to correct, use [''] for all\n",
"run = 10 # run to process, required\n",
"\n",
"# Source parameters\n",
@@ -45,17 +45,18 @@
"metadata": {},
"outputs": [],
"source": [
- "from logging import warning\n",
"from pathlib import Path\n",
+ "from time import perf_counter\n",
+ "\n",
+ "from calibration_client import CalibrationClient\n",
+ "from calibration_client.modules import CalibrationConstantVersion\n",
"\n",
- "from cal_tools.calcat_interface import LPD_CalibrationData\n",
"from cal_tools.tools import (\n",
" CalibrationMetadata,\n",
" calcat_creation_time,\n",
+ " save_constant_metadata,\n",
")\n",
- "from calibration_client import CalibrationClient\n",
- "from cal_tools.restful_config import restful_config\n",
- "from cal_tools.step_timing import StepTimer"
+ "from cal_tools.restful_config import restful_config"
]
},
{
@@ -75,21 +76,11 @@
"print(f'Using {creation_time.isoformat()} as creation time')\n",
"\n",
"# Pick all modules/aggregators or those selected.\n",
- "if karabo_da == ['all']:\n",
- " if modules == [-1]:\n",
+ "if not karabo_da or karabo_da == ['']:\n",
+ " if not modules or modules == [-1]:\n",
" modules = list(range(16))\n",
- " karabo_da = [f'LPD{i:02d}' for i in modules]\n",
- "else:\n",
- " modules = [int(x[-2:]) for x in karabo_da]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "step_timer = StepTimer()"
+ "\n",
+ " karabo_da = [f'LPD{i:02d}' for i in modules] "
]
},
{
@@ -118,23 +109,17 @@
"metadata": {},
"outputs": [],
"source": [
- "step_timer.start()\n",
- "lpd_cal = LPD_CalibrationData(\n",
- " detector_name=karabo_id,\n",
- " modules=karabo_da,\n",
- " sensor_bias_voltage=bias_voltage,\n",
- " memory_cells=mem_cells,\n",
- " feedback_capacitor=capacitor,\n",
- " source_energy=photon_energy,\n",
- " category=category,\n",
- " event_at=creation_time,\n",
- " snapshot_at=creation_time,\n",
- " client=client,\n",
- ")\n",
- "constant_names = [\n",
- " \"Offset\", \"BadPixelsDark\",\n",
- " \"BadPixelsFF\", \"GainAmpMap\",\n",
- " \"FFMap\", \"RelativeGain\",\n",
+ "dark_calibrations = {\n",
+ " 1: 'Offset',\n",
+ " 14: 'BadPixelsDark',\n",
+ "}\n",
+ "\n",
+ "dark_condition = [\n",
+ " dict(parameter_id=1, value=bias_voltage), # Sensor bias voltage\n",
+ " dict(parameter_id=7, value=mem_cells), # Memory cells\n",
+ " dict(parameter_id=15, value=capacitor), # Feedback capacitor\n",
+ " dict(parameter_id=13, value=256), # Pixels X\n",
+ " dict(parameter_id=14, value=256), # Pixels Y\n",
"]\n",
"\n",
"print('Querying calibration database', end='', flush=True)\n",
@@ -182,7 +167,9 @@
" retrieved_constants[mod] = mdata_dict\n",
"metadata.save()\n",
"\n",
- "step_timer.done_step(f\"Stored retrieved constants in {metadata.filename}.\")"
+ "total_time = perf_counter() - start\n",
+ "print(f'{total_time:.1f}s')\n",
+ "print(f\"Stored retrieved constants in {metadata.filename}\")"
]
}
],
diff --git a/notebooks/ePix100/Correction_ePix100_NBC.ipynb b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
index a18f7c655b17f6e643761fc8300a35eaf6579e11..b7ef354bff5d828dad118edfd7fde16ebf7cd12a 100644
--- a/notebooks/ePix100/Correction_ePix100_NBC.ipynb
+++ b/notebooks/ePix100/Correction_ePix100_NBC.ipynb
@@ -881,9 +881,9 @@
],
"metadata": {
"kernelspec": {
- "display_name": ".cal2_venv",
+ "display_name": "cal_venv",
"language": "python",
- "name": "python3"
+ "name": "cal_venv"
},
"language_info": {
"codemirror_mode": {