diff --git a/notebooks/LPDMini/LPD_Mini_Char_Darks_NBC.ipynb b/notebooks/LPDMini/LPD_Mini_Char_Darks_NBC.ipynb
index 86566a2e0c890981fdea1b5712ef10e4edda6fa9..d9f7d7a540157b37f9b35a54afb90b0648062013 100644
--- a/notebooks/LPDMini/LPD_Mini_Char_Darks_NBC.ipynb
+++ b/notebooks/LPDMini/LPD_Mini_Char_Darks_NBC.ipynb
@@ -32,6 +32,7 @@
"karabo_id = \"FXE_DET_LPD_MINI\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
"source_name = \"{}/DET/0CH0:xtdf\"\n",
+ "control_source_name = \"{}/FPGA/FEM\"\n",
"\n",
"creation_time = \"\" # Override the timestamp taken from the run (format '2023-03-30 15:04:31')\n",
"cal_db_interface = \"tcp://max-exfl016:8015#8025\" # the database interface to use\n",
@@ -40,7 +41,8 @@
"db_output = False # output constants to database\n",
"\n",
"capacitor_setting = 5 # capacitor_setting for which data was taken \n",
- "bias_voltage = 250 # detector bias voltage\n",
+ "bias_voltage_0 = -1 # bias voltage for minis 1, 3, 5, 7; Setting -1 will read the value from files\n",
+ "bias_voltage_1 = -1 # bias voltage for minis 2, 4, 6, 8; Setting -1 will read the value from files\n",
"thresholds_offset_sigma = 3. # bad pixel relative threshold in terms of n sigma offset\n",
"thresholds_offset_hard = [400, 1500] # bad pixel hard threshold\n",
"thresholds_noise_sigma = 7. # bad pixel relative threshold in terms of n sigma noise\n",
@@ -123,6 +125,16 @@
"creation_time = calcat_creation_time(in_folder, run_high, creation_time)\n",
"print(f\"Using {creation_time} as creation time\")\n",
"\n",
+ "source_name = source_name.format(karabo_id)\n",
+ "control_source_name = control_source_name.format(karabo_id)\n",
+ "\n",
+ "if -1 in {bias_voltage_0, bias_voltage_1}:\n",
+ " run_data = RunDirectory(os.path.join(in_folder, f\"r{run_high:04d}\"))\n",
+ " if bias_voltage_0 == -1:\n",
+ " bias_voltage_0 = run_data[control_source_name, 'sensorBiasVoltage0'].as_single_value(atol=5.)\n",
+ " if bias_voltage_1 == -1:\n",
+ " bias_voltage_1 = run_data[control_source_name, 'sensorBiasVoltage1'].as_single_value(atol=5.)\n",
+ "\n",
"run, prop, seq = run_prop_seq_from_path(in_folder)\n",
"\n",
"display(Markdown('## Evaluated parameters'))\n",
@@ -133,7 +145,7 @@
"print(f\"Using DB: {db_output}\")\n",
"print(f\"Input: {in_folder}\")\n",
"print(f\"Output: {out_folder}\")\n",
- "print(f\"Bias voltage: {bias_voltage}V\")"
+ "print(f\"Bias voltage: {bias_voltage_0}V & {bias_voltage_1}V\")"
]
},
{
@@ -187,7 +199,7 @@
"# the actual characterization\n",
"def characterize_module(run_path, gg):\n",
"\n",
- " run = RunDirectory(run_path)\n",
+ " run = RunDirectory(run_path, parallelize=False)\n",
" det_source = source_name.format(karabo_id)\n",
" data = run[det_source, 'image.data'].drop_empty_trains()\n",
" data = data[skip_first_ntrains : skip_first_ntrains + ntrains]\n",
@@ -347,6 +359,8 @@
" else:\n",
" mem_cell_order = None\n",
"\n",
+ " # mod_num is from 0 to 7, so b_v_0 applies to even numbers\n",
+ " bias_voltage = bias_voltage_0 if (mod_num % 2 == 0) else bias_voltage_1\n",
" condition = Conditions.Dark.LPD(\n",
" memory_cells=mem_cells,\n",
" pixels_x=256,\n",
@@ -434,6 +448,9 @@
" dconst = getattr(Constants.LPD, const_name)()\n",
" dconst.data = const_dict[mod_num]\n",
"\n",
+ " # mod_num is from 0 to 7, so b_v_0 applies to even numbers\n",
+ " bias_voltage = bias_voltage_0 if (mod_num % 2 == 0) else bias_voltage_1\n",
+ " \n",
" # set the operating condition\n",
" condition = Conditions.Dark.LPD(\n",
" memory_cells=mem_cells,\n",
diff --git a/notebooks/LPDMini/LPD_Mini_Correct.ipynb b/notebooks/LPDMini/LPD_Mini_Correct.ipynb
index 59b1471db62fd5200786c7365751013757e5c207..c541296565e244793b513c785b892eb0a57f9f13 100644
--- a/notebooks/LPDMini/LPD_Mini_Correct.ipynb
+++ b/notebooks/LPDMini/LPD_Mini_Correct.ipynb
@@ -25,7 +25,7 @@
"out_folder = \"/gpfs/exfel/data/scratch/kluyvert/correct-lpdmini-p4576-r48\" # the folder to output to, required\n",
"metadata_folder = '' # Directory containing calibration_metadata.yml when run by xfel-calibrate.\n",
"sequences = [-1] # Sequences to correct, use [-1] for all\n",
- "modules = [-1] # Modules indices to correct, use [-1] for all, only used when karabo_da is empty\n",
+ "modules = [-1] # Modules indices to correct, use [-1] for all\n",
"karabo_da = [''] # Data aggregators names to correct, use [''] for all\n",
"run = 48 # run to process, required\n",
"\n",
@@ -33,6 +33,7 @@
"karabo_id = 'FXE_DET_LPD_MINI' # Karabo domain for detector.\n",
"input_source = '{karabo_id}/DET/0CH0:xtdf' # Input fast data source.\n",
"output_source = '{karabo_id}/CORR/0CH0:output' # Output fast data source, empty to use same as input.\n",
+ "control_source = '{karabo_id}/FPGA/FEM' # Control source\n",
"\n",
"# CalCat parameters\n",
"creation_time = \"\" # The timestamp to use with Calibration DB. Required Format: \"YYYY-MM-DD hh:mm:ss\" e.g. 2019-07-04 11:02:41\n",
@@ -41,7 +42,8 @@
"cal_db_root = '/gpfs/exfel/d/cal/caldb_store'\n",
"\n",
"# Operating conditions\n",
- "bias_voltage = 250.0 # Detector bias voltage.\n",
+ "bias_voltage_0 = -1 # bias voltage for minis 1, 3, 5, 7; Setting -1 will read the value from files\n",
+ "bias_voltage_1 = -1 # bias voltage for minis 2, 4, 6, 8; Setting -1 will read the value from files\n",
"capacitor = '5pF' # Capacitor setting: 5pF or 50pF\n",
"photon_energy = 9.2 # Photon energy in keV.\n",
"use_cell_order = False # Whether to use memory cell order as a detector condition (not stored for older constants)\n",
@@ -97,6 +99,7 @@
"import extra_geom as xg\n",
"import pasha as psh\n",
"\n",
+ "from cal_tools.calcat_interface import CalCatApi\n",
"from cal_tools.lpdalgs import correct_lpd_frames\n",
"from cal_tools.lpdlib import get_mem_cell_order\n",
"from cal_tools.tools import CalibrationMetadata, calcat_creation_time\n",
@@ -125,6 +128,10 @@
"\n",
"output_source = output_source or input_source\n",
"\n",
+ "input_source = input_source.format(karabo_id=karabo_id)\n",
+ "output_source = output_source.format(karabo_id=karabo_id)\n",
+ "control_source = control_source.format(karabo_id=karabo_id)\n",
+ "\n",
"cal_db_root = Path(cal_db_root)\n",
"\n",
"creation_time = calcat_creation_time(in_folder, run, creation_time)\n",
@@ -133,15 +140,12 @@
"# Pick all modules/aggregators or those selected.\n",
"if not karabo_da or karabo_da == ['']:\n",
" karabo_da = ['LPDMINI00']\n",
- "\n",
- "if not modules or modules == [-1]:\n",
- " modules = list(range(8))\n",
" \n",
"# Pick all sequences or those selected.\n",
"if not sequences or sequences == [-1]:\n",
" do_sequence = lambda seq: True\n",
"else:\n",
- " do_sequence = [int(x) for x in sequences].__contains__ \n",
+ " do_sequence = [int(x) for x in sequences].__contains__\n",
" \n",
"# List of detector sources.\n",
"det_inp_sources = [input_source.format(karabo_id=karabo_id)]\n",
@@ -149,6 +153,22 @@
"mem_cells = 512"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "if -1 in {bias_voltage_0, bias_voltage_1}:\n",
+ " run_data = xd.RunDirectory(Path(in_folder, f\"r{run:04d}\"))\n",
+ " if bias_voltage_0 == -1:\n",
+ " bias_voltage_0 = run_data[control_source, 'sensorBiasVoltage0'].as_single_value(atol=5.)\n",
+ " if bias_voltage_1 == -1:\n",
+ " bias_voltage_1 = run_data[control_source, 'sensorBiasVoltage1'].as_single_value(atol=5.)\n",
+ "\n",
+ "print(f\"Using bias voltages {bias_voltage_0}V & {bias_voltage_1}V\")"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -187,6 +207,28 @@
"# Obtain and prepare calibration constants"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "calcat_client = calibration_client()\n",
+ "calcat = CalCatApi(client=calcat_client)\n",
+ "\n",
+ "# Look up PDUs\n",
+ "detector_id = calcat.detector(karabo_id)['id']\n",
+ "pdus_by_da = calcat.physical_detector_units(detector_id, pdu_snapshot_at=creation_time)\n",
+ "modnos_from_db = {int(x.split('/')[-1]) for x in pdus_by_da}\n",
+ "\n",
+ "if not modules or modules == [-1]:\n",
+ " modules = sorted(modnos_from_db)\n",
+ "else:\n",
+ " modules = sorted(set(modules) & modnos_from_db)\n",
+ "\n",
+ "print(\"Modules to correct:\", modules)"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -206,7 +248,7 @@
"}\n",
"\n",
"base_condition = [\n",
- " dict(parameter_id=1, value=bias_voltage), # Sensor bias voltage\n",
+ " # Bias voltage added below as it differs by module\n",
" dict(parameter_id=15, value=capacitor), # Feedback capacitor\n",
" dict(parameter_id=7, value=mem_cells), # Memory cells\n",
" dict(parameter_id=13, value=256), # Pixels X\n",
@@ -238,47 +280,49 @@
"print('Querying calibration database', end='', flush=True)\n",
"start = perf_counter()\n",
"\n",
- "client = calibration_client()\n",
+ "\n",
"for calibrations, condition in [\n",
" (dark_calibrations, dark_condition),\n",
" (illuminated_calibrations, illuminated_condition)\n",
"]:\n",
- " resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(\n",
- " client, karabo_id, list(calibrations.keys()),\n",
- " {'parameters_conditions_attributes': condition},\n",
- " karabo_da='', event_at=creation_time.isoformat()\n",
- " )\n",
- "\n",
- " if not resp['success']:\n",
- " raise RuntimeError(resp)\n",
- "\n",
- " for ccv in resp['data']:\n",
- " cc = ccv['calibration_constant']\n",
- " karabo_da = ccv['physical_detector_unit']['karabo_da']\n",
- " module_num = int(karabo_da.split('/')[1])\n",
- " if module_num not in modules:\n",
- " continue # Module not selected for correction\n",
- "\n",
- " calibration_name = calibrations[cc['calibration_id']]\n",
+ " for mod_num in modules:\n",
+ " # mod_num is from 0 to 7, so b_v_0 applies to even numbers\n",
+ " bias_voltage = bias_voltage_0 if mod_num % 2 == 0 else bias_voltage_1\n",
+ " condition_w_voltage = [dict(parameter_id=1, value=bias_voltage)] + condition\n",
+ " karabo_da_m = f'{karabo_da[0]}/{mod_num}'\n",
" \n",
- " mod_const_metadata = retrieved_consts.setdefault(karabo_da, {\n",
- " 'physical-name': ccv['physical_detector_unit']['physical_name']\n",
- " })\n",
- " mod_const_metadata.setdefault('constants', {})[calibration_name] = {\n",
- " \"path\": str(cal_db_root / ccv['path_to_file'] / ccv['file_name']),\n",
- " \"dataset\": ccv['data_set_name'],\n",
- " \"creation-time\": ccv[\"begin_validity_at\"],\n",
- " \"ccv-id\": ccv[\"id\"],\n",
- " }\n",
- "\n",
- " dtype = np.uint32 if calibration_name.startswith('BadPixels') else np.float32\n",
- "\n",
- " const_data[(module_num, calibration_name)] = dict(\n",
- " path=Path(ccv['path_to_file']) / ccv['file_name'],\n",
- " dataset=ccv['data_set_name'],\n",
- " data=const_load_mp.alloc(shape=module_const_shape, dtype=dtype)\n",
+ " resp = CalibrationConstantVersion.get_closest_by_time_by_detector_conditions(\n",
+ " calcat_client, karabo_id, list(calibrations.keys()),\n",
+ " {'parameters_conditions_attributes': condition_w_voltage},\n",
+ " karabo_da=karabo_da_m, event_at=creation_time.isoformat()\n",
" )\n",
- " print('.', end='', flush=True)\n",
+ "\n",
+ " if not resp['success']:\n",
+ " raise RuntimeError(resp)\n",
+ "\n",
+ " for ccv in resp['data']:\n",
+ " cc = ccv['calibration_constant']\n",
+ "\n",
+ " calibration_name = calibrations[cc['calibration_id']]\n",
+ "\n",
+ " mod_const_metadata = retrieved_consts.setdefault(karabo_da_m, {\n",
+ " 'physical-name': ccv['physical_detector_unit']['physical_name']\n",
+ " })\n",
+ " mod_const_metadata.setdefault('constants', {})[calibration_name] = {\n",
+ " \"path\": str(cal_db_root / ccv['path_to_file'] / ccv['file_name']),\n",
+ " \"dataset\": ccv['data_set_name'],\n",
+ " \"creation-time\": ccv[\"begin_validity_at\"],\n",
+ " \"ccv-id\": ccv[\"id\"],\n",
+ " }\n",
+ "\n",
+ " dtype = np.uint32 if calibration_name.startswith('BadPixels') else np.float32\n",
+ "\n",
+ " const_data[(mod_num, calibration_name)] = dict(\n",
+ " path=Path(ccv['path_to_file']) / ccv['file_name'],\n",
+ " dataset=ccv['data_set_name'],\n",
+ " data=const_load_mp.alloc(shape=module_const_shape, dtype=dtype)\n",
+ " )\n",
+ " print('.', end='', flush=True)\n",
" \n",
"total_time = perf_counter() - start\n",
"print(f'{total_time:.1f}s')"
@@ -373,7 +417,7 @@
" \n",
" start = perf_counter()\n",
" dc = xd.H5File(inp_path, inc_suspect_trains=False).select('*', 'image.*', require_all=True)\n",
- " inp_source = dc[input_source.format(karabo_id=karabo_id)]\n",
+ " inp_source = dc[input_source]\n",
" open_time = perf_counter() - start\n",
" \n",
" # Load raw data for this file.\n",
@@ -409,13 +453,11 @@
" \n",
" start = perf_counter()\n",
" if (not outp_path.exists() or overwrite) and image_counts.sum() > 0:\n",
- " outp_source_name = output_source.format(karabo_id=karabo_id)\n",
- "\n",
" with DataFile(outp_path, 'w') as outp_file: \n",
" outp_file.create_index(dc.train_ids, from_file=dc.files[0])\n",
- " outp_file.create_metadata(like=dc, instrument_channels=(f'{outp_source_name}/image',))\n",
+ " outp_file.create_metadata(like=dc, instrument_channels=(f'{output_source}/image',))\n",
" \n",
- " outp_source = outp_file.create_instrument_source(outp_source_name)\n",
+ " outp_source = outp_file.create_instrument_source(output_source)\n",
" \n",
" outp_source.create_index(image=image_counts)\n",
" outp_source.create_key('image.cellId', data=in_cell,\n",
@@ -427,6 +469,7 @@
" chunks=(min(chunks_data, out_data.shape[0]), nmods, 32, 256))\n",
" outp_source.create_compressed_key('image.gain', data=out_gain)\n",
" outp_source.create_compressed_key('image.mask', data=out_mask)\n",
+ "\n",
" write_time = perf_counter() - start\n",
" \n",
" total_time = open_time + read_time + correct_time + write_time\n",