From 47d51b87b48cde5230f866d087dfc9e21cc343d3 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Fri, 5 Jul 2024 09:57:30 +0200
Subject: [PATCH] feat[jungfrau][correct]: use new correct data source and link
to old data source - New corrected data source - Link to old data source -
rename instrument*src to input or output *src
---
...Jungfrau_Gain_Correct_and_Verify_NBC.ipynb | 44 +++++++++++--------
1 file changed, 26 insertions(+), 18 deletions(-)
diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
index d9d393e87..7ab1613d4 100644
--- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
@@ -28,7 +28,7 @@
"karabo_id = \"HED_IA1_JF500K4\" # karabo prefix of Jungfrau devices\n",
"karabo_da = ['JNGFR04'] # data aggregators\n",
"receiver_template = \"JNGFR{:02d}\" # Detector receiver template for accessing raw data files. e.g. \"JNGFR{:02d}\"\n",
- "instrument_source_template = '{}/DET/{}:daqOutput' # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput'\n",
+ "input_source_template = '{karabo_id}/DET/{receiver}:daqOutput' # template for source name (filled with karabo_id & receiver_id). e.g. 'SPB_IRDA_JF4M/DET/JNGFR01:daqOutput'\n",
"ctrl_source_template = '{}/DET/CONTROL' # template for control source name (filled with karabo_id_control)\n",
"karabo_id_control = \"\" # if control is on a different ID, set to empty string if it is the same a karabo-id\n",
"\n",
@@ -46,6 +46,7 @@
"chunks_data = 1 # HDF chunk size for pixel data in number of frames.\n",
"wrong_gain_pixels = [-1] # List of 5 values (e.g. [4, 0, 255, 896, 1024]) defining the module number (4 for JNGFR04). And using the indexes of the FEM row [pixel_x_0:pixel_x_1] and column [pixel_y_0:pixel_y_1]. Set to -1 to not pick pixels for gain replacement.\n",
"replace_wrong_gain_value = 0 # Force gain value into the chosen gain [0, 1, or 2] for pixels specified in `wrong_gain_pixels`. This has no effect if wrong_gain_pixels = [-1]\n",
+ "output_source_template = '{karabo_id}/CORR/{receiver}:daqOutput'\n",
"\n",
"# Parameters for retrieving calibration constants\n",
"integration_time = -1 # integration time in us. set to -1 to overwrite by value in file.\n",
@@ -82,6 +83,7 @@
"from logging import warning\n",
"from pathlib import Path\n",
"\n",
+ "import h5py\n",
"import matplotlib\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
@@ -120,12 +122,14 @@
"out_folder = Path(out_folder)\n",
"run_folder = in_folder / f'r{run:04d}'\n",
"run_dc = RunDirectory(run_folder)\n",
- "instrument_src = instrument_source_template.format(karabo_id, receiver_template)\n",
"\n",
+ "output_source_template = output_source_template or input_source_template\n",
+ "\n",
+ "input_src = input_source_template.format(karabo_id, receiver_template)\n",
+ "output_src = output_source_template.format(karabo_id, receiver_template)\n",
"out_folder.mkdir(parents=True, exist_ok=True)\n",
"\n",
"print(f\"Run is: {run}\")\n",
- "print(f\"Instrument H5File source: {instrument_src}\")\n",
"karabo_da = sorted(karabo_da)\n",
"print(f\"Process modules: {karabo_da}\")\n",
"\n",
@@ -602,8 +606,8 @@
"empty_seq = 0\n",
"corrected_files = []\n",
"for local_karabo_da, mapped_files_module in mapped_files.items():\n",
- " instrument_src_kda = instrument_src.format(int(local_karabo_da[-2:]))\n",
- "\n",
+ " input_src_kda = input_src.format(int(local_karabo_da[-2:]))\n",
+ " output_src_kda = output_src.format(int(local_karabo_da[-2:]))\n",
" for sequence_file in mapped_files_module:\n",
" # Save corrected data in an output file with name\n",
" # of corresponding raw sequence file.\n",
@@ -615,7 +619,7 @@
" # the shape for data to later created arrays of the same shape,\n",
" # and number of available trains to correct.\n",
" seq_dc = H5File(sequence_file)\n",
- " seq_dc_adc = seq_dc[instrument_src_kda, \"data.adc\"]\n",
+ " seq_dc_adc = seq_dc[input_src_kda, \"data.adc\"]\n",
" ishape = seq_dc_adc.shape # input shape.\n",
" corr_ntrains = ishape[0] # number of available trains to correct.\n",
" all_train_ids = seq_dc_adc.train_ids\n",
@@ -656,13 +660,13 @@
" step_timer.start()\n",
" # Overwrite seq_dc after eliminating empty trains or/and applying limited images.\n",
" seq_dc = seq_dc.select(\n",
- " instrument_src_kda, \"*\", require_all=True).select_trains(np.s_[:corr_ntrains])\n",
+ " input_src_kda, \"*\", require_all=True).select_trains(np.s_[:corr_ntrains])\n",
"\n",
" # Load raw images(adc), gain, memcells, and frame numbers. \n",
- " data = seq_dc[instrument_src_kda, \"data.adc\"].ndarray()\n",
- " gain = seq_dc[instrument_src_kda, \"data.gain\"].ndarray()\n",
- " memcells = seq_dc[instrument_src_kda, \"data.memoryCell\"].ndarray()\n",
- " frame_number = seq_dc[instrument_src_kda, \"data.frameNumber\"].ndarray()\n",
+ " data = seq_dc[input_src_kda, \"data.adc\"].ndarray()\n",
+ " gain = seq_dc[input_src_kda, \"data.gain\"].ndarray()\n",
+ " memcells = seq_dc[input_src_kda, \"data.memoryCell\"].ndarray()\n",
+ " frame_number = seq_dc[input_src_kda, \"data.frameNumber\"].ndarray()\n",
"\n",
" # Validate that the selected cell id to preview is available in raw data.\n",
" if memory_cells > 1:\n",
@@ -684,15 +688,21 @@
" step_timer.start()\n",
"\n",
" # Create CORR files and add corrected data sections.\n",
- " image_counts = seq_dc[instrument_src_kda, \"data.adc\"].data_counts(labelled=False)\n",
+ " image_counts = seq_dc[input_src_kda, \"data.adc\"].data_counts(labelled=False)\n",
"\n",
" seq_file = seq_dc.files[0] # FileAccess\n",
" with DataFile(out_file, 'w') as outp_file:\n",
" # Create INDEX datasets.\n",
" outp_file.create_index(seq_dc.train_ids, from_file=seq_file)\n",
"\n",
+ " # Create METDATA datasets\n",
+ " outp_file.create_metadata(\n",
+ " like=seq_dc,\n",
+ " sequence=seq_file.sequence,\n",
+ " instrument_channels=sorted({f'{output_src_kda}/data', f'{input_src_kda}/data'})\n",
+ " )\n",
" # Create Instrument section to later add corrected datasets.\n",
- " outp_source = outp_file.create_instrument_source(instrument_src_kda)\n",
+ " outp_source = outp_file.create_instrument_source(output_src_kda)\n",
"\n",
" # Create count/first datasets at INDEX source.\n",
" outp_source.create_index(data=image_counts)\n",
@@ -721,11 +731,9 @@
"\n",
" save_reduced_rois(outp_file, data_corr, mask_corr, local_karabo_da)\n",
"\n",
- " # Create METDATA datasets\n",
- " outp_file.create_metadata(\n",
- " like=seq_dc,\n",
- " sequence=seq_file.sequence,\n",
- " )\n",
+ " if output_src != input_src:\n",
+ " outp_file[f'/INSTRUMENT/{input_src_kda}'] = h5py.SoftLink(f'/INSTRUMENT/{output_src_kda}')\n",
+ " outp_file[f'/INDEX/{input_src_kda}'] = h5py.SoftLink(f'/INDEX/{output_src_kda}')\n",
"\n",
" step_timer.done_step(f'Saving data time.')\n",
"if empty_seq == sum([len(i) for i in mapped_files.values()]):\n",
--
GitLab