From 0a64f5a39813f94fc982649dd9ff8000d7be0105 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Tue, 18 Oct 2022 14:55:11 +0200
Subject: [PATCH] MR comments and update ROI part
---
...Jungfrau_Gain_Correct_and_Verify_NBC.ipynb | 37 ++++++++++---------
src/cal_tools/files.py | 2 +-
2 files changed, 20 insertions(+), 19 deletions(-)
diff --git a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
index 3df01b98e..30f66d5be 100644
--- a/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
+++ b/notebooks/Jungfrau/Jungfrau_Gain_Correct_and_Verify_NBC.ipynb
@@ -409,7 +409,7 @@
" rois_defined = 0\n",
" module_no = int(karabo_da[-2:])\n",
" params_source = f'{karabo_id}/ROIPROC/{karabo_da}'\n",
- " rois_source = f'{params_source}:output/data'\n",
+ " rois_source = f'{params_source}:output'\n",
" \n",
" for i in range(len(roi_definitions) // 6):\n",
" roi_module, a1, a2, b1, b2, mean_axis = roi_definitions[i*6 : (i+1)*6]\n",
@@ -419,22 +419,22 @@
" roi_data = data_corr[..., a1:a2, b1:b2].mean(\n",
" axis=mean_axis, where=(mask_corr[..., a1:a2, b1:b2] == 0)\n",
" )\n",
- " ofile.create_dataset(\n",
- " f'INSTRUMENT/{rois_source}/roi{rois_defined}/data',\n",
- " data=roi_data\n",
- " )\n",
- " ofile.require_group(f'CONTROL/{params_source}')\n",
- " params_grp = ofile.create_group(f'RUN/{params_source}/roi{rois_defined}')\n",
- " params_grp['region'] = np.array([[a1, a2, b1, b2]])\n",
- " params_grp['reduce_axis'] = np.array([mean_axis])\n",
+ "\n",
+ " # Create Instrument section to later add corrected datasets.\n",
+ " outp_source = ofile.create_instrument_source(rois_source)\n",
+ " outp_source.create_key(f'data.roi{rois_defined}.data', data=roi_data)\n",
+ "\n",
+ " ctrl_source = ofile.create_control_source(params_source)\n",
+ " ctrl_source.create_run_key('region', np.array([[a1, a2, b1, b2]]))\n",
+ " ctrl_source.create_run_key('reduce_axis', np.array([mean_axis]))\n",
" \n",
" if rois_defined:\n",
" # Copy the index for the new source\n",
+ " # Create count/first datasets at INDEX source.\n",
" ofile.copy(f'INDEX/{karabo_id}/DET/{karabo_da}:daqOutput/data',\n",
" f'INDEX/{rois_source}')\n",
" ntrains = ofile['INDEX/trainId'].shape[0]\n",
- " ofile.create_dataset(f'INDEX/{params_source}/count', shape=(ntrains,), dtype=np.uint64)\n",
- " ofile.create_dataset(f'INDEX/{params_source}/first', shape=(ntrains,), dtype=np.uint64)\n",
+ " ctrl_source.create_index(ntrains)\n",
" \n",
" # Add the new source to the list in METADATA\n",
" if 'dataSourceId' in ofile['METADATA']:\n",
@@ -493,9 +493,9 @@
" print(f\"\\t- WARNING: No trains to correct for {sequence_file.name}: \"\n",
" f\"Skipping the processing of this file.\")\n",
" continue\n",
- " elif len(all_train_ids) - corr_ntrains != 0:\n",
+ " elif len(all_train_ids) != corr_ntrains:\n",
" print(f\"{sequence_file.name} has {len(seq_dc_adc.train_ids) - corr_ntrains} \"\n",
- " \"trains with empty data.\")\n",
+ " \"trains with missing data.\")\n",
"\n",
" # For testing, limit corrected trains. i.e. Getting output faster.\n",
" if limit_trains > 0:\n",
@@ -542,21 +542,22 @@
"\n",
" step_timer.start()\n",
"\n",
- " # Create CORR files and add corrected data sources.\n",
+ " # Create CORR files and add corrected data sections.\n",
" sel_trains = np.isin(all_train_ids, seq_dc.train_ids)\n",
" image_counts = seq_dc[instrument_src_kda, \"data.adc\"].data_counts(labelled=False)\n",
"\n",
" with DataFile(out_file, 'w') as outp_file:\n",
- " # Create INDEX source.\n",
+ " # Create INDEX datasets.\n",
" outp_file.create_index(\n",
" train_ids=seq_dc.train_ids,\n",
" timestamps=seq_dc.files[0].file['INDEX/timestamp'][sel_trains],\n",
" flags=seq_dc.files[0].validity_flag[sel_trains])\n",
"\n",
- " # Create METDATA source\n",
- " outp_file.create_metadata(like=seq_dc, instrument_channels=(f'{instrument_src_kda}/data',))\n",
+ " # Create METDATA datasets\n",
+ " outp_file.create_metadata(\n",
+ " like=seq_dc, instrument_channels=(f'{instrument_src_kda}/data',))\n",
"\n",
- " # Create Instrument source to add corrected datasets.\n",
+ " # Create Instrument section to later add corrected datasets.\n",
" outp_source = outp_file.create_instrument_source(instrument_src_kda)\n",
"\n",
" # Create count/first datasets at INDEX source.\n",
diff --git a/src/cal_tools/files.py b/src/cal_tools/files.py
index 5c82f953f..35c4f3cd6 100644
--- a/src/cal_tools/files.py
+++ b/src/cal_tools/files.py
@@ -450,7 +450,7 @@ class ControlSource(h5py.Group):
self.__run_group.create_dataset(
f'{key}/value', data=value, shape=shape, dtype=dtype)
self.__run_group.create_dataset(
- f'{key}/timestamp', data=timestamp, shape=shape, dtype=np.uint64)
+ f'{key}/timestamp', data=timestamp, dtype=np.uint64)
def create_index(self, num_trains):
"""Create source-specific INDEX datasets.
--
GitLab