Refactor write_file on AgipdCorrections to create indexes using the DataFile...

Refactor write_file on AgipdCorrections to create indexes using the DataFile instead of copying them from the raw data. For compatibility, it still copies some xtdf groups

src/cal_tools/agipdlib.py

@@ -750,11 +750,7 @@ class AgipdCorrections:
         :param ofile_name: Name of output file including path
         :param i_proc: Index of shared memory array
         """
-
-        module_idx = int(file_name.split('/')[-1].split('-')[2][-2:])
-        agipd_base = f'INSTRUMENT/{self.h5_data_path}/'.format(module_idx)
-        idx_base = self.h5_index_path.format(module_idx)
-        data_path = f'{agipd_base}/image'
+        from .files import DataFile
 
         # Obtain a shallow copy of the pointer map to allow for local
         # changes in this method.
@@ -767,23 +763,75 @@ class AgipdCorrections:
         n_img = data_dict['nImg'][0]
         if n_img == 0:
             return
-        trains = data_dict['trainId'][:n_img]
-
-        # Re-cast fields in-place, i.e. using the same memory region.
-        for field, dtype in self.recast_image_fields.items():
-            data_dict[field] = cast_array_inplace(data_dict[field], dtype)
-
-        with h5py.File(ofile_name, "w") as outfile:
-            # Copy any other data from the input file.
-            # This includes indexes, so it's important that the corrected data
-            # we write is aligned with the raw data.
-            with h5py.File(file_name, "r") as infile:
-                self.copy_and_sanitize_non_cal_data(
-                    infile, outfile, agipd_base, idx_base, trains
-                )
+
+        dc = H5File(file_name)
+
+        # make index for corrected images
+        trains, count = np.unique(data_dict['trainId'][:n_img],
+                                  return_counts=True)
+
+        # backward compatibility BEGIN
+        # make index the samse as in raw file
+        raw_trains = dc.files[0].train_ids  # zero trainId are removed
+        image_count = np.zeros(len(raw_trains), dtype=int)
+        idx = np.searchsorted(raw_trains, trains,
+                              sorter=np.argsort(raw_trains))
+        image_count[idx] = count
+        # backward compatibility END
+
+        # parse filename and get parameters
+        out_folder, fname = os.path.split(ofile_name)
+        tokens = os.path.splitext(fname)[0].split('-')
+        runno = int(tokens[1][1:])
+        modno = int(tokens[2][-2:])
+        agg = tokens[2]
+        seqno = int(tokens[3][1:])
+
+        agipd_base = self.h5_data_path.format(modno)
+        karabo_id, _, channel = agipd_base.split('/')
+        agipd_corr_source = f"{karabo_id}/CORR/{channel}"
+
+        agipd_corr_channels = {
+            "image": image_count,
+        }
+
+        instrument_channels = [f"{agipd_corr_source}/{ch}"
+                               for ch in agipd_corr_channels]
+
+        # backward compatibility BEGIN
+        copy_channels = {
+            ch: dc.files[0].get_index(agipd_base, ch)[1]
+            for ch in ["detector", "header", "trailer"]
+        }
+        instrument_channels += [f"{agipd_base}/{ch}" for ch in copy_channels]
+        # backward compatibility END
+
+        with DataFile.from_details(out_folder, agg, runno, seqno) as outfile:
+            outfile.create_metadata(
+                like=dc, instrument_channels=instrument_channels)
+            outfile.create_index(raw_trains, from_file=dc.files[0])
+
+            agipd_src = outfile.create_instrument_source(agipd_corr_source)
+            agipd_src.create_index(**agipd_corr_channels)
 
             # All corrected data goes in a /INSTRUMENT/.../image group
-            image_grp = outfile[data_path]
+            image_grp = agipd_src.require_group("image")
+            image_index_grp = agipd_src.get_index_group("image")
+
+            # backward compatibility BEGIN
+            agipd_legacy = outfile.create_instrument_source(agipd_base)
+            # this two lines must go after `agipd_base` instrument source
+            # creation, overwise it tries to create existing h5 group
+            outfile[f"INDEX/{agipd_base}/image"] = image_index_grp
+            outfile[f"INSTRUMENT/{agipd_base}/image"] = image_grp
+
+            agipd_legacy.create_index(**copy_channels)
+            for ch in copy_channels:
+                dc.files[0].file.copy(
+                    f"INSTRUMENT/{agipd_base}/{ch}", agipd_legacy, ch,
+                    without_attrs=True
+                )
+            # backward compatibility END
 
             # Set up all the datasets before filling them. This puts the
             # metadata about the datasets together at the start of the file,