revert unrelated changes

cal_tools/cal_tools/agipdlib.py

@@ -1106,11 +1106,11 @@ class AgipdCorrections:
     def get_valid_image_idx(self):
         """ Return the indices of valid data
         """
-        idx_base = self.idx_base
+        agipd_base = self.idx_base
 
         if self.index_v == 2:
-            count = np.squeeze(self.infile[idx_base + "image/count"])
-            first = np.squeeze(self.infile[idx_base + "image/first"])
+            count = np.squeeze(self.infile[agipd_base + "image/count"])
+            first = np.squeeze(self.infile[agipd_base + "image/first"])
             if np.count_nonzero(count != 0) == 0:
                 raise IOError("File has no valid counts")
             valid = count != 0
@@ -1137,12 +1137,12 @@ class AgipdCorrections:
             self.valid_indices = np.squeeze(valid_indices).astype(np.int32)
 
         elif self.index_v == 1:
-            status = np.squeeze(self.infile[idx_base + "image/status"])
+            status = np.squeeze(self.infile[agipd_base + "image/status"])
             if np.count_nonzero(status != 0) == 0:
                 raise IOError("File {} has no valid counts".format(
                     self.infile))
-            last = np.squeeze(self.infile[idx_base + "image/last"])
-            first = np.squeeze(self.infile[idx_base + "image/first"])
+            last = np.squeeze(self.infile[agipd_base + "image/last"])
+            first = np.squeeze(self.infile[agipd_base + "image/first"])
             valid = status != 0
             last_index = int(last[status != 0][-1]) + 1
             first_index = int(first[status != 0][0])
@@ -1281,47 +1281,42 @@ class AgipdCorrections:
 
         # sanitize indices
         for do in ["image", ]:
-            # uq: INDEX/trainID
-            # fidxv: INDEX/.../image/first idx values
-            # cntsv: INDEX/.../image/counts values
 
-            # Extract parameters through identifying
-            # unique trains, index and numbers.
             uq, fidxv, cntsv = np.unique(alltrains[firange - firange[0]],
                                          return_index=True,
                                          return_counts=True)
 
-            # Validate calculated CORR INDEX contents by checking difference between
-            # trainId stored in RAW data and trains from
-            train_diff = np.isin(np.array(self.infile["/INDEX/trainId"]), uq, invert=True)
-
-            # Insert zeros for missing trains.
-            # fidxv and cntsv should have same length as
-            # raw INDEX/.../image/first and INDEX/.../image/count,
-            # respectively
-            for i, diff in enumerate(train_diff):
-                if diff:
-                    if i < len(cntsv):
-                        cntsv = np.insert(cntsv, i, 0)
-                        if i == 0:
-                            fidxv = np.insert(fidxv, i, 0)
-                        else:
-                            fidxv = np.insert(fidxv, i, fidxv[i-1])
-                    else:
-                        # append if at the end of the array
-                        cntsv = np.append(cntsv, 0)
-                        fidxv = np.append(fidxv, 0)
-
-            # save INDEX contents (first, count) in CORR files
+            duq = (uq[1:] - uq[:-1]).astype(np.int64)
+
+            cfidxv = [fidxv[0], ]
+            ccntsv = [cntsv[0], ]
+            for i, du in enumerate(duq.tolist()):
+                if du > 1000:
+                    du = 1
+                    cntsv[i] = 0
+                cfidxv += [0] * (du - 1) + [fidxv[i + 1], ]
+                ccntsv += [0] * (du - 1) + [cntsv[i + 1], ]
+
+            mv = len(cfidxv)
+            fidx = np.zeros(len(cfidxv), fidxv.dtype)
+            fidx[self.valid[:mv]] = np.array(cfidxv)[self.valid[:mv]]
+
+            for i in range(len(fidx) - 1, 2, -1):
+                if fidx[i - 1] == 0 and fidx[i] != 0:
+                    fidx[i - 1] = fidx[i]
+
+            cnts = np.zeros(len(cfidxv), cntsv.dtype)
+            cnts[self.valid[:mv]] = np.array(ccntsv)[self.valid[:mv]]
+
             self.outfile.create_dataset(idx_base + "{}/first".format(do),
-                                        fidxv.shape,
-                                        dtype=fidxv.dtype,
-                                        data=fidxv,
+                                        fidx.shape,
+                                        dtype=fidx.dtype,
+                                        data=fidx,
                                         fletcher32=True)
             self.outfile.create_dataset(idx_base + "{}/count".format(do),
-                                        cntsv.shape,
-                                        dtype=cntsv.dtype,
-                                        data=cntsv,
+                                        cnts.shape,
+                                        dtype=cnts.dtype,
+                                        data=cnts,
                                         fletcher32=True)
 
     def create_output_datasets(self):