From 4ad688551a6699d316a37928262d8a4caf8d1dee Mon Sep 17 00:00:00 2001
From: Karim Ahmed <karim.ahmed@xfel.eu>
Date: Mon, 9 Dec 2019 09:52:08 +0100
Subject: [PATCH] fill INDEX differently to account missing trains and have
consistency
---
cal_tools/cal_tools/agipdlib.py | 45 ++++++++++++++++++---------------
1 file changed, 25 insertions(+), 20 deletions(-)
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index f5445ec0d..182a1e616 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -1106,11 +1106,11 @@ class AgipdCorrections:
def get_valid_image_idx(self):
""" Return the indices of valid data
"""
- agipd_base = self.idx_base
+ idx_base = self.idx_base
if self.index_v == 2:
- count = np.squeeze(self.infile[agipd_base + "image/count"])
- first = np.squeeze(self.infile[agipd_base + "image/first"])
+ count = np.squeeze(self.infile[idx_base + "image/count"])
+ first = np.squeeze(self.infile[idx_base + "image/first"])
if np.count_nonzero(count != 0) == 0:
raise IOError("File has no valid counts")
valid = count != 0
@@ -1137,12 +1137,12 @@ class AgipdCorrections:
self.valid_indices = np.squeeze(valid_indices).astype(np.int32)
elif self.index_v == 1:
- status = np.squeeze(self.infile[agipd_base + "image/status"])
+ status = np.squeeze(self.infile[idx_base + "image/status"])
if np.count_nonzero(status != 0) == 0:
raise IOError("File {} has no valid counts".format(
self.infile))
- last = np.squeeze(self.infile[agipd_base + "image/last"])
- first = np.squeeze(self.infile[agipd_base + "image/first"])
+ last = np.squeeze(self.infile[idx_base + "image/last"])
+ first = np.squeeze(self.infile[idx_base + "image/first"])
valid = status != 0
last_index = int(last[status != 0][-1]) + 1
first_index = int(first[status != 0][0])
@@ -1281,33 +1281,38 @@ class AgipdCorrections:
# sanitize indices
for do in ["image", ]:
+ # uq: INDEX/trainID
+ # fidxv: INDEX/.../image/first idx values
+ # cntsv: INDEX/.../image/counts values
uq, fidxv, cntsv = np.unique(alltrains[firange - firange[0]],
return_index=True,
return_counts=True)
- duq = (uq[1:] - uq[:-1]).astype(np.int64)
+ # Validate calculated CORR INDEX contents
+ train_diff = np.isin(np.array(self.infile["/INDEX/trainId"]), uq, invert=True)
- cfidxv = [fidxv[0], ]
- ccntsv = [cntsv[0], ]
- for i, du in enumerate(duq.tolist()):
- if du > 1000:
- du = 1
- cntsv[i] = 0
- cfidxv += [0] * (du - 1) + [fidxv[i + 1], ]
- ccntsv += [0] * (du - 1) + [cntsv[i + 1], ]
+ for i, diff in enumerate(train_diff):
+ if diff:
+ if i < len(cntsv):
+ cntsv = np.insert(cntsv, i, 0)
+ fidxv = np.insert(fidxv, i, 0)
+ else:
+ cntsv = np.append(cntsv, 0)
+ fidxv = np.append(fidxv, 0)
- mv = len(cfidxv)
- fidx = np.zeros(len(cfidxv), fidxv.dtype)
- fidx[self.valid[:mv]] = np.array(cfidxv)[self.valid[:mv]]
+ mv = len(fidxv)
+ fidx = np.zeros(len(fidxv), fidxv.dtype)
+ fidx[self.valid[:mv]] = np.array(fidxv)[self.valid[:mv]]
for i in range(len(fidx) - 1, 2, -1):
if fidx[i - 1] == 0 and fidx[i] != 0:
fidx[i - 1] = fidx[i]
- cnts = np.zeros(len(cfidxv), cntsv.dtype)
- cnts[self.valid[:mv]] = np.array(ccntsv)[self.valid[:mv]]
+ cnts = np.zeros(len(fidxv), cntsv.dtype)
+ cnts[self.valid[:mv]] = np.array(cntsv)[self.valid[:mv]]
+ # add INDEX contents in CORR files
self.outfile.create_dataset(idx_base + "{}/first".format(do),
fidx.shape,
dtype=fidx.dtype,
--
GitLab