Skip to content
Snippets Groups Projects
Commit 8a5704e2 authored by Steffen Hauf's avatar Steffen Hauf
Browse files

Speed inprovements and only available memory cells

parent 65a6dfc4
No related branches found
No related tags found
1 merge request!5Clean
Hide whitespace changes
Inline Side-by-side
LPD/correct_lpd_batch.py
+ 96
− 48
View file @ 8a5704e2
...@@ -68,7 +68,8 @@ cells = np.arange(max_cells) ...@@ -68,7 +68,8 @@ cells = np.arange(max_cells)
QUADRANTS = 4 QUADRANTS = 4
MODULES_PER_QUAD = 4 MODULES_PER_QUAD = 4
DET_FILE_INSET = "LPD" DET_FILE_INSET = "LPD"
CHUNK_SIZE = 512
MAX_PAR = 32
if in_folder[-1] == "/": if in_folder[-1] == "/":
in_folder = in_folder[:-1] in_folder = in_folder[:-1]
...@@ -80,6 +81,7 @@ if not os.path.exists(out_folder): ...@@ -80,6 +81,7 @@ if not os.path.exists(out_folder):
elif not overwrite: elif not overwrite:
raise AttributeError("Output path exists! Exiting") raise AttributeError("Output path exists! Exiting")
max_cells_db = 128
# In[42]: # In[42]:
...@@ -202,7 +204,7 @@ if True: ...@@ -202,7 +204,7 @@ if True:
metadata.calibration_constant = offset metadata.calibration_constant = offset
# set the operating condition # set the operating condition
condition = Conditions.Dark.LPD(memory_cells=max_cells, bias_voltage=bias_voltage) condition = Conditions.Dark.LPD(memory_cells=max_cells_db, bias_voltage=bias_voltage)
metadata.detector_condition = condition metadata.detector_condition = condition
# specify the a version for this constant # specify the a version for this constant
...@@ -216,9 +218,11 @@ if True: ...@@ -216,9 +218,11 @@ if True:
try: try:
metadata.retrieve(cal_db_interface) metadata.retrieve(cal_db_interface)
offsets.append(copy.copy(offset.data)) offsets.append(copy.copy(offset.data))
except: except Exception as e:
print("Could not retrieve offset from db for {}: {}".format(qm, e))
offsets.append(np.zeros((256,256,max_cells,3))) offsets.append(np.zeros((256,256,max_cells,3)))
else: else:
print("Could not retrieve offset from db for {}".format(qm))
offsets.append(np.zeros((256,256,max_cells,3))) offsets.append(np.zeros((256,256,max_cells,3)))
""" """
metadata = ConstantMetaData() metadata = ConstantMetaData()
...@@ -362,8 +366,9 @@ def map_modules_from_files(filelist): ...@@ -362,8 +366,9 @@ def map_modules_from_files(filelist):
return module_files, mod_ids return module_files, mod_ids
dirlist = os.listdir(in_folder) dirlist = sorted(os.listdir(in_folder))
file_list = [] file_list = []
for entry in dirlist: for entry in dirlist:
#only h5 file #only h5 file
abs_entry = "{}/{}".format(in_folder, entry) abs_entry = "{}/{}".format(in_folder, entry)
...@@ -378,13 +383,13 @@ for entry in dirlist: ...@@ -378,13 +383,13 @@ for entry in dirlist:
mapped_files, mod_ids = map_modules_from_files(file_list) mapped_files, mod_ids = map_modules_from_files(file_list)
print(file_list)
# In[45]: # In[45]:
import copy import copy
from functools import partial from functools import partial
def correct_module(cells, do_ff, index_v, inp): def correct_module(max_cells, do_ff, index_v, CHUNK_SIZE, inp):
import numpy as np import numpy as np
import copy import copy
import h5py import h5py
...@@ -419,10 +424,16 @@ def correct_module(cells, do_ff, index_v, inp): ...@@ -419,10 +424,16 @@ def correct_module(cells, do_ff, index_v, inp):
last = np.squeeze(infile["/INDEX/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/last".format(channel)]) last = np.squeeze(infile["/INDEX/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/last".format(channel)])
last_index = int(last[status != 0][-1]) last_index = int(last[status != 0][-1])
first_index = int(last[status != 0][0]) first_index = int(last[status != 0][0])
im = np.array(infile["/INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/data".format(channel)][first_index:last_index, ...]) allcells = np.squeeze(np.array(infile["/INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/cellId".format(channel)][first_index:last_index, ...]))
cells = np.squeeze(np.array(infile["/INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/cellId".format(channel)][first_index:last_index, ...])) single_image = np.array(np.array(infile["/INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/data".format(channel)][first_index, ...]))
can_calibrate = allcells < max_cells
dont_copy = ["data",] if np.count_nonzero(can_calibrate) == 0:
return
allcells = allcells[can_calibrate]
firange = np.arange(first_index, last_index)
firange = firange[can_calibrate]
dont_copy = ["data", "cellId", "trainId", "pulseId", "status", "length"]
dont_copy = ["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/{}".format(channel, do) dont_copy = ["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/{}".format(channel, do)
for do in dont_copy] for do in dont_copy]
...@@ -438,52 +449,86 @@ def correct_module(cells, do_ff, index_v, inp): ...@@ -438,52 +449,86 @@ def correct_module(cells, do_ff, index_v, inp):
infile.visititems(visitor) infile.visititems(visitor)
outfile.flush() outfile.flush()
outfile.flush()
infile.close()
im, gain = splitOffGainLPD(im[:,0,...])
im = im.astype(np.float32) oshape = (firange.size, single_image.shape[2], single_image.shape[1])
im[gain > 2] = np.nan
gain[gain > 2] = 0
im = np.rollaxis(im, 2) outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/data".format(channel)] = np.zeros(oshape, np.float32)
im = np.rollaxis(im, 2, 1) outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/gain".format(channel)] = np.zeros(oshape, np.uint8)
outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/mask".format(channel)] = np.zeros(oshape, np.uint32)
gain = np.rollaxis(gain, 2)
gain = np.rollaxis(gain, 2, 1) outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/cellId".format(channel)] = np.zeros(firange.size, np.uint16)
outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/trainId".format(channel)] = np.zeros(firange.size, np.uint64)
om = offset[...,cells,:] outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/pulseId".format(channel)] = np.zeros(firange.size, np.uint64)
rc = rel_gain[...,cells,:] outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/status".format(channel)] = np.zeros(firange.size, np.uint16)
rbc = rel_gain_b[...,cells,:]
og = np.choose(gain, (om[...,0], om[...,1], om[...,2]))
rg = np.choose(gain, (rc[...,0], rc[...,1], rc[...,2]))
rgb = np.choose(gain, (rbc[...,0], rbc[...,1], rbc[...,2]))
mskg = mask[...,cells,:] outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/length".format(channel)] = np.zeros(firange.size, np.uint32)
msk = np.choose(gain, (mskg[...,0], mskg[...,1], mskg[...,2]))
im -= og
im = (im-rgb)/rg #
if do_ff: cidx = 0
im /= flatfield[:,:,None] for irange in np.array_split(firange, firange.size//CHUNK_SIZE):
im = np.array(infile["/INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/data".format(channel)][irange, ...])
trainId = np.squeeze(infile["/INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/trainId".format(channel)][irange, ...])
pulseId = np.squeeze(infile["/INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/pulseId".format(channel)][irange, ...])
status = np.squeeze(infile["/INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/status".format(channel)][irange, ...])
outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/data".format(channel)] = np.rollaxis(np.rollaxis(im,1), 2) cells = np.squeeze(np.array(infile["/INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/cellId".format(channel)][irange, ...]))
outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/gain".format(channel)] = np.rollaxis(np.rollaxis(gain,1), 2)
outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/mask".format(channel)] = np.rollaxis(np.rollaxis(msk,1), 2)
length = np.squeeze(np.array(infile["/INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/length".format(channel)][irange, ...]))
im, gain = splitOffGainLPD(im[:,0,...])
im = im.astype(np.float32)
im[gain > 2] = np.nan
gain[gain > 2] = 0
im = np.rollaxis(im, 2)
im = np.rollaxis(im, 2, 1)
gain = np.rollaxis(gain, 2)
gain = np.rollaxis(gain, 2, 1)
om = offset[...,cells,:]
rc = rel_gain[...,cells,:]
rbc = rel_gain_b[...,cells,:]
og = np.choose(gain, (om[...,0], om[...,1], om[...,2]))
rg = np.choose(gain, (rc[...,0], rc[...,1], rc[...,2]))
rgb = np.choose(gain, (rbc[...,0], rbc[...,1], rbc[...,2]))
mskg = mask[...,cells,:]
msk = np.choose(gain, (mskg[...,0], mskg[...,1], mskg[...,2]))
im -= og
im = (im-rgb)/rg
if do_ff:
im /= flatfield[:,:,None]
nidx = int(cidx+irange.size)
outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/data".format(channel)][cidx:nidx,...] = np.rollaxis(np.rollaxis(im,1), 2)
outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/gain".format(channel)][cidx:nidx,...] = np.rollaxis(np.rollaxis(gain,1), 2)
outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/mask".format(channel)][cidx:nidx,...] = np.rollaxis(np.rollaxis(msk,1), 2)
outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/cellId".format(channel)][cidx:nidx] = cells
outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/trainId".format(channel)][cidx:nidx] = trainId
outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/pulseId".format(channel)][cidx:nidx] = pulseId
outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/status".format(channel)][cidx:nidx] = status
outfile["INSTRUMENT/FXE_DET_LPD1M-1/DET/{}CH0:xtdf/image/length".format(channel)][cidx:nidx] = length
cidx = nidx
outfile.close() outfile.close()
infile.close()
except Exception as e: except Exception as e:
print(e) print(e)
pass pass
done = False done = False
first_files = [] first_files = []
inp = []
while not done: while not done:
inp = []
dones = [] dones = []
first = True first = True
for i in range(16): for i in range(16):
...@@ -502,13 +547,16 @@ while not done: ...@@ -502,13 +547,16 @@ while not done:
rel_gains[i][...,:max_cells,:], bad_pixels[i][...,:max_cells,:], rel_gains[i][...,:max_cells,:], bad_pixels[i][...,:max_cells,:],
flat_fields[i], rel_gains_b[i][...,:max_cells,:])) flat_fields[i], rel_gains_b[i][...,:max_cells,:]))
first = False first = False
p = partial(correct_module, max_cells, do_ff, index_v) if len(inp) > MAX_PAR:
print("Running {} tasks parallel".format(len(inp)))
p = partial(correct_module, max_cells, do_ff, index_v, CHUNK_SIZE)
r = view.map_sync(p, inp)
inp = []
r = view.map_sync(p, inp)
#r = list(map(p, inp)) #r = list(map(p, inp))
done = all(dones) done = all(dones)
#r.wait()
# In[46]: # In[46]:
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment

Legal Notice | Contacts