diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 573588c58d948df3967afb0bc6ff0a61632b2d03..9ee105b9993396873817c928bfa5ef12ee154f7a 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -266,32 +266,39 @@ class AgipdCorrections:
module_idx = int(file_name.split('/')[-1].split('-')[2][-2:])
agipd_base = self.h5_data_path.format(module_idx)
idx_base = self.h5_index_path.format(module_idx)
- data_path = f'{agipd_base}/image'
data_dict = self.shared_dict[i_proc]
-
+ data_dict['moduleIdx'][0] = module_idx
try:
f = h5py.File(file_name, 'r')
- group = f[data_path]
+ group = f[agipd_base]["image"]
+
+ (_, first_index, last_index,
+ _, valid_indices) = self.get_valid_image_idx(idx_base, f)
- _, first_index, last_index, __, valid_indices = \
- self.get_valid_image_idx(idx_base, f)
firange = self.gen_valid_range(first_index, last_index,
self.max_cells, agipd_base, f,
valid_indices)
n_img = firange.shape[0]
+ data_dict['nImg'][0] = n_img
+ if np.all(np.diff(firange) == 1):
+ # if firange consists of contiguous indices
+ # convert firange from fancy indexing to slicing
+ firange = slice(firange[0], firange[-1]+1)
+ raw_data = group['data'][firange]
+ else:
+ # Avoid very slow performance using fancing indexing,
+ # if firange consists of non-contiguous indices.
+ raw_data = group['data'][:][firange]
+ data_dict['data'][:n_img] = raw_data[:, 0]
+ data_dict['rawgain'][:n_img] = raw_data[:, 1]
+ # All below arrays are one dimensional and they were not
+ # affected much by the non-contiguous fancy indexing.
data_dict['cellId'][:n_img] = np.squeeze(group['cellId'][firange])
data_dict['pulseId'][:n_img] = np.squeeze(
group['pulseId'][firange])
data_dict['trainId'][:n_img] = np.squeeze(
group['trainId'][firange])
- data_dict['moduleIdx'][0] = module_idx
- data_dict['nImg'][0] = n_img
-
- raw_data = group['data'][firange]
- data_dict['data'][:n_img] = raw_data[:, 0]
- data_dict['rawgain'][:n_img] = raw_data[:, 1]
-
except Exception as e:
print(f'Error during reading data from file {file_name}: {e}')
print(f'Error traceback: {traceback.format_exc()}')
@@ -609,7 +616,6 @@ class AgipdCorrections:
def get_valid_image_idx(self, idx_base, infile, index_v=2):
""" Return the indices of valid data
"""
-
if index_v == 2:
count = np.squeeze(infile[idx_base + "image/count"])
first = np.squeeze(infile[idx_base + "image/first"])
@@ -636,8 +642,7 @@ class AgipdCorrections:
elif index_v == 1:
status = np.squeeze(infile[idx_base + "image/status"])
if np.count_nonzero(status != 0) == 0:
- raise IOError("File {} has no valid counts".format(
- infile))
+ raise IOError(f"File {infile} has no valid counts")
last = np.squeeze(infile[idx_base + "image/last"])
first = np.squeeze(infile[idx_base + "image/first"])
valid = status != 0
@@ -652,9 +657,10 @@ class AgipdCorrections:
valid_indices = None
else:
raise AttributeError(
- "Not a known raw format version: {}".format(index_v))
+ f"Not a known raw format version: {index_v}")
- return valid, first_index, last_index, idxtrains, valid_indices
+ return (valid, first_index, last_index, idxtrains,
+ valid_indices)
def gen_valid_range(self, first_index, last_index, max_cells, agipd_base,
infile, valid_indices=None):
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index 9d396f967bd3d1a38abc5bcdba9ba086388c52a3..e8e5e52265eca66111d76c8bd63109f0a2bc1c5e 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -22,21 +22,21 @@
},
"outputs": [],
"source": [
- "in_folder = \"/gpfs/exfel/exp/MID/202030/p900137/raw\" # the folder to read data from, required\n",
- "out_folder = \"/gpfs/exfel/exp/MID/202030/p900137/scratch/karnem/r449_v06\" # the folder to output to, required\n",
+ "in_folder = \"/gpfs/exfel/exp/HED/202031/p900174/raw\" # the folder to read data from, required\n",
+ "out_folder = \"/gpfs/exfel/data/scratch/ahmedk/test/hibef_agipd2\" # the folder to output to, required\n",
"sequences = [-1] # sequences to correct, set to -1 for all, range allowed\n",
"modules = [-1] # modules to correct, set to -1 for all, range allowed\n",
- "run = 449 # runs to process, required\n",
+ "run = 155 # runs to process, required\n",
"\n",
- "karabo_id = \"MID_DET_AGIPD1M-1\" # karabo karabo_id\n",
+ "karabo_id = \"HED_DET_AGIPD500K2G\" # karabo karabo_id\n",
"karabo_da = ['-1'] # a list of data aggregators names, Default [-1] for selecting all data aggregators\n",
"receiver_id = \"{}CH0\" # inset for receiver devices\n",
"path_template = 'RAW-R{:04d}-{}-S{:05d}.h5' # the template to use to access data\n",
"h5path = 'INSTRUMENT/{}/DET/{}:xtdf/' # path in the HDF5 file to images\n",
"h5path_idx = 'INDEX/{}/DET/{}:xtdf/' # path in the HDF5 file to images\n",
- "h5path_ctrl = '/CONTROL/{}/MDL/FPGA_COMP_TEST' # path to control information\n",
- "karabo_id_control = \"MID_IRU_AGIPD1M1\" # karabo-id for control device\n",
- "karabo_da_control = 'AGIPD1MCTRL00' # karabo DA for control infromation\n",
+ "h5path_ctrl = '/CONTROL/{}/MDL/FPGA_COMP' # path to control information\n",
+ "karabo_id_control = \"HED_EXP_AGIPD500K2G\" # karabo-id for control device\n",
+ "karabo_da_control = 'AGIPD500K2G00' # karabo DA for control infromation\n",
"\n",
"use_dir_creation_date = True # use the creation data of the input dir for database queries\n",
"cal_db_interface = \"tcp://max-exfl016:8015#8045\" # the database interface to use\n",
@@ -120,7 +120,7 @@
"warnings.filterwarnings('ignore')\n",
"import yaml\n",
"\n",
- "from extra_geom import AGIPD_1MGeometry\n",
+ "from extra_geom import AGIPD_1MGeometry, AGIPD_500K2GGeometry\n",
"from extra_data import RunDirectory, stack_detector_data\n",
"from iCalibrationDB import Detectors\n",
"from mpl_toolkits.mplot3d import Axes3D\n",
@@ -143,7 +143,12 @@
"from cal_tools.ana_tools import get_range\n",
"from cal_tools.enums import BadPixels\n",
"from cal_tools.tools import get_dir_creation_date, map_modules_from_folder\n",
- "from cal_tools.step_timing import StepTimer"
+ "from cal_tools.step_timing import StepTimer\n",
+ "\n",
+ "import seaborn as sns\n",
+ "sns.set()\n",
+ "sns.set_context(\"paper\", font_scale=1.4)\n",
+ "sns.set_style(\"ticks\")"
]
},
{
@@ -227,13 +232,19 @@
"instrument = karabo_id.split(\"_\")[0]\n",
"if instrument == \"SPB\":\n",
" dinstance = \"AGIPD1M1\"\n",
- "else:\n",
+ " nmods = 16\n",
+ "elif instrument == \"MID\":\n",
" dinstance = \"AGIPD1M2\"\n",
+ " nmods = 16\n",
+ "# TODO: Remove DETLAB\n",
+ "elif instrument == \"HED\" or instrument == \"DETLAB\":\n",
+ " dinstance = \"AGIPD500K\"\n",
+ " nmods = 8\n",
"\n",
"# Evaluate requested modules\n",
"if karabo_da[0] == '-1':\n",
" if modules[0] == -1:\n",
- " modules = list(range(16))\n",
+ " modules = list(range(nmods))\n",
" karabo_da = [\"AGIPD{:02d}\".format(i) for i in modules]\n",
"else:\n",
" modules = [int(x[-2:]) for x in karabo_da]\n",
@@ -319,8 +330,6 @@
"# Evaluate aquisition rate\n",
"if acq_rate == 0:\n",
" acq_rate = get_acq_rate((filename, karabo_id, channel))\n",
- "else:\n",
- " acq_rate = None\n",
"\n",
"print(f\"Maximum memory cells to calibrate: {max_cells}\")"
]
@@ -433,7 +442,7 @@
"\n",
"\n",
"ts = perf_counter()\n",
- "with Pool(processes=16) as pool:\n",
+ "with Pool(processes=len(modules)) as pool:\n",
" const_out = pool.map(retrieve_constants, modules)\n",
"print(f\"Constants were loaded in {perf_counter()-ts:.01f}s\")"
]
@@ -651,7 +660,7 @@
"metadata": {},
"outputs": [],
"source": [
- "def get_trains_data(run_folder, source, include, tid=None, path='*/DET/*'):\n",
+ "def get_trains_data(run_folder, source, include, tid=None, path='*/DET/*', modules=16, fillvalue=np.nan):\n",
" \"\"\"\n",
" Load single train for all module\n",
" \n",
@@ -665,10 +674,10 @@
" run_data = RunDirectory(run_folder, include)\n",
" if tid:\n",
" tid, data = run_data.select('*/DET/*', source).train_from_id(tid)\n",
- " return tid, stack_detector_data(data, source)\n",
+ " return tid, stack_detector_data(train=data, data=source, fillvalue=fillvalue, modules=modules)\n",
" else:\n",
" for tid, data in run_data.select('*/DET/*', source).trains(require_all=True):\n",
- " return tid, stack_detector_data(data, source)\n",
+ " return tid, stack_detector_data(train=data, data=source, fillvalue=fillvalue, modules=modules)\n",
" return None, None"
]
},
@@ -678,12 +687,15 @@
"metadata": {},
"outputs": [],
"source": [
- "geom = AGIPD_1MGeometry.from_quad_positions(quad_pos=[\n",
- " (-525, 625),\n",
- " (-550, -10),\n",
- " (520, -160),\n",
- " (542.5, 475),\n",
- "])"
+ "if dinstance == \"AGIPD500K\":\n",
+ " geom = AGIPD_500K2GGeometry.from_origin()\n",
+ "else:\n",
+ " geom = AGIPD_1MGeometry.from_quad_positions(quad_pos=[\n",
+ " (-525, 625),\n",
+ " (-550, -10),\n",
+ " (520, -160),\n",
+ " (542.5, 475),\n",
+ " ])"
]
},
{
@@ -693,13 +705,14 @@
"outputs": [],
"source": [
"include = '*S00000*' if sequences is None else f'*S{sequences[0]:05d}*'\n",
- "tid, corrected = get_trains_data(f'{out_folder}/', 'image.data', include)\n",
- "_, gains = get_trains_data(f'{out_folder}/', 'image.gain', include, tid)\n",
- "_, mask = get_trains_data(f'{out_folder}/', 'image.mask', include, tid)\n",
- "_, blshift = get_trains_data(f'{out_folder}/', 'image.blShift', include, tid)\n",
- "_, cellId = get_trains_data(f'{out_folder}/', 'image.cellId', include, tid)\n",
- "_, pulseId = get_trains_data(f'{out_folder}/', 'image.pulseId', include, tid)\n",
- "_, raw = get_trains_data(f'{in_folder}/r{run:04d}/', 'image.data', include, tid)"
+ "tid, corrected = get_trains_data(f'{out_folder}/', 'image.data', include, modules=nmods)\n",
+ "_, gains = get_trains_data(f'{out_folder}/', 'image.gain', include, tid, modules=nmods)\n",
+ "_, mask = get_trains_data(f'{out_folder}/', 'image.mask', include, tid, modules=nmods)\n",
+ "_, blshift = get_trains_data(f'{out_folder}/', 'image.blShift', include, tid, modules=nmods)\n",
+ "_, cellId = get_trains_data(f'{out_folder}/', 'image.cellId', include, tid, modules=nmods)\n",
+ "_, pulseId = get_trains_data(f'{out_folder}/', 'image.pulseId', include, tid,\n",
+ " modules=nmods, fillvalue=0)\n",
+ "_, raw = get_trains_data(f'{in_folder}/r{run:04d}/', 'image.data', include, tid, modules=nmods)"
]
},
{
diff --git a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
index 3e639c349a6b0ef6471cc5490323f11afab1b12d..87bb034efbd1cb504fb6cf6a4b407dd38a168d2d 100644
--- a/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
+++ b/notebooks/AGIPD/AGIPD_Retrieve_Constants_Precorrection.ipynb
@@ -172,9 +172,28 @@
"print(f\"Gain setting: {gain_setting}\")\n",
"print(f\"Detector in use is {karabo_id}\")\n",
"\n",
+ "\n",
+ "# Extracting Instrument string\n",
+ "instrument = karabo_id.split(\"_\")[0]\n",
+ "# Evaluate detector instance for mapping\n",
+ "if instrument == \"SPB\":\n",
+ " dinstance = \"AGIPD1M1\"\n",
+ " nmods = 16\n",
+ "elif instrument == \"MID\":\n",
+ " dinstance = \"AGIPD1M2\"\n",
+ " nmods = 16\n",
+ "# TODO: Remove DETLAB\n",
+ "elif instrument == \"HED\" or instrument == \"DETLAB\":\n",
+ " dinstance = \"AGIPD500K\"\n",
+ " nmods = 8\n",
+ "\n",
+ "print(f\"Instrument {instrument}\")\n",
+ "print(f\"Detector instance {dinstance}\")\n",
+ "\n",
+ "\n",
"if karabo_da[0] == '-1':\n",
" if modules[0] == -1:\n",
- " modules = list(range(16))\n",
+ " modules = list(range(nmods))\n",
" karabo_da = [\"AGIPD{:02d}\".format(i) for i in modules]\n",
"else:\n",
" modules = [int(x[-2:]) for x in karabo_da]"
@@ -255,7 +274,7 @@
" qm_files, qm, dev, idx = inp\n",
" # get number of memory cells from a sequence file with image data\n",
" for f in qm_files:\n",
- " if max_cells == 0:\n",
+ " if not max_cells:\n",
" max_cells = get_num_cells(f, karabo_id, idx)\n",
" if max_cells is None:\n",
" if f != qm_files[-1]:\n",
@@ -270,8 +289,6 @@
"\n",
" if acq_rate == 0.:\n",
" acq_rate = get_acq_rate((f, karabo_id, idx))\n",
- " else:\n",
- " acq_rate = None\n",
"\n",
" print(f\"Set memory cells to {max_cells}\")\n",
" print(f\"Set acquistion rate cells to {acq_rate} MHz\")\n",
@@ -316,16 +333,6 @@
" corr_bools.get('blc_stripes'),\n",
" melt_snow]\n",
"\n",
- "# Extracting Instrument string\n",
- "instrument = karabo_id.split(\"_\")[0]\n",
- "if instrument == \"SPB\":\n",
- " dinstance = \"AGIPD1M1\"\n",
- "else:\n",
- " dinstance = \"AGIPD1M2\"\n",
- "\n",
- "print(f\"Instrument {instrument}\")\n",
- "print(f\"Detector instance {dinstance}\")\n",
- "\n",
"inp = []\n",
"only_dark = False\n",
"nodb_with_dark = False\n",