From 32ec3759284718cd5b653bfd14e3282c91bcab15 Mon Sep 17 00:00:00 2001
From: David Hammer <dhammer@mailbox.org>
Date: Fri, 19 Mar 2021 16:06:45 +0100
Subject: [PATCH] Formatting, absolute multiprocessing import, typos
---
cal_tools/cal_tools/agipdlib.py | 68 ++++++++-----------
cal_tools/cal_tools/agipdutils.py | 4 +-
cal_tools/cal_tools/tools.py | 3 +-
.../AGIPD/AGIPD_Correct_and_Verify.ipynb | 6 +-
4 files changed, 36 insertions(+), 45 deletions(-)
diff --git a/cal_tools/cal_tools/agipdlib.py b/cal_tools/cal_tools/agipdlib.py
index 6eb9d6f60..e22b228d7 100644
--- a/cal_tools/cal_tools/agipdlib.py
+++ b/cal_tools/cal_tools/agipdlib.py
@@ -961,13 +961,11 @@ class AgipdCorrections:
# Extract parameters through identifying
# unique trains, index and numbers.
- uq, fidxv, cntsv = np.unique(trains, return_index=True,
- return_counts=True)
+ uq, fidxv, cntsv = np.unique(trains, return_index=True, return_counts=True)
# Validate calculated CORR INDEX contents by checking
# difference between trainId stored in RAW data and trains from
- train_diff = np.isin(np.array(infile["/INDEX/trainId"]), uq,
- invert=True)
+ train_diff = np.isin(np.array(infile["/INDEX/trainId"]), uq, invert=True)
# Insert zeros for missing trains.
# fidxv and cntsv should have same length as
@@ -1260,10 +1258,6 @@ class AgipdCorrections:
for cname, mdata in const_yaml[karabo_da]["constants"].items():
when[cname] = mdata["creation-time"]
if when[cname]:
- # This path is only used when testing new flat fields from
- # file during development: it takes ages to test using all
- # cells. Consequently, the shape needs to be fixed when less
- # cells are used.
with h5py.File(mdata["file-path"], "r") as cf:
cons_data[cname] = np.copy(cf[f"{db_module}/{cname}/0/data"]) # noqa
else:
@@ -1334,15 +1328,21 @@ class AgipdCorrections:
"""
- const_dict = \
- assemble_constant_dict(self.corr_bools, self.pc_bools,
- memory_cells, bias_voltage, gain_setting,
- acquisition_rate, photon_energy,
- beam_energy=None, only_dark=only_dark)
+ const_dict = assemble_constant_dict(
+ self.corr_bools,
+ self.pc_bools,
+ memory_cells,
+ bias_voltage,
+ gain_setting,
+ acquisition_rate,
+ photon_energy,
+ beam_energy=None,
+ only_dark=only_dark,
+ )
- cons_data, when = \
- self.retrieve_constant_and_time(karabo_id, karabo_da, const_dict,
- cal_db_interface, creation_time)
+ cons_data, when = self.retrieve_constant_and_time(
+ karabo_id, karabo_da, const_dict, cal_db_interface, creation_time
+ )
self.init_constants(cons_data, when, module_idx)
@@ -1380,26 +1380,18 @@ class AgipdCorrections:
self.shared_dict = []
for i in range(n_cores_files):
self.shared_dict.append({})
- self.shared_dict[i]['cellId'] = sharedmem.empty(shape[0],
- dtype='u2')
- self.shared_dict[i]['pulseId'] = sharedmem.empty(shape[0],
- dtype='u8')
- self.shared_dict[i]['trainId'] = sharedmem.empty(shape[0],
- dtype='u8')
- self.shared_dict[i]['moduleIdx'] = sharedmem.empty(1, dtype='i4')
- self.shared_dict[i]['nImg'] = sharedmem.empty(1, dtype='i4')
- self.shared_dict[i]['mask'] = sharedmem.empty(shape, dtype='u4')
- self.shared_dict[i]['data'] = sharedmem.empty(shape, dtype='f4')
- self.shared_dict[i]['rawgain'] = sharedmem.empty(shape,
- dtype='u2')
- self.shared_dict[i]['gain'] = sharedmem.empty(shape, dtype='u1')
- self.shared_dict[i]['blShift'] = sharedmem.empty(shape[0],
- dtype='f4')
+ self.shared_dict[i]["cellId"] = sharedmem.empty(shape[0], dtype="u2")
+ self.shared_dict[i]["pulseId"] = sharedmem.empty(shape[0], dtype="u8")
+ self.shared_dict[i]["trainId"] = sharedmem.empty(shape[0], dtype="u8")
+ self.shared_dict[i]["moduleIdx"] = sharedmem.empty(1, dtype="i4")
+ self.shared_dict[i]["nImg"] = sharedmem.empty(1, dtype="i4")
+ self.shared_dict[i]["mask"] = sharedmem.empty(shape, dtype="u4")
+ self.shared_dict[i]["data"] = sharedmem.empty(shape, dtype="f4")
+ self.shared_dict[i]["rawgain"] = sharedmem.empty(shape, dtype="u2")
+ self.shared_dict[i]["gain"] = sharedmem.empty(shape, dtype="u1")
+ self.shared_dict[i]["blShift"] = sharedmem.empty(shape[0], dtype="f4")
# Parameters shared between image-wise correction functions
- self.shared_dict[i]['msk'] = sharedmem.empty(shape, dtype='i4')
- self.shared_dict[i]['raw_data'] = sharedmem.empty(shape,
- dtype='f4')
- self.shared_dict[i]['rel_corr'] = sharedmem.empty(shape,
- dtype='f4')
- self.shared_dict[i]['t0_rgain'] = sharedmem.empty(shape,
- dtype='u2')
+ self.shared_dict[i]["msk"] = sharedmem.empty(shape, dtype="i4")
+ self.shared_dict[i]["raw_data"] = sharedmem.empty(shape, dtype="f4")
+ self.shared_dict[i]["rel_corr"] = sharedmem.empty(shape, dtype="f4")
+ self.shared_dict[i]["t0_rgain"] = sharedmem.empty(shape, dtype="u2")
diff --git a/cal_tools/cal_tools/agipdutils.py b/cal_tools/cal_tools/agipdutils.py
index aec289dd6..44cac4c87 100644
--- a/cal_tools/cal_tools/agipdutils.py
+++ b/cal_tools/cal_tools/agipdutils.py
@@ -32,11 +32,11 @@ def assemble_constant_dict(
:param bias_voltage: (Int) Bias Voltage
:param gain_setting: (Float) Gain setting
:param acquisition_rate: (Float) Acquisition rate
- :param photon_energy: (Float) Photong energy
+ :param photon_energy: (Float) Photon energy
:param beam_energy: (Float) Beam Energy
:param only_dark: (Bool) Indicating a retrieval for dark constants only from db
:param gain_mode: Operation mode of the detector (default to adaptive gain)
- :return: const_dict: (Dict) An assembeld dictionary that can be used
+ :return: const_dict: (Dict) An assembled dictionary that can be used
to retrieve the required constants
"""
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index a6aa3f5e8..4e5575146 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -539,8 +539,7 @@ def get_from_db(karabo_id: str, karabo_da: str,
if ntries > 0:
if load_data and meta_only:
mdata_const = metadata.calibration_constant_version
- fpath = Path(mdata_const.hdf5path,
- mdata_const.filename)
+ fpath = Path(mdata_const.hdf5path, mdata_const.filename)
with h5py.File(fpath, "r") as f:
arr = f[f"{mdata_const.h5path}/data"][()]
metadata.calibration_constant.data = arr
diff --git a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
index c66a23a15..25abd5e19 100644
--- a/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
+++ b/notebooks/AGIPD/AGIPD_Correct_and_Verify.ipynb
@@ -101,11 +101,11 @@
"source": [
"import itertools\n",
"import math\n",
+ "import multiprocessing\n",
"import re\n",
"import traceback\n",
"import warnings\n",
"from datetime import timedelta\n",
- "from multiprocessing import Pool\n",
"from pathlib import Path\n",
"from time import perf_counter\n",
"\n",
@@ -486,7 +486,7 @@
"\n",
"\n",
"ts = perf_counter()\n",
- "with Pool(processes=len(modules)) as pool:\n",
+ "with multiprocessing.Pool(processes=len(modules)) as pool:\n",
" const_out = pool.map(retrieve_constants, modules)\n",
"print(f\"Constants were loaded in {perf_counter()-ts:.01f}s\")"
]
@@ -549,7 +549,7 @@
"metadata": {},
"outputs": [],
"source": [
- "with Pool() as pool:\n",
+ "with multiprocessing.Pool() as pool:\n",
" for file_batch in batches(file_list, n_cores_files):\n",
" # TODO: Move some printed output to logging or similar\n",
" print(f\"Processing next {len(file_batch)} files:\")\n",
--
GitLab