From 27a9ca24e24b3c45b585ca3efb3feb3c7b107b00 Mon Sep 17 00:00:00 2001
From: Rafael Gort <rafael.gort@xfel.eu>
Date: Fri, 25 Sep 2020 18:29:18 +0200
Subject: [PATCH 1/3] Adaptation to processing routines such that they avoid
return values. Test suites to be updated
---
src/toolbox_scs/detectors/__init__.py | 3 ++-
src/toolbox_scs/detectors/dssc.py | 10 ++++-----
src/toolbox_scs/detectors/dssc_data.py | 8 +++++++
src/toolbox_scs/detectors/dssc_processing.py | 22 +++++++++++---------
4 files changed, 27 insertions(+), 16 deletions(-)
diff --git a/src/toolbox_scs/detectors/__init__.py b/src/toolbox_scs/detectors/__init__.py
index 545a784..8189827 100644
--- a/src/toolbox_scs/detectors/__init__.py
+++ b/src/toolbox_scs/detectors/__init__.py
@@ -3,7 +3,7 @@ from .xgm import (
from .tim import (
load_TIM,)
from .dssc_data import (
- save_xarray, load_xarray, get_data_formatted)
+ save_xarray, load_xarray, get_data_formatted, save_attributes_h5)
from .dssc_misc import (
load_dssc_info, create_dssc_bins, quickmask_DSSC_ASIC,
get_xgm_formatted, load_mask)
@@ -27,6 +27,7 @@ __all__ = (
"save_xarray",
"load_xarray",
"get_data_formatted",
+ "save_attributes_h5",
"quickmask_DSSC_ASIC",
"load_mask",
# Classes
diff --git a/src/toolbox_scs/detectors/dssc.py b/src/toolbox_scs/detectors/dssc.py
index 2c3e726..333abe4 100644
--- a/src/toolbox_scs/detectors/dssc.py
+++ b/src/toolbox_scs/detectors/dssc.py
@@ -182,7 +182,7 @@ class DSSCBinner:
# -------------------------------------------------------------------------
# Data processing
# -------------------------------------------------------------------------
- def process_data(self, modules=[],
+ def process_data(self, modules=[], filepath='./',
chunksize=512, backend='loky', n_jobs=None,
dark_image=None,
xgm_normalization=False, normevery=1
@@ -240,6 +240,7 @@ class DSSCBinner:
run_nr=self.runnr,
module=m,
chunksize=chunksize,
+ path=filepath,
info=self.info,
dssc_binners=self.binners,
pulsemask=self.pulsemask,
@@ -252,14 +253,13 @@ class DSSCBinner:
data = None
log.info(f'using parallelization backend {backend}')
- data = joblib.Parallel(n_jobs=njobs, backend=backend) \
+ joblib.Parallel(n_jobs=njobs, backend=backend) \
(joblib.delayed(process_dssc_data)(**module_jobs[i]) \
for i in range(len(mod_list)))
- data = xr.concat(data, dim='module')
- data = data.assign_coords(module=("module", np.array(mod_list)))
+ #data = xr.concat(data, dim='module')
+ #data = data.assign_coords(module=("module", np.array(mod_list)))
log.info(f'Binning done')
- return data
class DSSCFormatter:
diff --git a/src/toolbox_scs/detectors/dssc_data.py b/src/toolbox_scs/detectors/dssc_data.py
index f38afe6..fa918bb 100644
--- a/src/toolbox_scs/detectors/dssc_data.py
+++ b/src/toolbox_scs/detectors/dssc_data.py
@@ -157,6 +157,11 @@ def get_data_formatted(filenames=[], data_list=[]):
elif any(data_list) is True:
data = data_list
+ mod_list = []
+ for d in data:
+ if 'module' in d.attrs.keys():
+ mod_list.append(d.attrs['module'])
+
if type(data[0]).__module__ == 'xarray.core.dataset':
data = xr.concat(data, dim='module')
elif type(data[0]).__module__ == 'pandas.core.frame':
@@ -164,7 +169,10 @@ def get_data_formatted(filenames=[], data_list=[]):
elif type(data[0]).__module__ == 'dask.dataframe.core':
pass
+ if mod_list is not None:
+ data = data.assign_coords(module=("module", mod_list))
data = data.sortby("module")
+ data.attrs.clear()
return data.transpose('trainId', 'pulse', 'module', 'x', 'y')
diff --git a/src/toolbox_scs/detectors/dssc_processing.py b/src/toolbox_scs/detectors/dssc_processing.py
index fb8eb0d..256e13f 100644
--- a/src/toolbox_scs/detectors/dssc_processing.py
+++ b/src/toolbox_scs/detectors/dssc_processing.py
@@ -4,6 +4,7 @@
comment: contributions should comply with pep8 code structure guidelines.
"""
import logging
+import os
from tqdm import tqdm
import numpy as np
@@ -14,6 +15,7 @@ import extra_data as ed
from .dssc_misc import get_xgm_formatted
from ..constants import mnemonics as _mnemonics
+from .dssc_data import save_xarray
log = logging.getLogger(__name__)
@@ -134,11 +136,12 @@ def _load_chunk_xgm(sel, xgm_mnemonic, stride):
return d
-def process_dssc_data(proposal, run_nr, module, chunksize, info, dssc_binners,
+def process_dssc_data(proposal, run_nr, module, chunksize, info, dssc_binners,
+ path='./',
pulsemask=None,
dark_image=None,
xgm_normalization=False,
- xgm_mnemonic='SCS_SA3',
+ xgm_mnemonic='SCS_SA3',
normevery=1
):
"""
@@ -195,10 +198,6 @@ def process_dssc_data(proposal, run_nr, module, chunksize, info, dssc_binners,
# create empty dataset for dssc data to be filled iteratively
module_data = create_empty_dataset(info, dssc_binners)
- # progress bar for module 15
- if module == 15:
- pbar = tqdm(total=len(chunks))
-
# load data chunk by chunk and merge result into prepared empty dataset
for chunk in chunks:
log.debug(f"Module {module}: "
@@ -263,12 +262,15 @@ def process_dssc_data(proposal, run_nr, module, chunksize, info, dssc_binners,
log.debug(f"Module {module}: "
f"processed trains {chunk}:{chunk + chunksize}")
- if module == 15:
- pbar.update(1)
-
log.debug(f'Module {module}: calculate mean')
module_data['data'] = module_data['data'] / module_data['hist']
module_data = module_data.transpose('trainId', 'pulse', 'x', 'y')
+ module_data.attrs['module'] = module
+
+ log.info(f'saving module {module}')
+ if not os.path.isdir(path):
+ os.mkdir(path)
+ fname = f'run_{run_nr}_module{module}.h5'
+ save_xarray(path+fname, module_data, mode='a')
log.info(f"processing module {module}: done")
- return module_data
--
GitLab
From 90c4a286f0ca2e041299b9e659b010464bb94532 Mon Sep 17 00:00:00 2001
From: Rafael Gort <rafael.gort@xfel.eu>
Date: Fri, 25 Sep 2020 19:51:55 +0200
Subject: [PATCH 2/3] Removed outdated test suites
---
src/toolbox_scs/test/test_detectors_common.py | 121 ------
src/toolbox_scs/test/test_dssc_cls.py | 153 +-------
src/toolbox_scs/test/test_dssc_methods.py | 353 ------------------
3 files changed, 1 insertion(+), 626 deletions(-)
delete mode 100644 src/toolbox_scs/test/test_detectors_common.py
delete mode 100644 src/toolbox_scs/test/test_dssc_methods.py
diff --git a/src/toolbox_scs/test/test_detectors_common.py b/src/toolbox_scs/test/test_detectors_common.py
deleted file mode 100644
index 7e54d49..0000000
--- a/src/toolbox_scs/test/test_detectors_common.py
+++ /dev/null
@@ -1,121 +0,0 @@
-import unittest
-import logging
-import os
-import sys
-import argparse
-
-
-import toolbox_scs as tb
-import toolbox_scs.detectors as tbdet
-from toolbox_scs.util.exceptions import *
-
-logging.basicConfig(level=logging.DEBUG)
-log_root = logging.getLogger(__name__)
-
-suites = {"packaging": (
- "test_init",),
- "xgm": (
- "test_loadxgm",
- "test_cleanxgm",
- "test_matchxgmtim",),
- "tim": (
- "test_loadtim",)
- }
-
-
-def list_suites():
- print("""\nPossible test suites:\n-------------------------""")
- for key in suites:
- print(key)
- print("-------------------------\n")
-
-
-class TestDetectors(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
- log_root.info("Start global setup.")
- cls._run = tb.load_run(2212, 235)
-
- fields = ["sase1", "sase3", "npulses_sase3",
- "npulses_sase1", "MCP2apd", "SCS_SA3", "nrj"]
- cls._tb_data = tb.load(fields, 235, 2212)
-
- log_root.info("Finished global setup, start tests.")
-
- @classmethod
- def tearDownClass(cls):
- pass
-
- def setUp(self):
- pass
-
- def tearDown(self):
- pass
-
- def test_init(self):
- self.assertEqual(tbdet.__name__, "toolbox_scs.detectors")
-
- def test_loadxgm(self):
- cls = self.__class__
- xgm_data = tbdet.load_xgm(cls._run)
- self.assertTrue(xgm_data.values[0][-1])
-
- def test_cleanxgm(self):
- cls = self.__class__
- data = tbdet.cleanXGMdata(cls._tb_data)
- self.assertEqual(data['sa3_pId'].values[-1], 19)
-
- def test_matchxgmtim(self):
- cls = self.__class__
- data = tbdet.matchXgmTimPulseId(cls._tb_data)
- self.assertEqual(data['npulses_sase3'].values[0], 20)
-
- def test_loadtim(self):
- cls = self.__class__
- data = tbdet.load_TIM(cls._run)
- self.assertEqual(data.name, 'MCP2apd')
-
-
-
-def suite(*tests):
- suite = unittest.TestSuite()
- for test in tests:
- suite.addTest(TestDetectors(test))
- return suite
-
-
-def main(*cliargs):
- try:
- for test_suite in cliargs:
- if test_suite in suites:
- runner = unittest.TextTestRunner(verbosity=2)
- runner.run(suite(*suites[test_suite]))
- else:
- log_root.warning(
- "Unknown suite: '{}'".format(test_suite))
- pass
- except Exception as err:
- log_root.error("Unecpected error: {}".format(err),
- exc_info=True)
- pass
-
-
-
-
-
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser()
- parser.add_argument('--list-suites',
- action='store_true',
- help='list possible test suites')
- parser.add_argument('--run-suites', metavar='S',
- nargs='+', action='store',
- help='a list of valid test suites')
- args = parser.parse_args()
-
- if args.list_suites:
- list_suites()
-
- if args.run_suites:
- main(*args.run_suites)
diff --git a/src/toolbox_scs/test/test_dssc_cls.py b/src/toolbox_scs/test/test_dssc_cls.py
index f1bc984..c43b45f 100644
--- a/src/toolbox_scs/test/test_dssc_cls.py
+++ b/src/toolbox_scs/test/test_dssc_cls.py
@@ -19,14 +19,8 @@ log_root = logging.getLogger(__name__)
suites = {"no-processing": (
"test_create",
),
- "processing-quick": (
- "test_binning_quick",
- ),
"processing": (
- #"test_process_masking",
- #"test_process_normalization",
- "test_normalization_all",
- #"test_normalization_all2",
+ "test_normalization_all2",
)
}
@@ -102,151 +96,6 @@ class TestDSSC(unittest.TestCase):
self.assertEqual(str(cm.exception), err_msg)
- def test_binning_quick(self):
- # dark
- proposal_nb = 2212
- run_nb = 232
- run = tb.load_run(proposal_nb, run_nb, include='*DA*')
- run_info = tbdet.load_dssc_info(proposal_nb, run_nb)
- bins_trainId = tb.get_array(run,
- 'PP800_PhaseShifter',
- 0.03)
- bins_pulse = ['pumped', 'unpumped'] * 10
-
- binner1 = tbdet.create_dssc_bins("trainId",
- run_info['trainIds'],
- bins_trainId.values)
- binner2 = tbdet.create_dssc_bins("pulse",
- np.linspace(0,19,20, dtype=int),
- bins_pulse)
- binners = {'trainId': binner1, 'pulse': binner2}
- run232 = tbdet.DSSCBinner(proposal_nb, run_nb, binners=binners)
- mod_list = [0, 15]
- data = run232.process_data(modules=mod_list, chunksize=256)
- self.assertIsNotNone(data.data)
- #data = run232.process_data(backend='multiprocessing', n_jobs=10,
- # modules=mod_list,
- # chunksize=128)
- self.assertIsNotNone(data.data)
-
- tbdet.save_xarray('./tmp/run232.h5', data)
- data = tbdet.load_xarray('./tmp/run232.h5')
- self.assertIsNotNone(data.data)
-
-
- def test_process_masking(self):
- proposal_nb = 2212
- run_nb = 235
-
- run_info = tbdet.load_dssc_info(proposal_nb, run_nb)
- fpt = run_info['frames_per_train']
- n_trains = run_info['number_of_trains']
- trainIds = run_info['trainIds']
-
- run_obj = ed.open_run(proposal_nb,run_nb)
- del_pos = tb.get_array(run_obj, 'PP800_PhaseShifter', 0.03)
-
- buckets_train = del_pos.values
- buckets_pulse = ['pumped', 'unpumped'] * 10
-
- binner1 = tbdet.create_dssc_bins("trainId", trainIds, buckets_train)
- binner2 = tbdet.create_dssc_bins("pulse",
- np.linspace(0,fpt-1,fpt, dtype=int),
- buckets_pulse)
- binners = {'trainId': binner1, 'pulse': binner2}
- bin_obj = tbdet.DSSCBinner(proposal_nb, run_nb, binners)
-
- dssc_norm_every = 1
- xgm_threshold=(300.0, 8000.0)
- bin_obj.create_xgm_mask(xgm_threshold, dssc_norm_every)
-
- data = bin_obj.process_data(modules=[3], chunksize=248)
- self.assertIsNotNone(dark_data.data)
-
-
- def test_process_normalization(self):
- proposal_nb = 2212
- run_nb = 235
-
- run_info = tbdet.load_dssc_info(proposal_nb, run_nb)
- fpt = run_info['frames_per_train']
- n_trains = run_info['number_of_trains']
- trainIds = run_info['trainIds']
-
- run_obj = ed.open_run(proposal_nb,run_nb)
- del_pos = tb.get_array(run_obj, 'PP800_PhaseShifter', 0.03)
-
- buckets_train = del_pos.values
- buckets_pulse = ['pumped', 'unpumped'] * 10
-
- binner1 = tbdet.create_dssc_bins("trainId", trainIds, buckets_train)
- binner2 = tbdet.create_dssc_bins("pulse",
- np.linspace(0,fpt-1,fpt, dtype=int),
- buckets_pulse)
- binners = {'trainId': binner1, 'pulse': binner2}
- bin_obj = tbdet.DSSCBinner(proposal_nb, run_nb, binners)
-
- dssc_norm_every = 1
- xgm_threshold=(300.0, 8000.0)
- bin_obj.create_xgm_mask(xgm_threshold, dssc_norm_every)
-
- data = bin_obj.process_data(modules=[3,15], chunksize=248,
- xgm_normalization=True,
- normevery=dssc_norm_every)
- self.assertIsNotNone(data.data)
-
- def test_normalization_all(self):
- proposal_nb = 2212
- run_nb = 232
-
- run_info = tbdet.load_dssc_info(proposal_nb, run_nb)
- fpt = run_info['frames_per_train']
- n_trains = run_info['number_of_trains']
- trainIds = run_info['trainIds']
-
- buckets_train = np.zeros(n_trains)
- buckets_pulse = ['dark'] * fpt
-
- binner1 = tbdet.create_dssc_bins("trainId", trainIds, buckets_train)
- binner2 = tbdet.create_dssc_bins("pulse",
- np.linspace(0,fpt-1,fpt, dtype=int),
- buckets_pulse)
- binners = {'trainId': binner1, 'pulse': binner2}
- bin_obj = tbdet.DSSCBinner(proposal_nb, run_nb, binners)
-
- dark_data = bin_obj.process_data(modules=[2,3,4,15], chunksize=248)
- self.assertIsNotNone(dark_data.data)
-
- run_nb = 235
-
- run_info = tbdet.load_dssc_info(proposal_nb, run_nb)
- fpt = run_info['frames_per_train']
- n_trains = run_info['number_of_trains']
- trainIds = run_info['trainIds']
-
- run_obj = ed.open_run(proposal_nb,run_nb)
- del_pos = tb.get_array(run_obj, 'PP800_PhaseShifter', 0.03)
-
- buckets_train = del_pos.values
- buckets_pulse = ['pumped', 'unpumped'] * 10
-
- binner1 = tbdet.create_dssc_bins("trainId", trainIds, buckets_train)
- binner2 = tbdet.create_dssc_bins("pulse",
- np.linspace(0,fpt-1,fpt, dtype=int),
- buckets_pulse)
- binners = {'trainId': binner1, 'pulse': binner2}
- bin_obj = tbdet.DSSCBinner(proposal_nb, run_nb, binners)
-
- dssc_norm_every = 1
- xgm_threshold=(300.0, 8000.0)
- bin_obj.create_xgm_mask(xgm_threshold, dssc_norm_every)
-
- data = bin_obj.process_data(modules=[2,3,4,15], chunksize=248,
- dark_image=dark_data['data'],
- xgm_normalization=True,
- normevery=dssc_norm_every)
- self.assertIsNotNone(data.data)
-
def test_normalization_all2(self):
proposal_nb = 2530
diff --git a/src/toolbox_scs/test/test_dssc_methods.py b/src/toolbox_scs/test/test_dssc_methods.py
deleted file mode 100644
index 6b5739a..0000000
--- a/src/toolbox_scs/test/test_dssc_methods.py
+++ /dev/null
@@ -1,353 +0,0 @@
-import unittest
-import logging
-import os
-import argparse
-import shutil
-import joblib
-from time import strftime
-
-import numpy as np
-import xarray as xr
-
-import extra_data as ed
-import toolbox_scs as tb
-import toolbox_scs.detectors as tbdet
-from toolbox_scs.detectors.dssc_processing import (load_chunk_data,
- create_empty_dataset)
-from toolbox_scs.util.exceptions import ToolBoxFileError
-
-logging.basicConfig(level=logging.DEBUG)
-log_root = logging.getLogger(__name__)
-
-
-suites = {"metafunctions": (
- "test_info",
- "test_binners",
- #"test_createpulsemask",
- "test_createempty",
- #"test_loadmergechunk",
- #"test_storage",
- ),
- "binning-normalize": (
- "normalize_xgm",
- )
- }
-
-
-_temp_dirs = ['tmp']
-
-
-def setup_tmp_dir():
- for d in _temp_dirs:
- if not os.path.isdir(d):
- os.mkdir(d)
-
-
-def cleanup_tmp_dir():
- for d in _temp_dirs:
- shutil.rmtree(d, ignore_errors=True)
- log_root.info(f'remove {d}')
-
-
-class TestDSSC(unittest.TestCase):
- @classmethod
- def setUpClass(cls):
- log_root.info("Start global setup")
- # ---------------------------------------------------------------------
- # global test settings
- # ---------------------------------------------------------------------
-
- setup_tmp_dir()
-
- # ---------------------------------------------------------------------
- log_root.info("Finished global setup, start tests")
-
-
- @classmethod
- def tearDownClass(cls):
- log_root.info("Clean up test environment....")
- cleanup_tmp_dir()
-
-
- def test_info(self):
- proposal = 2212
- run_nr = 235
- info = tbdet.load_dssc_info(proposal, run_nr)
- self.assertEqual(info['frames_per_train'], 20)
-
-
- def test_binners(self):
- cls = self.__class__
- proposal = 2212
- run_nr = 235
- cls._dssc_info_235 = tbdet.load_dssc_info(proposal, run_nr)
- cls._run_235 = tb.load_run(proposal, run_nr, include='*DA*')
-
- # create 3 different binners manually
- bins_trainId_name = 'PP800_PhaseShifter'
- stepsize = .04
- bins_trainId = tb.get_array(cls._run_235,
- 'PP800_PhaseShifter',
- stepsize)
- bins_pulse = ['pumped', 'unpumped'] * 10
-
-
- bin1 = tbdet.create_dssc_bins("trainId",
- cls._dssc_info_235['trainIds'],
- bins_trainId.values)
- bin2 = tbdet.create_dssc_bins("pulse",
- np.linspace(0,19,20, dtype=int),
- bins_pulse)
- bin3 = tbdet.create_dssc_bins("x",
- np.linspace(1,128,128, dtype=int),
- np.linspace(1,128,128, dtype=int))
- # create binner with invalid name
- with self.assertRaises(ValueError) as err:
- bin2 = tbdet.create_dssc_bins("blabla",
- np.linspace(0,19,20, dtype=int),
- bins_pulse)
-
- # format binner dictionary. The 4th binner is constructed automatically
- binners = {"trainId": bin1,
- "pulse": bin2,
- "x": bin3}
- self.assertIsNotNone(binners)
- cls._binners_235 = {'trainId':bin1,'pulse':bin2}
-
-
- def test_createpulsemask(self):
- cls = self.__class__
- xgm_min = 0
- xgm_max = np.inf
-
- data = np.ones([len(cls._run_235.train_ids),
- cls._dssc_info_235['frames_per_train']], dtype=bool)
- dimensions = ['trainId', 'pulse']
- coordinates = {'trainId': cls._run_235.train_ids,
- 'pulse': range(cls._dssc_info_235['frames_per_train'])}
- pulsemask = xr.DataArray(data, dims=dimensions, coords=coordinates)
-
- valid = (cls._xgm > xgm_min) * (cls._xgm < xgm_max)
- cls._pulsemask = valid.combine_first(pulsemask).astype(bool)
-
- self.assertIsNotNone(cls._pulsemask)
-
-
- def test_createempty(self):
- cls = self.__class__
-
- # standard dset
- empty_data = create_empty_dataset(cls._dssc_info_235, cls._binners_235)
- self.assertIsNotNone(empty_data.dims['trainId'])
-
- # bin along pulse dimension only
- bins_pulse = ['pumped', 'unpumped', 'pumped_dark', 'unpumped_dark'] * 5
- binner_pulse = tbdet.create_dssc_bins("pulse",
- np.linspace(0,19,20, dtype=int),
- bins_pulse)
- empty_data = create_empty_dataset(cls._dssc_info_235,
- {'pulse':binner_pulse})
- self.assertEqual(empty_data.pulse.values[1], 'pumped_dark')
-
-
- def test_loadmergechunk(self):
- cls = self.__class__
- # load single chunk
- proposal = 2212
- run_nr = 235
- module = 1
- chunksize = 512
- sourcename = f'SCS_DET_DSSC1M-1/DET/{module}CH0:xtdf'
-
- collection = ed.open_run(proposal, run_nr,
- include=f'*DSSC{module:02d}*')
-
- binners = cls._binners_235
- info = cls._dssc_info_235
- pulsemask = cls._pulsemask
-
- ntrains = len(collection.train_ids)
- chunks = np.arange(ntrains, step=chunksize)
- start_index = chunks[0]
-
- module_data = create_empty_dataset(info, binners)
-
- for chunk in chunks[0:2]:
- sel = collection.select_trains(
- ed.by_index[chunk:chunk + chunksize])
- log_root.debug(f"Module {module}: "
- f"loading trains {chunk}:{chunk + chunksize}")
- chunk_data = load_chunk_data(sel, sourcename)
- self.assertIsNotNone(chunk_data)
-
- # pulse masking, and creation of related hist subset.
- chunk_hist = xr.full_like(chunk_data[:,:,0,0], fill_value=1)
- if pulsemask is not None:
- chunk_data = chunk_data.where(pulsemask)
- chunk_hist = chunk_hist.where(pulsemask)
- chunk_data = chunk_data.to_dataset(name='data')
- chunk_data['hist'] = chunk_hist
-
- # apply predefined binning
- log_root.debug(f'Module {module}: apply binning to chunk_data')
- for b in binners:
- chunk_data[b+"_binned"] = binners[b]
- chunk_data = chunk_data.groupby(b+"_binned").sum(b)
- chunk_data = chunk_data.rename(name_dict={b+"_binned":b})
- # ToDo: Avoid creation of unnecessary data when binning along x,y
-
- # merge dsets and format
- log_root.debug(f'Module {module}: merge data into prepared dset')
- for var in ['data', 'hist']:
- module_data[var] = xr.concat([module_data[var],
- chunk_data[var]],
- dim='tmp').sum('tmp')
-
- #module_data = module_data.transpose('trainId', 'pulse', 'x', 'y')
- cls._module_data = module_data
- self.assertIsNotNone(module_data)
-
-
- def normalize_xgm(self):
- cls = self.__class__
- # load single chunk
- proposal = 2530
- run_nr = 50
- module = 1
- chunksize = 128
- sourcename = f'SCS_DET_DSSC1M-1/DET/{module}CH0:xtdf'
-
- collection = ed.open_run(proposal, run_nr,
- include=f'*DSSC{module:02d}*')
- collection_DA1 = ed.open_run(proposal, run_nr, include='*DA01*')
-
- dssc_info = tbdet.load_dssc_info(proposal, run_nr)
- fpt = dssc_info['frames_per_train']
-
- buckets_train = np.zeros(len(collection.train_ids))
- binner1 = tbdet.create_dssc_bins(
- "trainId",collection.train_ids,buckets_train)
-
- buckets_pulse = ['image', 'dark'] * 99 + ['image_last']
- binner2 = tbdet.create_dssc_bins(
- "pulse",np.linspace(0,fpt-1,fpt, dtype=int),buckets_pulse)
- binners = {'trainId':binner1, 'pulse':binner2}
-
-
- ntrains = len(collection.train_ids)
- chunks = np.arange(ntrains, step=chunksize)
- start_index = chunks[0]
-
- module_data = create_empty_dataset(dssc_info, binners)
-
- for chunk in chunks[0:1]:
- sel = collection.select_trains(
- ed.by_index[chunk:chunk + chunksize])
- sel_DA1 = collection_DA1.select_trains(
- ed.by_index[chunk:chunk + chunksize])
- log_root.debug(f"Module {module}: "
- f"loading trains {chunk}:{chunk + chunksize}")
- chunk_data = load_chunk_data(sel, sourcename)
- self.assertIsNotNone(chunk_data)
-
- # pulse masking, and creation of related hist subset.
- chunk_hist = xr.full_like(chunk_data[:,:,0,0], fill_value=1)
- chunk_data = chunk_data.to_dataset(name='data')
- chunk_data['hist'] = chunk_hist
-
- log_root.debug('load and format xgm data')
- xgm_mnem = tb.mnemonics['SCS_SA3'].values()
- data_xgm = sel_DA1.get_array(*xgm_mnem)
- data_xgm = data_xgm.rename(
- new_name_or_name_dict={'XGMbunchId':'pulse'})
-
- xgm_shape = data_xgm.shape[1]
- norm_every = int(2)
- xgm_coord_arr = np.linspace(0,
- norm_every*(xgm_shape-1),
- xgm_shape, dtype=int)
-
- data_xgm = data_xgm.assign_coords({'pulse':xgm_coord_arr})
- log_root.debug('normalize chunk_data using xgm')
- chunk_data['data'][:,::norm_every,:,:] =\
- chunk_data['data'][:,::norm_every,:,:] /\
- data_xgm[:,0:int(np.ceil(fpt/norm_every))]
-
- # apply predefined binning
- log_root.debug(f'Module {module}: apply binning to chunk_data')
- for b in binners:
- chunk_data[b+"_binned"] = binners[b]
- chunk_data = chunk_data.groupby(b+"_binned").sum(b)
- chunk_data = chunk_data.rename(name_dict={b+"_binned":b})
-
- # merge dsets and format
- log_root.debug(f'Module {module}: merge data into prepared dset')
- for var in ['data', 'hist']:
- module_data[var] = xr.concat([module_data[var],
- chunk_data[var]],
- dim='tmp').sum('tmp')
-
- #module_data = module_data.transpose('trainId', 'pulse', 'x', 'y')
-
-
- def test_storage(self):
- cls = self.__class__
-
- tbdet.save_xarray('./tmp/run235.h5', cls._module_data)
- tbdet.save_xarray('./tmp/run235.h5', cls._module_data,
- mode = 'w')
- run235 = tbdet.load_xarray('./tmp/run235.h5')
-
- self.assertIsNotNone(run235)
- #run235.close()
-
- with self.assertRaises(ToolBoxFileError) as cm:
- tbdet.save_xarray('./tmp/run235.h5',
- cls._module_data.isel(pulse=0),
- mode = 'a')
-
-def list_suites():
- print("\nPossible test suites:\n" + "-" * 79)
- for key in suites:
- print(key)
- print("-" * 79 + "\n")
-
-
-def suite(*tests):
- suite = unittest.TestSuite()
- for test in tests:
- suite.addTest(TestDSSC(test))
- return suite
-
-
-def main(*cliargs):
- try:
- for test_suite in cliargs:
- if test_suite in suites:
- runner = unittest.TextTestRunner(verbosity=2)
- runner.run(suite(*suites[test_suite]))
- else:
- log_root.warning(
- "Unknown suite: '{}'".format(test_suite))
- pass
- except Exception as err:
- log_root.error("Unecpected error: {}".format(err),
- exc_info=True)
- pass
-
-
-if __name__ == '__main__':
- parser = argparse.ArgumentParser()
- parser.add_argument('--list-suites',
- action='store_true',
- help='list possible test suites')
- parser.add_argument('--run-suites', metavar='S',
- nargs='+', action='store',
- help='a list of valid test suites')
- args = parser.parse_args()
-
- if args.list_suites:
- list_suites()
-
- if args.run_suites:
- main(*args.run_suites)
--
GitLab
From def53c7d988927839070a733348dbec4a480c25f Mon Sep 17 00:00:00 2001
From: Rafael Gort <rafael.gort@xfel.eu>
Date: Fri, 25 Sep 2020 20:23:27 +0200
Subject: [PATCH 3/3] removed outdated comments
---
src/toolbox_scs/detectors/dssc.py | 2 --
1 file changed, 2 deletions(-)
diff --git a/src/toolbox_scs/detectors/dssc.py b/src/toolbox_scs/detectors/dssc.py
index 333abe4..a5aa64b 100644
--- a/src/toolbox_scs/detectors/dssc.py
+++ b/src/toolbox_scs/detectors/dssc.py
@@ -257,8 +257,6 @@ class DSSCBinner:
(joblib.delayed(process_dssc_data)(**module_jobs[i]) \
for i in range(len(mod_list)))
- #data = xr.concat(data, dim='module')
- #data = data.assign_coords(module=("module", np.array(mod_list)))
log.info(f'Binning done')
--
GitLab