From d5ef591e9e35a0814f48ea1e0b126a3104bb9a1d Mon Sep 17 00:00:00 2001
From: Rafael Gort <rafael.gort@xfel.eu>
Date: Wed, 12 Aug 2020 22:12:43 +0200
Subject: [PATCH] Started assembly of dssc formatter
---
src/toolbox_scs/detectors/__init__.py | 3 +-
src/toolbox_scs/detectors/dssc.py | 110 +++++++++++++++++++------
src/toolbox_scs/detectors/dssc_data.py | 72 +++++++++++++++-
src/toolbox_scs/util/exceptions.py | 2 +-
4 files changed, 159 insertions(+), 28 deletions(-)
diff --git a/src/toolbox_scs/detectors/__init__.py b/src/toolbox_scs/detectors/__init__.py
index 5a0df3e..831e750 100644
--- a/src/toolbox_scs/detectors/__init__.py
+++ b/src/toolbox_scs/detectors/__init__.py
@@ -3,7 +3,7 @@ from .xgm import (
from .tim import (
load_TIM,)
from .dssc_data import (
- save_to_file, load_from_file)
+ save_to_file, load_from_file, get_data_formatted)
from .dssc_misc import (
load_dssc_info, create_dssc_bins, load_geom, quickmask_DSSC_ASIC,
calc_xgm_frame_indices, get_xgm_formatted, load_mask)
@@ -27,6 +27,7 @@ __all__ = (
"bin_data",
"save_to_file",
"load_from_file",
+ "get_data_formatted",
"load_geom",
"quickmask_DSSC_ASIC",
"load_mask",
diff --git a/src/toolbox_scs/detectors/dssc.py b/src/toolbox_scs/detectors/dssc.py
index 65db106..145f092 100644
--- a/src/toolbox_scs/detectors/dssc.py
+++ b/src/toolbox_scs/detectors/dssc.py
@@ -21,22 +21,17 @@ import numpy as np
import xarray as xr
from ..load import load_run, get_array
-from .dssc_data import save_to_file
+from ..util.exceptions import ToolBoxValueError, ToolBoxFileError
+from .dssc_data import save_to_file, search_files, get_data_formatted
from .dssc_misc import (
- load_dssc_info, get_xgm_formatted)
+ load_dssc_info, get_xgm_formatted, load_geom)
from .dssc_processing import (
- bin_data, bin_data_multipr)
+ bin_data, bin_data_multipr, create_empty_dataset)
__all__ = ["DSSCBinner", "DSSCFormatter", "DSSCAnalyzer"]
log = logging.getLogger(__name__)
-def _setup_dir():
- for f in ['processed_runs']:
- if not os.path.isdir(f):
- os.mkdir(f)
-
-
class DSSCBinner:
def __init__(self, proposal_nr, run_nr,
binners = {},
@@ -81,14 +76,13 @@ class DSSCBinner:
2.) detailed -> docs
"""
- _setup_dir()
# ---------------------------------------------------------------------
# object (run) properties
# ---------------------------------------------------------------------
self.proposal = proposal_nr
self.runnr = run_nr
self.info = load_dssc_info(proposal_nr, run_nr)
- self.run = load_run(proposal_nr, run_nr, include='*DA*')
+ self.run = load_run(proposal_nr, run_nr)
self.xgmthreshold = xgm_threshold
self.binners = binners
@@ -119,6 +113,36 @@ class DSSCBinner:
def __del__(self):
pass
+
+ def add_binner(self, name, binner):
+ """
+ Add additional binner to internal dictionary
+
+ Parameters
+ ----------
+ ToDo ...
+
+ Raises
+ ------
+ ToDo....
+ """
+ if name in ['trainId','pulse','x','y']:
+ self.binners[name] = binner
+ else:
+ msg = "Invalid binner name"
+ log.info(msg+", no binner created")
+ raise ToolBoxValueError(msg, name)
+
+
+ def get_info(self):
+ if any(self.binners):
+ empty = create_empty_dataset(self.info, self.binners)
+ return(empty.dims)
+ else:
+ log.info("no binner defined yet.")
+ pass
+
+
def get_xgm_binned(self):
"""
Bin the xgm data according to the binners of the dssc data. The result
@@ -202,25 +226,46 @@ class DSSCBinner:
class DSSCFormatter:
- def __init__(self):
+ def __init__(self, filepath):
"""
- The DSSC formatter class brings data, produced by the DSSC binner class
- into a standardized format, such that the following analysis software
- is unaffected by changes of the underlying data reduction.
- """
- pass
+ Class that will handle formatting and detector geometry related
+ aspects.
- def merge_modules_xarray(self, filelist):
- pass
+ Parameters
+ ----------
+ path_to_processed: str
+ location of processed files.
- def add_xgm_data():
- pass
+ Returns
+ -------
+ ToDo ...
- def data_from_filelist():
- pass
+ Raises
+ ------
+ ToDo....
- def get_data_formatted():
- pass
+ Example
+ -------
+ """
+ self._filenames = []
+ if os.path.exists(filepath):
+ try:
+ self.filenames = search_files(filepath)
+ except ToolBoxFileError as err:
+ log.info("path did not contain any files")
+ else:
+ log.info("path did not exist")
+ self.data = None
+
+ self.geom = load_geom()
+
+ def combine_files(self, filenames=[]):
+ if any(filenames) is True:
+ self.filenames = filenames
+ if self.filenames is not None:
+ self.data = get_data_formatted(self.filenames)
+ else:
+ log.info("No matching data found.")
class DSSCAnalyzer:
@@ -228,6 +273,21 @@ class DSSCAnalyzer:
"""
Placeholder for future class handling image operation/manipulation
(the actual data evaluation).
+
+ Parameters
+ ----------
+ ToDo ...
+
+ Returns
+ -------
+ ToDo ...
+
+ Raises
+ ------
+ ToDo....
+
+ Example
+ -------
"""
# self.distance = distance
# self.pxpitchh = 236
diff --git a/src/toolbox_scs/detectors/dssc_data.py b/src/toolbox_scs/detectors/dssc_data.py
index e6c9995..a1084bd 100644
--- a/src/toolbox_scs/detectors/dssc_data.py
+++ b/src/toolbox_scs/detectors/dssc_data.py
@@ -55,4 +55,74 @@ def load_from_file(fname):
data engine that is malfunctioning due to related software installed in the
exfel_anaconda3 environment. May be dropped at some point.
"""
- return xr.open_dataset(fname, 'data', engine='h5netcdf')
+ f_exists = os.path.isfile(fname)
+ if f_exists:
+ return xr.open_dataset(fname, 'data', engine='h5netcdf')
+ else:
+ msg = "File does not exists."
+ raise ToolBoxFileError(msg, fname)
+
+
+def _data_from_list(filenames):
+ """
+ ToDo
+
+ Parameters
+ ----------
+
+ Returns
+ -------
+ """
+ data = []
+ for name in filenames:
+ f_exists = os.path.isfile(name)
+ if f_exists:
+ data.append(load_from_file(name))
+ else:
+ msg = "File does not exists."
+ raise ToolBoxFileError(msg, name)
+ return data
+
+
+def get_data_formatted(filenames=[], data_list=[]):
+ """
+ ToDo
+
+ Parameters
+ ----------
+
+ Returns
+ -------
+ """
+ if any(filenames) is True:
+ data = _data_from_list(filenames)
+ elif any(data_list) is True:
+ data = data_list
+
+ if type(data[0]).__module__ == 'xarray.core.dataset':
+ data = xr.concat(data, dim='module')
+ elif type(data[0]).__module__ == 'pandas.core.frame':
+ pass
+ elif type(data[0]).__module__ == 'dask.dataframe.core':
+ pass
+
+ data = data.sortby("module")
+ return data.transpose('trainId','pulse','module','x','y')
+
+
+def search_files(run_folder):
+ """
+ ToDo
+
+ Parameters
+ ----------
+
+ Returns
+ -------
+ """
+ try:
+ filenames = os.listdir(run_folder)
+ return [run_folder+name for name in filenames if ".h5" in name]
+ except:
+ msg = "No files in folder"
+ raise ToolBoxFileError(msg, run_folder)
diff --git a/src/toolbox_scs/util/exceptions.py b/src/toolbox_scs/util/exceptions.py
index 9b8049a..52aa3c5 100644
--- a/src/toolbox_scs/util/exceptions.py
+++ b/src/toolbox_scs/util/exceptions.py
@@ -20,7 +20,7 @@ class ToolBoxTypeError(ToolBoxError):
class ToolBoxValueError(ToolBoxError):
def __init__(self, msg = "", val = None):
self.value = val
- self.message = msg + " unknown value: " + val
+ self.message = msg + " unknown value: " + str(val)
class ToolBoxFileError(ToolBoxError):
--
GitLab