From ae87d29208a754cd9f027694fd002bbe114f3be4 Mon Sep 17 00:00:00 2001
From: Rafael Gort <rafael.gort@xfel.eu>
Date: Thu, 30 Apr 2020 22:11:39 +0200
Subject: [PATCH] Started assembly of dssc related sub-routines based on ms
inital package. tested
---
src/toolbox_scs/__init__.py | 7 +-
src/toolbox_scs/detectors/dssc_process.py | 35 --------
src/toolbox_scs/load.py | 99 ++++++++++++++++++++-
src/toolbox_scs/misc/data_access.py | 61 +++++++++++++
src/toolbox_scs/test/test_data_access.py | 102 ++++++++++++++++++++++
src/toolbox_scs/test/test_top_level.py | 66 +++++++++++---
src/toolbox_scs/util/exceptions.py | 21 +++++
7 files changed, 343 insertions(+), 48 deletions(-)
create mode 100644 src/toolbox_scs/misc/data_access.py
create mode 100644 src/toolbox_scs/test/test_data_access.py
create mode 100644 src/toolbox_scs/util/exceptions.py
diff --git a/src/toolbox_scs/__init__.py b/src/toolbox_scs/__init__.py
index 9b44a26..7fe650e 100644
--- a/src/toolbox_scs/__init__.py
+++ b/src/toolbox_scs/__init__.py
@@ -1,10 +1,15 @@
-from .load import load, concatenateRuns
+from .load import (load, concatenateRuns, load_scan_variable,
+ run_by_proposal, run_by_path)
+
from .constants import mnemonics
__all__ = (
# Top level functions
"load",
"concatenateRuns",
+ "load_scan_variable",
+ "run_by_proposal",
+ "run_by_path",
# Classes
# Variables
"mnemonics",
diff --git a/src/toolbox_scs/detectors/dssc_process.py b/src/toolbox_scs/detectors/dssc_process.py
index ee6981b..4c0ef47 100644
--- a/src/toolbox_scs/detectors/dssc_process.py
+++ b/src/toolbox_scs/detectors/dssc_process.py
@@ -20,41 +20,6 @@ import xarray as xr
import pandas as pd
import extra_data as ed
-from extra_data.read_machinery import find_proposal
-
-
-def find_run_dir(proposal, run):
- '''returns the raw data folder for given (integer) proposal and run number'''
- proposal_dir = find_proposal(f'p{proposal:06d}')
- return os.path.join(proposal_dir, f'raw/r{run:04d}')
-
-
-def load_scan_variable(run, scan_variable, stepsize=None):
- '''
- Loads the given scan variable and rounds scan positions to integer multiples of "stepsize"
- for consistent grouping (except for stepsize=None).
- Returns a dummy scan if scan_variable is set to None.
- Parameters:
- run : (karabo_data.DataCollection) RunDirectory instance
- scan_variable : (tuple of str) ("source name", "value path"), examples:
- ('SCS_ILH_LAS/PHASESHIFTER/DOOCS', 'actualPosition.value')
- ('SCS_ILH_LAS/DOOCS/PPL_OPT_DELAY', 'actualPosition.value')
- ('SA3_XTD10_MONO/MDL/PHOTON_ENERGY', 'actualEnergy.value')
- None creates a dummy file to average over all trains of the run
- stepsize : (float) nominal stepsize of the scan - values of scan_variable will be
- rounded to integer multiples of this value
- '''
- if scan_variable is not None:
- source, path = scan_variable
- scan = run.get_array(source, path)
- if stepsize is not None:
- scan = stepsize * np.round(scan / stepsize)
- else:
- # dummy scan variable - this will average over all trains
- scan = xr.DataArray(np.ones(len(run.train_ids), dtype=np.int16),
- dims=['trainId'], coords={'trainId': run.train_ids})
- scan.name = 'scan_variable'
- return scan
def load_xgm(run, print_info=False):
diff --git a/src/toolbox_scs/load.py b/src/toolbox_scs/load.py
index 89acc64..172917d 100644
--- a/src/toolbox_scs/load.py
+++ b/src/toolbox_scs/load.py
@@ -6,14 +6,18 @@
Copyright (2019) SCS Team.
"""
import os
+import logging
import numpy as np
import xarray as xr
-from extra_data import by_index, RunDirectory
+from extra_data import by_index, RunDirectory, open_run
from extra_data.read_machinery import find_proposal
from .misc.bunch_pattern import extractBunchPattern
from .constants import mnemonics as _mnemonics_ld
+from .util.exceptions import *
+
+log = logging.getLogger(__name__)
def load(fields, runNB, proposalNB, subFolder='raw', display=False, validate=False,
@@ -144,3 +148,96 @@ def concatenateRuns(runs):
result.attrs[k] = [run.attrs[k] for run in orderedRuns]
return result
+
+def run_by_proposal(proposal, run):
+ """
+ Get run in given proposal
+
+ Wraps the extra_data open_run routine, to ease its use for the
+ scs-toolbox user.
+
+ Parameters
+ ----------
+ proposal: str, int
+ Proposal number
+ run: str, int
+ Run number
+
+ Returns
+ -------
+ run : extra_data.DataCollection
+ DataCollection object containing information about the specified
+ run. Data can be loaded using built-in class methods.
+ """
+ return open_run(proposal=proposal, run=run)
+
+
+def run_by_path(path):
+ """
+ Return specified run
+
+ Wraps the extra_data RunDirectory routine, to ease its use for the
+ scs-toolbox user.
+
+ Parameters
+ ----------
+ path: str
+ path to the run directory
+
+ Returns
+ -------
+ run : extra_data.DataCollection
+ DataCollection object containing information about the specified
+ run. Data can be loaded using built-in class methods.
+ """
+ return RunDirectory(path)
+
+
+def load_scan_variable(run, mnemonic, stepsize=None):
+ """
+ Loads the given scan variable and rounds scan positions to integer
+ multiples of stepsize for consistent grouping (except for
+ stepsize=None).
+ Returns a dummy scan if scan_variable is set to None.
+
+ Parameters
+ ----------
+ run: karabo_data.DataCollection
+ path to the run directory
+ mnemonic: dic
+ single entry of mnemonics collection. None creates a dummy file
+ to average over all trains of the run
+ stepsize : float
+ nominal stepsize of the scan - values of scan_variable will be
+ rounded to integer multiples of this value
+
+ Returns
+ -------
+ scan : xarray.DataArray
+ xarray DataArray containing the specified scan variable using
+ the trainId as coordinate.
+
+ Example
+ -------
+ >>> import toolbox_scs as tb
+ >>> run = tb.run_by_proposal(2212, 235)
+ >>> mnemonic = 'PP800_PhaseShifter'
+ >>> scan_variable = tb.load_scan_variable(
+ self.ed_run, mnemonic, 0.5)
+ """
+
+ try:
+ if mnemonic not in _mnemonics_ld:
+ raise ToolBoxValueError("Invalid mnemonic given", mnemonic)
+ mnem = _mnemonics_ld[mnemonic]
+ data = run.get_array(mnem['source'],
+ mnem['key'], mnem['dim'])
+ if stepsize is not None:
+ data = stepsize * np.round(data / stepsize)
+ data.name = 'scan_variable'
+ log.debug(f"Constructed scan variable for {mnemonic}")
+ except ToolBoxValueError:
+ log.error("Invalid mnemonic, raise ToolBoxValueError.")
+ raise
+
+ return data
\ No newline at end of file
diff --git a/src/toolbox_scs/misc/data_access.py b/src/toolbox_scs/misc/data_access.py
new file mode 100644
index 0000000..b088bac
--- /dev/null
+++ b/src/toolbox_scs/misc/data_access.py
@@ -0,0 +1,61 @@
+'''
+Extensions to the extra_data package.
+
+contributions should comply with pep8 code structure guidelines.
+'''
+
+import os
+import logging
+
+import extra_data as ed
+from extra_data.read_machinery import find_proposal
+
+from ..util.exceptions import ToolBoxPathError
+
+log = logging.getLogger(__name__)
+
+def find_run_dir(proposal, run):
+ """
+ Get run directory for given run.
+
+ This method is an extension to the extra_data method
+ 'find_proposal' and should eventually be transferred over.
+
+ Parameters
+ ----------
+ proposal: str, int
+ Proposal number
+ run: str, int
+ Run number
+
+ Returns
+ -------
+ rdir : str
+ Run directory as a string
+
+ Raises
+ ------
+ ToolBoxPathError: Exception
+ Error raised if the constructed path does not exist. This may
+ happen when entering a non-valid run number, or the folder has
+ been renamed/removed.
+
+ """
+ rdir = None
+
+ try:
+ pdir = find_proposal(f'p{proposal:06d}')
+ rdir = os.path.join(pdir, f'raw/r{run:04d}')
+ if os.path.isdir(rdir) is False:
+ log.warning("Invalid directory: raise ToolBoxPathError.")
+ msg = f"The constructed path '{rdir}' does not exist"
+ raise ToolBoxPathError(msg, rdir)
+
+ except ToolBoxPathError:
+ raise
+ except Exception as err:
+ log.error("Unexpected error:", exc_info=True)
+ log.warning("Unexpected error orrured, return None")
+ pass
+
+ return rdir
\ No newline at end of file
diff --git a/src/toolbox_scs/test/test_data_access.py b/src/toolbox_scs/test/test_data_access.py
new file mode 100644
index 0000000..5440836
--- /dev/null
+++ b/src/toolbox_scs/test/test_data_access.py
@@ -0,0 +1,102 @@
+import unittest
+import logging
+import os
+import sys
+import argparse
+
+
+from toolbox_scs.misc.data_access import (
+ find_run_dir,
+ )
+from toolbox_scs.util.exceptions import ToolBoxPathError
+
+suites = {"proposal-handlers": (
+ "test_rundir1",
+ "test_rundir2",
+ "test_rundir3",
+ )
+ }
+
+
+def list_suites():
+ print("""\nPossible test suites:\n-------------------------""")
+ for key in suites:
+ print(key)
+ print("-------------------------\n")
+
+
+class TestDataAccess(unittest.TestCase):
+ def setUp(self):
+ pass
+
+ def tearDown(self):
+ pass
+
+ def test_rundir1(self):
+ Proposal = 2212
+ Run = 235
+ Dir = find_run_dir(Proposal, Run)
+ self.assertEqual(Dir,
+ "/gpfs/exfel/exp/SCS/201901/p002212/raw/r0235")
+
+ def test_rundir2(self):
+ Proposal = 23678
+ Run = 235
+ Dir = find_run_dir(Proposal, Run)
+ self.assertEqual(Dir, None)
+
+ def test_rundir3(self):
+ Proposal = 2212
+ Run = 2325
+ with self.assertRaises(ToolBoxPathError) as cm:
+ find_run_dir(Proposal, Run)
+ the_exception = cm.exception
+ path = '/gpfs/exfel/exp/SCS/201901/p002212/raw/r2325'
+ err_msg = f"The constructed path '{path}' does not exist"
+ self.assertEqual(the_exception.message, err_msg)
+
+
+def suite(*tests):
+ suite = unittest.TestSuite()
+ for test in tests:
+ suite.addTest(TestDataAccess(test))
+ return suite
+
+
+def main(*cliargs):
+ logging.basicConfig(level=logging.DEBUG)
+ log_root = logging.getLogger(__name__)
+ try:
+ for test_suite in cliargs:
+ if test_suite in suites:
+ runner = unittest.TextTestRunner(verbosity=2)
+ runner.run(suite(*suites[test_suite]))
+ else:
+ log_root.warning(
+ "Unknown suite: '{}'".format(test_suite))
+ pass
+ except Exception as err:
+ log_root.error("Unecpected error: {}".format(err),
+ exc_info=True)
+ pass
+
+
+
+
+
+
+if __name__ == '__main__':
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--list-suites',
+ action='store_true',
+ help='list possible test suites')
+ parser.add_argument('--run-suites', metavar='S',
+ nargs='+', action='store',
+ help='a list of valid test suites')
+ args = parser.parse_args()
+
+ if args.list_suites:
+ list_suites()
+
+ if args.run_suites:
+ main(*args.run_suites)
diff --git a/src/toolbox_scs/test/test_top_level.py b/src/toolbox_scs/test/test_top_level.py
index 5703c30..d696f43 100644
--- a/src/toolbox_scs/test/test_top_level.py
+++ b/src/toolbox_scs/test/test_top_level.py
@@ -6,10 +6,19 @@ import argparse
import toolbox_scs as tb
-
-suites = {"suite-1": (
+from toolbox_scs.util.exceptions import *
+import extra_data as ed
+
+suites = {"packaging": (
+ "test_constant",
+
+ ),
+ "load": (
"test_load",
- "test_load"
+ "test_openrun",
+ #"test_openrunpath",
+ "test_loadscanvariable1",
+ "test_loadscanvariable2",
)
}
@@ -21,22 +30,57 @@ def list_suites():
print("-------------------------\n")
-class TemplateTest(unittest.TestCase):
+class TestToolbox(unittest.TestCase):
def setUp(self):
- pass
+ self.mnentry = 'SCS_RR_UTC/MDL/BUNCH_DECODER'
+ self.ed_run = ed.open_run(2212, 235)
def tearDown(self):
pass
+ def test_constant(self):
+ self.assertEqual(tb.mnemonics['sase3']['source'],self.mnentry)
+
def test_load(self):
- #tb.load(....)
- pass
+ proposalNB = 2511
+ runNB = 176
+ fields = ["SCS_XGM"]
+ run_tb = tb.load(fields, runNB, proposalNB,
+ validate=False, display=False)
+ self.assertEqual(run_tb['npulses_sase3'].values[0], 42)
+
+ def test_openrun(self):
+ self.run = tb.run_by_proposal(2212, 235)
+ src = 'SCS_DET_DSSC1M-1/DET/0CH0:xtdf'
+ self.assertTrue(src in self.run.all_sources)
+
+ def test_openrunpath(self):
+ run = tb.run_by_path(
+ "/gpfs/exfel/exp/SCS/201901/p002212/raw/r0235")
+ src = 'SCS_DET_DSSC1M-1/DET/0CH0:xtdf'
+ self.assertTrue(src in run.all_sources)
+
+ def test_loadscanvariable1(self):
+ mnemonic = 'PP800_PhaseShifter'
+ scan_variable = tb.load_scan_variable(self.ed_run, mnemonic, 0.5)
+ self.assertTrue = (scan_variable)
+
+ def test_loadscanvariable2(self):
+ mnemonic = 'blabla'
+ scan_variable = None
+ with self.assertRaises(ToolBoxValueError) as cm:
+ scan_variable = tb.load_scan_variable(self.ed_run, mnemonic, 0.5)
+ excp = cm.exception
+ err_msg = "Invalid mnemonic given"
+ self.assertEqual(excp.message, err_msg)
+ self.assertFalse(scan_variable)
+
def suite(*tests):
suite = unittest.TestSuite()
for test in tests:
- suite.addTest(TemplateTest(test))
+ suite.addTest(TestToolbox(test))
return suite
@@ -67,7 +111,7 @@ if __name__ == '__main__':
parser.add_argument('--list-suites',
action='store_true',
help='list possible test suites')
- parser.add_argument('--test-suites', metavar='S',
+ parser.add_argument('--run-suites', metavar='S',
nargs='+', action='store',
help='a list of valid test suites')
args = parser.parse_args()
@@ -75,5 +119,5 @@ if __name__ == '__main__':
if args.list_suites:
list_suites()
- if args.test_suites:
- main(*args.test_suites)
+ if args.run_suites:
+ main(*args.run_suites)
diff --git a/src/toolbox_scs/util/exceptions.py b/src/toolbox_scs/util/exceptions.py
new file mode 100644
index 0000000..1aa8858
--- /dev/null
+++ b/src/toolbox_scs/util/exceptions.py
@@ -0,0 +1,21 @@
+class ToolBoxError(Exception):
+ """ Parent Toolbox exception."""
+ pass
+
+
+class ToolBoxPathError(ToolBoxError):
+ """ Raise in case of error related to a file path. """
+ def __init__(self, message = "", path = ""):
+ self.path = path
+ self.message = message
+
+class ToolBoxInputError(ToolBoxError):
+ """ Raise in case of error related to a file path. """
+ def __init__(self, message = ""):
+ self.message = message
+
+class ToolBoxValueError(ToolBoxError):
+ """ Raise in case of error related to a file path. """
+ def __init__(self, msg = "", val = None):
+ self.value = val
+ self.message = msg
\ No newline at end of file
--
GitLab