From 7a862108f7fb47b2936c8f02d5bb982a398141a3 Mon Sep 17 00:00:00 2001
From: Cyril Danilevski <cyril.danilevski@xfel.eu>
Date: Tue, 17 Nov 2020 22:56:17 +0100
Subject: [PATCH] Silently retrieve the creation date from directory
If the information isn't available neither from MyMDC nor from the dataset, as might be the case for data prior to 2020, then the directory's creation date is used automatically
---
cal_tools/cal_tools/tools.py | 18 ++++++------------
tests/test_cal_tools.py | 10 +++-------
2 files changed, 9 insertions(+), 19 deletions(-)
diff --git a/cal_tools/cal_tools/tools.py b/cal_tools/cal_tools/tools.py
index 77b1ebcc2..5f1170d52 100644
--- a/cal_tools/cal_tools/tools.py
+++ b/cal_tools/cal_tools/tools.py
@@ -231,19 +231,18 @@ def get_run_info(proposal, run):
def get_dir_creation_date(directory: str, run: int,
- use_dir: Optional[bool] = False,
verbosity: Optional[int] = 0) -> datetime.datetime:
"""
Return run start time from MyDC.
If not available from MyMDC, retrieve the data from the dataset's metadata
- in [directory]/[run], or from the folder on disk if `use_dir` is set.
+ in [directory]/[run] or, if the dataset is older than 2020, from the
+ directory's creation time.
- If the files are not available, this function will raise a ValueError.
+ If the data is not available from either source, this function will raise a
+ ValueError.
:param directory: path to directory which contains runs
:param run: run number
- :param use_dir: get the creation time from the directory, needed for older
- datasets.
:param verbosity: Level of verbosity (0 - silent)
:return: (datetime) modification time
@@ -265,8 +264,6 @@ def get_dir_creation_date(directory: str, run: int,
ntries = 100
while ntries > 0:
try:
- if use_dir:
- return datetime.datetime.fromtimestamp(directory.stat().st_ctime) # noqa
dates = []
for f in directory.glob('*.h5'):
with h5py.File(f, 'r') as fin:
@@ -274,13 +271,10 @@ def get_dir_creation_date(directory: str, run: int,
cdate = datetime.datetime.strptime(cdate, "%Y%m%dT%H%M%SZ")
dates.append(cdate)
return min(dates)
-
except (IOError, ValueError):
ntries -= 1
- except KeyError:
- msg = ("It seems that you're working on old data. Try setting "
- "`use_dir=True` to get file creation date")
- raise ValueError(msg) from None
+ except KeyError: # The files are here, but it's an older dataset
+ return datetime.datetime.fromtimestamp(directory.stat().st_ctime)
msg = 'Could not get the creation time from the directory'
raise ValueError(msg, directory)
diff --git a/tests/test_cal_tools.py b/tests/test_cal_tools.py
index 5aef404be..865eb90c4 100644
--- a/tests/test_cal_tools.py
+++ b/tests/test_cal_tools.py
@@ -6,7 +6,7 @@ from cal_tools.tools import get_dir_creation_date
def test_dir_creation_date():
- folder = "/gpfs/exfel/exp/DETLAB/202031/p900172/raw"
+ folder = '/gpfs/exfel/exp/DETLAB/202031/p900172/raw'
date = get_dir_creation_date(folder, 10)
assert isinstance(date, datetime)
@@ -17,12 +17,8 @@ def test_dir_creation_date():
assert e.value.args[1] == Path(folder) / 'r0004'
# The following data predates the addition of creation_time in metadata
- folder = "/gpfs/exfel/exp/SQS/201930/p900075/raw/"
+ folder = '/gpfs/exfel/exp/SQS/201930/p900075/raw/'
- with pytest.raises(ValueError) as e:
- get_dir_creation_date(folder, 365)
- assert "It seems that you're working on old data." in e.value.args[0]
-
- date = get_dir_creation_date(folder, 365, use_dir=True)
+ date = get_dir_creation_date(folder, 365)
assert isinstance(date, datetime)
assert str(date) == '2019-07-04 11:02:41.280000'
--
GitLab