Fix get_dir_creation_time

We cannot rely on the directory creation date, as it might get modified with filesystem upgrades. File modification dates are conserved though (not creation date), so we must use these when the information is not available from run metadata

cal_tools/cal_tools/tools.py

@@ -8,7 +8,7 @@ from os.path import isfile
 from pathlib import Path
 from queue import Queue
 from time import sleep
-from typing import Optional, Union
+from typing import Union
 from urllib.parse import urljoin
 
 import dateutil.parser
@@ -237,8 +237,8 @@ def get_dir_creation_date(directory: Union[str, Path], run: int,
     """
     Return run start time from MyDC.
     If not available from MyMDC, retrieve the data from the dataset's metadata
-    in [directory]/[run] or, if the dataset is older than 2020, from the
-    directory's creation time.
+    in [directory]/[run] or, if the dataset is older than 2020, from the oldest
+    file's modified time.
 
     If the data is not available from either source, this function will raise a
     ValueError.
@@ -267,18 +267,18 @@ def get_dir_creation_date(directory: Union[str, Path], run: int,
     while ntries > 0:
         try:
             rfiles = list(directory.glob('*.h5'))
-            rfiles.sort(key=path.getmtime)
             # get creation time for oldest file,
             # as creation time between run files
-            # should be different only within few seconds
-            with h5py.File(rfiles[0], 'r') as fin:
+            # should differ by a few seconds only.
+            rfile = sorted(rfiles, key=path.getmtime)[0]
+            with h5py.File(rfile, 'r') as fin:
                 cdate = fin['METADATA/creationDate'][0].decode()
                 cdate = datetime.datetime.strptime(cdate, "%Y%m%dT%H%M%SZ")
             return cdate
         except (IndexError, IOError, ValueError):
             ntries -= 1
         except KeyError:  # The files are here, but it's an older dataset
-            return datetime.datetime.fromtimestamp(directory.stat().st_ctime)
+            return datetime.datetime.fromtimestamp(rfile.stat().st_mtime)
 
     msg = 'Could not get the creation time from the directory'
     raise ValueError(msg, directory)

tests/test_cal_tools.py

@@ -29,4 +29,4 @@ def test_dir_creation_date():
     # The following data predates the addition of creation_time in metadata
     date = get_dir_creation_date(folder, 9999)
     assert isinstance(date, datetime)
-    assert str(date) == '2021-01-25 20:30:52.818820'
+    assert str(date) == '2019-12-16 08:52:25.196603'