From 99df6d18c16e95345e298d568e2b6349722213e9 Mon Sep 17 00:00:00 2001
From: ahmedk <karim.ahmed@xfel.eu>
Date: Tue, 22 Feb 2022 13:48:50 +0100
Subject: [PATCH] flake8
---
src/cal_tools/tools.py | 71 ++++++++++++++++++++++++++++-------------
tests/conftest.py | 7 ++--
tests/test_cal_tools.py | 24 +++++++++++---
3 files changed, 74 insertions(+), 28 deletions(-)
diff --git a/src/cal_tools/tools.py b/src/cal_tools/tools.py
index c837fa5da..2c13b1aa0 100644
--- a/src/cal_tools/tools.py
+++ b/src/cal_tools/tools.py
@@ -12,7 +12,6 @@ from queue import Queue
from time import sleep
from typing import List, Optional, Tuple, Union
from urllib.parse import urljoin
-from extra_data import RunDirectory
import h5py
import ipykernel
@@ -20,6 +19,7 @@ import numpy as np
import requests
import yaml
import zmq
+from extra_data import RunDirectory
from iCalibrationDB import ConstantMetaData, Versions
from metadata_client.metadata_client import MetadataClient
from notebook.notebookapp import list_running_servers
@@ -246,8 +246,7 @@ def get_notebook_name():
def get_run_info(proposal, run):
- """
- Return information about run from the MDC
+ """Return information about run from the MDC
:param proposal: proposal number
:param run: run number
@@ -265,32 +264,42 @@ def get_run_info(proposal, run):
base_api_url=mdc_config['base-api-url'],
)
- runs = mdc.get_proposal_runs(proposal_number=proposal,
- run_number=run)
- if len(runs['data']) != 0:
- run_id = runs['data']['runs'][0]['id']
- return mdc.get_run_by_id_api(run_id).json()
+ mdc_response = mdc.get_proposal_runs(
+ proposal_number=proposal, run_number=run)
+
+ if mdc_response["success"]:
+ return mdc_response
else: # empty dictionary for wrong proposal or run.
- raise KeyError(runs['app_info'])
+ raise KeyError(mdc_response['app_info'])
def creation_date_metadata_client(
- proposal: int, run: int) -> datetime.datetime:
- """ Get run directory creation date from myMDC using metadata client.
+ proposal: int, run: int
+) -> datetime.datetime:
+ """Get run directory creation date from myMDC using metadata client.
using method `get_proposal_runs`.
+
+ :param proposal: proposal number e.g. 2656 or 900113.
+ :param run: run number.
+ :return Optional[datetime.datetime]: Run creation date.
"""
run_info = get_run_info(proposal, run)
return datetime.datetime.strptime(
- run_info['begin_at'], "%Y-%m-%dT%H:%M:%S.%f%z",
+ run_info['data']['runs'][0]['begin_at'],
+ "%Y-%m-%dT%H:%M:%S.%f%z",
).astimezone(datetime.timezone.utc)
+# TODO: update after DAQ store same date as myMDC.
def creation_date_file_metadata(
dc: RunDirectory
) -> Optional[datetime.datetime]:
- """ Get run directory creation date from
+ """Get run directory creation date from
METADATA/CreationDate of the oldest file using EXtra-data.
+
+ :param dc: EXtra-data DataCollection for the run directory.
+ :return Optional[datetime.datetime]: Run creation date.
"""
md_dict = dc.run_metadata()
@@ -299,16 +308,34 @@ def creation_date_file_metadata(
dc.files, key=lambda x: x.metadata()["creationDate"])[0]
return datetime.datetime.strptime(
oldest_file.metadata()["creationDate"],
- "%Y%m%dT%H%M%SZ",
- ).replace(tzinfo=datetime.timezone.utc)
+ "%Y%m%dT%H%M%S%z",
+ )
else:
print("WARNING: input files contains old datasets. "
- "No `METADATA/creationDate` to read.")
+ "No `METADATA/creationDate` to read.")
+
+
+def creation_date_train_timestamp(
+ dc: RunDirectory
+) -> Optional[datetime.datetime]:
+ """Get creation date from the timestamp of the first train.
+
+ :param dc: EXtra-data DataCollection for the run directory.
+ :return Optional[datetime.datetime]: Run creation date.
+ """
+
+ creation_date = np.datetime64(
+ dc.select_trains(np.s_[0]).train_timestamps()[0], 'us').item()
+ if creation_date is None:
+ print("WARNING: input files contains old datasets without"
+ " trains timestamps.")
+ return None
+ return creation_date.replace(tzinfo=datetime.timezone.utc)
def get_dir_creation_date(directory: Union[str, Path], run: int,
verbosity: int = 0) -> datetime.datetime:
- """ Get the directory creation data based on 3 different methods.
+ """Get the directory creation data based on 3 different methods.
1) Return run start time from myMDC. (get_runtime_metadata_client)
2) If myMDC connection is not set,
@@ -335,7 +362,8 @@ def get_dir_creation_date(directory: Union[str, Path], run: int,
dc = RunDirectory(directory)
except FileNotFoundError as e:
raise FileNotFoundError(
- f"- Failed to read creation time, wrong input folder", directory) from e
+ "- Failed to read creation time, wrong input folder",
+ directory) from e
try:
return creation_date_metadata_client(proposal, run)
@@ -343,15 +371,15 @@ def get_dir_creation_date(directory: Union[str, Path], run: int,
if verbosity > 0:
print(e)
- cdate = creation_date_file_metadata(dc)
+ cdate = creation_date_train_timestamp(dc)
if cdate is not None:
# Exposing the method used for reading the creation_date.
print("Reading creation_date from input files metadata"
- " `METADATA/creationDate`")
+ " `METADATA/creationDate`")
else: # It's an older dataset.
print("Reading creation_date from last modification data "
- "for the oldest input file.")
+ "for the oldest input file.")
cdate = datetime.datetime.fromtimestamp(
sorted(
[Path(f.filename) for f in dc.files],
@@ -359,7 +387,6 @@ def get_dir_creation_date(directory: Union[str, Path], run: int,
)[0].stat().st_mtime,
tz=datetime.timezone.utc,
)
-
return cdate
diff --git a/tests/conftest.py b/tests/conftest.py
index 8b1e2ac4c..fd71bd602 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -59,5 +59,8 @@ def pytest_runtest_setup(item):
if list(item.iter_markers(name="requires_caldb")) and not server_reachable():
pytest.skip("caldb not available")
- if list(item.iter_markers(name="requires_mdc")) and item.config.getoption("--no-mdc"):
- pytest.skip("myMDC not available")
\ No newline at end of file
+ if (
+ list(item.iter_markers(name="requires_mdc")) and
+ item.config.getoption("--no-mdc")
+ ):
+ pytest.skip("myMDC not available")
diff --git a/tests/test_cal_tools.py b/tests/test_cal_tools.py
index 243f6f617..577055ec1 100644
--- a/tests/test_cal_tools.py
+++ b/tests/test_cal_tools.py
@@ -13,6 +13,7 @@ from cal_tools.plotting import show_processed_modules
from cal_tools.tools import (
creation_date_file_metadata,
creation_date_metadata_client,
+ creation_date_train_timestamp,
get_dir_creation_date,
get_from_db,
get_pdu_from_db,
@@ -67,7 +68,7 @@ def test_dir_creation_date():
date = get_dir_creation_date(folder, 9983)
assert isinstance(date, datetime)
- assert str(date) == '2020-09-23 13:30:50+00:00'
+ assert str(date) == '2020-09-23 13:30:45.821262+00:00'
# The following data predates the addition of creation_time in metadata
date = get_dir_creation_date(folder, 9999)
@@ -81,9 +82,9 @@ def test_raise_dir_creation_date():
with pytest.raises(FileNotFoundError) as e:
get_dir_creation_date(folder, 4)
- print(e.value)
assert e.value.args[1] == Path(folder) / 'r0004'
+
@pytest.mark.requires_mdc
def test_creation_date_metadata_client():
@@ -101,6 +102,21 @@ def test_creation_date_file_metadata():
assert isinstance(date, datetime)
assert str(date) == '2020-09-23 13:30:50+00:00'
+
+@pytest.mark.parametrize("run,cdate", [
+ (9983, '2020-09-23 13:30:45.821262+00:00'),
+ (9999, 'None'),
+])
+@pytest.mark.requires_gpfs
+def test_creation_date_train_timestamp(run, cdate):
+
+ folder = f'/gpfs/exfel/exp/CALLAB/202031/p900113/raw/r{run}'
+
+ date = creation_date_train_timestamp(RunDirectory(folder))
+
+ assert str(date) == cdate
+
+
def _call_get_from_db(
constant,
condition,
@@ -160,7 +176,7 @@ def _call_send_to_db(
return metadata
-# TODO add a marker for accessing zmq end_point
+@pytest.mark.requires_caldb
@pytest.mark.requires_gpfs
def test_get_from_db_load_data(_agipd_const_cond):
""" Test retrieving calibration constants with get_from_db
@@ -197,7 +213,7 @@ def test_get_from_db_load_data(_agipd_const_cond):
assert isinstance(md, ConstantMetaData)
-# TODO add a marker for accessing zmq end_point
+@pytest.mark.requires_caldb
@pytest.mark.requires_gpfs
def test_raise_get_from_db(_agipd_const_cond):
""" Test error raised scenarios for get_from_db:"""
--
GitLab