From 67ee77e25a0b4128ecbb63cd70c4662551a870dd Mon Sep 17 00:00:00 2001
From: Thomas Kluyver <thomas.kluyver@xfel.eu>
Date: Wed, 2 Oct 2024 10:48:02 +0100
Subject: [PATCH] Change default logs path, rotate logs weekly
---
webservice/common.py | 24 +++++++++++++++++++++---
webservice/job_monitor.py | 2 +-
webservice/webservice.py | 2 +-
3 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/webservice/common.py b/webservice/common.py
index f047979b4..be69b7c12 100644
--- a/webservice/common.py
+++ b/webservice/common.py
@@ -8,7 +8,10 @@ import errno
import gzip
import logging
import os
+import shutil
import socket
+from datetime import time
+from logging.handlers import TimedRotatingFileHandler
from pathlib import Path
def notify(message):
@@ -34,11 +37,26 @@ def notify_ready():
notify(b"READY=1")
-def file_and_stderr_logs(filename):
+def file_and_stderr_logs(path: Path):
"""Make logging handlers for files & stderr"""
- Path(filename).parent.mkdir(parents=True, exist_ok=True)
+ path = path.expanduser()
+ path.parent.mkdir(parents=True, exist_ok=True)
- filehandler = logging.FileHandler(filename)
+ # From Python logging cookbook: compress logs when they're rotated
+ def rotator(source, dest):
+ with open(source, 'rb') as f_in:
+ with gzip.open(dest, 'wb') as f_out:
+ shutil.copyfileobj(f_in, f_out)
+ os.remove(source)
+
+ filehandler = TimedRotatingFileHandler(
+ path,
+ when='W1', # W1 = Tuesday
+ atTime=time(12, 0), # At midday (after experiments finish)
+ backupCount=8, # weeks of logs to keep
+ )
+ filehandler.namer = lambda name: name + ".gz"
+ filehandler.rotator = rotator
filehandler.setFormatter(logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - [%(filename)s:%(lineno)d] %(message)s' # noqa
))
diff --git a/webservice/job_monitor.py b/webservice/job_monitor.py
index 6af2b6643..a1bfe41ac 100644
--- a/webservice/job_monitor.py
+++ b/webservice/job_monitor.py
@@ -398,7 +398,7 @@ def main(argv=None):
description='Start the calibration webservice'
)
parser.add_argument('--config-file', type=str, default=None)
- parser.add_argument('--log-file', type=str, default='./monitor.log')
+ parser.add_argument('--log-file', type=Path, default='~/webservice-logs/monitor.log')
parser.add_argument(
'--log-level', type=str, default="INFO", choices=['INFO', 'DEBUG', 'ERROR'] # noqa
)
diff --git a/webservice/webservice.py b/webservice/webservice.py
index 9444d49dd..7971b12fd 100644
--- a/webservice/webservice.py
+++ b/webservice/webservice.py
@@ -1610,7 +1610,7 @@ def main(argv: Optional[List[str]] = None):
)
parser.add_argument('--config-file', type=str, default=None)
parser.add_argument('--mode', type=str, default="sim", choices=['sim', 'prod']) # noqa
- parser.add_argument('--log-file', type=str, default='./web.log')
+ parser.add_argument('--log-file', type=Path, default='~/webservice-logs/web.log')
parser.add_argument(
'--log-level', type=str, default="INFO", choices=['INFO', 'DEBUG', 'ERROR'] # noqa
)
--
GitLab