From ddb81ffb67d045b893955817bbbe0085ad3e5c4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rodrigo=20P=C3=A9rez?= Date: Sun, 10 May 2026 01:20:26 -0400 Subject: [PATCH] feat: reopen log files on SIGUSR2 for logrotate Add reopen_file_handlers() and register SIGUSR2 after logging setup in main and parrot. Document create/postrotate/kill -USR2 in monitoring guide (EN/ES). SIGUSR2 only reopens file handlers; does not reload YAML. --- docs/en/server/user-guide/monitoring.md | 33 +++++++++++++++++++ docs/es/server/user-guide/monitoring.md | 33 +++++++++++++++++++ src/adn_server/infrastructure/__init__.py | 4 +-- .../infrastructure/logging_config.py | 29 ++++++++++++++++ src/adn_server/main.py | 8 ++++- src/adn_server/parrot_main.py | 8 ++++- 6 files changed, 111 insertions(+), 4 deletions(-) diff --git a/docs/en/server/user-guide/monitoring.md b/docs/en/server/user-guide/monitoring.md index 545dfe4..5a242f3 100644 --- a/docs/en/server/user-guide/monitoring.md +++ b/docs/en/server/user-guide/monitoring.md @@ -36,6 +36,39 @@ At **WARNING**: invalid HELLO JSON (`(REPORT) HELLO payload not valid JSON`), or The dashboard shows **operational** state from **START** (canonical); the **Monitor** log shows **INGRESS** plus **START** for troubleshooting mesh duplicates. +## Log file rotation (logrotate) + +After **logrotate** renames or moves a log file (common pattern: **`create`** so the old path is rotated away and a **new empty file** appears at the configured path), the process may still hold an open file descriptor on the **previous inode**. Logs then appear “missing” from the current path until the process **reopens** its file handlers. + +**Recommended:** use **`create`** (not **`copytruncate`** when the service supports signaling): **`copytruncate`** can race with concurrent writes and drop lines; **`WatchedFileHandler`** avoids signals but adds overhead per log record. + +These processes handle **`SIGUSR2`** by reopening **`logging.FileHandler`** streams only — **they do not reload YAML**, databases, or Twisted configuration. + +| Process | Typical config keys | +|---------|---------------------| +| **`adn-server`** / **`adn-parrot`** | **`LOGGER.LOG_FILE`** (see `adn-server.example.yaml`) | +| **`adn-proxy`** | **`LOG.PATH`** + **`LOG.LOG_FILE`** in `adn-proxy.yaml` | +| **`adn-monitor`** | **`LOG.PATH`** + **`LOG.LOG_FILE`** in `adn-monitor.yaml` | + +Example **`/etc/logrotate.d/adn`** fragment (adjust paths and service names): + +```text +/var/log/adn-server/adn-server.log { + weekly + rotate 12 + compress + delaycompress + missingok + notifempty + create 0640 adn adn + postrotate + /bin/kill -USR2 "$(systemctl show adn-server.service -p MainPID --value)" 2>/dev/null || true + endscript +} +``` + +Repeat **`postrotate`** with **`kill -USR2`** for **`adn-parrot`**, **`adn-proxy`**, and **`adn-monitor`** units if those logs are rotated on the same host. Use the correct **PID** (systemd **`MainPID`**, a pidfile, or **`kill`** targeting the process you manage). + ## Requirements - Network reachability from the **monitor host** to the server’s **`REPORTS.REPORT_PORT`** (and the server’s **`REPORT_CLIENTS`** allow list must include the monitor if used). diff --git a/docs/es/server/user-guide/monitoring.md b/docs/es/server/user-guide/monitoring.md index 875ac9c..9e6719c 100644 --- a/docs/es/server/user-guide/monitoring.md +++ b/docs/es/server/user-guide/monitoring.md @@ -36,6 +36,39 @@ En **WARNING**: JSON HELLO inválido (`(REPORT) HELLO payload not valid JSON`), El panel muestra el estado **operativo** desde **START** (canónico); el **log del Monitor** muestra **INGRESS** más **START** para depurar duplicados en malla. +## Rotación de logs (logrotate) + +Tras que **logrotate** renombre o mueva el fichero de log (patrón habitual: **`create`** — el fichero antiguo rota y aparece uno **nuevo vacío** en la ruta configurada), el proceso puede seguir con el descriptor abierto sobre el **inodo anterior**. Los logs parecen “no escribirse” en la ruta actual hasta que el proceso **reabra** los `FileHandler`. + +**Recomendado:** usar **`create`** (evitar **`copytruncate`** si el servicio admite señal): **`copytruncate`** puede competir con escrituras concurrentes y **perder líneas**; **`WatchedFileHandler`** evita la señal pero tiene coste **por cada línea** de log. + +Estos procesos tratan **`SIGUSR2`** solo para **reabrir** los ficheros de log (`logging.FileHandler`). **No recargan YAML**, bases de datos ni la configuración de Twisted. + +| Proceso | Claves típicas de configuración | +|---------|-----------------------------------| +| **`adn-server`** / **`adn-parrot`** | **`LOGGER.LOG_FILE`** (ver `adn-server.example.yaml`) | +| **`adn-proxy`** | **`LOG.PATH`** + **`LOG.LOG_FILE`** en `adn-proxy.yaml` | +| **`adn-monitor`** | **`LOG.PATH`** + **`LOG.LOG_FILE`** en `adn-monitor.yaml` | + +Ejemplo de fragmento en **`/etc/logrotate.d/adn`** (adaptar rutas y nombres de unidad): + +```text +/var/log/adn-server/adn-server.log { + weekly + rotate 12 + compress + delaycompress + missingok + notifempty + create 0640 adn adn + postrotate + /bin/kill -USR2 "$(systemctl show adn-server.service -p MainPID --value)" 2>/dev/null || true + endscript +} +``` + +Repite **`postrotate`** con **`kill -USR2`** para las unidades **`adn-parrot`**, **`adn-proxy`** y **`adn-monitor`** si rotas sus logs en el mismo host. Usa el **PID** correcto (**`MainPID`** de systemd, pidfile, o el proceso que gestiones). + ## Requisitos - Conectividad de red desde el **host del monitor** al **`REPORTS.REPORT_PORT`** del servidor (y la lista **`REPORT_CLIENTS`** del servidor debe incluir al monitor si se usa). diff --git a/src/adn_server/infrastructure/__init__.py b/src/adn_server/infrastructure/__init__.py index 2b287c9..1d67bd1 100644 --- a/src/adn_server/infrastructure/__init__.py +++ b/src/adn_server/infrastructure/__init__.py @@ -22,6 +22,6 @@ ############################################################################### from .config_loader import YamlConfigLoader -from .logging_config import setup_logging +from .logging_config import reopen_file_handlers, setup_logging -__all__ = ["YamlConfigLoader", "setup_logging"] +__all__ = ["YamlConfigLoader", "reopen_file_handlers", "setup_logging"] diff --git a/src/adn_server/infrastructure/logging_config.py b/src/adn_server/infrastructure/logging_config.py index f615380..3528586 100644 --- a/src/adn_server/infrastructure/logging_config.py +++ b/src/adn_server/infrastructure/logging_config.py @@ -31,6 +31,35 @@ from functools import partial, partialmethod from typing import Any +def reopen_file_handlers(logger: logging.Logger | None = None) -> int: + """Reopen all :class:`logging.FileHandler` streams on *logger* (default: root). + + Use after **logrotate** moves/renames the log file (``create`` + ``postrotate``), + so new writes go to the current path. Typically invoked from **SIGUSR2**. + + Does not reload YAML or change log level. Returns the number of handlers reopened. + """ + target = logger if logger is not None else logging.root + count = 0 + for handler in target.handlers: + if not isinstance(handler, logging.FileHandler): + continue + handler.acquire() + try: + handler.flush() + if handler.stream: + handler.stream.close() + handler.stream = handler._open() + count += 1 + except OSError as e: + sys.stderr.write( + "(LOGGER) Could not reopen log file %s: %s\n" % (getattr(handler, "baseFilename", "?"), e) + ) + finally: + handler.release() + return count + + def setup_logging(log_config: dict[str, Any]) -> logging.Logger: """Configure logging from CONFIG['LOGGER']. Returns root logger.""" level = getattr(logging, (log_config.get("LOG_LEVEL", "INFO")).upper(), logging.INFO) diff --git a/src/adn_server/main.py b/src/adn_server/main.py index 22e5eb1..94672c6 100644 --- a/src/adn_server/main.py +++ b/src/adn_server/main.py @@ -48,7 +48,7 @@ if str(_ROOT) not in sys.path: from twisted.internet import reactor, task, threads from .domain import bytes_3 -from .infrastructure import YamlConfigLoader, setup_logging +from .infrastructure import YamlConfigLoader, reopen_file_handlers, setup_logging from .infrastructure.config_normalizer import ( expand_generator as _expand_generator, ensure_system_runtime_config as _ensure_system_runtime_config, @@ -416,8 +416,14 @@ def main() -> None: if reactor.running: reactor.stop() + def sigusr2_reopen_logs(_sig, _frame): + """Logrotate: reopen file log handlers (does not reload config).""" + n = reopen_file_handlers() + logger.info("(LOGGER) Reopened %s file log handler(s) after SIGUSR2", n) + signal.signal(signal.SIGTERM, sig_handler) signal.signal(signal.SIGINT, sig_handler) + signal.signal(signal.SIGUSR2, sigusr2_reopen_logs) reactor.addSystemEventTrigger("before", "shutdown", shutdown_handler) # Voice config reload (15s): re-read adn-voice.yaml, start/stop announcement LoopingCalls on change diff --git a/src/adn_server/parrot_main.py b/src/adn_server/parrot_main.py index c5722c4..5c8b6a4 100644 --- a/src/adn_server/parrot_main.py +++ b/src/adn_server/parrot_main.py @@ -48,7 +48,7 @@ if str(_ROOT) not in sys.path: from twisted.internet import reactor, task -from .infrastructure import YamlConfigLoader, setup_logging +from .infrastructure import YamlConfigLoader, reopen_file_handlers, setup_logging from .infrastructure.config_normalizer import ( ensure_system_runtime_config, normalize_peer_config, @@ -122,8 +122,14 @@ def main() -> None: if reactor.running: reactor.stop() + def sigusr2_reopen_logs(_sig, _frame): + """Logrotate: reopen file log handlers (does not reload config).""" + n = reopen_file_handlers() + logger.info("(LOGGER) Reopened %s file log handler(s) after SIGUSR2", n) + signal.signal(signal.SIGTERM, sig_handler) signal.signal(signal.SIGINT, sig_handler) + signal.signal(signal.SIGUSR2, sigusr2_reopen_logs) logger.info("ADN Parrot -- SYSTEM STARTING...") for system_name, sys_cfg in systems_cfg.items():