✨ chore(logs): add system-wide logging alongside module-local logs

- introduce central system_logs directory for aggregated branch logs - update logger setup to include dual handlers for system and local logs - modify discovery and watcher to track system log file paths - enhance status reporting to show system logs directory - update .gitignore to exclude *.json files alongside backups
2026-03-07 16:27:57 -08:00
parent fe187be8cd
commit 6bd7395763
8 changed files with 88 additions and 42 deletions
@@ -102,4 +102,5 @@ notepad.md
 !src/commons/apps/handlers/artifacts/
 !src/commons/apps/handlers/artifacts/*.py

-.backup
+.backup
+*.json
@@ -64,8 +64,8 @@ _system_logs_dir_cache: Path | None = None
 def get_system_logs_dir() -> Path:
    """Lazily resolve and create system_logs directory (package-relative).

-    DEPRECATED: Use get_module_logs_dir(module_name) for per-module logging.
-    Kept for monitoring code that scans a central directory.
+    Central aggregation: all branches log here for system-wide monitoring.
+    Per-module logs use get_module_logs_dir() for local debugging.
    """
    global _system_logs_dir_cache
    if _system_logs_dir_cache is None:
@@ -30,6 +30,7 @@ from typing import Dict, Any
 # Import from prax config
 from aipass.prax.apps.handlers.config.load import (
    ECOSYSTEM_ROOT,
+    get_system_logs_dir,
    get_module_logs_dir
 )

@@ -62,6 +63,7 @@ def scan_directory_safely(directory: Path, modules: Dict, max_depth: int = 10):
                modules[module_name] = {
                    "file_path": str(item),
                    "relative_path": str(relative_path),
+                    "system_log_file": str(get_system_logs_dir() / f"prax_{module_name}.log"),
                    "log_file": str(get_module_logs_dir("prax") / f"{module_name}.log"),
                    "discovered_time": datetime.now(timezone.utc).isoformat(),
                    "size": item.stat().st_size,
@@ -32,6 +32,7 @@ from watchdog.events import FileSystemEventHandler
 # Import from prax config
 from aipass.prax.apps.handlers.config.load import (
    ECOSYSTEM_ROOT,
+    get_system_logs_dir,
    get_module_logs_dir
 )

@@ -75,6 +76,7 @@ class PythonFileWatcher(FileSystemEventHandler):
            modules[module_name] = {
                "file_path": str(py_file),
                "relative_path": str(relative_path),
+                "system_log_file": str(get_system_logs_dir() / f"prax_{module_name}.log"),
                "log_file": str(get_module_logs_dir("prax") / f"{module_name}.log"),
                "discovered_time": datetime.now(timezone.utc).isoformat(),
                "size": py_file.stat().st_size,
@@ -49,6 +49,7 @@ from pathlib import Path
 from typing import Dict, Optional, Tuple

 from aipass.prax.apps.handlers.config.load import (
+    get_system_logs_dir,
    get_module_logs_dir,
    DEFAULT_LOG_LEVEL,
    load_log_config,
@@ -98,9 +99,9 @@ def _create_direct_logger(
    branch_name: str,
    branch_path: Optional[str]
 ) -> logging.Logger:
-    """Create a standalone logger with a RotatingFileHandler.
+    """Create a standalone logger with dual RotatingFileHandlers.

-    Same logging setup as setup_individual_logger but with NO
+    Same dual-logging setup as setup_individual_logger but with NO
    connection to the event pipeline. Uses logging.Logger internally
    only for RotatingFileHandler management - no root logger propagation.

@@ -110,7 +111,7 @@ def _create_direct_logger(
        branch_path: Module/branch name (e.g., 'prax') or None

    Returns:
-        Configured logger with file handler and no propagation.
+        Configured logger with dual file handlers and no propagation.
    """
    logger_key = f"direct_{branch_name}_{module_name}"
    logger = logging.getLogger(logger_key)
@@ -124,19 +125,31 @@ def _create_direct_logger(
        config['date_format']
    )

-    # Module-local log: each module logs to <module>/logs/
    target = branch_name if branch_path else "prax"
-    logs_dir = get_module_logs_dir(target)
-    log_file = logs_dir / f"{module_name}.log"
-    limits = config['system_logs']
-    handler = RotatingFileHandler(
-        log_file,
-        maxBytes=lines_to_bytes(limits['max_lines']),
-        backupCount=limits['backup_count'],
+
+    # Handler 1: System-wide log (central aggregation)
+    sys_log_file = get_system_logs_dir() / f"{target}_{module_name}.log"
+    sys_limits = config['system_logs']
+    sys_handler = RotatingFileHandler(
+        sys_log_file,
+        maxBytes=lines_to_bytes(sys_limits['max_lines']),
+        backupCount=sys_limits['backup_count'],
        encoding='utf-8'
    )
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
+    sys_handler.setFormatter(formatter)
+    logger.addHandler(sys_handler)
+
+    # Handler 2: Module-local log (local debugging)
+    local_log_file = get_module_logs_dir(target) / f"{module_name}.log"
+    local_limits = config['local_logs']
+    local_handler = RotatingFileHandler(
+        local_log_file,
+        maxBytes=lines_to_bytes(local_limits['max_lines']),
+        backupCount=local_limits['backup_count'],
+        encoding='utf-8'
+    )
+    local_handler.setFormatter(formatter)
+    logger.addHandler(local_handler)

    return logger

@@ -24,6 +24,7 @@ from logging.handlers import RotatingFileHandler

 # Import from prax config
 from aipass.prax.apps.handlers.config.load import (
+    get_system_logs_dir,
    get_module_logs_dir,
    DEFAULT_LOG_LEVEL,
    load_log_config,
@@ -57,6 +58,7 @@ def setup_individual_logger(module_name: str) -> logging.Logger:
    """Setup individual logger for a specific module with dual logging support

    Creates:
+    - System-wide log: {repo_root}/system_logs/{branch}_{module_name}.log
    - Module-local log: src/aipass/{branch}/logs/{module_name}.log
    - Terminal handler (if enabled)

@@ -99,22 +101,35 @@ def setup_individual_logger(module_name: str) -> logging.Logger:
        log_config['date_format']
    )

-    # Module-local log: each module logs to <module>/logs/
-    logs_dir = get_module_logs_dir(branch_name)
-    log_file = logs_dir / f"{module_name}.log"
-    limits = log_config['system_logs']
-    max_bytes = lines_to_bytes(limits['max_lines'])
-    handler = RotatingFileHandler(
-        log_file,
-        maxBytes=max_bytes,
-        backupCount=limits['backup_count'],
+    # HANDLER 1: System-wide log (central aggregation)
+    system_log_file = get_system_logs_dir() / f"{branch_name}_{module_name}.log"
+    system_limits = log_config['system_logs']
+    system_max_bytes = lines_to_bytes(system_limits['max_lines'])
+    system_handler = RotatingFileHandler(
+        system_log_file,
+        maxBytes=system_max_bytes,
+        backupCount=system_limits['backup_count'],
        encoding='utf-8'
    )
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
+    system_handler.setFormatter(formatter)
+    logger.addHandler(system_handler)
+
+    # HANDLER 2: Module-local log (local debugging)
+    module_logs_dir = get_module_logs_dir(branch_name)
+    module_log_file = module_logs_dir / f"{module_name}.log"
+    local_limits = log_config['local_logs']
+    local_max_bytes = lines_to_bytes(local_limits['max_lines'])
+    local_handler = RotatingFileHandler(
+        module_log_file,
+        maxBytes=local_max_bytes,
+        backupCount=local_limits['backup_count'],
+        encoding='utf-8'
+    )
+    local_handler.setFormatter(formatter)
+    logger.addHandler(local_handler)

    if _system_logger:
-        _system_logger.info(f"Logger created for {module_name} → {log_file} ({limits['max_lines']} lines)")
+        _system_logger.info(f"Logger created for {module_name} → system: {system_log_file} ({system_limits['max_lines']} lines), local: {module_log_file} ({local_limits['max_lines']} lines)")

    # HANDLER 3: Terminal output (if enabled)
    if _terminal_output_enabled and _terminal_module_available:
@@ -148,30 +163,41 @@ def setup_system_logger() -> logging.Logger:
    _system_logger.setLevel(DEFAULT_LOG_LEVEL)
    _system_logger.handlers.clear()

-    # Create prax_logger's own log file (in prax/logs/)
-    log_file = get_module_logs_dir("prax") / "prax_logger.log"
+    # Formatter shared by both handlers
+    log_config_fmt = logging.Formatter(
+        log_config['log_format'],
+        log_config['date_format']
+    )

-    # Create rotating file handler with config-driven limits
+    # HANDLER 1: System-wide log (central aggregation)
+    system_log_file = get_system_logs_dir() / "prax_logger.log"
    system_limits = log_config['system_logs']
    system_max_bytes = lines_to_bytes(system_limits['max_lines'])
-    handler = RotatingFileHandler(
-        log_file,
+    system_handler = RotatingFileHandler(
+        system_log_file,
        maxBytes=system_max_bytes,
        backupCount=system_limits['backup_count'],
        encoding='utf-8'
    )
+    system_handler.setFormatter(log_config_fmt)
+    _system_logger.addHandler(system_handler)

-    # Set formatter
-    formatter = logging.Formatter(
-        log_config['log_format'],
-        log_config['date_format']
+    # HANDLER 2: Module-local log (local debugging)
+    local_log_file = get_module_logs_dir("prax") / "prax_logger.log"
+    local_limits = log_config['local_logs']
+    local_max_bytes = lines_to_bytes(local_limits['max_lines'])
+    local_handler = RotatingFileHandler(
+        local_log_file,
+        maxBytes=local_max_bytes,
+        backupCount=local_limits['backup_count'],
+        encoding='utf-8'
    )
-    handler.setFormatter(formatter)
-    _system_logger.addHandler(handler)
+    local_handler.setFormatter(log_config_fmt)
+    _system_logger.addHandler(local_handler)

    # Log system logger creation
    _system_logger.info("Prax system logger initialized successfully")
-    _system_logger.info(f"System logger writing to: {log_file} ({system_limits['max_lines']} lines max)")
+    _system_logger.info(f"System logger writing to: {system_log_file} + {local_log_file}")

    return _system_logger

@@ -65,7 +65,7 @@ from aipass.prax.apps.handlers.discovery.watcher import (
 )
 from aipass.prax.apps.handlers.registry.save import save_module_registry
 from aipass.prax.apps.handlers.registry.load import load_module_registry
-from aipass.prax.apps.handlers.config.load import get_module_logs_dir, PRAX_JSON_DIR
+from aipass.prax.apps.handlers.config.load import get_system_logs_dir, get_module_logs_dir, PRAX_JSON_DIR
 from aipass.prax.apps.handlers.logging.direct import (
    get_direct_logger,
    direct_log,
@@ -162,7 +162,7 @@ def initialize_logging_system():

    # Log system startup
    system_logger_instance.info("Prax logging system initialized")
-    system_logger_instance.info(f"Module logs directory: {get_module_logs_dir('prax')}")
+    system_logger_instance.info(f"System logs: {get_system_logs_dir()}, Module logs: {get_module_logs_dir('prax')}")
    system_logger_instance.info(f"Found {len(modules)} modules for logging setup")

    # Install logger override
@@ -252,6 +252,7 @@ def get_system_status() -> Dict[str, Any]:
    return {
        "total_modules": len(modules),
        "individual_loggers": get_captured_loggers_count(),
+        "system_logs_dir": str(get_system_logs_dir()),
        "module_logs_dir": str(get_module_logs_dir("prax")),
        "registry_file": str(DATA_FILE),
        "file_watcher_active": is_file_watcher_active(),
@@ -65,6 +65,7 @@ def handle_command(command: str, args: List[str]) -> bool:
    console.print("=" * 60)
    console.print(f"Total Modules: {status['total_modules']}")
    console.print(f"Active Loggers: {status['individual_loggers']}")
+    console.print(f"System Logs Dir: {status['system_logs_dir']}")
    console.print(f"Module Logs Dir: {status['module_logs_dir']}")
    console.print(f"Registry File: {status['registry_file']}")
    console.print(f"File Watcher: {'🟢 Active' if status['file_watcher_active'] else '🔴 Inactive'}")