From 0e658baf8544b863f7f7a8bf2efc34b1eac2ad4b Mon Sep 17 00:00:00 2001 From: AIPass Date: Sat, 14 Mar 2026 23:07:03 -0700 Subject: [PATCH] feat(backup): stderr routing, error masking fix, mtime pre-check, ignore patterns split (#44) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Route all error/warning prints to stderr via error()/warning() (backup_core, google_drive_sync) - Fix route_command error masking — show real errors instead of "Unknown command" - Add versioned backup mtime pre-check to skip unchanged files - Extract ignore patterns from config_handler to editable JSON (FPLAN-0037) - New: ignore_patterns.py loads from json_templates/ignore_patterns.json - config_handler.py v3.0.0 re-exports for backwards compatibility - 132 patterns, 14/14 tests pass Co-authored-by: @backup Co-authored-by: Claude Opus 4.6 (1M context) --- src/aipass/backup/apps/backup.py | 6 +- .../apps/handlers/config/config_handler.py | 431 +----------------- .../apps/handlers/config/ignore_patterns.py | 212 +++++++++ .../apps/json_templates/ignore_patterns.json | 282 ++++++++++++ src/aipass/backup/apps/modules/backup_core.py | 12 +- .../backup/apps/modules/google_drive_sync.py | 36 +- 6 files changed, 538 insertions(+), 441 deletions(-) create mode 100644 src/aipass/backup/apps/handlers/config/ignore_patterns.py create mode 100644 src/aipass/backup/apps/json_templates/ignore_patterns.json diff --git a/src/aipass/backup/apps/backup.py b/src/aipass/backup/apps/backup.py index 6ea32c12..a77f4c41 100755 --- a/src/aipass/backup/apps/backup.py +++ b/src/aipass/backup/apps/backup.py @@ -62,7 +62,11 @@ def route_command(args: argparse.Namespace, modules: List[Any]) -> bool: if module.handle_command(args): return True except Exception as e: - logger.error(f"Module error: {e}") + # Show the actual error — don't mask it behind "Unknown command" + module_name = getattr(module, '__name__', str(module)).split('.')[-1] + logger.error(f"Module error in {module_name}: {e}") + error(f"{module_name} failed: {e}") + return True # Command was handled (by crashing) — don't show "Unknown command" return False diff --git a/src/aipass/backup/apps/handlers/config/config_handler.py b/src/aipass/backup/apps/handlers/config/config_handler.py index 2dbb9d21..f7d6c952 100755 --- a/src/aipass/backup/apps/handlers/config/config_handler.py +++ b/src/aipass/backup/apps/handlers/config/config_handler.py @@ -1,17 +1,20 @@ # =================== AIPass ==================== # Name: config_handler.py -# Description: Backup system configuration and patterns -# Version: 2.0.2 +# Description: Backup system configuration — modes and destinations +# Version: 3.0.0 # Created: 2025-11-23 -# Modified: 2026-03-09 +# Modified: 2026-03-14 # ============================================= """ Backup System Configuration Handler Centralized configuration management for backup system. -Contains all backup modes, ignore patterns, and configuration constants. -Pure configuration with helper functions for pattern access and filtering. +Contains backup modes, destinations, and configuration constants. + +Ignore patterns and pattern-matching functions have been extracted to +ignore_patterns.py (FPLAN-0037) and are re-exported here for backwards +compatibility. """ # ============================================= @@ -21,6 +24,13 @@ Pure configuration with helper functions for pattern access and filtering. from pathlib import Path from typing import Dict, Set, List, Optional +# Re-export from ignore_patterns for backwards compatibility +from aipass.backup.apps.handlers.config.ignore_patterns import ( + GLOBAL_IGNORE_PATTERNS, IGNORE_EXCEPTIONS, should_ignore, + filter_tracked_items, get_ignore_patterns, get_cli_tracking_patterns, + DIFF_IGNORE_PATTERNS, DIFF_INCLUDE_PATTERNS, CLI_TRACKING_PATTERNS +) + # ============================================= # CONFIGURATION CONSTANTS # ============================================= @@ -35,286 +45,6 @@ BACKUP_DESTINATIONS = { "versioned_backup": f"{BASE_BACKUP_DIR}", } -# ============================================= -# IGNORE PATTERNS -# ============================================= - -# Global ignore patterns (what NOT to backup across all systems) -GLOBAL_IGNORE_PATTERNS = [ - # Python cache and temp files - "__pycache__", - "*.pyc", - "*.pyo", - - # Virtual environments - ".venv", - "venv", - - # Node.js - "node_modules", - "npm-debug.log", - "yarn-error.log", - - # Operating system files - ".DS_Store", - "Thumbs.db", - "desktop.ini", - "nul", # Windows reserved filename issue - - # Temporary files - #"temp", - ".temp", - "tmp", - ".tmp", - "*.tmp", - "*.temp", - - # Build and distribution directories - "build", - "dist", - "install", - "lib", - "bin", - - # IDE and editor files - "*.swp", - "*.swo", - "*~", - - # Backup directories (prevent recursive backups) - CRITICAL! - "backups", # Matches any directory named "backups" - "backup_system/backups", # Explicit path to our backup directory - "*/backups", # Any backups subdirectory - "system_snapshot", # Snapshot backup folder name - "versioned_backup", # Versioned backup folder name - "deleted_branches", # Deleted/archived branches (trash folder) - - # System backups and trash - "TimeShift*", - "timeshift*", - ".local/share/Trash", - "Trash", - - # Linux system directories (don't backup user settings/cache) - ".local", - ".cache", - ".config", - ".mozilla", - ".gnupg", - ".ssh", - ".vscode/cli", # VS Code CLI binary (large, regenerates) - ".vscode/extensions", # VS Code extensions (large, regenerates) - ".vscode-server", # VS Code remote server - ".npm-global", - ".pki", - ".dotnet", - ".var", # System application data (Flatpak, etc) - ".backup", # Old backup directories - ".antigravity", # AI assistant cache/data - ".gemini", # AI assistant cache - "snap", # Snap package directories (broken symlinks, system-managed) - - # AIPass internal state (runtime, not user data) - ".trinity", # Branch identity/memory files (passport, local, observations) - ".ai_mail.local", # Branch mailbox directories - ".archive", # Branch archive directories - "backup_data", # Backup runtime state - "backup_json", # Backup metadata tracking - "DASHBOARD.local.json", # Branch dashboard state - "CLOSED_PLANS.local.json", # Flow plan close records - "STATUS.local.md", # Branch status boards - - # Version control (huge number of files!) - ".git", # Git repositories are version controlled elsewhere - - # IDE and editor directories - ".idea", # IntelliJ IDEA settings - ".eclipse", # Eclipse settings - - # Claude Code session files (change every session) - synced from .gitignore - #".claude/projects", - ".claude/todos", - ".claude/shell-snapshots", - ".claude/ide", - ".claude/statsig", - #".claude/plugins", - ".claude/.credentials.json", - ".claude/debug", - ".claude/file-history", - ".claude/history.jsonl", - ".claude/.update.lock", - ".claude/.projects", - #".claude/hooks/.last_diagnostics_file", - ".serena/logs", - ".code", - - # Development cache/build directories - ".npm", - ".cargo", - ".rustup", - ".gem", - ".gradle", - ".m2", - - # Application data we don't need in backups - ".thunderbird", - ".wine", - ".steam", - ".zoom", - - # User directories that shouldn't be backed up - "Downloads", - #"Music", - "Videos", - "Pictures", - "Dropbox", - "system_logs", - "external_repos", # External git repos - version controlled elsewhere - "mcp_servers/context7", # External MCP server repo - "mcp_servers/playwright-mcp", # External MCP server repo - "mcp_servers/serena", # External MCP server repo - "mcp_servers/servers", # External MCP server repo - "mcp_servers/dropbox", # External MCP server repo - - # Archive and compressed files - "*.zip", - "*.tar", - "*.gz", - "*.bz2", - "*.rar", - "*.7z", - "*.whl", # Python wheel files (long filenames in archives) - - # Large binary/image files (VMs, disk images) - "*.img", # Disk images (sandbox.img, etc - can be 50GB+) - "*.iso", # ISO files - "*.vmdk", # Virtual machine disk - "*.vdi", # VirtualBox disk image - "*.qcow2", # QEMU disk image - - # Image/media files (screenshots, icons, etc.) - "*.png", - "*.jpg", - "*.jpeg", - "*.gif", - "*.svg", - "*.webp", - "*.ico", - "*.bmp", - "*.tiff", - "*.tif", - - # Log files - "*.log", - "logs", - - # Most JSON files (frequent changes, not human-readable diffs) - "*_data.json", - "*_log.json", - "*_registry.json", - "snapshot_backup.json", - "snapshot_backup_changelog.json", - - # Miscellaneous - "*.db", - "*.bashrc", - "*.bash_history", - "*.bash_logout", - #"*.claude.json.backup", - #"*.env", - "*.lesshst", - "*.npmrc", - "*.sudo_as_admin_successful", -] - -# Notable patterns to highlight when skipped -CLI_TRACKING_PATTERNS = [] - -# Files that should be backed up but NOT generate diff files -DIFF_IGNORE_PATTERNS = [ - # Log files (append-only, huge diffs) - "*.log", - "*.logs", - "system_logs/*.log", - - # Most JSON files (frequent changes, not human-readable diffs) - #"*_config.json", - #"*_data.json", - #"*_log.json", - #"*_registry.json", - - # Python cache/compiled - "*.pyc", - "*.pyo", - "*.pyd", - "__pycache__/*", - - # Database files - "*.db", - "*.sqlite", - "*.sqlite3", - - # Binary/media files - "*.exe", - "*.dll", - "*.so", - "*.dylib", - - # Pickle/cache files - "*.pkl", - "*.pickle", - - # Temporary files - "*.tmp", - "*.temp", - "*.bak", - "~*", -] - -# Files that should be backed up despite matching ignore patterns -# Synced from .gitignore exceptions (! prefix patterns) -IGNORE_EXCEPTIONS = [ - ".gitignore", # Root .gitignore file should be backed up - "*.local.md", # AIPass session tracking files (caught by .local pattern but should be backed up) - ".vscode/settings.json", # VS Code settings - "*/.claude/settings.local.json", # Claude local settings in any directory - "tools/cleanup_configs/*.json", # Cleanup config files - "*_config.json", # All config JSON files - "drone/commands/global/*.json", # Drone global commands - ".claude.json", # Claude config - ".mcp.json", # MCP config - ".config/nerd-dictation/*", # Nerd dictation config - ".commands.json", # Commands config - "BRANCH_REGISTRY.json", # Core ecosystem registry - vital file - - # === TEMPLATES: FULL EXCEPTION - EVERY FILE === - "templates/**", # FULL EXCEPTION: Include EVERY file in templates directory - "*/templates/**", # Templates in any subdirectory - "templates/*/**", # All subdirectories and files in templates - "*/templates/*/**", # All subdirectories and files in templates anywhere - - # === TEMPLATE SUBDIRECTORIES (explicit) === - "templates/ai_branch_setup_template/ai_mail.local/**", - "templates/ai_branch_setup_template/logs/**", - "templates/ai_branch_setup_template/standards.local/**", - "templates/ai_branch_setup_template/.claude/**", - "templates/ai_branch_setup_template/.archive/**", - - # === MARKERS === - ".gitkeep", # Include all .gitkeep marker files (especially in templates) - ".gitattributes", # Git attributes files -] - -# Files that SHOULD have diffs created (exceptions to ignore patterns) -DIFF_INCLUDE_PATTERNS = [ - "profile.json", - "pyrightconfig.json", - "package.json", - ".mcp.json", - "settings.json", - "settings.local.json", -] - # ============================================= # BACKUP MODE CONFIGURATIONS # ============================================= @@ -342,24 +72,6 @@ BACKUP_MODES = { # HELPER FUNCTIONS # ============================================= -def get_ignore_patterns() -> List[str]: - """Get the global ignore patterns list - - Returns: - Copy of global ignore patterns list - """ - return GLOBAL_IGNORE_PATTERNS.copy() - - -def get_cli_tracking_patterns() -> List[str]: - """Get the CLI tracking patterns list - - Returns: - Copy of CLI tracking patterns list - """ - return CLI_TRACKING_PATTERNS.copy() - - def get_backup_destination(system_name: str) -> str: """Get backup destination for a specific backup system @@ -371,119 +83,6 @@ def get_backup_destination(system_name: str) -> str: """ return BACKUP_DESTINATIONS.get(system_name, BASE_BACKUP_DIR) - -def filter_tracked_items(skipped_items: Dict[str, Set[str]]) -> Dict[str, Set[str]]: - """Filter skipped items to only show project-specific items - - Uses CLI tracking patterns to identify important items worth showing to user. - - Args: - skipped_items: Dictionary with 'directories' and 'files' sets - - Returns: - Filtered dictionary with only tracked items - """ - tracking_patterns = get_cli_tracking_patterns() - filtered_items = { - "directories": set(), - "files": set() - } - - def matches_pattern(item_path: str, patterns: List[str]) -> bool: - """Check if item path matches any tracking pattern""" - for pattern in patterns: - if pattern in item_path or item_path.startswith(pattern): - return True - # Check wildcard patterns - if pattern.startswith('*') and item_path.endswith(pattern[1:]): - return True - return False - - # Filter directories - for directory in skipped_items.get("directories", set()): - if matches_pattern(directory, tracking_patterns): - filtered_items["directories"].add(directory) - - # Filter files - for file_path in skipped_items.get("files", set()): - if matches_pattern(file_path, tracking_patterns): - filtered_items["files"].add(file_path) - - return filtered_items - - -def should_ignore(path: Path, ignore_patterns: Optional[List[str]] = None, - exceptions: Optional[List[str]] = None, - backup_dest: Optional[Path] = None) -> bool: - """Check if a file/folder should be ignored based on patterns. - - Centralizes ignore pattern matching logic used during backup scanning. - Checks exceptions first (files that should NOT be ignored), then patterns. - - Args: - path: Path object to check - ignore_patterns: List of patterns to ignore (defaults to GLOBAL_IGNORE_PATTERNS) - exceptions: List of exception patterns (defaults to IGNORE_EXCEPTIONS) - backup_dest: Optional backup destination to always ignore - - Returns: - True if path should be ignored, False otherwise - - Example: - # From backup engine - should_ignore(Path("/home/user/file.pyc")) # True - should_ignore(Path("/home/user/.gitignore")) # False (exception) - """ - import os - - # Use defaults if not provided - if ignore_patterns is None: - ignore_patterns = GLOBAL_IGNORE_PATTERNS - if exceptions is None: - exceptions = IGNORE_EXCEPTIONS - - path_str = str(path) - parts = set(path_str.split(os.sep)) - name = path.name - - # Always ignore backup destination if provided - if backup_dest and str(backup_dest) in path_str: - return True - - # Ignore paths containing 'Backups' - if 'Backups' in parts: - return True - - # Check exceptions first - files that should NOT be ignored - for exception in exceptions: - # Full path matching for template exceptions - if "**" in exception: - # Convert glob pattern to regex-like check - exception_parts = exception.split("/**")[0] # Get everything before /** - if exception_parts in path_str or exception_parts in "/".join(parts): - return False # Matches exception pattern - don't ignore - elif exception.startswith('*') and name.endswith(exception[1:]): - return False # Matches wildcard exception pattern - elif exception == name: - return False # Exact match - elif exception in path_str: - return False # Exception pattern is in the full path - - # Check ignore patterns - for pattern in ignore_patterns: - # Special case: "backups" should only match directory names, not filenames - if pattern == "backups": - if "backups" in parts: # Only ignore if "backups" is a directory in the path - return True - elif pattern == name: - return True - elif pattern.startswith('*') and name.endswith(pattern[1:]): - return True - elif pattern in parts or pattern in path_str: - return True - - return False - # ============================================= # MODULE INITIALIZATION # ============================================= diff --git a/src/aipass/backup/apps/handlers/config/ignore_patterns.py b/src/aipass/backup/apps/handlers/config/ignore_patterns.py new file mode 100644 index 00000000..da0941ed --- /dev/null +++ b/src/aipass/backup/apps/handlers/config/ignore_patterns.py @@ -0,0 +1,212 @@ +# =================== AIPass ==================== +# Name: ignore_patterns.py +# Description: Ignore pattern loading and matching from JSON config +# Version: 1.0.0 +# Created: 2026-03-14 +# Modified: 2026-03-14 +# ============================================= + +""" +Ignore Patterns Handler + +Loads ignore/exception patterns from apps/json_templates/ignore_patterns.json +and provides pattern matching functions for backup file filtering. + +Extracted from config_handler.py (FPLAN-0037) to separate data (JSON) from logic. +""" + +# ============================================= +# IMPORTS +# ============================================= + +import json +import os +from pathlib import Path +from typing import Dict, Set, List, Optional + +from aipass.prax import logger + +# ============================================= +# PATTERN LOADING +# ============================================= + +_PATTERNS_JSON = Path(__file__).parents[2] / "json_templates" / "ignore_patterns.json" + + +def load_patterns() -> dict: + """Load all pattern sets from the ignore_patterns.json file. + + Returns: + Dictionary with all pattern sections from the JSON file. + + Raises: + FileNotFoundError: If the JSON file is missing (fail loud). + json.JSONDecodeError: If the JSON file is malformed. + """ + if not _PATTERNS_JSON.exists(): + raise FileNotFoundError( + f"[ignore_patterns] CRITICAL: Pattern file missing: {_PATTERNS_JSON}\n" + f"Expected at: apps/json_templates/ignore_patterns.json\n" + f"Cannot proceed without ignore patterns — backup would copy everything." + ) + + with open(_PATTERNS_JSON, "r", encoding="utf-8") as f: + data = json.load(f) + + logger.info(f"[ignore_patterns] Loaded patterns from {_PATTERNS_JSON}") + return data + + +# ============================================= +# LOAD PATTERN CONSTANTS +# ============================================= + +_data = load_patterns() + +GLOBAL_IGNORE_PATTERNS: List[str] = _data["global_ignore_patterns"]["patterns"] +IGNORE_EXCEPTIONS: List[str] = _data["ignore_exceptions"]["patterns"] +CLI_TRACKING_PATTERNS: List[str] = _data["cli_tracking_patterns"]["patterns"] +DIFF_IGNORE_PATTERNS: List[str] = _data["diff_ignore_patterns"]["patterns"] +DIFF_INCLUDE_PATTERNS: List[str] = _data["diff_include_patterns"]["patterns"] + +# ============================================= +# HELPER FUNCTIONS +# ============================================= + + +def get_ignore_patterns() -> List[str]: + """Get the global ignore patterns list. + + Returns: + Copy of global ignore patterns list + """ + return GLOBAL_IGNORE_PATTERNS.copy() + + +def get_cli_tracking_patterns() -> List[str]: + """Get the CLI tracking patterns list. + + Returns: + Copy of CLI tracking patterns list + """ + return CLI_TRACKING_PATTERNS.copy() + + +def filter_tracked_items(skipped_items: Dict[str, Set[str]]) -> Dict[str, Set[str]]: + """Filter skipped items to only show project-specific items. + + Uses CLI tracking patterns to identify important items worth showing to user. + + Args: + skipped_items: Dictionary with 'directories' and 'files' sets + + Returns: + Filtered dictionary with only tracked items + """ + tracking_patterns = get_cli_tracking_patterns() + filtered_items = { + "directories": set(), + "files": set() + } + + def matches_pattern(item_path: str, patterns: List[str]) -> bool: + """Check if item path matches any tracking pattern.""" + for pattern in patterns: + if pattern in item_path or item_path.startswith(pattern): + return True + # Check wildcard patterns + if pattern.startswith('*') and item_path.endswith(pattern[1:]): + return True + return False + + # Filter directories + for directory in skipped_items.get("directories", set()): + if matches_pattern(directory, tracking_patterns): + filtered_items["directories"].add(directory) + + # Filter files + for file_path in skipped_items.get("files", set()): + if matches_pattern(file_path, tracking_patterns): + filtered_items["files"].add(file_path) + + return filtered_items + + +def should_ignore(path: Path, ignore_patterns: Optional[List[str]] = None, + exceptions: Optional[List[str]] = None, + backup_dest: Optional[Path] = None) -> bool: + """Check if a file/folder should be ignored based on patterns. + + Centralizes ignore pattern matching logic used during backup scanning. + Checks exceptions first (files that should NOT be ignored), then patterns. + + Args: + path: Path object to check + ignore_patterns: List of patterns to ignore (defaults to GLOBAL_IGNORE_PATTERNS) + exceptions: List of exception patterns (defaults to IGNORE_EXCEPTIONS) + backup_dest: Optional backup destination to always ignore + + Returns: + True if path should be ignored, False otherwise + + Example: + # From backup engine + should_ignore(Path("/home/user/file.pyc")) # True + should_ignore(Path("/home/user/.gitignore")) # False (exception) + """ + # Use defaults if not provided + if ignore_patterns is None: + ignore_patterns = GLOBAL_IGNORE_PATTERNS + if exceptions is None: + exceptions = IGNORE_EXCEPTIONS + + path_str = str(path) + parts = set(path_str.split(os.sep)) + name = path.name + + # Always ignore backup destination if provided + if backup_dest and str(backup_dest) in path_str: + return True + + # Ignore paths containing 'Backups' + if 'Backups' in parts: + return True + + # Check exceptions first - files that should NOT be ignored + for exception in exceptions: + # Full path matching for template exceptions + if "**" in exception: + # Convert glob pattern to regex-like check + exception_parts = exception.split("/**")[0] # Get everything before /** + if exception_parts in path_str or exception_parts in "/".join(parts): + return False # Matches exception pattern - don't ignore + elif exception.startswith('*') and name.endswith(exception[1:]): + return False # Matches wildcard exception pattern + elif exception == name: + return False # Exact match + elif exception in path_str: + return False # Exception pattern is in the full path + + # Check ignore patterns + for pattern in ignore_patterns: + # Special case: "backups" should only match directory names, not filenames + if pattern == "backups": + if "backups" in parts: # Only ignore if "backups" is a directory in the path + return True + elif pattern == name: + return True + elif pattern.startswith('*') and name.endswith(pattern[1:]): + return True + elif pattern in parts or pattern in path_str: + return True + + return False + + +# ============================================= +# MODULE INITIALIZATION +# ============================================= + +logger.info(f"[ignore_patterns] Module loaded — {len(GLOBAL_IGNORE_PATTERNS)} global patterns, " + f"{len(IGNORE_EXCEPTIONS)} exceptions, {len(DIFF_IGNORE_PATTERNS)} diff-ignore, " + f"{len(DIFF_INCLUDE_PATTERNS)} diff-include") diff --git a/src/aipass/backup/apps/json_templates/ignore_patterns.json b/src/aipass/backup/apps/json_templates/ignore_patterns.json new file mode 100644 index 00000000..a2d92798 --- /dev/null +++ b/src/aipass/backup/apps/json_templates/ignore_patterns.json @@ -0,0 +1,282 @@ +{ + "global_ignore_patterns": { + "comments": "Global ignore patterns — what NOT to backup across all systems. Each category documents WHY patterns exist.", + "patterns": [ + "__pycache__", + "*.pyc", + "*.pyo", + + ".venv", + "venv", + + "node_modules", + "npm-debug.log", + "yarn-error.log", + + ".DS_Store", + "Thumbs.db", + "desktop.ini", + "nul", + + ".temp", + "tmp", + ".tmp", + "*.tmp", + "*.temp", + + "build", + "dist", + "install", + "lib", + "bin", + + "*.swp", + "*.swo", + "*~", + + "backups", + "backup_system/backups", + "*/backups", + "system_snapshot", + "versioned_backup", + "deleted_branches", + + "TimeShift*", + "timeshift*", + ".local/share/Trash", + "Trash", + + ".local", + ".cache", + ".config", + ".mozilla", + ".gnupg", + ".ssh", + ".vscode/cli", + ".vscode/extensions", + ".vscode-server", + ".npm-global", + ".pki", + ".dotnet", + ".var", + ".backup", + ".antigravity", + ".gemini", + "snap", + + ".trinity", + ".ai_mail.local", + ".archive", + "backup_data", + "backup_json", + "DASHBOARD.local.json", + "CLOSED_PLANS.local.json", + "STATUS.local.md", + + ".git", + + ".idea", + ".eclipse", + + ".claude/todos", + ".claude/shell-snapshots", + ".claude/ide", + ".claude/statsig", + ".claude/.credentials.json", + ".claude/debug", + ".claude/file-history", + ".claude/history.jsonl", + ".claude/.update.lock", + ".claude/.projects", + ".serena/logs", + ".code", + + ".npm", + ".cargo", + ".rustup", + ".gem", + ".gradle", + ".m2", + + ".thunderbird", + ".wine", + ".steam", + ".zoom", + + "Downloads", + "Videos", + "Pictures", + "Dropbox", + "system_logs", + "external_repos", + "mcp_servers/context7", + "mcp_servers/playwright-mcp", + "mcp_servers/serena", + "mcp_servers/servers", + "mcp_servers/dropbox", + + "*.zip", + "*.tar", + "*.gz", + "*.bz2", + "*.rar", + "*.7z", + "*.whl", + + "*.img", + "*.iso", + "*.vmdk", + "*.vdi", + "*.qcow2", + + "*.png", + "*.jpg", + "*.jpeg", + "*.gif", + "*.svg", + "*.webp", + "*.ico", + "*.bmp", + "*.tiff", + "*.tif", + + "*.log", + "logs", + + "*_data.json", + "*_log.json", + "*_registry.json", + "snapshot_backup.json", + "snapshot_backup_changelog.json", + + "*.db", + "*.bashrc", + "*.bash_history", + "*.bash_logout", + "*.lesshst", + "*.npmrc", + "*.sudo_as_admin_successful" + ], + "category_comments": { + "python_cache": "Python cache and temp files: __pycache__, *.pyc, *.pyo", + "virtual_environments": "Virtual environments: .venv, venv", + "nodejs": "Node.js: node_modules, npm-debug.log, yarn-error.log", + "os_files": "Operating system files: .DS_Store, Thumbs.db, desktop.ini, nul (Windows reserved filename issue)", + "temp_files": "Temporary files: .temp, tmp, .tmp, *.tmp, *.temp", + "build_dirs": "Build and distribution directories: build, dist, install, lib, bin", + "editor_files": "IDE and editor files: *.swp, *.swo, *~", + "backup_dirs": "Backup directories (prevent recursive backups) — CRITICAL! backups matches any directory named 'backups'. Also system_snapshot, versioned_backup folder names, deleted_branches (trash folder)", + "system_backups": "System backups and trash: TimeShift*, timeshift*, .local/share/Trash, Trash", + "linux_system": "Linux system directories (don't backup user settings/cache): .local, .cache, .config, .mozilla, .gnupg, .ssh, .vscode/cli (large, regenerates), .vscode/extensions (large, regenerates), .vscode-server, .npm-global, .pki, .dotnet, .var (Flatpak etc), .backup, .antigravity (AI assistant cache/data), .gemini (AI assistant cache), snap (broken symlinks, system-managed)", + "aipass_internal": "AIPass internal state (runtime, not user data): .trinity (passport, local, observations), .ai_mail.local (mailbox dirs), .archive, backup_data (runtime state), backup_json (metadata tracking), DASHBOARD.local.json, CLOSED_PLANS.local.json, STATUS.local.md", + "version_control": "Version control (huge number of files!): .git repos are version controlled elsewhere", + "ide_dirs": "IDE and editor directories: .idea (IntelliJ), .eclipse", + "claude_session": "Claude Code session files (change every session) — synced from .gitignore: .claude/todos, .claude/shell-snapshots, .claude/ide, .claude/statsig, .claude/.credentials.json, .claude/debug, .claude/file-history, .claude/history.jsonl, .claude/.update.lock, .claude/.projects, .serena/logs, .code", + "dev_cache": "Development cache/build directories: .npm, .cargo, .rustup, .gem, .gradle, .m2", + "app_data": "Application data we don't need in backups: .thunderbird, .wine, .steam, .zoom", + "user_dirs": "User directories that shouldn't be backed up: Downloads, Videos, Pictures, Dropbox, system_logs, external_repos (version controlled elsewhere), mcp_servers/* (external MCP server repos)", + "archives": "Archive and compressed files: *.zip, *.tar, *.gz, *.bz2, *.rar, *.7z, *.whl (Python wheel files — long filenames)", + "large_binaries": "Large binary/image files (VMs, disk images): *.img (can be 50GB+), *.iso, *.vmdk, *.vdi, *.qcow2", + "media_files": "Image/media files (screenshots, icons, etc.): *.png, *.jpg, *.jpeg, *.gif, *.svg, *.webp, *.ico, *.bmp, *.tiff, *.tif", + "log_files": "Log files: *.log, logs", + "json_files": "Most JSON files (frequent changes, not human-readable diffs): *_data.json, *_log.json, *_registry.json, snapshot_backup.json, snapshot_backup_changelog.json", + "misc": "Miscellaneous: *.db, *.bashrc, *.bash_history, *.bash_logout, *.lesshst, *.npmrc, *.sudo_as_admin_successful" + } + }, + "ignore_exceptions": { + "comments": "Files that should be backed up despite matching ignore patterns. Synced from .gitignore exceptions (! prefix patterns).", + "patterns": [ + ".gitignore", + "*.local.md", + ".vscode/settings.json", + "*/.claude/settings.local.json", + "tools/cleanup_configs/*.json", + "*_config.json", + "drone/commands/global/*.json", + ".claude.json", + ".mcp.json", + ".config/nerd-dictation/*", + ".commands.json", + "BRANCH_REGISTRY.json", + + "templates/**", + "*/templates/**", + "templates/*/**", + "*/templates/*/**", + + "templates/ai_branch_setup_template/ai_mail.local/**", + "templates/ai_branch_setup_template/logs/**", + "templates/ai_branch_setup_template/standards.local/**", + "templates/ai_branch_setup_template/.claude/**", + "templates/ai_branch_setup_template/.archive/**", + + ".gitkeep", + ".gitattributes" + ], + "category_comments": { + "root_files": ".gitignore — Root .gitignore file should be backed up", + "aipass_session": "*.local.md — AIPass session tracking files (caught by .local pattern but should be backed up)", + "vscode": ".vscode/settings.json — VS Code settings", + "claude_settings": "*/.claude/settings.local.json — Claude local settings in any directory", + "config_json": "*_config.json, tools/cleanup_configs/*.json, drone/commands/global/*.json, .claude.json, .mcp.json, .commands.json — Various config JSON files", + "nerd_dictation": ".config/nerd-dictation/* — Nerd dictation config", + "branch_registry": "BRANCH_REGISTRY.json — Core ecosystem registry — vital file", + "templates_full_exception": "FULL EXCEPTION: Include EVERY file in templates directory — templates/**, */templates/**, templates/*/** , */templates/*/**", + "template_subdirs_explicit": "Explicit template subdirectories: ai_branch_setup_template/ai_mail.local, logs, standards.local, .claude, .archive", + "markers": ".gitkeep — Include all .gitkeep marker files (especially in templates). .gitattributes — Git attributes files" + } + }, + "cli_tracking_patterns": { + "comments": "Notable patterns to highlight when skipped. Currently empty — reserved for future use.", + "patterns": [] + }, + "diff_ignore_patterns": { + "comments": "Files that should be backed up but NOT generate diff files.", + "patterns": [ + "*.log", + "*.logs", + "system_logs/*.log", + + "*.pyc", + "*.pyo", + "*.pyd", + "__pycache__/*", + + "*.db", + "*.sqlite", + "*.sqlite3", + + "*.exe", + "*.dll", + "*.so", + "*.dylib", + + "*.pkl", + "*.pickle", + + "*.tmp", + "*.temp", + "*.bak", + "~*" + ], + "category_comments": { + "log_files": "Log files (append-only, huge diffs): *.log, *.logs, system_logs/*.log", + "python_cache": "Python cache/compiled: *.pyc, *.pyo, *.pyd, __pycache__/*", + "database_files": "Database files: *.db, *.sqlite, *.sqlite3", + "binary_files": "Binary/media files: *.exe, *.dll, *.so, *.dylib", + "pickle_cache": "Pickle/cache files: *.pkl, *.pickle", + "temp_files": "Temporary files: *.tmp, *.temp, *.bak, ~*" + } + }, + "diff_include_patterns": { + "comments": "Files that SHOULD have diffs created (exceptions to ignore patterns).", + "patterns": [ + "profile.json", + "pyrightconfig.json", + "package.json", + ".mcp.json", + "settings.json", + "settings.local.json" + ] + } +} diff --git a/src/aipass/backup/apps/modules/backup_core.py b/src/aipass/backup/apps/modules/backup_core.py index c6e9af20..fecce03b 100755 --- a/src/aipass/backup/apps/modules/backup_core.py +++ b/src/aipass/backup/apps/modules/backup_core.py @@ -371,16 +371,13 @@ class BackupEngine: # Display dry-run warning if in test mode if self.dry_run: console.print() - console.print("╭" + "─" * 68 + "╮", style="yellow") - console.print("│" + " " * 18 + "[bold yellow]🔍 DRY-RUN MODE ACTIVE[/bold yellow]" + " " * 18 + "│", style="yellow") - console.print("│" + " " * 10 + "[yellow]Files will be scanned but NOT copied or deleted[/yellow]" + " " * 10 + "│", style="yellow") - console.print("╰" + "─" * 68 + "╯", style="yellow") + warning("DRY-RUN MODE ACTIVE — Files will be scanned but NOT copied or deleted") console.print() header(f"AIPass {self.mode_config['name']} - {self.mode_config['description']}") # Ensure backup directory exists if not self.ensure_backup_directory(result): - console.print(f"\nBACKUP FAILED: Could not create backup directory") + error("BACKUP FAILED: Could not create backup directory") return result # Load previous backup info @@ -449,7 +446,10 @@ class BackupEngine: else: result.files_skipped += 1 # Only skip if unchanged elif self.mode_config['behavior'] == 'versioned': - if copy_versioned_file(file_path, backup_file, self.backup_path, result): + # Skip unchanged files (mtime pre-check avoids expensive copy_versioned_file overhead) + if backup_file.exists() and backup_file.stat().st_mtime == file_path.stat().st_mtime: + result.files_skipped += 1 + elif copy_versioned_file(file_path, backup_file, self.backup_path, result): result.files_copied += 1 elif self.file_needs_backup(file_path, backup_file, last_timestamps): if copy_file_with_structure(file_path, backup_file, self.backup_path, result): diff --git a/src/aipass/backup/apps/modules/google_drive_sync.py b/src/aipass/backup/apps/modules/google_drive_sync.py index c82af138..677e2195 100644 --- a/src/aipass/backup/apps/modules/google_drive_sync.py +++ b/src/aipass/backup/apps/modules/google_drive_sync.py @@ -105,7 +105,7 @@ def _show_file_tracker_stats() -> bool: console.print(f" ... and {remaining} more files") return True except Exception as e: - console.print(f"Error showing tracker stats: {e}") + error(f"Error showing tracker stats: {e}") logger.error(f"Error showing tracker stats: {e}") return False @@ -123,7 +123,7 @@ def _clear_file_tracker() -> bool: console.print("File tracker already empty or clear failed") return success except Exception as e: - console.print(f"Error clearing file tracker: {e}") + error(f"Error clearing file tracker: {e}") logger.error(f"Error clearing file tracker: {e}") return False @@ -140,10 +140,10 @@ def _test_drive_sync() -> bool: folder_id = sync.get_or_create_backup_folder() console.print(f"Backup folder ready: {folder_id}") else: - console.print("Failed to create backup folder") + error("Failed to create backup folder") return result except Exception as e: - console.print(f"Test failed: {e}") + error(f"Test failed: {e}") logger.error(f"Test failed: {e}") return False @@ -279,7 +279,7 @@ def handle_command(args) -> bool: # Default to snapshot backup directory backup_path = _BACKUP_ROOT / "backups" / "system_snapshot" if not backup_path.exists(): - console.print(f"Error: Backup directory not found: {backup_path}") + error(f"Backup directory not found: {backup_path}") return False project = getattr(args, 'project', 'AIPass') or 'AIPass' @@ -288,7 +288,7 @@ def handle_command(args) -> bool: sync = GoogleDriveSync() if not sync.authenticate(): - console.print("[red]FAILED: Could not authenticate with Google Drive[/red]") + error("FAILED: Could not authenticate with Google Drive") return False limit = getattr(args, 'limit', 0) or 0 @@ -308,13 +308,13 @@ def handle_command(args) -> bool: folder_id = sync.get_or_create_project_folder(project) if not folder_id: error_msg = sync.last_error or "Unknown error" - console.print(f"[red]FAILED: {error_msg}[/red]") - console.print("[red]Sync aborted - no files uploaded[/red]") + error(f"FAILED: {error_msg}") + error("Sync aborted - no files uploaded") return False console.print(f" Drive folder ready: {project}") if sync.tracker_was_reset: - console.print(f"[yellow] Drive folder is new - tracker reset, full re-sync needed[/yellow]") + warning("Drive folder is new - tracker reset, full re-sync needed") # Phase 1: Scan (local only, fast) - runs AFTER folder check so tracker is accurate console.print(f"\nScanning {backup_path}...") @@ -362,8 +362,8 @@ def handle_command(args) -> bool: # Check for errors if result.get("error"): - console.print(f"\n[red]FAILED: {result['error']}[/red]") - console.print("[red]Sync aborted - no files uploaded[/red]") + error(f"FAILED: {result['error']}") + error("Sync aborted - no files uploaded") return False # Summary @@ -379,7 +379,7 @@ def handle_command(args) -> bool: console.print(f" [dim]Versioned: {format_age(ts.get('versioned'))}[/dim]") console.print(f" [dim]Drive sync: {format_age(ts.get('drive_sync'))}[/dim]") else: - console.print(f"[red]Sync failed: {result['uploaded']} uploaded, {result['failed']} failed, {result['skipped']} unchanged[/red]") + error(f"Sync failed: {result['uploaded']} uploaded, {result['failed']} failed, {result['skipped']} unchanged") return result["success"] @@ -464,7 +464,7 @@ EXAMPLES: # Check if module is enabled config = _load_config() if not config.get("config", {}).get("enabled", True): - console.print("Warning: Google Drive sync is disabled") + warning("Google Drive sync is disabled") sys.exit(0) if args.command == 'clear-tracker': @@ -472,7 +472,7 @@ EXAMPLES: console.print("File tracker cleared successfully") sys.exit(0) else: - console.print("Failed to clear file tracker") + error("Failed to clear file tracker") sys.exit(1) elif args.command == 'show-stats': @@ -483,19 +483,19 @@ EXAMPLES: elif args.command == 'sync': if not args.path: - console.print("Error: sync command requires a path argument") + error("sync command requires a path argument") console.print("Usage: python3 google_drive_sync.py sync /path/to/backups") sys.exit(1) backup_path = Path(args.path) if not backup_path.exists(): - console.print(f"Error: Backup directory not found: {backup_path}") + error(f"Backup directory not found: {backup_path}") sys.exit(1) sync = GoogleDriveSync() if not sync.authenticate(): - console.print("[red]Failed to authenticate with Google Drive[/red]") + error("Failed to authenticate with Google Drive") sys.exit(1) # Phase 1: Scan @@ -531,7 +531,7 @@ EXAMPLES: console.print(f"[green]Sync complete: {result['uploaded']} uploaded, {result['skipped']} unchanged[/green]") sys.exit(0) else: - console.print(f"[yellow]Sync finished: {result['uploaded']} uploaded, {result['failed']} failed, {result['skipped']} unchanged[/yellow]") + warning(f"Sync finished: {result['uploaded']} uploaded, {result['failed']} failed, {result['skipped']} unchanged") sys.exit(1) elif args.command == 'sync-test':