feat(backup): stderr routing, error masking fix, mtime pre-check, ignore patterns split (#44)

- Route all error/warning prints to stderr via error()/warning() (backup_core, google_drive_sync)
- Fix route_command error masking — show real errors instead of "Unknown command"
- Add versioned backup mtime pre-check to skip unchanged files
- Extract ignore patterns from config_handler to editable JSON (FPLAN-0037)
  - New: ignore_patterns.py loads from json_templates/ignore_patterns.json
  - config_handler.py v3.0.0 re-exports for backwards compatibility
  - 132 patterns, 14/14 tests pass

Co-authored-by: @backup <backup@aipass>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
AIPass
2026-03-14 23:07:03 -07:00
committed by GitHub
co-authored by @backup Claude Opus 4.6
parent 9ab96013e7
commit 0e658baf85
6 changed files with 538 additions and 441 deletions
+5 -1
View File
@@ -62,7 +62,11 @@ def route_command(args: argparse.Namespace, modules: List[Any]) -> bool:
if module.handle_command(args):
return True
except Exception as e:
logger.error(f"Module error: {e}")
# Show the actual error — don't mask it behind "Unknown command"
module_name = getattr(module, '__name__', str(module)).split('.')[-1]
logger.error(f"Module error in {module_name}: {e}")
error(f"{module_name} failed: {e}")
return True # Command was handled (by crashing) — don't show "Unknown command"
return False
@@ -1,17 +1,20 @@
# =================== AIPass ====================
# Name: config_handler.py
# Description: Backup system configuration and patterns
# Version: 2.0.2
# Description: Backup system configuration — modes and destinations
# Version: 3.0.0
# Created: 2025-11-23
# Modified: 2026-03-09
# Modified: 2026-03-14
# =============================================
"""
Backup System Configuration Handler
Centralized configuration management for backup system.
Contains all backup modes, ignore patterns, and configuration constants.
Pure configuration with helper functions for pattern access and filtering.
Contains backup modes, destinations, and configuration constants.
Ignore patterns and pattern-matching functions have been extracted to
ignore_patterns.py (FPLAN-0037) and are re-exported here for backwards
compatibility.
"""
# =============================================
@@ -21,6 +24,13 @@ Pure configuration with helper functions for pattern access and filtering.
from pathlib import Path
from typing import Dict, Set, List, Optional
# Re-export from ignore_patterns for backwards compatibility
from aipass.backup.apps.handlers.config.ignore_patterns import (
GLOBAL_IGNORE_PATTERNS, IGNORE_EXCEPTIONS, should_ignore,
filter_tracked_items, get_ignore_patterns, get_cli_tracking_patterns,
DIFF_IGNORE_PATTERNS, DIFF_INCLUDE_PATTERNS, CLI_TRACKING_PATTERNS
)
# =============================================
# CONFIGURATION CONSTANTS
# =============================================
@@ -35,286 +45,6 @@ BACKUP_DESTINATIONS = {
"versioned_backup": f"{BASE_BACKUP_DIR}",
}
# =============================================
# IGNORE PATTERNS
# =============================================
# Global ignore patterns (what NOT to backup across all systems)
GLOBAL_IGNORE_PATTERNS = [
# Python cache and temp files
"__pycache__",
"*.pyc",
"*.pyo",
# Virtual environments
".venv",
"venv",
# Node.js
"node_modules",
"npm-debug.log",
"yarn-error.log",
# Operating system files
".DS_Store",
"Thumbs.db",
"desktop.ini",
"nul", # Windows reserved filename issue
# Temporary files
#"temp",
".temp",
"tmp",
".tmp",
"*.tmp",
"*.temp",
# Build and distribution directories
"build",
"dist",
"install",
"lib",
"bin",
# IDE and editor files
"*.swp",
"*.swo",
"*~",
# Backup directories (prevent recursive backups) - CRITICAL!
"backups", # Matches any directory named "backups"
"backup_system/backups", # Explicit path to our backup directory
"*/backups", # Any backups subdirectory
"system_snapshot", # Snapshot backup folder name
"versioned_backup", # Versioned backup folder name
"deleted_branches", # Deleted/archived branches (trash folder)
# System backups and trash
"TimeShift*",
"timeshift*",
".local/share/Trash",
"Trash",
# Linux system directories (don't backup user settings/cache)
".local",
".cache",
".config",
".mozilla",
".gnupg",
".ssh",
".vscode/cli", # VS Code CLI binary (large, regenerates)
".vscode/extensions", # VS Code extensions (large, regenerates)
".vscode-server", # VS Code remote server
".npm-global",
".pki",
".dotnet",
".var", # System application data (Flatpak, etc)
".backup", # Old backup directories
".antigravity", # AI assistant cache/data
".gemini", # AI assistant cache
"snap", # Snap package directories (broken symlinks, system-managed)
# AIPass internal state (runtime, not user data)
".trinity", # Branch identity/memory files (passport, local, observations)
".ai_mail.local", # Branch mailbox directories
".archive", # Branch archive directories
"backup_data", # Backup runtime state
"backup_json", # Backup metadata tracking
"DASHBOARD.local.json", # Branch dashboard state
"CLOSED_PLANS.local.json", # Flow plan close records
"STATUS.local.md", # Branch status boards
# Version control (huge number of files!)
".git", # Git repositories are version controlled elsewhere
# IDE and editor directories
".idea", # IntelliJ IDEA settings
".eclipse", # Eclipse settings
# Claude Code session files (change every session) - synced from .gitignore
#".claude/projects",
".claude/todos",
".claude/shell-snapshots",
".claude/ide",
".claude/statsig",
#".claude/plugins",
".claude/.credentials.json",
".claude/debug",
".claude/file-history",
".claude/history.jsonl",
".claude/.update.lock",
".claude/.projects",
#".claude/hooks/.last_diagnostics_file",
".serena/logs",
".code",
# Development cache/build directories
".npm",
".cargo",
".rustup",
".gem",
".gradle",
".m2",
# Application data we don't need in backups
".thunderbird",
".wine",
".steam",
".zoom",
# User directories that shouldn't be backed up
"Downloads",
#"Music",
"Videos",
"Pictures",
"Dropbox",
"system_logs",
"external_repos", # External git repos - version controlled elsewhere
"mcp_servers/context7", # External MCP server repo
"mcp_servers/playwright-mcp", # External MCP server repo
"mcp_servers/serena", # External MCP server repo
"mcp_servers/servers", # External MCP server repo
"mcp_servers/dropbox", # External MCP server repo
# Archive and compressed files
"*.zip",
"*.tar",
"*.gz",
"*.bz2",
"*.rar",
"*.7z",
"*.whl", # Python wheel files (long filenames in archives)
# Large binary/image files (VMs, disk images)
"*.img", # Disk images (sandbox.img, etc - can be 50GB+)
"*.iso", # ISO files
"*.vmdk", # Virtual machine disk
"*.vdi", # VirtualBox disk image
"*.qcow2", # QEMU disk image
# Image/media files (screenshots, icons, etc.)
"*.png",
"*.jpg",
"*.jpeg",
"*.gif",
"*.svg",
"*.webp",
"*.ico",
"*.bmp",
"*.tiff",
"*.tif",
# Log files
"*.log",
"logs",
# Most JSON files (frequent changes, not human-readable diffs)
"*_data.json",
"*_log.json",
"*_registry.json",
"snapshot_backup.json",
"snapshot_backup_changelog.json",
# Miscellaneous
"*.db",
"*.bashrc",
"*.bash_history",
"*.bash_logout",
#"*.claude.json.backup",
#"*.env",
"*.lesshst",
"*.npmrc",
"*.sudo_as_admin_successful",
]
# Notable patterns to highlight when skipped
CLI_TRACKING_PATTERNS = []
# Files that should be backed up but NOT generate diff files
DIFF_IGNORE_PATTERNS = [
# Log files (append-only, huge diffs)
"*.log",
"*.logs",
"system_logs/*.log",
# Most JSON files (frequent changes, not human-readable diffs)
#"*_config.json",
#"*_data.json",
#"*_log.json",
#"*_registry.json",
# Python cache/compiled
"*.pyc",
"*.pyo",
"*.pyd",
"__pycache__/*",
# Database files
"*.db",
"*.sqlite",
"*.sqlite3",
# Binary/media files
"*.exe",
"*.dll",
"*.so",
"*.dylib",
# Pickle/cache files
"*.pkl",
"*.pickle",
# Temporary files
"*.tmp",
"*.temp",
"*.bak",
"~*",
]
# Files that should be backed up despite matching ignore patterns
# Synced from .gitignore exceptions (! prefix patterns)
IGNORE_EXCEPTIONS = [
".gitignore", # Root .gitignore file should be backed up
"*.local.md", # AIPass session tracking files (caught by .local pattern but should be backed up)
".vscode/settings.json", # VS Code settings
"*/.claude/settings.local.json", # Claude local settings in any directory
"tools/cleanup_configs/*.json", # Cleanup config files
"*_config.json", # All config JSON files
"drone/commands/global/*.json", # Drone global commands
".claude.json", # Claude config
".mcp.json", # MCP config
".config/nerd-dictation/*", # Nerd dictation config
".commands.json", # Commands config
"BRANCH_REGISTRY.json", # Core ecosystem registry - vital file
# === TEMPLATES: FULL EXCEPTION - EVERY FILE ===
"templates/**", # FULL EXCEPTION: Include EVERY file in templates directory
"*/templates/**", # Templates in any subdirectory
"templates/*/**", # All subdirectories and files in templates
"*/templates/*/**", # All subdirectories and files in templates anywhere
# === TEMPLATE SUBDIRECTORIES (explicit) ===
"templates/ai_branch_setup_template/ai_mail.local/**",
"templates/ai_branch_setup_template/logs/**",
"templates/ai_branch_setup_template/standards.local/**",
"templates/ai_branch_setup_template/.claude/**",
"templates/ai_branch_setup_template/.archive/**",
# === MARKERS ===
".gitkeep", # Include all .gitkeep marker files (especially in templates)
".gitattributes", # Git attributes files
]
# Files that SHOULD have diffs created (exceptions to ignore patterns)
DIFF_INCLUDE_PATTERNS = [
"profile.json",
"pyrightconfig.json",
"package.json",
".mcp.json",
"settings.json",
"settings.local.json",
]
# =============================================
# BACKUP MODE CONFIGURATIONS
# =============================================
@@ -342,24 +72,6 @@ BACKUP_MODES = {
# HELPER FUNCTIONS
# =============================================
def get_ignore_patterns() -> List[str]:
"""Get the global ignore patterns list
Returns:
Copy of global ignore patterns list
"""
return GLOBAL_IGNORE_PATTERNS.copy()
def get_cli_tracking_patterns() -> List[str]:
"""Get the CLI tracking patterns list
Returns:
Copy of CLI tracking patterns list
"""
return CLI_TRACKING_PATTERNS.copy()
def get_backup_destination(system_name: str) -> str:
"""Get backup destination for a specific backup system
@@ -371,119 +83,6 @@ def get_backup_destination(system_name: str) -> str:
"""
return BACKUP_DESTINATIONS.get(system_name, BASE_BACKUP_DIR)
def filter_tracked_items(skipped_items: Dict[str, Set[str]]) -> Dict[str, Set[str]]:
"""Filter skipped items to only show project-specific items
Uses CLI tracking patterns to identify important items worth showing to user.
Args:
skipped_items: Dictionary with 'directories' and 'files' sets
Returns:
Filtered dictionary with only tracked items
"""
tracking_patterns = get_cli_tracking_patterns()
filtered_items = {
"directories": set(),
"files": set()
}
def matches_pattern(item_path: str, patterns: List[str]) -> bool:
"""Check if item path matches any tracking pattern"""
for pattern in patterns:
if pattern in item_path or item_path.startswith(pattern):
return True
# Check wildcard patterns
if pattern.startswith('*') and item_path.endswith(pattern[1:]):
return True
return False
# Filter directories
for directory in skipped_items.get("directories", set()):
if matches_pattern(directory, tracking_patterns):
filtered_items["directories"].add(directory)
# Filter files
for file_path in skipped_items.get("files", set()):
if matches_pattern(file_path, tracking_patterns):
filtered_items["files"].add(file_path)
return filtered_items
def should_ignore(path: Path, ignore_patterns: Optional[List[str]] = None,
exceptions: Optional[List[str]] = None,
backup_dest: Optional[Path] = None) -> bool:
"""Check if a file/folder should be ignored based on patterns.
Centralizes ignore pattern matching logic used during backup scanning.
Checks exceptions first (files that should NOT be ignored), then patterns.
Args:
path: Path object to check
ignore_patterns: List of patterns to ignore (defaults to GLOBAL_IGNORE_PATTERNS)
exceptions: List of exception patterns (defaults to IGNORE_EXCEPTIONS)
backup_dest: Optional backup destination to always ignore
Returns:
True if path should be ignored, False otherwise
Example:
# From backup engine
should_ignore(Path("/home/user/file.pyc")) # True
should_ignore(Path("/home/user/.gitignore")) # False (exception)
"""
import os
# Use defaults if not provided
if ignore_patterns is None:
ignore_patterns = GLOBAL_IGNORE_PATTERNS
if exceptions is None:
exceptions = IGNORE_EXCEPTIONS
path_str = str(path)
parts = set(path_str.split(os.sep))
name = path.name
# Always ignore backup destination if provided
if backup_dest and str(backup_dest) in path_str:
return True
# Ignore paths containing 'Backups'
if 'Backups' in parts:
return True
# Check exceptions first - files that should NOT be ignored
for exception in exceptions:
# Full path matching for template exceptions
if "**" in exception:
# Convert glob pattern to regex-like check
exception_parts = exception.split("/**")[0] # Get everything before /**
if exception_parts in path_str or exception_parts in "/".join(parts):
return False # Matches exception pattern - don't ignore
elif exception.startswith('*') and name.endswith(exception[1:]):
return False # Matches wildcard exception pattern
elif exception == name:
return False # Exact match
elif exception in path_str:
return False # Exception pattern is in the full path
# Check ignore patterns
for pattern in ignore_patterns:
# Special case: "backups" should only match directory names, not filenames
if pattern == "backups":
if "backups" in parts: # Only ignore if "backups" is a directory in the path
return True
elif pattern == name:
return True
elif pattern.startswith('*') and name.endswith(pattern[1:]):
return True
elif pattern in parts or pattern in path_str:
return True
return False
# =============================================
# MODULE INITIALIZATION
# =============================================
@@ -0,0 +1,212 @@
# =================== AIPass ====================
# Name: ignore_patterns.py
# Description: Ignore pattern loading and matching from JSON config
# Version: 1.0.0
# Created: 2026-03-14
# Modified: 2026-03-14
# =============================================
"""
Ignore Patterns Handler
Loads ignore/exception patterns from apps/json_templates/ignore_patterns.json
and provides pattern matching functions for backup file filtering.
Extracted from config_handler.py (FPLAN-0037) to separate data (JSON) from logic.
"""
# =============================================
# IMPORTS
# =============================================
import json
import os
from pathlib import Path
from typing import Dict, Set, List, Optional
from aipass.prax import logger
# =============================================
# PATTERN LOADING
# =============================================
_PATTERNS_JSON = Path(__file__).parents[2] / "json_templates" / "ignore_patterns.json"
def load_patterns() -> dict:
"""Load all pattern sets from the ignore_patterns.json file.
Returns:
Dictionary with all pattern sections from the JSON file.
Raises:
FileNotFoundError: If the JSON file is missing (fail loud).
json.JSONDecodeError: If the JSON file is malformed.
"""
if not _PATTERNS_JSON.exists():
raise FileNotFoundError(
f"[ignore_patterns] CRITICAL: Pattern file missing: {_PATTERNS_JSON}\n"
f"Expected at: apps/json_templates/ignore_patterns.json\n"
f"Cannot proceed without ignore patterns — backup would copy everything."
)
with open(_PATTERNS_JSON, "r", encoding="utf-8") as f:
data = json.load(f)
logger.info(f"[ignore_patterns] Loaded patterns from {_PATTERNS_JSON}")
return data
# =============================================
# LOAD PATTERN CONSTANTS
# =============================================
_data = load_patterns()
GLOBAL_IGNORE_PATTERNS: List[str] = _data["global_ignore_patterns"]["patterns"]
IGNORE_EXCEPTIONS: List[str] = _data["ignore_exceptions"]["patterns"]
CLI_TRACKING_PATTERNS: List[str] = _data["cli_tracking_patterns"]["patterns"]
DIFF_IGNORE_PATTERNS: List[str] = _data["diff_ignore_patterns"]["patterns"]
DIFF_INCLUDE_PATTERNS: List[str] = _data["diff_include_patterns"]["patterns"]
# =============================================
# HELPER FUNCTIONS
# =============================================
def get_ignore_patterns() -> List[str]:
"""Get the global ignore patterns list.
Returns:
Copy of global ignore patterns list
"""
return GLOBAL_IGNORE_PATTERNS.copy()
def get_cli_tracking_patterns() -> List[str]:
"""Get the CLI tracking patterns list.
Returns:
Copy of CLI tracking patterns list
"""
return CLI_TRACKING_PATTERNS.copy()
def filter_tracked_items(skipped_items: Dict[str, Set[str]]) -> Dict[str, Set[str]]:
"""Filter skipped items to only show project-specific items.
Uses CLI tracking patterns to identify important items worth showing to user.
Args:
skipped_items: Dictionary with 'directories' and 'files' sets
Returns:
Filtered dictionary with only tracked items
"""
tracking_patterns = get_cli_tracking_patterns()
filtered_items = {
"directories": set(),
"files": set()
}
def matches_pattern(item_path: str, patterns: List[str]) -> bool:
"""Check if item path matches any tracking pattern."""
for pattern in patterns:
if pattern in item_path or item_path.startswith(pattern):
return True
# Check wildcard patterns
if pattern.startswith('*') and item_path.endswith(pattern[1:]):
return True
return False
# Filter directories
for directory in skipped_items.get("directories", set()):
if matches_pattern(directory, tracking_patterns):
filtered_items["directories"].add(directory)
# Filter files
for file_path in skipped_items.get("files", set()):
if matches_pattern(file_path, tracking_patterns):
filtered_items["files"].add(file_path)
return filtered_items
def should_ignore(path: Path, ignore_patterns: Optional[List[str]] = None,
exceptions: Optional[List[str]] = None,
backup_dest: Optional[Path] = None) -> bool:
"""Check if a file/folder should be ignored based on patterns.
Centralizes ignore pattern matching logic used during backup scanning.
Checks exceptions first (files that should NOT be ignored), then patterns.
Args:
path: Path object to check
ignore_patterns: List of patterns to ignore (defaults to GLOBAL_IGNORE_PATTERNS)
exceptions: List of exception patterns (defaults to IGNORE_EXCEPTIONS)
backup_dest: Optional backup destination to always ignore
Returns:
True if path should be ignored, False otherwise
Example:
# From backup engine
should_ignore(Path("/home/user/file.pyc")) # True
should_ignore(Path("/home/user/.gitignore")) # False (exception)
"""
# Use defaults if not provided
if ignore_patterns is None:
ignore_patterns = GLOBAL_IGNORE_PATTERNS
if exceptions is None:
exceptions = IGNORE_EXCEPTIONS
path_str = str(path)
parts = set(path_str.split(os.sep))
name = path.name
# Always ignore backup destination if provided
if backup_dest and str(backup_dest) in path_str:
return True
# Ignore paths containing 'Backups'
if 'Backups' in parts:
return True
# Check exceptions first - files that should NOT be ignored
for exception in exceptions:
# Full path matching for template exceptions
if "**" in exception:
# Convert glob pattern to regex-like check
exception_parts = exception.split("/**")[0] # Get everything before /**
if exception_parts in path_str or exception_parts in "/".join(parts):
return False # Matches exception pattern - don't ignore
elif exception.startswith('*') and name.endswith(exception[1:]):
return False # Matches wildcard exception pattern
elif exception == name:
return False # Exact match
elif exception in path_str:
return False # Exception pattern is in the full path
# Check ignore patterns
for pattern in ignore_patterns:
# Special case: "backups" should only match directory names, not filenames
if pattern == "backups":
if "backups" in parts: # Only ignore if "backups" is a directory in the path
return True
elif pattern == name:
return True
elif pattern.startswith('*') and name.endswith(pattern[1:]):
return True
elif pattern in parts or pattern in path_str:
return True
return False
# =============================================
# MODULE INITIALIZATION
# =============================================
logger.info(f"[ignore_patterns] Module loaded — {len(GLOBAL_IGNORE_PATTERNS)} global patterns, "
f"{len(IGNORE_EXCEPTIONS)} exceptions, {len(DIFF_IGNORE_PATTERNS)} diff-ignore, "
f"{len(DIFF_INCLUDE_PATTERNS)} diff-include")
@@ -0,0 +1,282 @@
{
"global_ignore_patterns": {
"comments": "Global ignore patterns — what NOT to backup across all systems. Each category documents WHY patterns exist.",
"patterns": [
"__pycache__",
"*.pyc",
"*.pyo",
".venv",
"venv",
"node_modules",
"npm-debug.log",
"yarn-error.log",
".DS_Store",
"Thumbs.db",
"desktop.ini",
"nul",
".temp",
"tmp",
".tmp",
"*.tmp",
"*.temp",
"build",
"dist",
"install",
"lib",
"bin",
"*.swp",
"*.swo",
"*~",
"backups",
"backup_system/backups",
"*/backups",
"system_snapshot",
"versioned_backup",
"deleted_branches",
"TimeShift*",
"timeshift*",
".local/share/Trash",
"Trash",
".local",
".cache",
".config",
".mozilla",
".gnupg",
".ssh",
".vscode/cli",
".vscode/extensions",
".vscode-server",
".npm-global",
".pki",
".dotnet",
".var",
".backup",
".antigravity",
".gemini",
"snap",
".trinity",
".ai_mail.local",
".archive",
"backup_data",
"backup_json",
"DASHBOARD.local.json",
"CLOSED_PLANS.local.json",
"STATUS.local.md",
".git",
".idea",
".eclipse",
".claude/todos",
".claude/shell-snapshots",
".claude/ide",
".claude/statsig",
".claude/.credentials.json",
".claude/debug",
".claude/file-history",
".claude/history.jsonl",
".claude/.update.lock",
".claude/.projects",
".serena/logs",
".code",
".npm",
".cargo",
".rustup",
".gem",
".gradle",
".m2",
".thunderbird",
".wine",
".steam",
".zoom",
"Downloads",
"Videos",
"Pictures",
"Dropbox",
"system_logs",
"external_repos",
"mcp_servers/context7",
"mcp_servers/playwright-mcp",
"mcp_servers/serena",
"mcp_servers/servers",
"mcp_servers/dropbox",
"*.zip",
"*.tar",
"*.gz",
"*.bz2",
"*.rar",
"*.7z",
"*.whl",
"*.img",
"*.iso",
"*.vmdk",
"*.vdi",
"*.qcow2",
"*.png",
"*.jpg",
"*.jpeg",
"*.gif",
"*.svg",
"*.webp",
"*.ico",
"*.bmp",
"*.tiff",
"*.tif",
"*.log",
"logs",
"*_data.json",
"*_log.json",
"*_registry.json",
"snapshot_backup.json",
"snapshot_backup_changelog.json",
"*.db",
"*.bashrc",
"*.bash_history",
"*.bash_logout",
"*.lesshst",
"*.npmrc",
"*.sudo_as_admin_successful"
],
"category_comments": {
"python_cache": "Python cache and temp files: __pycache__, *.pyc, *.pyo",
"virtual_environments": "Virtual environments: .venv, venv",
"nodejs": "Node.js: node_modules, npm-debug.log, yarn-error.log",
"os_files": "Operating system files: .DS_Store, Thumbs.db, desktop.ini, nul (Windows reserved filename issue)",
"temp_files": "Temporary files: .temp, tmp, .tmp, *.tmp, *.temp",
"build_dirs": "Build and distribution directories: build, dist, install, lib, bin",
"editor_files": "IDE and editor files: *.swp, *.swo, *~",
"backup_dirs": "Backup directories (prevent recursive backups) — CRITICAL! backups matches any directory named 'backups'. Also system_snapshot, versioned_backup folder names, deleted_branches (trash folder)",
"system_backups": "System backups and trash: TimeShift*, timeshift*, .local/share/Trash, Trash",
"linux_system": "Linux system directories (don't backup user settings/cache): .local, .cache, .config, .mozilla, .gnupg, .ssh, .vscode/cli (large, regenerates), .vscode/extensions (large, regenerates), .vscode-server, .npm-global, .pki, .dotnet, .var (Flatpak etc), .backup, .antigravity (AI assistant cache/data), .gemini (AI assistant cache), snap (broken symlinks, system-managed)",
"aipass_internal": "AIPass internal state (runtime, not user data): .trinity (passport, local, observations), .ai_mail.local (mailbox dirs), .archive, backup_data (runtime state), backup_json (metadata tracking), DASHBOARD.local.json, CLOSED_PLANS.local.json, STATUS.local.md",
"version_control": "Version control (huge number of files!): .git repos are version controlled elsewhere",
"ide_dirs": "IDE and editor directories: .idea (IntelliJ), .eclipse",
"claude_session": "Claude Code session files (change every session) — synced from .gitignore: .claude/todos, .claude/shell-snapshots, .claude/ide, .claude/statsig, .claude/.credentials.json, .claude/debug, .claude/file-history, .claude/history.jsonl, .claude/.update.lock, .claude/.projects, .serena/logs, .code",
"dev_cache": "Development cache/build directories: .npm, .cargo, .rustup, .gem, .gradle, .m2",
"app_data": "Application data we don't need in backups: .thunderbird, .wine, .steam, .zoom",
"user_dirs": "User directories that shouldn't be backed up: Downloads, Videos, Pictures, Dropbox, system_logs, external_repos (version controlled elsewhere), mcp_servers/* (external MCP server repos)",
"archives": "Archive and compressed files: *.zip, *.tar, *.gz, *.bz2, *.rar, *.7z, *.whl (Python wheel files — long filenames)",
"large_binaries": "Large binary/image files (VMs, disk images): *.img (can be 50GB+), *.iso, *.vmdk, *.vdi, *.qcow2",
"media_files": "Image/media files (screenshots, icons, etc.): *.png, *.jpg, *.jpeg, *.gif, *.svg, *.webp, *.ico, *.bmp, *.tiff, *.tif",
"log_files": "Log files: *.log, logs",
"json_files": "Most JSON files (frequent changes, not human-readable diffs): *_data.json, *_log.json, *_registry.json, snapshot_backup.json, snapshot_backup_changelog.json",
"misc": "Miscellaneous: *.db, *.bashrc, *.bash_history, *.bash_logout, *.lesshst, *.npmrc, *.sudo_as_admin_successful"
}
},
"ignore_exceptions": {
"comments": "Files that should be backed up despite matching ignore patterns. Synced from .gitignore exceptions (! prefix patterns).",
"patterns": [
".gitignore",
"*.local.md",
".vscode/settings.json",
"*/.claude/settings.local.json",
"tools/cleanup_configs/*.json",
"*_config.json",
"drone/commands/global/*.json",
".claude.json",
".mcp.json",
".config/nerd-dictation/*",
".commands.json",
"BRANCH_REGISTRY.json",
"templates/**",
"*/templates/**",
"templates/*/**",
"*/templates/*/**",
"templates/ai_branch_setup_template/ai_mail.local/**",
"templates/ai_branch_setup_template/logs/**",
"templates/ai_branch_setup_template/standards.local/**",
"templates/ai_branch_setup_template/.claude/**",
"templates/ai_branch_setup_template/.archive/**",
".gitkeep",
".gitattributes"
],
"category_comments": {
"root_files": ".gitignore — Root .gitignore file should be backed up",
"aipass_session": "*.local.md — AIPass session tracking files (caught by .local pattern but should be backed up)",
"vscode": ".vscode/settings.json — VS Code settings",
"claude_settings": "*/.claude/settings.local.json — Claude local settings in any directory",
"config_json": "*_config.json, tools/cleanup_configs/*.json, drone/commands/global/*.json, .claude.json, .mcp.json, .commands.json — Various config JSON files",
"nerd_dictation": ".config/nerd-dictation/* — Nerd dictation config",
"branch_registry": "BRANCH_REGISTRY.json — Core ecosystem registry — vital file",
"templates_full_exception": "FULL EXCEPTION: Include EVERY file in templates directory — templates/**, */templates/**, templates/*/** , */templates/*/**",
"template_subdirs_explicit": "Explicit template subdirectories: ai_branch_setup_template/ai_mail.local, logs, standards.local, .claude, .archive",
"markers": ".gitkeep — Include all .gitkeep marker files (especially in templates). .gitattributes — Git attributes files"
}
},
"cli_tracking_patterns": {
"comments": "Notable patterns to highlight when skipped. Currently empty — reserved for future use.",
"patterns": []
},
"diff_ignore_patterns": {
"comments": "Files that should be backed up but NOT generate diff files.",
"patterns": [
"*.log",
"*.logs",
"system_logs/*.log",
"*.pyc",
"*.pyo",
"*.pyd",
"__pycache__/*",
"*.db",
"*.sqlite",
"*.sqlite3",
"*.exe",
"*.dll",
"*.so",
"*.dylib",
"*.pkl",
"*.pickle",
"*.tmp",
"*.temp",
"*.bak",
"~*"
],
"category_comments": {
"log_files": "Log files (append-only, huge diffs): *.log, *.logs, system_logs/*.log",
"python_cache": "Python cache/compiled: *.pyc, *.pyo, *.pyd, __pycache__/*",
"database_files": "Database files: *.db, *.sqlite, *.sqlite3",
"binary_files": "Binary/media files: *.exe, *.dll, *.so, *.dylib",
"pickle_cache": "Pickle/cache files: *.pkl, *.pickle",
"temp_files": "Temporary files: *.tmp, *.temp, *.bak, ~*"
}
},
"diff_include_patterns": {
"comments": "Files that SHOULD have diffs created (exceptions to ignore patterns).",
"patterns": [
"profile.json",
"pyrightconfig.json",
"package.json",
".mcp.json",
"settings.json",
"settings.local.json"
]
}
}
@@ -371,16 +371,13 @@ class BackupEngine:
# Display dry-run warning if in test mode
if self.dry_run:
console.print()
console.print("╭" + "─" * 68 + "╮", style="yellow")
console.print("│" + " " * 18 + "[bold yellow]🔍 DRY-RUN MODE ACTIVE[/bold yellow]" + " " * 18 + "│", style="yellow")
console.print("│" + " " * 10 + "[yellow]Files will be scanned but NOT copied or deleted[/yellow]" + " " * 10 + "│", style="yellow")
console.print("╰" + "─" * 68 + "╯", style="yellow")
warning("DRY-RUN MODE ACTIVE — Files will be scanned but NOT copied or deleted")
console.print()
header(f"AIPass {self.mode_config['name']} - {self.mode_config['description']}")
# Ensure backup directory exists
if not self.ensure_backup_directory(result):
console.print(f"\nBACKUP FAILED: Could not create backup directory")
error("BACKUP FAILED: Could not create backup directory")
return result
# Load previous backup info
@@ -449,7 +446,10 @@ class BackupEngine:
else:
result.files_skipped += 1 # Only skip if unchanged
elif self.mode_config['behavior'] == 'versioned':
if copy_versioned_file(file_path, backup_file, self.backup_path, result):
# Skip unchanged files (mtime pre-check avoids expensive copy_versioned_file overhead)
if backup_file.exists() and backup_file.stat().st_mtime == file_path.stat().st_mtime:
result.files_skipped += 1
elif copy_versioned_file(file_path, backup_file, self.backup_path, result):
result.files_copied += 1
elif self.file_needs_backup(file_path, backup_file, last_timestamps):
if copy_file_with_structure(file_path, backup_file, self.backup_path, result):
@@ -105,7 +105,7 @@ def _show_file_tracker_stats() -> bool:
console.print(f" ... and {remaining} more files")
return True
except Exception as e:
console.print(f"Error showing tracker stats: {e}")
error(f"Error showing tracker stats: {e}")
logger.error(f"Error showing tracker stats: {e}")
return False
@@ -123,7 +123,7 @@ def _clear_file_tracker() -> bool:
console.print("File tracker already empty or clear failed")
return success
except Exception as e:
console.print(f"Error clearing file tracker: {e}")
error(f"Error clearing file tracker: {e}")
logger.error(f"Error clearing file tracker: {e}")
return False
@@ -140,10 +140,10 @@ def _test_drive_sync() -> bool:
folder_id = sync.get_or_create_backup_folder()
console.print(f"Backup folder ready: {folder_id}")
else:
console.print("Failed to create backup folder")
error("Failed to create backup folder")
return result
except Exception as e:
console.print(f"Test failed: {e}")
error(f"Test failed: {e}")
logger.error(f"Test failed: {e}")
return False
@@ -279,7 +279,7 @@ def handle_command(args) -> bool:
# Default to snapshot backup directory
backup_path = _BACKUP_ROOT / "backups" / "system_snapshot"
if not backup_path.exists():
console.print(f"Error: Backup directory not found: {backup_path}")
error(f"Backup directory not found: {backup_path}")
return False
project = getattr(args, 'project', 'AIPass') or 'AIPass'
@@ -288,7 +288,7 @@ def handle_command(args) -> bool:
sync = GoogleDriveSync()
if not sync.authenticate():
console.print("[red]FAILED: Could not authenticate with Google Drive[/red]")
error("FAILED: Could not authenticate with Google Drive")
return False
limit = getattr(args, 'limit', 0) or 0
@@ -308,13 +308,13 @@ def handle_command(args) -> bool:
folder_id = sync.get_or_create_project_folder(project)
if not folder_id:
error_msg = sync.last_error or "Unknown error"
console.print(f"[red]FAILED: {error_msg}[/red]")
console.print("[red]Sync aborted - no files uploaded[/red]")
error(f"FAILED: {error_msg}")
error("Sync aborted - no files uploaded")
return False
console.print(f" Drive folder ready: {project}")
if sync.tracker_was_reset:
console.print(f"[yellow] Drive folder is new - tracker reset, full re-sync needed[/yellow]")
warning("Drive folder is new - tracker reset, full re-sync needed")
# Phase 1: Scan (local only, fast) - runs AFTER folder check so tracker is accurate
console.print(f"\nScanning {backup_path}...")
@@ -362,8 +362,8 @@ def handle_command(args) -> bool:
# Check for errors
if result.get("error"):
console.print(f"\n[red]FAILED: {result['error']}[/red]")
console.print("[red]Sync aborted - no files uploaded[/red]")
error(f"FAILED: {result['error']}")
error("Sync aborted - no files uploaded")
return False
# Summary
@@ -379,7 +379,7 @@ def handle_command(args) -> bool:
console.print(f" [dim]Versioned: {format_age(ts.get('versioned'))}[/dim]")
console.print(f" [dim]Drive sync: {format_age(ts.get('drive_sync'))}[/dim]")
else:
console.print(f"[red]Sync failed: {result['uploaded']} uploaded, {result['failed']} failed, {result['skipped']} unchanged[/red]")
error(f"Sync failed: {result['uploaded']} uploaded, {result['failed']} failed, {result['skipped']} unchanged")
return result["success"]
@@ -464,7 +464,7 @@ EXAMPLES:
# Check if module is enabled
config = _load_config()
if not config.get("config", {}).get("enabled", True):
console.print("Warning: Google Drive sync is disabled")
warning("Google Drive sync is disabled")
sys.exit(0)
if args.command == 'clear-tracker':
@@ -472,7 +472,7 @@ EXAMPLES:
console.print("File tracker cleared successfully")
sys.exit(0)
else:
console.print("Failed to clear file tracker")
error("Failed to clear file tracker")
sys.exit(1)
elif args.command == 'show-stats':
@@ -483,19 +483,19 @@ EXAMPLES:
elif args.command == 'sync':
if not args.path:
console.print("Error: sync command requires a path argument")
error("sync command requires a path argument")
console.print("Usage: python3 google_drive_sync.py sync /path/to/backups")
sys.exit(1)
backup_path = Path(args.path)
if not backup_path.exists():
console.print(f"Error: Backup directory not found: {backup_path}")
error(f"Backup directory not found: {backup_path}")
sys.exit(1)
sync = GoogleDriveSync()
if not sync.authenticate():
console.print("[red]Failed to authenticate with Google Drive[/red]")
error("Failed to authenticate with Google Drive")
sys.exit(1)
# Phase 1: Scan
@@ -531,7 +531,7 @@ EXAMPLES:
console.print(f"[green]Sync complete: {result['uploaded']} uploaded, {result['skipped']} unchanged[/green]")
sys.exit(0)
else:
console.print(f"[yellow]Sync finished: {result['uploaded']} uploaded, {result['failed']} failed, {result['skipped']} unchanged[/yellow]")
warning(f"Sync finished: {result['uploaded']} uploaded, {result['failed']} failed, {result['skipped']} unchanged")
sys.exit(1)
elif args.command == 'sync-test':