feat(system): seedgo v2 operational, full system audit, 14/15 branches at 99%
Three days of intensive work bringing seedgo to full operational status and driving all branches through comprehensive standards compliance. Seedgo v2.0.0: - 22 checkers active (up from 20), standards pack fully operational - New introspection standard researched from Dev-Pass, FPLAN-0017 open - Bypass system for false positives (.seedgo config) - Standards query and audit commands fully functional System-wide audit (FPLAN-0016): - All 14 auditable branches at 99%+ compliance - CLI imports standardized across all branches (console from cli.apps.modules) - handle_command(command, args) → bool contract added to all modules - print_help() function naming fixed for checker pattern matching - Handler extraction: large modules split, file I/O moved to handler layer - New handlers created across ai_mail, backup, daemon, flow, skills, spawn, seedgo Branch-specific highlights: - ai_mail: email.py split 840→420 lines, 4 new handlers - flow: dplan_flow.py 688→591 lines, 4 new handlers - seedgo: massive restructure — standards moved to handlers/aipass_standards/, old standards/ tree removed, bypass system added, diagnostics module - commons: database module added, CLI imports fixed - skills: 5 handle_commands added, help function renamed - trigger: error reporter handler, handle_command routing - All branches: consistent architecture, clean drone routing Culture doc (CLAUDE.md) added — documents AIPass philosophy, identity, memory system, and collaboration principles. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
AIOSAI
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
09e759a8a4
commit
babedd9c64
@@ -1,19 +1,9 @@
|
||||
|
||||
# ===================AIPASS====================
|
||||
# META DATA HEADER
|
||||
# Name: rollover.py - Rollover Orchestration Module
|
||||
# Date: 2025-11-16
|
||||
# Version: 0.2.0
|
||||
# Category: memory/modules
|
||||
#
|
||||
# CHANGELOG (Max 5 entries):
|
||||
# - v0.2.0 (2026-03-06): Adapted for AIPass public repo - removed internal deps
|
||||
# - v0.1.0 (2025-11-16): Initial version - orchestrate rollover workflow
|
||||
#
|
||||
# CODE STANDARDS:
|
||||
# - Thin orchestration: Delegate all logic to handlers
|
||||
# - No business logic: Only coordinate workflow
|
||||
# - handle_command() pattern
|
||||
# =================== AIPass ====================
|
||||
# Name: rollover.py
|
||||
# Description: Rollover Orchestration Module
|
||||
# Version: 0.5.0
|
||||
# Created: 2025-11-16
|
||||
# Modified: 2026-03-08
|
||||
# =============================================
|
||||
|
||||
"""
|
||||
@@ -31,100 +21,24 @@ Purpose:
|
||||
"""
|
||||
|
||||
import sys
|
||||
import logging
|
||||
import subprocess
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import List, Dict
|
||||
from typing import List
|
||||
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich import box
|
||||
|
||||
from aipass.prax import logger
|
||||
from aipass.cli.apps.modules import console
|
||||
|
||||
# =============================================================================
|
||||
# INFRASTRUCTURE SETUP
|
||||
# =============================================================================
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
console = Console()
|
||||
|
||||
# Handler imports (relative within the memory package)
|
||||
# Handler imports
|
||||
from ..handlers.monitor import detector
|
||||
from ..handlers.rollover import extractor
|
||||
from ..handlers.vector import embedder
|
||||
from ..handlers.tracking import line_counter
|
||||
|
||||
# ChromaDB storage via subprocess
|
||||
# Resolve paths relative to the handlers directory
|
||||
_HANDLERS_DIR = Path(__file__).resolve().parent.parent / "handlers"
|
||||
CHROMA_SUBPROCESS_SCRIPT = _HANDLERS_DIR / "storage" / "chroma_subprocess.py"
|
||||
|
||||
# Use system python by default; can be overridden via environment variable
|
||||
import os
|
||||
MEMORY_PYTHON = os.environ.get("AIPASS_MEMORY_PYTHON", sys.executable)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# REPO ROOT DISCOVERY
|
||||
# =============================================================================
|
||||
|
||||
def _find_repo_root() -> Path:
|
||||
"""Walk up from this file to find the repo root (contains AIPASS_REGISTRY.json)."""
|
||||
current = Path(__file__).resolve().parent
|
||||
for parent in [current] + list(current.parents):
|
||||
if (parent / "AIPASS_REGISTRY.json").exists():
|
||||
return parent
|
||||
return Path.cwd()
|
||||
|
||||
|
||||
_REPO_ROOT = _find_repo_root()
|
||||
|
||||
|
||||
# No other module imports (modules don't import modules)
|
||||
|
||||
|
||||
def _store_vectors_subprocess(branch: str, memory_type: str, embeddings: list,
|
||||
documents: list, metadatas: list, db_path: str | Path | None = None) -> dict:
|
||||
"""
|
||||
Store vectors via subprocess.
|
||||
|
||||
This ensures ChromaDB compatibility regardless of calling Python version.
|
||||
"""
|
||||
# Convert numpy arrays to lists for JSON serialization
|
||||
embeddings_serializable = [
|
||||
emb.tolist() if hasattr(emb, 'tolist') else emb
|
||||
for emb in embeddings
|
||||
]
|
||||
|
||||
input_data = {
|
||||
'operation': 'store_vectors',
|
||||
'branch': branch,
|
||||
'memory_type': memory_type,
|
||||
'embeddings': embeddings_serializable,
|
||||
'documents': documents,
|
||||
'metadatas': metadatas,
|
||||
'db_path': str(db_path) if db_path else None
|
||||
}
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[str(MEMORY_PYTHON), str(CHROMA_SUBPROCESS_SCRIPT)],
|
||||
input=json.dumps(input_data),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
return {'success': False, 'error': result.stderr or 'Subprocess failed'}
|
||||
|
||||
return json.loads(result.stdout)
|
||||
except subprocess.TimeoutExpired:
|
||||
return {'success': False, 'error': 'Storage operation timed out'}
|
||||
except json.JSONDecodeError as e:
|
||||
return {'success': False, 'error': f'Invalid JSON response: {e}'}
|
||||
except Exception as e:
|
||||
return {'success': False, 'error': str(e)}
|
||||
from ..handlers.rollover.orchestrator import (
|
||||
execute_rollover as _handler_execute_rollover,
|
||||
sync_line_counts as _handler_sync_line_counts,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -153,7 +67,7 @@ def handle_command(command: str, args: List[str]) -> bool: # noqa: ARG001
|
||||
return True
|
||||
|
||||
if command == 'rollover':
|
||||
execute_rollover()
|
||||
run_rollover()
|
||||
return True
|
||||
|
||||
elif command == 'status':
|
||||
@@ -202,17 +116,11 @@ def print_help() -> None:
|
||||
# ROLLOVER ORCHESTRATION
|
||||
# =============================================================================
|
||||
|
||||
def execute_rollover() -> bool:
|
||||
def run_rollover() -> bool:
|
||||
"""
|
||||
Execute rollover workflow for all triggered branches
|
||||
Execute rollover workflow for all triggered branches.
|
||||
|
||||
Workflow:
|
||||
1. Check all branches for triggers
|
||||
2. For each trigger:
|
||||
- Extract oldest 100 entries
|
||||
- Generate embeddings
|
||||
- Store in Chroma
|
||||
3. Report results
|
||||
Delegates to handler and renders results with Rich.
|
||||
"""
|
||||
console.print()
|
||||
console.print(Panel.fit(
|
||||
@@ -222,267 +130,47 @@ def execute_rollover() -> bool:
|
||||
))
|
||||
console.print()
|
||||
|
||||
# Step 1: Detect triggers
|
||||
console.print("[cyan]Checking for rollover triggers...[/cyan]")
|
||||
triggers_result = detector.check_all_branches()
|
||||
|
||||
if not triggers_result['success']:
|
||||
logger.error(f"[rollover] Failed to check branches: {triggers_result.get('error', 'Unknown error')}")
|
||||
console.print("[red]x[/red] Failed to check for rollover triggers")
|
||||
result = _handler_execute_rollover()
|
||||
|
||||
if not result.get('success') and result.get('error'):
|
||||
console.print(f"[red]x[/red] {result['error']}")
|
||||
return False
|
||||
|
||||
triggers = triggers_result.get('triggers', [])
|
||||
if not triggers:
|
||||
triggers_count = result.get('triggers_count', 0)
|
||||
if triggers_count == 0:
|
||||
console.print("[green]>[/green] No files need rollover")
|
||||
logger.info("[rollover] No rollover triggers detected")
|
||||
return True
|
||||
|
||||
console.print(f"[green]>[/green] Found {len(triggers)} files ready for rollover")
|
||||
logger.info(f"[rollover] Found {len(triggers)} files ready for rollover")
|
||||
console.print(f"[green]>[/green] Found {triggers_count} files ready for rollover")
|
||||
console.print()
|
||||
|
||||
# Process each trigger
|
||||
success_count = 0
|
||||
failed = []
|
||||
|
||||
for trigger in triggers:
|
||||
console.print(f"[yellow]Processing:[/yellow] {trigger}")
|
||||
|
||||
# Step 1: CREATE BACKUP (safety net)
|
||||
backup_result = extractor.create_rollover_backup(trigger.file_path)
|
||||
|
||||
if not backup_result['success']:
|
||||
error_msg = backup_result.get('error', 'Backup failed')
|
||||
logger.error(f"[rollover] Backup failed for {trigger}: {error_msg}")
|
||||
failed.append((trigger, "backup", error_msg))
|
||||
continue # Don't proceed without backup
|
||||
|
||||
logger.info(f"[rollover] {backup_result.get('message')}")
|
||||
|
||||
# Step 2: Extract memories (auto-calculates percentage)
|
||||
extract_result = extractor.extract_with_metadata(trigger.file_path)
|
||||
|
||||
if not extract_result['success']:
|
||||
error_msg = extract_result.get('error', 'Unknown error')
|
||||
logger.error(f"[rollover] Extraction failed for {trigger}: {error_msg}")
|
||||
|
||||
# RESTORE from backup
|
||||
restore_result = extractor.restore_from_backup(trigger.file_path)
|
||||
if restore_result['success']:
|
||||
logger.info("[rollover] Restored from backup after extraction failure")
|
||||
|
||||
failed.append((trigger, "extraction", error_msg))
|
||||
continue
|
||||
|
||||
memories = extract_result.get('entries', [])
|
||||
branch = extract_result.get('branch', '')
|
||||
memory_type = extract_result.get('type', 'unknown')
|
||||
old_lines = extract_result.get('old_lines', 0)
|
||||
new_lines = extract_result.get('new_lines', 0)
|
||||
|
||||
if not branch:
|
||||
logger.error(f"[rollover] No branch found in extraction result for {trigger}")
|
||||
failed.append((trigger, "extraction", "No branch in result"))
|
||||
continue
|
||||
|
||||
logger.info(f"[rollover] Extracted {len(memories)} items from {trigger} ({old_lines} -> {new_lines} lines)")
|
||||
|
||||
# Convert memory items to text for vectorization
|
||||
texts = _extract_text_from_memories(memories)
|
||||
|
||||
# Step 3: Generate embeddings
|
||||
embed_result = embedder.encode_batch(texts)
|
||||
|
||||
if not embed_result['success']:
|
||||
error_msg = embed_result.get('error', 'Unknown error')
|
||||
logger.error(f"[rollover] Embedding failed for {trigger}: {error_msg}")
|
||||
|
||||
# RESTORE from backup
|
||||
restore_result = extractor.restore_from_backup(trigger.file_path)
|
||||
if restore_result['success']:
|
||||
logger.info("[rollover] Restored from backup after embedding failure")
|
||||
|
||||
failed.append((trigger, "embedding", error_msg))
|
||||
continue
|
||||
|
||||
embeddings = embed_result.get('embeddings', [])
|
||||
if not embeddings:
|
||||
logger.error(f"[rollover] No embeddings generated for {trigger}")
|
||||
failed.append((trigger, "embedding", "No embeddings in result"))
|
||||
continue
|
||||
|
||||
logger.info(f"[rollover] Generated {len(embeddings)} embeddings for {trigger}")
|
||||
|
||||
# Step 4: Prepare metadata for vectorization
|
||||
metadatas = []
|
||||
for memory in memories:
|
||||
metadata = memory.get('_metadata', {})
|
||||
metadata['timestamp'] = memory.get('timestamp', '')
|
||||
metadatas.append(metadata)
|
||||
|
||||
# Step 5: Store in LOCAL branch Chroma (via subprocess)
|
||||
# Type assertions for Pylance (validated above with early returns)
|
||||
branch_str: str = branch
|
||||
memory_type_str: str = memory_type
|
||||
embeddings_list: list = embeddings
|
||||
|
||||
local_chroma_path = _get_branch_local_chroma_path(branch_str)
|
||||
local_store_result = None
|
||||
|
||||
if local_chroma_path:
|
||||
local_store_result = _store_vectors_subprocess(
|
||||
branch=branch_str,
|
||||
memory_type=memory_type_str,
|
||||
embeddings=embeddings_list,
|
||||
documents=texts,
|
||||
metadatas=metadatas,
|
||||
db_path=str(local_chroma_path)
|
||||
)
|
||||
|
||||
if not local_store_result['success']:
|
||||
logger.warning(f"[rollover] Local storage failed for {branch}: {local_store_result.get('error')}")
|
||||
# Continue anyway - global storage is primary
|
||||
else:
|
||||
logger.info(f"[rollover] Stored {len(embeddings)} vectors in local Chroma for {branch}")
|
||||
|
||||
# Step 6: Store in GLOBAL Memory Chroma (via subprocess)
|
||||
global_store_result = _store_vectors_subprocess(
|
||||
branch=branch_str,
|
||||
memory_type=memory_type_str,
|
||||
embeddings=embeddings_list,
|
||||
documents=texts,
|
||||
metadatas=metadatas
|
||||
# db_path=None means global
|
||||
)
|
||||
|
||||
if not global_store_result['success']:
|
||||
error_msg = global_store_result.get('error', 'Unknown error')
|
||||
logger.error(f"[rollover] Global storage failed for {trigger}: {error_msg}")
|
||||
|
||||
# RESTORE from backup (CRITICAL - file was modified but storage failed)
|
||||
restore_result = extractor.restore_from_backup(trigger.file_path)
|
||||
if restore_result['success']:
|
||||
logger.info("[rollover] Restored from backup after storage failure")
|
||||
else:
|
||||
logger.error(f"[rollover] CRITICAL: Failed to restore from backup: {restore_result.get('error')}")
|
||||
|
||||
failed.append((trigger, "global_storage", error_msg))
|
||||
continue
|
||||
|
||||
logger.info(f"[rollover] Stored {len(embeddings)} vectors in global Chroma for {branch}")
|
||||
|
||||
# Step 7: Update line count metadata
|
||||
update_result = line_counter.update_line_count(trigger.file_path)
|
||||
if update_result['success']:
|
||||
logger.info(f"[rollover] Updated line count metadata for {trigger.file_path.name}")
|
||||
else:
|
||||
logger.warning(f"[rollover] Failed to update line count for {trigger.file_path.name}: {update_result.get('error')}")
|
||||
|
||||
# Success!
|
||||
success_count += 1
|
||||
global_collection = global_store_result.get('collection')
|
||||
global_total = global_store_result.get('total_vectors')
|
||||
|
||||
# Report both local and global storage
|
||||
local_status = "> local" if local_store_result and local_store_result['success'] else "x local"
|
||||
# Display individual results
|
||||
for item in result.get('results', []):
|
||||
local_status = "> local" if item.get('local_stored') else "x local"
|
||||
console.print(
|
||||
f" [green]>[/green] Rolled over {len(memories)} items -> {global_collection} "
|
||||
f"({old_lines} -> {new_lines} lines, global: {global_total} vectors, {local_status})"
|
||||
f" [green]>[/green] Rolled over {item['memories_count']} items -> {item['global_collection']} "
|
||||
f"({item['old_lines']} -> {item['new_lines']} lines, global: {item['global_total']} vectors, {local_status})"
|
||||
)
|
||||
logger.info(f"[rollover] Successfully rolled over {trigger}: {len(memories)} items, {old_lines} -> {new_lines} lines")
|
||||
|
||||
# Report results
|
||||
success_count = result.get('success_count', 0)
|
||||
failed = result.get('failed', [])
|
||||
|
||||
console.print()
|
||||
if success_count > 0:
|
||||
console.print(f"[green]>[/green] Rollover complete: {success_count}/{len(triggers)} successful")
|
||||
logger.info(f"[rollover] Rollover complete: {success_count}/{len(triggers)} successful")
|
||||
console.print(f"[green]>[/green] Rollover complete: {success_count}/{triggers_count} successful")
|
||||
|
||||
if failed:
|
||||
console.print()
|
||||
console.print("[red]Failed operations:[/red]")
|
||||
for trigger, stage, err in failed:
|
||||
console.print(f" [red]x[/red] {trigger} - {stage}: {err}")
|
||||
logger.error(f"[rollover] {len(failed)} operations failed")
|
||||
for fail in failed:
|
||||
console.print(f" [red]x[/red] {fail['trigger']} - {fail['stage']}: {fail['error']}")
|
||||
|
||||
return success_count > 0
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# PATH HELPERS
|
||||
# =============================================================================
|
||||
|
||||
def _get_branch_local_chroma_path(branch_name: str) -> Path | None:
|
||||
"""
|
||||
Get local .chroma path for branch
|
||||
|
||||
Args:
|
||||
branch_name: Branch name (e.g., "SEED", "AIPASS")
|
||||
|
||||
Returns:
|
||||
Path to branch's local .chroma directory, or None if branch not found
|
||||
"""
|
||||
# Read registry to get branch path
|
||||
if not branch_name:
|
||||
return None
|
||||
|
||||
registry = detector._read_registry()
|
||||
|
||||
for branch in registry:
|
||||
if branch.get('name', '').upper() == branch_name.upper():
|
||||
branch_path = Path(branch.get('path', ''))
|
||||
if branch_path.exists():
|
||||
chroma_path = branch_path / '.chroma'
|
||||
# Auto-create .chroma directory if missing
|
||||
if not chroma_path.exists():
|
||||
chroma_path.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"[rollover] Created local .chroma directory for {branch_name}")
|
||||
return chroma_path
|
||||
|
||||
logger.warning(f"[rollover] Branch {branch_name} not found in registry")
|
||||
return None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEXT EXTRACTION HELPERS
|
||||
# =============================================================================
|
||||
|
||||
def _extract_text_from_memories(memories: List[Dict]) -> List[str]:
|
||||
"""
|
||||
Extract text content from memory items for vectorization
|
||||
|
||||
Memory items have different structures:
|
||||
- sessions: 'activities' array (join into text)
|
||||
- observations: might have 'content' or 'text' field
|
||||
- generic: convert to JSON string
|
||||
|
||||
Args:
|
||||
memories: List of memory items
|
||||
|
||||
Returns:
|
||||
List of text strings for embedding
|
||||
"""
|
||||
texts = []
|
||||
|
||||
for memory in memories:
|
||||
# Try common text fields
|
||||
if 'activities' in memory and isinstance(memory['activities'], list):
|
||||
# Sessions type - join activities
|
||||
text = '\n'.join(str(a) for a in memory['activities'])
|
||||
elif 'content' in memory:
|
||||
text = str(memory['content'])
|
||||
elif 'text' in memory:
|
||||
text = str(memory['text'])
|
||||
elif 'message' in memory:
|
||||
text = str(memory['message'])
|
||||
else:
|
||||
# Fallback - convert to string representation
|
||||
text = str(memory)
|
||||
|
||||
texts.append(text)
|
||||
|
||||
return texts
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# LINE COUNT SYNC
|
||||
# =============================================================================
|
||||
@@ -491,8 +179,7 @@ def sync_line_counts() -> None:
|
||||
"""
|
||||
Update line count metadata for all branch memory files.
|
||||
|
||||
Reads actual line counts and updates document_metadata.status.current_lines
|
||||
for all *.local.json and *.observations.json files in AIPASS_REGISTRY.
|
||||
Delegates to handler and renders results with Rich.
|
||||
"""
|
||||
console.print()
|
||||
console.print(Panel.fit(
|
||||
@@ -505,7 +192,7 @@ def sync_line_counts() -> None:
|
||||
console.print("[cyan]Updating line counts for all memory files...[/cyan]")
|
||||
console.print()
|
||||
|
||||
result = line_counter.update_all_memory_files()
|
||||
result = _handler_sync_line_counts()
|
||||
|
||||
if result['success']:
|
||||
console.print(f"[green]>[/green] Updated {result['updated']} files")
|
||||
@@ -513,10 +200,8 @@ def sync_line_counts() -> None:
|
||||
console.print(f"[yellow]![/yellow] {result['failed']} files failed:")
|
||||
for branch, mem_type, error in result.get('failures', []):
|
||||
console.print(f" [red]x[/red] {branch}.{mem_type}: {error}")
|
||||
logger.info(f"[rollover] Synced line counts: {result['updated']} updated, {result['failed']} failed")
|
||||
else:
|
||||
console.print("[red]x[/red] Failed to sync line counts")
|
||||
logger.error("[rollover] Failed to sync line counts")
|
||||
|
||||
console.print()
|
||||
|
||||
@@ -525,11 +210,6 @@ def sync_line_counts() -> None:
|
||||
# STATUS & CHECKING
|
||||
# =============================================================================
|
||||
|
||||
def get_rollover_stats() -> dict:
|
||||
"""Return rollover stats from detector (module-layer passthrough)."""
|
||||
return detector.get_rollover_stats()
|
||||
|
||||
|
||||
def show_status() -> None:
|
||||
"""
|
||||
Show rollover statistics for all branches
|
||||
@@ -625,6 +305,26 @@ def check_triggers() -> None:
|
||||
console.print()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# INTROSPECTION
|
||||
# =============================================================================
|
||||
|
||||
def print_introspection():
|
||||
"""Display module introspection info."""
|
||||
console.print()
|
||||
console.print("rollover Module")
|
||||
console.print("Orchestrates memory rollover workflow: trigger detection, extraction, embedding, and vector storage")
|
||||
console.print()
|
||||
console.print("Connected Handlers:")
|
||||
console.print(" handlers/monitor/")
|
||||
console.print(" - detector.py (check_all_branches — detect branches exceeding rollover threshold)")
|
||||
console.print(" - detector.py (get_rollover_stats — retrieve rollover statistics for all branches)")
|
||||
console.print(" handlers/rollover/")
|
||||
console.print(" - orchestrator.py (execute_rollover — run full rollover pipeline for triggered branches)")
|
||||
console.print(" - orchestrator.py (sync_line_counts — update line count metadata for all memory files)")
|
||||
console.print()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# STANDALONE EXECUTION
|
||||
# =============================================================================
|
||||
|
||||
@@ -1,19 +1,9 @@
|
||||
|
||||
# ===================AIPASS====================
|
||||
# META DATA HEADER
|
||||
# Name: search.py - Search Orchestration Module
|
||||
# Date: 2025-11-27
|
||||
# Version: 0.2.0
|
||||
# Category: memory/modules
|
||||
#
|
||||
# CHANGELOG (Max 5 entries):
|
||||
# - v0.2.0 (2026-03-06): Adapted for AIPass public repo - removed internal deps
|
||||
# - v0.1.0 (2025-11-27): Initial version - orchestrate semantic search
|
||||
#
|
||||
# CODE STANDARDS:
|
||||
# - Thin orchestration: Delegate all logic to handlers
|
||||
# - No business logic: Only coordinate workflow
|
||||
# - handle_command() pattern
|
||||
# =================== AIPass ====================
|
||||
# Name: search.py
|
||||
# Description: Search Orchestration Module
|
||||
# Version: 0.4.0
|
||||
# Created: 2025-11-27
|
||||
# Modified: 2026-03-08
|
||||
# =============================================
|
||||
|
||||
"""
|
||||
@@ -30,75 +20,22 @@ Purpose:
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import logging
|
||||
import subprocess
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich import box
|
||||
|
||||
from aipass.prax import logger
|
||||
from aipass.cli.apps.modules import console
|
||||
|
||||
# =============================================================================
|
||||
# INFRASTRUCTURE SETUP
|
||||
# =============================================================================
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
console = Console()
|
||||
|
||||
# Handler imports (relative within the memory package)
|
||||
from ..handlers.vector import embedder
|
||||
|
||||
# ChromaDB search via subprocess
|
||||
_HANDLERS_DIR = Path(__file__).resolve().parent.parent / "handlers"
|
||||
CHROMA_SUBPROCESS_SCRIPT = _HANDLERS_DIR / "storage" / "chroma_subprocess.py"
|
||||
|
||||
# Use system python by default; can be overridden via environment variable
|
||||
MEMORY_PYTHON = os.environ.get("AIPASS_MEMORY_PYTHON", sys.executable)
|
||||
|
||||
|
||||
def _search_vectors_subprocess(
|
||||
query_embedding: list,
|
||||
branch: str | None = None,
|
||||
memory_type: str | None = None,
|
||||
n_results: int = 5,
|
||||
db_path: str | Path | None = None
|
||||
) -> dict:
|
||||
"""
|
||||
Search vectors via subprocess.
|
||||
|
||||
This ensures ChromaDB compatibility regardless of calling Python version.
|
||||
"""
|
||||
input_data = {
|
||||
'operation': 'search_vectors',
|
||||
'query_embedding': query_embedding,
|
||||
'branch': branch,
|
||||
'memory_type': memory_type,
|
||||
'n_results': n_results,
|
||||
'db_path': str(db_path) if db_path else None
|
||||
}
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[str(MEMORY_PYTHON), str(CHROMA_SUBPROCESS_SCRIPT)],
|
||||
input=json.dumps(input_data),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=60
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
return {'success': False, 'error': result.stderr or 'Subprocess failed'}
|
||||
|
||||
return json.loads(result.stdout)
|
||||
except subprocess.TimeoutExpired:
|
||||
return {'success': False, 'error': 'Search operation timed out'}
|
||||
except json.JSONDecodeError as e:
|
||||
return {'success': False, 'error': f'Invalid JSON response: {e}'}
|
||||
except Exception as e:
|
||||
return {'success': False, 'error': str(e)}
|
||||
# Handler imports
|
||||
from aipass.memory.apps.handlers.search.query_executor import (
|
||||
execute_search as _handler_execute_search,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -160,7 +97,7 @@ def handle_command(command: str, args: List[str]) -> bool:
|
||||
console.print("[red]Error:[/red] Search query required")
|
||||
return True
|
||||
|
||||
execute_search(query, branch=branch, memory_type=memory_type, n_results=n_results)
|
||||
show_search_results(query, branch=branch, memory_type=memory_type, n_results=n_results)
|
||||
return True
|
||||
|
||||
return False
|
||||
@@ -205,17 +142,17 @@ def print_help() -> None:
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# SEARCH ORCHESTRATION
|
||||
# SEARCH RESULTS DISPLAY
|
||||
# =============================================================================
|
||||
|
||||
def execute_search(query: str, branch: str | None = None, memory_type: str | None = None, n_results: int = 5) -> bool:
|
||||
def show_search_results(
|
||||
query: str,
|
||||
branch: str | None = None,
|
||||
memory_type: str | None = None,
|
||||
n_results: int = 5
|
||||
) -> bool:
|
||||
"""
|
||||
Execute semantic search and display results
|
||||
|
||||
Workflow:
|
||||
1. Encode query to embedding vector
|
||||
2. Search ChromaDB via subprocess
|
||||
3. Format and display results with Rich
|
||||
Execute semantic search via handler and display results with Rich.
|
||||
|
||||
Args:
|
||||
query: Search query text
|
||||
@@ -234,7 +171,7 @@ def execute_search(query: str, branch: str | None = None, memory_type: str | Non
|
||||
))
|
||||
console.print()
|
||||
|
||||
# Step 1: Encode query
|
||||
# Display query info
|
||||
console.print(f"[cyan]Query:[/cyan] {query}")
|
||||
if branch:
|
||||
console.print(f"[cyan]Branch:[/cyan] {branch}")
|
||||
@@ -243,52 +180,29 @@ def execute_search(query: str, branch: str | None = None, memory_type: str | Non
|
||||
console.print()
|
||||
|
||||
console.print("[dim]Encoding query...[/dim]")
|
||||
embed_result = embedder.encode_batch([query])
|
||||
|
||||
if not embed_result['success']:
|
||||
error_msg = embed_result.get('error', 'Unknown error')
|
||||
logger.error(f"[search] Failed to encode query: {error_msg}")
|
||||
console.print(f"[red]x[/red] Failed to encode query: {error_msg}")
|
||||
return False
|
||||
|
||||
embeddings = embed_result.get('embeddings', [])
|
||||
if not embeddings:
|
||||
console.print("[red]x[/red] No embedding generated")
|
||||
return False
|
||||
|
||||
query_embedding = embeddings[0]
|
||||
# Convert numpy array to list for JSON serialization
|
||||
if hasattr(query_embedding, 'tolist'):
|
||||
query_embedding = query_embedding.tolist()
|
||||
|
||||
logger.info(f"[search] Encoded query to {len(query_embedding)}-dim vector")
|
||||
|
||||
# Step 2: Search via subprocess
|
||||
console.print("[dim]Searching collections...[/dim]")
|
||||
search_result = _search_vectors_subprocess(
|
||||
query_embedding=query_embedding,
|
||||
|
||||
# Delegate to handler
|
||||
result = _handler_execute_search(
|
||||
query=query,
|
||||
branch=branch,
|
||||
memory_type=memory_type,
|
||||
n_results=n_results
|
||||
)
|
||||
|
||||
if not search_result['success']:
|
||||
error_msg = search_result.get('error', 'Unknown error')
|
||||
logger.error(f"[search] Search failed: {error_msg}")
|
||||
console.print(f"[red]x[/red] Search failed: {error_msg}")
|
||||
if not result['success']:
|
||||
console.print(f"[red]x[/red] {result.get('error', 'Unknown error')}")
|
||||
return False
|
||||
|
||||
results = search_result.get('results', [])
|
||||
collections_searched = search_result.get('collections_searched', 0)
|
||||
total_results = search_result.get('total_results', 0)
|
||||
collections_searched = result.get('collections_searched', 0)
|
||||
total_results = result.get('total_results', 0)
|
||||
filtered_results = result.get('results', [])
|
||||
|
||||
logger.info(f"[search] Found {total_results} results across {collections_searched} collections")
|
||||
|
||||
# Step 3: Display results
|
||||
# Display summary
|
||||
console.print(f"[green]>[/green] Found {total_results} results in {collections_searched} collections")
|
||||
console.print()
|
||||
|
||||
if not results:
|
||||
if not filtered_results and total_results == 0:
|
||||
console.print("[yellow]No matching memories found[/yellow]")
|
||||
console.print()
|
||||
console.print("[dim]Try:[/dim]")
|
||||
@@ -297,27 +211,6 @@ def execute_search(query: str, branch: str | None = None, memory_type: str | Non
|
||||
console.print(" * Check if memories have been rolled over (drone @memory status)")
|
||||
return True
|
||||
|
||||
# Minimum similarity threshold - filter out irrelevant results
|
||||
MIN_SIMILARITY_THRESHOLD = 0.40 # 40% minimum relevance
|
||||
|
||||
# Filter and process results
|
||||
filtered_results = []
|
||||
for result in results[:n_results]:
|
||||
document = result.get('document', '')
|
||||
distance = result.get('distance', 0)
|
||||
|
||||
# Calculate similarity (ChromaDB L2 distance: 0=identical, ~2=very different)
|
||||
similarity = max(0, 1 - (distance / 2))
|
||||
|
||||
# Skip empty documents and low-relevance results
|
||||
if not document or not document.strip():
|
||||
continue
|
||||
if similarity < MIN_SIMILARITY_THRESHOLD:
|
||||
continue
|
||||
|
||||
result['similarity'] = similarity
|
||||
filtered_results.append(result)
|
||||
|
||||
if not filtered_results:
|
||||
console.print("[yellow]No relevant memories found[/yellow]")
|
||||
console.print()
|
||||
@@ -325,11 +218,11 @@ def execute_search(query: str, branch: str | None = None, memory_type: str | Non
|
||||
console.print("[dim]Try more specific search terms related to your AIPass work.[/dim]")
|
||||
return True
|
||||
|
||||
for i, result in enumerate(filtered_results, 1):
|
||||
collection = result.get('collection', 'unknown')
|
||||
document = result.get('document', '')
|
||||
metadata = result.get('metadata', {})
|
||||
similarity = result.get('similarity', 0)
|
||||
for i, item in enumerate(filtered_results, 1):
|
||||
collection = item.get('collection', 'unknown')
|
||||
document = item.get('document', '')
|
||||
metadata = item.get('metadata', {})
|
||||
similarity = item.get('similarity', 0)
|
||||
|
||||
# Parse collection name
|
||||
parts = collection.split('_')
|
||||
@@ -360,11 +253,25 @@ def execute_search(query: str, branch: str | None = None, memory_type: str | Non
|
||||
))
|
||||
|
||||
console.print()
|
||||
logger.info(f"[search] Displayed {len(filtered_results)} results")
|
||||
|
||||
return True
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# INTROSPECTION
|
||||
# =============================================================================
|
||||
|
||||
def print_introspection():
|
||||
"""Display module introspection info."""
|
||||
console.print()
|
||||
console.print("search Module")
|
||||
console.print("Orchestrates semantic search across memory collections via vector embeddings and ChromaDB")
|
||||
console.print()
|
||||
console.print("Connected Handlers:")
|
||||
console.print(" handlers/search/")
|
||||
console.print(" - query_executor.py (execute_search — encode query, search collections, filter results by similarity)")
|
||||
console.print()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# STANDALONE EXECUTION
|
||||
# =============================================================================
|
||||
|
||||
Reference in New Issue
Block a user