feat(memory): FPLAN-0276 — unify memory limits into config single-source + cleanup
- memory.config.json is the single source of truth: char caps (entry_limits, global) + rollover counts (per_branch, materialized from registry); .trinity files stripped to a one-line _usage header - changed_entries gate now matches by content identity, not array position — prepending a new entry never re-flags unchanged legacy entries (old=old, new=new; no trimming required on a cap change) - rollover push command + top-level 'drone @memory push' alias; corrected config _note + --help (rollover push surfaced, labeled destructive system-wide reset) - removed 3 dead functions: seed_per_branch, its orphaned write_config, add_learning + its tests - spawn birthright/builder + LOCAL/OBSERVATIONS templates aligned to the stripped shape - 966 tests, seedgo 100% Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
6c675e9b78
commit
72659eccbd
@@ -40,6 +40,14 @@ DEFAULT_CONFIG: dict[str, Any] = {
|
||||
"consumers": ["intake/pool_processor.py", "intake/auto_process.py", "monitor/memory_watcher.py"],
|
||||
"purpose": "Vectorize files dropped in memory_pool/, archive beyond keep_recent",
|
||||
},
|
||||
"entry_limits": {
|
||||
"consumers": ["json/entry_limits.py", "modules/lint.py"],
|
||||
"purpose": "Per-entry char caps on .trinity writes (warn-first baseline)",
|
||||
},
|
||||
"plans": {
|
||||
"consumers": ["intake/plans_processor.py", "monitor/memory_watcher.py"],
|
||||
"purpose": "Vectorize closed plan .md files into ChromaDB",
|
||||
},
|
||||
"rollover": {
|
||||
"consumers": [
|
||||
"monitor/detector.py",
|
||||
@@ -47,15 +55,7 @@ DEFAULT_CONFIG: dict[str, Any] = {
|
||||
"rollover/extractor.py",
|
||||
"templates/pusher.py",
|
||||
],
|
||||
"purpose": "Line/entry thresholds that trigger .trinity rollover",
|
||||
},
|
||||
"plans": {
|
||||
"consumers": ["intake/plans_processor.py", "monitor/memory_watcher.py"],
|
||||
"purpose": "Vectorize closed plan .md files into ChromaDB",
|
||||
},
|
||||
"entry_limits": {
|
||||
"consumers": ["json/entry_limits.py", "modules/lint.py"],
|
||||
"purpose": "Per-entry char caps on .trinity writes (warn-first baseline)",
|
||||
"purpose": "Entry-count thresholds that trigger .trinity rollover",
|
||||
},
|
||||
},
|
||||
"memory_pool": {
|
||||
@@ -68,19 +68,6 @@ DEFAULT_CONFIG: dict[str, Any] = {
|
||||
"chunk_overlap": 100,
|
||||
"archive_path": "memory_pool_archive",
|
||||
},
|
||||
"rollover": {
|
||||
"defaults": {
|
||||
"max_lines": 500,
|
||||
"archive_oldest": 100,
|
||||
},
|
||||
"per_branch": {},
|
||||
},
|
||||
"plans": {
|
||||
"enabled": True,
|
||||
"path": ".backup/processed_plans",
|
||||
"collection_name": "plans",
|
||||
"supported_extensions": [".md"],
|
||||
},
|
||||
"entry_limits": {
|
||||
"enabled": True,
|
||||
"enforce": False,
|
||||
@@ -116,6 +103,27 @@ DEFAULT_CONFIG: dict[str, Any] = {
|
||||
},
|
||||
"per_branch": {},
|
||||
},
|
||||
"plans": {
|
||||
"enabled": True,
|
||||
"path": ".backup/processed_plans",
|
||||
"collection_name": "plans",
|
||||
"supported_extensions": [".md"],
|
||||
},
|
||||
"rollover": {
|
||||
"defaults": {
|
||||
"local": {
|
||||
"sessions": {"count": 20},
|
||||
"key_learnings": {"count": 25},
|
||||
"todos": {"count": 10},
|
||||
},
|
||||
"observations": {
|
||||
"observations": {"count": 25},
|
||||
},
|
||||
"_note": "DEFAULTS — edit then `drone @memory rollover push` to apply system-wide."
|
||||
" Char caps live in entry_limits.",
|
||||
},
|
||||
"per_branch": {},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -184,3 +192,69 @@ def load(self_heal: bool = True) -> dict[str, Any]:
|
||||
def section(name: str) -> dict[str, Any]:
|
||||
"""Return a single top-level section from the config, or empty dict."""
|
||||
return load().get(name, {})
|
||||
|
||||
|
||||
def _find_repo_root() -> Path:
|
||||
"""Walk up from this file to find repo root (contains AIPASS_REGISTRY.json)."""
|
||||
current = Path(__file__).resolve().parent
|
||||
for parent in [current] + list(current.parents):
|
||||
if (parent / "AIPASS_REGISTRY.json").exists():
|
||||
return parent
|
||||
return Path.cwd()
|
||||
|
||||
|
||||
def materialize_per_branch() -> dict[str, Any]:
|
||||
"""Build per_branch from AIPASS_REGISTRY.json, seeded from rollover.defaults."""
|
||||
repo_root = _find_repo_root()
|
||||
registry_path = repo_root / "AIPASS_REGISTRY.json"
|
||||
if not registry_path.exists():
|
||||
logger.warning("[config_loader] AIPASS_REGISTRY.json not found")
|
||||
return {}
|
||||
|
||||
try:
|
||||
registry = json.loads(registry_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
logger.warning(f"[config_loader] Failed to load registry: {e}")
|
||||
return {}
|
||||
|
||||
cfg = load()
|
||||
defaults = cfg.get("rollover", {}).get("defaults", {})
|
||||
limits_only = {k: v for k, v in defaults.items() if k != "_note"}
|
||||
|
||||
branches = registry.get("branches", [])
|
||||
active = [b for b in branches if b.get("status") == "active"]
|
||||
|
||||
per_branch: dict[str, Any] = {}
|
||||
for branch in active:
|
||||
name = branch.get("name", "").lower()
|
||||
if not name:
|
||||
continue
|
||||
entry = copy.deepcopy(limits_only)
|
||||
entry["_note"] = f"Limits for @{name}. Manual edits persist until next push."
|
||||
per_branch[name] = entry
|
||||
|
||||
return per_branch
|
||||
|
||||
|
||||
def push_defaults_to_per_branch() -> dict[str, Any]:
|
||||
"""Overwrite every per_branch entry with defaults (full replacement, not merge).
|
||||
|
||||
Returns:
|
||||
Dict with branch count and the new per_branch data.
|
||||
"""
|
||||
per_branch = materialize_per_branch()
|
||||
if not per_branch:
|
||||
return {"success": False, "error": "No branches found in registry"}
|
||||
|
||||
current: dict = {}
|
||||
if _CONFIG_PATH.exists():
|
||||
try:
|
||||
current = json.loads(_CONFIG_PATH.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
logger.warning("[config_loader] Malformed config on disk, starting fresh")
|
||||
|
||||
current.setdefault("rollover", {})["per_branch"] = per_branch
|
||||
_CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
_CONFIG_PATH.write_text(json.dumps(current, indent=2) + "\n", encoding="utf-8")
|
||||
|
||||
return {"success": True, "branches": len(per_branch), "per_branch": per_branch}
|
||||
|
||||
@@ -87,13 +87,15 @@ def load_entry_limits(branch: str) -> dict[str, Any]:
|
||||
Returns:
|
||||
Dict with keys: enabled, enforce, entry_types.
|
||||
"""
|
||||
branch_key = branch.lower()
|
||||
|
||||
cfg = config_loader.load()
|
||||
section = cfg.get("entry_limits")
|
||||
if not isinstance(section, dict):
|
||||
logger.warning("[entry_limits] No valid 'entry_limits' section in config, returning safe defaults")
|
||||
json_handler.log_operation(
|
||||
"load_entry_limits",
|
||||
{"branch": branch, "fallback": "missing_section"},
|
||||
{"branch": branch_key, "fallback": "missing_section"},
|
||||
module_name="entry_limits",
|
||||
)
|
||||
section = config_loader.DEFAULT_CONFIG["entry_limits"]
|
||||
@@ -103,7 +105,7 @@ def load_entry_limits(branch: str) -> dict[str, Any]:
|
||||
base_types = section.get("entry_types", {})
|
||||
|
||||
per_branch = section.get("per_branch", {})
|
||||
branch_overrides = per_branch.get(branch, {})
|
||||
branch_overrides = per_branch.get(branch_key, {})
|
||||
|
||||
if branch_overrides:
|
||||
effective_types = _deep_merge_entry_types(base_types, branch_overrides)
|
||||
@@ -118,7 +120,7 @@ def load_entry_limits(branch: str) -> dict[str, Any]:
|
||||
|
||||
json_handler.log_operation(
|
||||
"load_entry_limits",
|
||||
{"branch": branch, "types_count": len(effective_types)},
|
||||
{"branch": branch_key, "types_count": len(effective_types)},
|
||||
module_name="entry_limits",
|
||||
)
|
||||
|
||||
@@ -275,12 +277,13 @@ def _check_list_container(
|
||||
if not isinstance(after_container, list):
|
||||
return []
|
||||
before_list = before_container if isinstance(before_container, list) else []
|
||||
before_texts = {_extract_text(item, field) for item in before_list}
|
||||
hits: list[dict[str, Any]] = []
|
||||
|
||||
for idx, after_item in enumerate(after_container):
|
||||
after_text = _extract_text(after_item, field)
|
||||
if idx < len(before_list) and after_text == _extract_text(before_list[idx], field):
|
||||
continue # Unchanged — skip even if over-limit
|
||||
if after_text in before_texts:
|
||||
continue # Already on disk — not new/changed; skip even if over cap
|
||||
verdict = check_entry(type_name, after_text, limits)
|
||||
if not verdict["ok"]:
|
||||
hits.append(
|
||||
|
||||
@@ -753,85 +753,6 @@ def enforce_limit_completed(file_path: Path) -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def add_learning(file_path: Path, key: str, value: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Add or update a key_learning entry.
|
||||
|
||||
Automatically adds timestamp and enforces limit after adding.
|
||||
|
||||
Args:
|
||||
file_path: Path to .local.json file
|
||||
key: Learning key (snake_case recommended)
|
||||
value: Learning value (without timestamp - will be added)
|
||||
|
||||
Returns:
|
||||
Dict with add status
|
||||
"""
|
||||
if not file_path.exists():
|
||||
return {"success": False, "error": f"File not found: {file_path}"}
|
||||
|
||||
try:
|
||||
data = read_memory_file_data(file_path)
|
||||
if data is None:
|
||||
return {"success": False, "error": f"Failed to parse file: {file_path.name}"}
|
||||
except Exception as e:
|
||||
logger.warning(f"[learnings_manager] Failed to read file: {e}")
|
||||
return {"success": False, "error": f"Failed to read file: {e}"}
|
||||
|
||||
learnings = _get_learnings(data)
|
||||
|
||||
if isinstance(learnings, list):
|
||||
# v3 list: find existing entry by key, or insert new at front
|
||||
is_update = False
|
||||
for entry in learnings:
|
||||
if isinstance(entry, dict) and entry.get("key") == key:
|
||||
entry["value"] = value
|
||||
entry["date"] = datetime.now().strftime("%Y-%m-%d")
|
||||
is_update = True
|
||||
break
|
||||
if not is_update:
|
||||
max_num = max((e.get("number", 0) for e in learnings if isinstance(e, dict)), default=0)
|
||||
learnings.insert(
|
||||
0,
|
||||
{
|
||||
"number": max_num + 1,
|
||||
"date": datetime.now().strftime("%Y-%m-%d"),
|
||||
"key": key,
|
||||
"value": value,
|
||||
},
|
||||
)
|
||||
timestamped_value = value
|
||||
_set_learnings(data, learnings)
|
||||
else:
|
||||
if not learnings:
|
||||
learnings = {}
|
||||
timestamped_value = add_timestamp(value)
|
||||
is_update = key in learnings
|
||||
learnings[key] = timestamped_value
|
||||
_set_learnings(data, learnings)
|
||||
|
||||
try:
|
||||
write_memory_file_simple(file_path, data)
|
||||
except Exception as e:
|
||||
logger.warning(f"[learnings_manager] Failed to write file: {e}")
|
||||
return {"success": False, "error": f"Failed to write file: {e}"}
|
||||
|
||||
# Enforce limit after adding
|
||||
enforce_result = enforce_limit(file_path)
|
||||
|
||||
json_handler.log_operation(
|
||||
"add_learning", {"key": key, "action": "updated" if is_update else "added", "success": True}
|
||||
)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"action": "updated" if is_update else "added",
|
||||
"key": key,
|
||||
"value": timestamped_value,
|
||||
"limit_enforced": enforce_result.get("removed", 0) > 0,
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# STATUS COUNT UPDATES
|
||||
# =============================================================================
|
||||
|
||||
@@ -21,6 +21,7 @@ Independence:
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any
|
||||
from dataclasses import dataclass
|
||||
@@ -198,6 +199,60 @@ def _count_file_lines(file_path: Path) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
_TEMPLATES_DIR = Path(__file__).resolve().parents[3] / "templates"
|
||||
_TEMPLATE_MAP = {
|
||||
"local": _TEMPLATES_DIR / "LOCAL.template.json",
|
||||
"observations": _TEMPLATES_DIR / "OBSERVATIONS.template.json",
|
||||
}
|
||||
|
||||
|
||||
def _recreate_trinity_file(branch_path: Path, branch_name: str, memory_type: str) -> Path | None:
|
||||
"""Recreate a missing .trinity file from canonical template."""
|
||||
template_path = _TEMPLATE_MAP.get(memory_type)
|
||||
if not template_path or not template_path.exists():
|
||||
logger.warning(f"[detector] No template for {memory_type}")
|
||||
return None
|
||||
|
||||
try:
|
||||
template = json.loads(template_path.read_text(encoding="utf-8"))
|
||||
except (json.JSONDecodeError, OSError) as e:
|
||||
logger.warning(f"[detector] Failed to read template {template_path}: {e}")
|
||||
return None
|
||||
|
||||
today = datetime.now().strftime("%Y-%m-%d")
|
||||
upper_name = branch_name.upper()
|
||||
|
||||
def _walk(val):
|
||||
if isinstance(val, str):
|
||||
return val.replace("{{BRANCHNAME}}", upper_name).replace("{{DATE}}", today)
|
||||
if isinstance(val, list):
|
||||
return [_walk(item) for item in val]
|
||||
if isinstance(val, dict):
|
||||
return {k: _walk(v) for k, v in val.items()}
|
||||
return val
|
||||
|
||||
data = _walk(template)
|
||||
|
||||
trinity_dir = branch_path / ".trinity"
|
||||
trinity_dir.mkdir(parents=True, exist_ok=True)
|
||||
file_path = trinity_dir / f"{memory_type}.json"
|
||||
|
||||
try:
|
||||
file_path.write_text(
|
||||
json.dumps(data, indent=2, ensure_ascii=False) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
logger.info(f"[detector] Recreated {file_path}")
|
||||
json_handler.log_operation(
|
||||
"recreate_trinity_file",
|
||||
{"branch": branch_name, "type": memory_type, "path": str(file_path)},
|
||||
)
|
||||
return file_path
|
||||
except OSError as e:
|
||||
logger.warning(f"[detector] Failed to write {file_path}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def _get_max_lines(file_path: Path, branch_name: str | None = None) -> int:
|
||||
"""
|
||||
Get max_lines limit with priority: file metadata > branch config > default
|
||||
@@ -223,9 +278,13 @@ def _get_max_lines(file_path: Path, branch_name: str | None = None) -> int:
|
||||
|
||||
# 2. Try branch-level config (if branch_name provided or can be extracted)
|
||||
if branch_name is None:
|
||||
# Extract from filename (e.g., SEEDGO.local.json -> SEEDGO)
|
||||
parts = file_path.stem.split(".")
|
||||
branch_name = parts[0] if parts else None
|
||||
# Derive from directory: .trinity/local.json → parent of .trinity is the branch
|
||||
if file_path.parent.name == ".trinity":
|
||||
branch_name = file_path.parents[1].name.lower()
|
||||
else:
|
||||
# Legacy flat files: SEEDGO.local.json → "SEEDGO"
|
||||
parts = file_path.stem.split(".")
|
||||
branch_name = parts[0].lower() if parts else None
|
||||
|
||||
if branch_name:
|
||||
config = _load_config()
|
||||
@@ -273,24 +332,37 @@ def _should_rollover(file_path: Path) -> tuple[bool, int, int, str, str]:
|
||||
metadata = data.get("document_metadata", {})
|
||||
limits = metadata.get("limits", {})
|
||||
|
||||
# v2: entry-count based limits (checked when v2 limit keys are present, regardless of schema_version)
|
||||
v2_limit_keys = {"max_sessions", "max_key_learnings", "max_observations"}
|
||||
if v2_limit_keys & set(limits.keys()):
|
||||
# v2: entry-count based limits — read from config per_branch, not file metadata
|
||||
# Derive branch name from file path: .trinity/local.json → parent of .trinity
|
||||
if file_path.parent.name == ".trinity":
|
||||
branch_name = file_path.parents[1].name.lower()
|
||||
else:
|
||||
branch_name = file_path.stem.split(".")[0].lower()
|
||||
|
||||
# Determine file type from filename
|
||||
file_type = file_path.stem.split(".")[0] if file_path.parent.name == ".trinity" else file_path.stem.split(".")[-1]
|
||||
# .trinity/local.json → "local"; .trinity/observations.json → "observations"
|
||||
|
||||
cfg = config_loader.section("rollover")
|
||||
branch_limits = cfg.get("per_branch", {}).get(branch_name, {})
|
||||
file_limits = branch_limits.get(file_type, {})
|
||||
|
||||
if file_limits:
|
||||
reasons = []
|
||||
|
||||
max_sessions = limits.get("max_sessions")
|
||||
max_sessions = file_limits.get("sessions", {}).get("count")
|
||||
if max_sessions is not None:
|
||||
sessions = data.get("sessions", [])
|
||||
if isinstance(sessions, list) and len(sessions) >= max_sessions:
|
||||
reasons.append(f"{len(sessions)}/{max_sessions} sessions")
|
||||
|
||||
max_key_learnings = limits.get("max_key_learnings")
|
||||
max_key_learnings = file_limits.get("key_learnings", {}).get("count")
|
||||
if max_key_learnings is not None:
|
||||
key_learnings = data.get("key_learnings", [])
|
||||
if isinstance(key_learnings, (list, dict)) and len(key_learnings) >= max_key_learnings:
|
||||
reasons.append(f"{len(key_learnings)}/{max_key_learnings} key_learnings")
|
||||
|
||||
max_observations = limits.get("max_observations")
|
||||
max_observations = file_limits.get("observations", {}).get("count")
|
||||
if max_observations is not None:
|
||||
observations = data.get("observations", [])
|
||||
if isinstance(observations, list) and len(observations) >= max_observations:
|
||||
@@ -333,7 +405,11 @@ def check_all_branches() -> Dict[str, Any]:
|
||||
file_path = _get_memory_file_path(branch, memory_type)
|
||||
|
||||
if file_path is None:
|
||||
continue # File doesn't exist, skip
|
||||
branch_path = Path(branch.get("path", ""))
|
||||
if branch_path.exists():
|
||||
file_path = _recreate_trinity_file(branch_path, branch_name, memory_type)
|
||||
if file_path is None:
|
||||
continue
|
||||
|
||||
should_trigger, current_lines, max_lines, schema_ver, v2_reason = _should_rollover(file_path)
|
||||
|
||||
@@ -377,10 +453,15 @@ def check_single_file(file_path: Path) -> Dict[str, Any]:
|
||||
should_trigger, current_lines, max_lines, schema_ver, v2_reason = _should_rollover(file_path)
|
||||
|
||||
if should_trigger:
|
||||
# Extract branch and type from filename (e.g., SEEDGO.observations.json)
|
||||
parts = file_path.stem.split(".")
|
||||
branch_name = parts[0] if len(parts) > 0 else "UNKNOWN"
|
||||
memory_type = parts[1] if len(parts) > 1 else "unknown"
|
||||
# Extract branch and type from file path
|
||||
if file_path.parent.name == ".trinity":
|
||||
branch_name = file_path.parents[1].name
|
||||
memory_type = file_path.stem # "local" or "observations"
|
||||
else:
|
||||
# Legacy flat files: SEEDGO.observations.json
|
||||
parts = file_path.stem.split(".")
|
||||
branch_name = parts[0] if len(parts) > 0 else "UNKNOWN"
|
||||
memory_type = parts[1] if len(parts) > 1 else "unknown"
|
||||
|
||||
trigger = RolloverTrigger(
|
||||
branch=branch_name,
|
||||
|
||||
@@ -66,54 +66,6 @@ _observer: Any = None
|
||||
_startup_check_done = False
|
||||
|
||||
|
||||
def _get_rollover_threshold(branch_name: str, file_path: Path | None = None) -> int:
|
||||
"""
|
||||
Get rollover threshold for a memory file (line-based, v1 only).
|
||||
|
||||
For v2 files (schema_version >= 2.0.0), returns a very large number so
|
||||
line-based checks never trigger. v2 rollover is handled by the detector
|
||||
using entry-count limits.
|
||||
|
||||
Priority: file metadata > per_branch config > defaults > hardcoded 600
|
||||
|
||||
Args:
|
||||
branch_name: Branch name (uppercase, e.g., 'DEVPULSE')
|
||||
file_path: Optional path to memory file (checks file-level limits first)
|
||||
|
||||
Returns:
|
||||
Max lines threshold for rollover
|
||||
"""
|
||||
import json
|
||||
|
||||
# 1. Check file-level metadata first (highest priority)
|
||||
if file_path is not None:
|
||||
try:
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
metadata = data.get("document_metadata", {})
|
||||
|
||||
# v2 files use entry-count limits — return -1 so caller uses detector
|
||||
schema_version = metadata.get("schema_version", "1.0.0")
|
||||
if schema_version.startswith("2"):
|
||||
return -1
|
||||
|
||||
file_limit = metadata.get("limits", {}).get("max_lines")
|
||||
if file_limit is not None:
|
||||
return file_limit
|
||||
except Exception as e:
|
||||
logger.warning(f"[memory_watcher] Failed to read file-level threshold from {file_path}: {e}")
|
||||
|
||||
# 2. Check per-branch config override
|
||||
cfg = config_loader.load()
|
||||
branch_limits = cfg.get("rollover", {}).get("per_branch", {}).get(branch_name, {})
|
||||
if "max_lines" in branch_limits:
|
||||
return branch_limits["max_lines"]
|
||||
default_limit = cfg.get("rollover", {}).get("defaults", {}).get("max_lines")
|
||||
if default_limit is not None:
|
||||
return default_limit
|
||||
return 500
|
||||
|
||||
|
||||
def _check_vector_deps() -> bool:
|
||||
"""
|
||||
Check whether the memory venv has chromadb and numpy available.
|
||||
|
||||
@@ -271,13 +271,23 @@ def _extract_items_v2(file_path: Path, data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
Returns:
|
||||
Dict with extracted items and metadata
|
||||
"""
|
||||
limits = data.get("document_metadata", {}).get("limits", {})
|
||||
old_lines = _count_file_lines(file_path)
|
||||
|
||||
# Read limits from config per_branch instead of file metadata
|
||||
if file_path.parent.name == ".trinity":
|
||||
branch_key = file_path.parents[1].name.lower()
|
||||
file_type = file_path.stem # "local" or "observations"
|
||||
else:
|
||||
branch_key = file_path.parent.name.lower()
|
||||
file_type = file_path.stem.split(".")[-1]
|
||||
|
||||
cfg = config_loader.section("rollover")
|
||||
file_limits = cfg.get("per_branch", {}).get(branch_key, {}).get(file_type, {})
|
||||
|
||||
all_extracted = []
|
||||
|
||||
# Extract from sessions array (newest first, oldest at end)
|
||||
max_sessions = limits.get("max_sessions")
|
||||
max_sessions = file_limits.get("sessions", {}).get("count")
|
||||
if max_sessions is not None:
|
||||
sessions = data.get("sessions", [])
|
||||
if isinstance(sessions, list) and len(sessions) >= max_sessions:
|
||||
@@ -287,7 +297,7 @@ def _extract_items_v2(file_path: Path, data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
all_extracted.extend(extracted_sessions)
|
||||
|
||||
# Extract from key_learnings list (sorted newest-first; oldest at end)
|
||||
max_key_learnings = limits.get("max_key_learnings")
|
||||
max_key_learnings = file_limits.get("key_learnings", {}).get("count")
|
||||
if max_key_learnings is not None:
|
||||
key_learnings = data.get("key_learnings", [])
|
||||
if isinstance(key_learnings, list) and len(key_learnings) >= max_key_learnings:
|
||||
@@ -297,7 +307,7 @@ def _extract_items_v2(file_path: Path, data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
all_extracted.extend(extracted_kl)
|
||||
|
||||
# Extract from observations array (if v2 observations file)
|
||||
max_observations = limits.get("max_observations")
|
||||
max_observations = file_limits.get("observations", {}).get("count")
|
||||
if max_observations is not None:
|
||||
observations = data.get("observations", [])
|
||||
if isinstance(observations, list) and len(observations) >= max_observations:
|
||||
@@ -370,10 +380,16 @@ def extract_items(file_path: Path, percentage: int | None = None) -> Dict[str, A
|
||||
logger.warning(f"[extractor] Failed to read file {file_path}: {e}")
|
||||
return {"success": False, "error": f"Failed to read file: {e}"}
|
||||
|
||||
# v2: entry-count based extraction (when v2 limit keys are present, regardless of schema_version)
|
||||
ext_limits = data.get("document_metadata", {}).get("limits", {})
|
||||
v2_limit_keys = {"max_sessions", "max_key_learnings", "max_observations"}
|
||||
if v2_limit_keys & set(ext_limits.keys()):
|
||||
# v2: entry-count based extraction — triggered when config has per_branch counts
|
||||
if file_path.parent.name == ".trinity":
|
||||
_ext_branch = file_path.parents[1].name.lower()
|
||||
_ext_ftype = file_path.stem
|
||||
else:
|
||||
_ext_branch = file_path.parent.name.lower()
|
||||
_ext_ftype = file_path.stem.split(".")[-1]
|
||||
_ext_cfg = config_loader.section("rollover")
|
||||
_ext_file_limits = _ext_cfg.get("per_branch", {}).get(_ext_branch, {}).get(_ext_ftype, {})
|
||||
if _ext_file_limits:
|
||||
return _extract_items_v2(file_path, data)
|
||||
|
||||
# v1: line-count based extraction
|
||||
@@ -383,7 +399,7 @@ def extract_items(file_path: Path, percentage: int | None = None) -> Dict[str, A
|
||||
return {"success": False, "error": f"No growing array found in {file_path.name}"}
|
||||
|
||||
# Get metadata
|
||||
_cfg_max = config_loader.section("rollover").get("defaults", {}).get("max_lines", 500)
|
||||
_cfg_max = config_loader.section("rollover").get("defaults", {}).get("max_lines", 600)
|
||||
max_lines = data.get("document_metadata", {}).get("limits", {}).get("max_lines", _cfg_max)
|
||||
|
||||
# Check if under limit
|
||||
|
||||
@@ -30,7 +30,7 @@ from typing import Dict, Any, List, Tuple, Optional
|
||||
from datetime import datetime
|
||||
|
||||
from aipass.prax import logger
|
||||
from aipass.memory.apps.handlers.json import json_handler, config_loader
|
||||
from aipass.memory.apps.handlers.json import json_handler
|
||||
|
||||
# Handler imports (same-branch allowed per handler boundaries)
|
||||
from aipass.memory.apps.handlers.json.memory_files import read_memory_file_data, write_memory_file_simple
|
||||
@@ -67,7 +67,7 @@ VERSION_FILE_PATH = TEMPLATES_DIR / ".template_version.json"
|
||||
|
||||
# Deprecated sections to REMOVE during push
|
||||
DEPRECATED_METADATA_KEYS = ["allowed_emojis"]
|
||||
DEPRECATED_LIMIT_KEYS = ["max_word_count", "max_token_count"]
|
||||
DEPRECATED_LIMIT_KEYS = ["max_word_count", "max_token_count", "max_lines", "archive_oldest"]
|
||||
DEPRECATED_STATUS_KEYS = ["auto_compress_at"]
|
||||
DEPRECATED_NOTES_KEYS = ["formatting_reference", "slash_command_tracking"]
|
||||
DEPRECATED_GUIDELINES_KEYS = ["emoji_usage", "high_value_patterns", "low_value_patterns"]
|
||||
@@ -166,18 +166,10 @@ def _merge_metadata(curr_meta: dict, tmpl_meta: dict) -> List[str]:
|
||||
curr_meta["tags"] = tmpl_tags
|
||||
changes.append("document_metadata.tags: updated to template tags")
|
||||
|
||||
# Limits (preserve per-branch max_lines override)
|
||||
tmpl_limits = tmpl_meta.get("limits", {})
|
||||
curr_limits = curr_meta.get("limits", {})
|
||||
branch_max_lines = curr_limits.get("max_lines")
|
||||
_cfg_max = config_loader.section("rollover").get("defaults", {}).get("max_lines", 500)
|
||||
tmpl_max_lines = tmpl_limits.get("max_lines", _cfg_max)
|
||||
new_limits = copy.deepcopy(tmpl_limits)
|
||||
if branch_max_lines is not None and branch_max_lines != tmpl_max_lines:
|
||||
new_limits["max_lines"] = branch_max_lines
|
||||
if curr_limits != new_limits:
|
||||
curr_meta["limits"] = new_limits
|
||||
changes.append("document_metadata.limits: updated from template")
|
||||
# Limits live in memory.config.json now — strip from files if still present
|
||||
if "limits" in curr_meta:
|
||||
del curr_meta["limits"]
|
||||
changes.append("document_metadata.limits: removed (lives in memory.config.json)")
|
||||
|
||||
# Status (add missing fields, preserve current values)
|
||||
curr_status = curr_meta.setdefault("status", {})
|
||||
|
||||
@@ -115,6 +115,7 @@ def print_help():
|
||||
table.add_row("rollover status", "Show rollover statistics")
|
||||
table.add_row("rollover check", "Dry run — check what needs rollover")
|
||||
table.add_row("rollover sync-lines", "Update line count metadata")
|
||||
table.add_row("rollover push", "⚠ Reset ALL per_branch limits to defaults (system-wide)")
|
||||
table.add_row("search <query>", "Semantic search across all branch memories")
|
||||
table.add_row("symbolic <subcommand>", "Symbolic/fragmented memory extraction and search")
|
||||
table.add_row("templates <subcommand>", "Living template push, diff, and status")
|
||||
@@ -168,7 +169,7 @@ def print_help():
|
||||
console.print()
|
||||
|
||||
console.print(
|
||||
"Commands: search, rollover [run|status|check|sync-lines], lint,"
|
||||
"Commands: search, rollover [run|status|check|sync-lines|push], lint,"
|
||||
" pool [process|status], symbolic, templates, verify, watch"
|
||||
)
|
||||
console.print()
|
||||
@@ -234,6 +235,9 @@ def route_command(command: str, args: List[str], modules: List[Any]) -> bool:
|
||||
start_watch()
|
||||
return True
|
||||
|
||||
if command == "push":
|
||||
return route_command("rollover", ["push"], modules)
|
||||
|
||||
for module in modules:
|
||||
try:
|
||||
if module.handle_command(command, args):
|
||||
|
||||
@@ -52,6 +52,7 @@ _SUBCOMMANDS = {
|
||||
"status": "Show rollover statistics for all branches",
|
||||
"check": "Check which files need rollover (dry run)",
|
||||
"sync-lines": "Update line count metadata for all branches",
|
||||
"push": "Overwrite all per_branch limits to defaults (system-wide reset)",
|
||||
}
|
||||
|
||||
|
||||
@@ -112,6 +113,10 @@ def handle_command(command: str, args: List[str]) -> bool:
|
||||
sync_line_counts()
|
||||
return True
|
||||
|
||||
if sub == "push":
|
||||
push_defaults()
|
||||
return True
|
||||
|
||||
# Unknown subcommand
|
||||
error(
|
||||
f"Unknown subcommand: '{sub}'",
|
||||
@@ -158,6 +163,7 @@ def print_help() -> None:
|
||||
console.print(" [cyan]status[/cyan] Show rollover statistics for all branches")
|
||||
console.print(" [cyan]check[/cyan] Check which files need rollover (dry run)")
|
||||
console.print(" [cyan]sync-lines[/cyan] Update line count metadata for all branches")
|
||||
console.print(" [cyan]push[/cyan] ⚠ Reset ALL per_branch limits to defaults (system-wide)")
|
||||
console.print(" [cyan]help[/cyan] Show this help message")
|
||||
console.print()
|
||||
console.print("[bold]LIMITS:[/bold]")
|
||||
@@ -213,7 +219,8 @@ def run_rollover() -> bool:
|
||||
local_status = "> local" if item.get("local_stored") else "x local"
|
||||
console.print(
|
||||
f" [green]>[/green] Rolled over {item['memories_count']} items -> {item['global_collection']} "
|
||||
f"({item['old_lines']} -> {item['new_lines']} lines, global: {item['global_total']} vectors, {local_status})"
|
||||
f"({item['old_lines']} -> {item['new_lines']} lines, "
|
||||
f"global: {item['global_total']} vectors, {local_status})"
|
||||
)
|
||||
|
||||
# Report results
|
||||
@@ -322,6 +329,34 @@ def sync_line_counts() -> None:
|
||||
console.print()
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# PUSH DEFAULTS
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def push_defaults() -> None:
|
||||
"""Overwrite every per_branch entry in memory.config.json with defaults."""
|
||||
from ..handlers.json import config_loader
|
||||
|
||||
console.print()
|
||||
console.print(Panel.fit("[bold cyan]Memory - Push Defaults[/bold cyan]", border_style="cyan", box=box.ROUNDED))
|
||||
console.print()
|
||||
|
||||
console.print("[cyan]Overwriting all per_branch limits with defaults...[/cyan]")
|
||||
console.print()
|
||||
|
||||
result = config_loader.push_defaults_to_per_branch()
|
||||
|
||||
if not result.get("success"):
|
||||
error(result.get("error", "Unknown error"))
|
||||
return
|
||||
|
||||
count = result.get("branches", 0)
|
||||
console.print(f"[green]>[/green] Pushed defaults to {count} branches")
|
||||
console.print()
|
||||
json_handler.log_operation("push_defaults", {"branches": count})
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# STATUS & CHECKING
|
||||
# =============================================================================
|
||||
|
||||
Reference in New Issue
Block a user