diff --git a/.agent/memory/memory_search.py b/.agent/memory/memory_search.py new file mode 100644 index 0000000..c3e9041 --- /dev/null +++ b/.agent/memory/memory_search.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +""" +Memory Search — SQLite FTS5 full-text search over .agent/memory/ files. + +Indexes all .md and .jsonl files under .agent/memory/ and provides ranked +keyword search. Falls back to grep if FTS5 is not available. + +Usage: + python3 memory_search.py Search memories by keyword + python3 memory_search.py --rebuild Force rebuild the index + python3 memory_search.py --status Show index status + +The index is stored at .agent/memory/.index/memory.db and auto-rebuilds +when any memory file changes. It is gitignored (derived data). +""" +import json +import sys +import sqlite3 +import subprocess +from pathlib import Path + +MEMORY_DIR = Path(__file__).resolve().parent +INDEX_DIR = MEMORY_DIR / ".index" +INDEX_PATH = INDEX_DIR / "memory.db" + + +def check_fts5() -> bool: + """Check if SQLite FTS5 extension is available.""" + try: + conn = sqlite3.connect(":memory:") + conn.execute("CREATE VIRTUAL TABLE _t USING fts5(c)") + conn.close() + return True + except Exception: + return False + + +def needs_rebuild() -> bool: + """Check if index is stale (any source file newer than the index).""" + if not INDEX_PATH.exists(): + return True + index_mtime = INDEX_PATH.stat().st_mtime + for f in MEMORY_DIR.rglob("*"): + if ".index" in str(f): + continue + if f.suffix in (".md", ".jsonl") and f.stat().st_mtime > index_mtime: + return True + return False + + +def _read_jsonl(path: Path) -> str: + """Read a .jsonl file and return a searchable text representation.""" + lines = [] + for raw in path.read_text(encoding="utf-8").splitlines(): + raw = raw.strip() + if not raw: + continue + try: + entry = json.loads(raw) + parts = [ + entry.get("action", ""), + entry.get("reflection", ""), + entry.get("detail", ""), + entry.get("skill", ""), + ] + lines.append(" ".join(p for p in parts if p)) + except json.JSONDecodeError: + continue + return "\n".join(lines) + + +def build_index() -> int: + """Build or rebuild the FTS5 index from all memory files.""" + INDEX_DIR.mkdir(exist_ok=True) + conn = sqlite3.connect(INDEX_PATH) + conn.execute("DROP TABLE IF EXISTS memories") + conn.execute(""" + CREATE VIRTUAL TABLE memories + USING fts5(filename, content, tokenize='porter unicode61') + """) + indexed = 0 + for f in MEMORY_DIR.rglob("*"): + if ".index" in str(f): + continue + try: + if f.suffix == ".md": + content = f.read_text(encoding="utf-8") + elif f.suffix == ".jsonl": + content = _read_jsonl(f) + else: + continue + rel_path = f.relative_to(MEMORY_DIR) + conn.execute("INSERT INTO memories VALUES (?, ?)", (str(rel_path), content)) + indexed += 1 + except Exception: + pass + conn.commit() + conn.close() + return indexed + + +def search_fts5(query: str) -> list[tuple[str, str]]: + """Search the FTS5 index. Returns (filename, snippet) pairs.""" + if needs_rebuild(): + build_index() + conn = sqlite3.connect(INDEX_PATH) + try: + rows = conn.execute( + """SELECT filename, + snippet(memories, 1, '>>>', '<<<', '...', 30) + FROM memories + WHERE memories MATCH ? + ORDER BY rank""", + (query,), + ).fetchall() + except sqlite3.OperationalError: + # Query syntax error — fall back to LIKE + rows = conn.execute( + "SELECT filename, substr(content, 1, 300) FROM memories WHERE content LIKE ?", + (f"%{query}%",), + ).fetchall() + conn.close() + return rows + + +def search_grep(query: str) -> list[tuple[str, str]]: + """Fallback search using grep when FTS5 is not available.""" + result = subprocess.run( + ["grep", "-ril", query, str(MEMORY_DIR)], + capture_output=True, + text=True, + ) + files = [ + f + for f in result.stdout.strip().split("\n") + if f and ".index" not in f + ] + return [(Path(f).relative_to(MEMORY_DIR), f"(match in {Path(f).name})") for f in files] + + +def cmd_rebuild(): + if not check_fts5(): + print("FTS5 not available — cannot build index.") + return + count = build_index() + print(f"Index rebuilt: {count} files indexed.") + + +def cmd_status(): + if not check_fts5(): + print("Mode: BASIC (grep fallback)") + print("Reason: SQLite FTS5 not available in this Python build.") + return + if not INDEX_PATH.exists(): + print("Mode: FTS5 (index not built yet — auto-builds on first search)") + return + conn = sqlite3.connect(INDEX_PATH) + count = conn.execute("SELECT COUNT(*) FROM memories").fetchone()[0] + conn.close() + size_kb = INDEX_PATH.stat().st_size // 1024 + print(f"Mode: FTS5") + print(f"Index: {count} files indexed ({size_kb} KB)") + print(f"Location: {INDEX_PATH}") + + +def main(): + args = sys.argv[1:] + + if not args or args[0] in ("-h", "--help"): + print("Usage:") + print(" memory_search.py Search memories by keyword") + print(" memory_search.py --rebuild Force rebuild index") + print(" memory_search.py --status Show index status") + sys.exit(0) + + if args[0] == "--rebuild": + cmd_rebuild() + return + + if args[0] == "--status": + cmd_status() + return + + query = " ".join(args) + use_fts5 = check_fts5() + + if use_fts5: + results = search_fts5(query) + mode = "FTS5" + else: + results = search_grep(query) + mode = "grep" + + if not results: + print(f"No results for: '{query}' [mode: {mode}]") + return + + print(f"Results for: '{query}' [mode: {mode}]\n") + for filename, snippet in results: + print(f" {filename}") + print(f" {snippet}\n") + + +if __name__ == "__main__": + main() diff --git a/.agent/skills/memory-manager/SKILL.md b/.agent/skills/memory-manager/SKILL.md index 2d09f13..e44f5a0 100644 --- a/.agent/skills/memory-manager/SKILL.md +++ b/.agent/skills/memory-manager/SKILL.md @@ -26,6 +26,19 @@ happened. Before important decisions, read the top entries from 5. Archive resolved working context to `memory/episodic/snapshots/`. 6. Commit via git so history is preserved: `git log memory/` is the agent's autobiography. +## Searching memory by keyword + +When you need to find a specific memory by topic or keyword (not just the +top-salience entries), use the FTS5 search tool: + +```bash +python3 .agent/memory/memory_search.py +``` + +This indexes all `.md` and `.jsonl` files under `.agent/memory/` and returns +ranked results with context snippets. Falls back to grep if FTS5 is not +available. The index auto-rebuilds when files change. + ## Anti-patterns - Do not auto-merge `personal/` into `semantic/` — user preferences are not general knowledge. - Do not delete entries to "clean up" memory. Archive them. diff --git a/.gitignore b/.gitignore index 089f82b..a8bf018 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,9 @@ venv/ .agent/memory/dream.log *.log +# derived search index (auto-rebuilt) +.agent/memory/.index/ + # do NOT ignore memory files — they are the point !.agent/memory/ !.agent/memory/**