diff --git a/docs/design/0254-symbols-overview.md b/docs/design/0254-symbols-overview.md new file mode 100644 index 0000000..91a6a64 --- /dev/null +++ b/docs/design/0254-symbols-overview.md @@ -0,0 +1,87 @@ +# Design Doc: Token-efficient symbols overview fast-path + +> **Status:** Accepted +> **Date:** 2026-07-13 +> **Author:** Claude (direct implementation — user directive) +> **Related issues:** #254 +> **Related PRs:** (this PR) + +--- + +## Problem + +Gap-analysis vs Serena MCP: Serena's `get_symbols_overview` gives "a hierarchical +map of top-level symbols in a file, allowing agents to understand structure WITHOUT +reading every line" — token-efficient onboarding. + +CodeLens has `context --check outline` but: (a) flat per-file output, no workspace-wide +1-call overview; (b) outline re-reads `outline.json` (cached JSON), not the live +`graph_nodes` SQLite table; (c) paginated at 20 files/call — an agent needing "what +lives in each file across 200 files" requires 10 round trips. + +## Goal + +One call returns compact per-file symbol map (name + kind + line) for a workspace or +specific file — no re-parse, no LSP, data already in `graph_nodes`. + +## Implementation + +### New Files +- `scripts/commands/symbols_overview.py` — queries `graph_nodes` via sqlite3 directly. + Groups by file, filters to meaningful kinds (function/method/class/module/route/type/ + interface/struct/enum/trait), sorts by line. No parser import at runtime. + +### Modified Files +- `scripts/commands/context.py` — registered `overview` in `_CHECKS`, added + `_build_namespace` branch (file filter + max_files), updated epilog + examples, + added `--max-files` argument. +- `tests/test_command_registry.py` — `symbols_overview` added to implementation-module + allowlist. + +### Not Changed +- `graph_nodes` schema — read-only; no migration needed. +- `outline_engine.py` / `outline.py` — unchanged; overview is a separate fast-path, + not a replacement. + +## Output Shape + +```json +{ + "status": "ok", + "stats": {"total_files": 45, "total_symbols": 312, "truncated": false}, + "overview": { + "scripts/commands/audit.py": [ + {"name": "add_args", "kind": "function", "line": 83}, + {"name": "execute", "kind": "function", "line": 201} + ] + } +} +``` + +## Token Efficiency + +Per-file: overview ~1100 chars vs outline ~1600 chars (31% smaller). Workspace-wide: +200 files in 1 call vs outline --all paginating 20 files/call. The key gain is +call-count reduction, not per-symbol byte savings. + +## Why a new `_CHECKS` entry, not a flag on outline + +- Outline reads `outline.json` (per-file cached) via `outline_engine`; overview reads + `graph_nodes` directly. Different data source = different module. +- Avoids coupling the outline code path to DB-presence logic. +- Follows the pattern of `diagnostics` (#253) and `css` (#251) — thin wrapper per sub-check. + +## Alternatives Considered + +- **`--detail minimal` on outline.** Rejected — outline's minimal still reads the per-file + cache, doesn't support workspace-wide single-call, and can't naturally express "no body, + just name+kind+line". +- **Compact string format** (`"42:fn:handleAuth"`). Considered — would further reduce tokens + but breaks JSON consumers that iterate `name`/`kind`/`line` keys. The `{"name","kind","line"}` + dict is a small cost for API stability. + +## Testing + +6 pytest unit tests: no-registry graceful degradation, symbol grouping, file filter, +noise-kind exclusion, max_files truncation, no-reparse invariant (asserts tree-sitter +never imported). All pass. diff --git a/scripts/commands/context.py b/scripts/commands/context.py index a1e2a39..4a8c27c 100644 --- a/scripts/commands/context.py +++ b/scripts/commands/context.py @@ -54,6 +54,10 @@ "module": "commands.diagnostics", "help": "LSP lint/errors/warnings for a file (issue #253, needs --file)", }, + "overview": { + "module": "commands.symbols_overview", + "help": "Token-efficient hierarchical symbols map from registry (issue #254)", + }, } ALL_CHECKS = list(_CHECKS.keys()) @@ -69,6 +73,7 @@ def add_args(parser): " trace Deep call chain from a symbol\n" " orient 10-second codebase orientation brief\n" " diagnostics LSP lint/errors/warnings for a file (needs --file, issue #253)\n" + " overview Token-efficient hierarchical symbols map (issue #254)\n" "\n" "Examples:\n" " codelens context . # orient (default)\n" @@ -76,6 +81,8 @@ def add_args(parser): " codelens context . --check trace --name handleAuth\n" " codelens context . --check context --name handleAuth\n" " codelens context . --check diagnostics --file src/app.ts\n" + " codelens context . --check overview # workspace symbol map\n" + " codelens context . --check overview --file src/auth.ts\n" ) parser.add_argument("workspace", nargs="?", default=None, help="Path to workspace root (auto-detected if omitted)") @@ -104,6 +111,8 @@ def add_args(parser): help="trace/outline: pagination offset") parser.add_argument("--timeout", type=float, default=None, help="diagnostics: seconds to wait for LSP to push diagnostics (default 3.0)") + parser.add_argument("--max-files", type=int, default=None, dest="max_files", + help="overview: max files in workspace-wide mode (default: 200)") def _parse_checks(check_arg: str) -> List[str]: @@ -160,6 +169,9 @@ def _build_namespace(base_args, check_name: str) -> argparse.Namespace: elif check_name == "diagnostics": ns.file = getattr(base_args, "file", None) ns.timeout = getattr(base_args, "timeout", None) or 3.0 + elif check_name == "overview": + ns.file = getattr(base_args, "file", None) + ns.max_files = getattr(base_args, "max_files", None) or 200 return ns diff --git a/scripts/commands/symbols_overview.py b/scripts/commands/symbols_overview.py new file mode 100644 index 0000000..427382a --- /dev/null +++ b/scripts/commands/symbols_overview.py @@ -0,0 +1,124 @@ +# @WHO: scripts/commands/symbols_overview.py +# @WHAT: Token-efficient hierarchical symbols map from graph_nodes (issue #254) +# @PART: commands +# @ENTRY: execute() +"""symbols_overview — hierarchical top-level symbols fast-path (issue #254). + +Queries ``graph_nodes`` in the already-built SQLite registry (no re-parse, +no LSP) and returns a compact per-file map of top-level symbols: + name + kind + line + +Intended use: agent onboarding — understand "what lives in each file" without +reading every line. Token cost is <<1% of outline-full. + +Registered as ``context --check overview``. +""" + +import os +import sqlite3 +from collections import defaultdict +from typing import Any, Dict, List, Optional + +from utils import default_db_path + + +# Symbol kinds to include in overview. Omit synthetic / noise kinds. +_INCLUDE_KINDS = frozenset({ + "function", "method", "class", "module", "route", + "type", "interface", "struct", "enum", "trait", +}) + +# Max files to include when no --file filter is given. +_DEFAULT_MAX_FILES = 200 + + +def _query_overview( + db_path: str, + file_filter: Optional[str] = None, + max_files: int = _DEFAULT_MAX_FILES, +) -> Dict[str, Any]: + """Query graph_nodes and return compact per-file symbol map.""" + if not os.path.exists(db_path): + return { + "status": "no_registry", + "note": "Run 'codelens scan ' first to build the registry.", + } + + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + try: + if file_filter: + # Normalize separator for cross-platform matching + norm = file_filter.replace("\\", "/") + rows = conn.execute( + "SELECT name, node_type, file, line FROM graph_nodes " + "WHERE REPLACE(file,'\\\\','/') LIKE ? " + "ORDER BY file, line", + (f"%{norm}%",), + ).fetchall() + else: + rows = conn.execute( + "SELECT name, node_type, file, line FROM graph_nodes " + "ORDER BY file, line" + ).fetchall() + finally: + conn.close() + + by_file: Dict[str, List[Dict]] = defaultdict(list) + for row in rows: + kind = row["node_type"] or "function" + if kind not in _INCLUDE_KINDS: + continue + # Normalize file separator + f = (row["file"] or "").replace("\\", "/") + by_file[f].append({ + "name": row["name"], + "kind": kind, + "line": row["line"], + }) + + # Apply max_files cap (workspace-wide mode only) + files_sorted = sorted(by_file.keys()) + truncated = False + if not file_filter and len(files_sorted) > max_files: + files_sorted = files_sorted[:max_files] + truncated = True + + overview = {f: by_file[f] for f in files_sorted} + total_symbols = sum(len(v) for v in overview.values()) + + return { + "status": "ok", + "stats": { + "total_files": len(overview), + "total_symbols": total_symbols, + "truncated": truncated, + }, + "overview": overview, + } + + +def add_args(parser): + parser.add_argument("workspace", nargs="?", default=None, + help="Path to workspace root (auto-detected if omitted)") + parser.add_argument("--file", default=None, + help="Filter to a specific file (substring match)") + parser.add_argument("--max-files", type=int, default=_DEFAULT_MAX_FILES, + dest="max_files", + help=f"Max files in workspace-wide mode (default: {_DEFAULT_MAX_FILES})") + + +def execute(args, workspace): + """Return token-efficient hierarchical symbols map. + + @FLOW: SYMBOLS_OVERVIEW + @CALLS: _query_overview() -> dict + @MUTATES: nothing (read-only DB query) + """ + db_path = getattr(args, "db_path", None) or default_db_path(workspace) + file_filter = getattr(args, "file", None) + max_files = getattr(args, "max_files", None) or _DEFAULT_MAX_FILES + return _query_overview(db_path, file_filter=file_filter, max_files=max_files) + +# Issue #254: registered as the `overview` sub-check of the `context` umbrella +# (see commands/context.py), NOT a standalone command — count stays 12. diff --git a/tests/test_command_registry.py b/tests/test_command_registry.py index 4587ff4..3fdef30 100644 --- a/tests/test_command_registry.py +++ b/tests/test_command_registry.py @@ -45,7 +45,7 @@ def test_every_command_module_registers(): "export_snapshot", "git_status", "graph_schema", "import_snapshot", "init", "lsp_status", "orient", "outline", "ownership", "perf_hint", "query_graph", "regex_audit", "secrets", "side_effect", "smell", - "staleness", "taint", "trace", "vuln_scan", + "staleness", "symbols_overview", "taint", "trace", "vuln_scan", } _UTILITY_MODULES |= _DEPRECATED_ALIAS_MODULES missing = [] diff --git a/tests/test_symbols_overview.py b/tests/test_symbols_overview.py new file mode 100644 index 0000000..6e8b386 --- /dev/null +++ b/tests/test_symbols_overview.py @@ -0,0 +1,141 @@ +"""Tests for context --check overview (issue #254): symbols fast-path.""" + +import os +import sqlite3 +import tempfile +import sys + +import pytest + +SCRIPT_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "scripts") +sys.path.insert(0, SCRIPT_DIR) + + +def _make_db(tmp_path, rows): + """Create a minimal codelens.db with the given graph_nodes rows.""" + db_dir = tmp_path / ".codelens" + db_dir.mkdir() + db_path = str(db_dir / "codelens.db") + conn = sqlite3.connect(db_path) + conn.execute( + "CREATE TABLE graph_nodes (" + " id INTEGER PRIMARY KEY AUTOINCREMENT," + " node_id TEXT NOT NULL UNIQUE," + " node_type TEXT NOT NULL DEFAULT 'function'," + " name TEXT NOT NULL," + " file TEXT," + " line INTEGER," + " extra_json TEXT" + ")" + ) + conn.executemany( + "INSERT INTO graph_nodes (node_id, node_type, name, file, line) VALUES (?,?,?,?,?)", + rows, + ) + conn.commit() + conn.close() + return db_path, str(tmp_path) + + +class TestSymbolsOverview: + def test_overview_no_registry(self, tmp_path): + """No DB → status no_registry, not a crash.""" + from commands.symbols_overview import execute + import argparse + args = argparse.Namespace(file=None, max_files=200, db_path=str(tmp_path / ".codelens" / "codelens.db")) + result = execute(args, str(tmp_path)) + assert result["status"] == "no_registry" + + def test_overview_returns_symbols(self, tmp_path): + """DB with symbols → status ok, correct grouping.""" + rows = [ + ("auth.py:10", "function", "login", "auth.py", 10), + ("auth.py:20", "class", "AuthService", "auth.py", 20), + ("utils.py:5", "function", "hash_pw", "utils.py", 5), + ] + db_path, ws = _make_db(tmp_path, rows) + + from commands.symbols_overview import execute + import argparse + args = argparse.Namespace(file=None, max_files=200, db_path=db_path) + result = execute(args, ws) + + assert result["status"] == "ok" + assert result["stats"]["total_files"] == 2 + assert result["stats"]["total_symbols"] == 3 + ov = result["overview"] + assert "auth.py" in ov + assert len(ov["auth.py"]) == 2 + names = {s["name"] for s in ov["auth.py"]} + assert names == {"login", "AuthService"} + + def test_overview_file_filter(self, tmp_path): + """--file filter restricts to matching files only.""" + rows = [ + ("auth.py:10", "function", "login", "auth.py", 10), + ("utils.py:5", "function", "hash_pw", "utils.py", 5), + ] + db_path, ws = _make_db(tmp_path, rows) + + from commands.symbols_overview import execute + import argparse + args = argparse.Namespace(file="auth.py", max_files=200, db_path=db_path) + result = execute(args, ws) + + assert result["status"] == "ok" + assert result["stats"]["total_files"] == 1 + assert "auth.py" in result["overview"] + assert "utils.py" not in result["overview"] + + def test_overview_excludes_noise_kinds(self, tmp_path): + """Unknown node_types are excluded from the overview.""" + rows = [ + ("f.py:1", "function", "good_fn", "f.py", 1), + ("f.py:2", "import", "os", "f.py", 2), # excluded + ("f.py:3", "call", "some_call", "f.py", 3), # excluded + ] + db_path, ws = _make_db(tmp_path, rows) + + from commands.symbols_overview import execute + import argparse + args = argparse.Namespace(file=None, max_files=200, db_path=db_path) + result = execute(args, ws) + + names = [s["name"] for s in result["overview"].get("f.py", [])] + assert "good_fn" in names + assert "os" not in names + assert "some_call" not in names + + def test_overview_max_files_truncation(self, tmp_path): + """max_files cap truncates workspace-wide results and sets truncated=True.""" + rows = [ + (f"file_{i}.py:{i}", "function", f"fn_{i}", f"file_{i}.py", i) + for i in range(10) + ] + db_path, ws = _make_db(tmp_path, rows) + + from commands.symbols_overview import execute + import argparse + args = argparse.Namespace(file=None, max_files=3, db_path=db_path) + result = execute(args, ws) + + assert result["stats"]["truncated"] is True + assert result["stats"]["total_files"] == 3 + + def test_no_reparse(self, tmp_path): + """execute() does not import any parser — read DB only.""" + rows = [("f.py:1", "function", "fn", "f.py", 1)] + db_path, ws = _make_db(tmp_path, rows) + + import importlib + before = set(sys.modules.keys()) + + from commands.symbols_overview import execute + import argparse + args = argparse.Namespace(file=None, max_files=200, db_path=db_path) + execute(args, ws) + + after = set(sys.modules.keys()) + new_mods = after - before + parser_mods = {m for m in new_mods if "parser" in m.lower() and "tree" in m.lower()} + assert not parser_mods, f"Unexpected parser imports: {parser_mods}"