From 42520c70bbb3a5770b7f7bbff9414849700505d9 Mon Sep 17 00:00:00 2001 From: EMRG Evolution Date: Thu, 13 Aug 2026 16:51:41 +0800 Subject: [PATCH] emrg: global cross-project session index (rant 2026-08-13T16:42:22) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add ~/.emrg/sessions_index.json mapping session_id → session directory so any session can locate and read another project's conversation. Minimal index (only the id→path map; everything else read on demand from meta.json / history.jsonl / memory/MEMORY.md). - emrg/sessions_index.py: upsert/remove/rebuild with atomic write (tmp + os.replace), corrupt-file tolerance, never raises; startup scan backfills sessions from registered projects + unregistered ones under ~/.emrg (prunes install/.git/node_modules/etc. to keep the walk fast). - Session._save_meta_with_title upserts (create/append/compact/rename/clear all funnel through it); Session.delete removes. Idempotent (skips rewrite when path unchanged). - daemon _build_system_prompt injects config_dir; serve() rebuilds the index at startup; system.j2 documents the discovery flow. - conftest redirects the index to a per-test tmp file (hermeticity). - +20 tests (779→799). --- Agent.md | 2 +- emrg/server/daemon.py | 38 +++++++ emrg/server/prompts/system.j2 | 15 +++ emrg/session.py | 8 ++ emrg/sessions_index.py | 181 +++++++++++++++++++++++++++++++++ tests/conftest.py | 18 ++++ tests/test_sessions_index.py | 182 ++++++++++++++++++++++++++++++++++ 7 files changed, 443 insertions(+), 1 deletion(-) create mode 100644 emrg/sessions_index.py create mode 100644 tests/test_sessions_index.py diff --git a/Agent.md b/Agent.md index 3b7dc64..82d4e6e 100644 --- a/Agent.md +++ b/Agent.md @@ -118,7 +118,7 @@ Community needs voiced in HN agent-UI discussions map directly to EMRG's design: pkill -f "emrg.server"; rm -f ~/.emrg/emrgd.port; python -m emrg ``` -Python: `uv run pytest tests/ -v` (779) — import check: `uv run python -c "from emrg.client.app import run_client"` +Python: `uv run pytest tests/ -v` (799) — import check: `uv run python -c "from emrg.client.app import run_client"` GUI: `cd emrg/gui && npm test` (236: 44 daemon_client + 19 conn-manager + 22 app-commands + 114 renderer smoke + 15 i18n + 7 integration + 3 commands + 5 build-config + 7 gui-state) — syntax: `node --check main.js preload.js daemon_client.js renderer/js/*.js` CI: `uv run pytest` (ubuntu + **windows-2025 matrix** — Windows pytest 回归在 PR CI 即失败，v0.2.29 教训 #725) + GUI tests + **actionlint workflow lint** (`rhysd/actionlint@v1.7.12` gate, #444 — workflow 解析错误在 PR CI 即失败，如 `if:` secrets 上下文) Re-trigger: `scripts/re-trigger-ci.sh [branch]` (workflow_dispatch, #527 — 替代空 commit 重触发：Actions outage 会整段丢弃 push 事件，dispatch 走 API 路径不受影响) diff --git a/emrg/server/daemon.py b/emrg/server/daemon.py index deb1836..efb3092 100644 --- a/emrg/server/daemon.py +++ b/emrg/server/daemon.py @@ -290,6 +290,13 @@ async def serve(self) -> None: self._scheduler = TaskScheduler(self.identity) self._scheduler.load_and_start() + # Global cross-project session index (rant 2026-08-13T16:42:22): + # backfill the index from every on-disk session (registered projects + + # unregistered ones under ~/.emrg) so sessions created before this + # feature are discoverable by other projects. Best-effort — never + # crashes startup. + self._rebuild_sessions_index() + # Background deterministic skill update check (rant 2026-08-08T10:14:29): # runs at startup + every 24h — refreshes managed skills to their # latest GitHub releases. Never installs a CLI silently, never touches @@ -361,6 +368,34 @@ def _assert_port_file(self, port: int) -> None: mode=0o600, ) + def _rebuild_sessions_index(self) -> None: + """Backfill the global cross-project session index at startup. + + Rant 2026-08-13T16:42:22: sessions created before this feature (or in + projects not registered in projects.yml) would otherwise be invisible + to other projects. Best-effort — failures are logged at debug level + and never crash the daemon. + """ + from emrg.sessions_index import rebuild_sessions_index + + try: + project_paths: list[str] = [] + if self._projects_log.exists(): + try: + data = yaml.safe_load(self._projects_log.read_text(encoding="utf-8")) + if isinstance(data, list): + project_paths = [ + e.get("path", "") + for e in data + if isinstance(e, dict) and e.get("path") + ] + except (yaml.YAMLError, OSError): + pass + count = rebuild_sessions_index(config_dir(), project_paths) + logger.info("sessions index rebuilt: %d sessions indexed", count) + except Exception: + logger.debug("sessions index rebuild failed", exc_info=True) + async def _skills_ttl_loop(self) -> None: """Background deterministic skill update check (startup + every 24h). @@ -1042,6 +1077,9 @@ def _build_system_prompt(self, session: Session | None = None) -> str: ctx["current_time"] = datetime.now().astimezone().isoformat(timespec="seconds") ctx["os_name"] = platform.system() ctx["platform_detail"] = platform.platform() + # Global config dir (~/.emrg) — injected so system.j2 can reference the + # cross-project sessions index and other global data files by path. + ctx["config_dir"] = str(config_dir()) # ── Working Directory ── if session: diff --git a/emrg/server/prompts/system.j2 b/emrg/server/prompts/system.j2 index 083d076..8b56aa4 100644 --- a/emrg/server/prompts/system.j2 +++ b/emrg/server/prompts/system.j2 @@ -83,6 +83,21 @@ Each line is a JSON record with `type`, `role`, `content`, `timestamp` fields. Message records: `type=message`, tool calls: `type=tool_call`/`tool_result`, compacted summaries: `type=summary`. {% endif %} +{% if config_dir %} +## Cross-Session Discovery (read other projects' sessions) + +A global index of all sessions across all projects lives at `{{ config_dir }}/sessions_index.json`. +It maps `session_id` → absolute session directory path (one JSON object). + +To learn what another project's session has been discussing: +1. read the index file to find the session's directory path +2. read `/meta.json` for basics (title, message_count, updated_at) +3. read `/history.jsonl` (or history_YYMMDD.jsonl) for the actual conversation +4. read `/memory/MEMORY.md` for that session's memory summary + +Use this whenever the host asks you to read or understand another session's (or another project's) conversation. +{% endif %} + ## Memory Management After each response, briefly consider whether anything from this exchange should be remembered. If so, create or update a memory file in the appropriate memory directory. diff --git a/emrg/session.py b/emrg/session.py index b32019d..b55f07d 100644 --- a/emrg/session.py +++ b/emrg/session.py @@ -22,6 +22,7 @@ from pathlib import Path from emrg.memory import SessionMemoryStore +from emrg.sessions_index import remove_session_index, upsert_session_index logger = logging.getLogger(__name__) @@ -490,6 +491,11 @@ def _save_meta_with_title(self, title: str | None) -> None: except (json.JSONDecodeError, OSError): pass self._meta_path.write_text(json.dumps(meta, indent=2, ensure_ascii=False), encoding="utf-8") + # Global cross-project index (rant 2026-08-13T16:42:22): record this + # session so other projects can locate it. Idempotent (no-op when the + # path is unchanged) and never raises — a failed index write must not + # break session creation or message persistence. + upsert_session_index(self.session_id, self._dir) # ── Clear ────────────────────────────────────────────────── @@ -526,6 +532,8 @@ def delete(session_id: str, cwd: Path) -> bool: if session_dir.exists(): shutil.rmtree(session_dir) logger.info("session deleted: %s", session_id) + # Global index (rant 2026-08-13T16:42:22): drop the deleted session. + remove_session_index(session_id) return True logger.warning("session not found for deletion: %s", session_id) return False diff --git a/emrg/sessions_index.py b/emrg/sessions_index.py new file mode 100644 index 0000000..0ff5b36 --- /dev/null +++ b/emrg/sessions_index.py @@ -0,0 +1,181 @@ +"""Global cross-project session index (rant 2026-08-13T16:42:22). + +Maintains a single JSON map ``session_id -> absolute session directory`` at +``~/.emrg/sessions_index.json`` so that any session (or the agent in any +session) can locate and read another project's conversation records. + +Design (host-finalized, minimal index): +- The index stores ONLY the session_id → directory mapping. Everything else + (title, message_count, updated_at, history, memory) is read on demand from + the target session's meta.json / history.jsonl / memory/MEMORY.md. +- Write hooks live in ``Session._save_meta_with_title`` (create/append/compact/ + rename/clear all funnel through it) and ``Session.delete``. +- A startup scan in the daemon (``rebuild_sessions_index``) backfills sessions + that predate this feature, including unregistered projects under ~/.emrg. +""" + +from __future__ import annotations + +import json +import logging +import os +import tempfile +from pathlib import Path + +from emrg.config import config_dir + +logger = logging.getLogger(__name__) + +_INDEX_FILENAME = "sessions_index.json" + +# Subtrees that never contain session dirs but are large/irrelevant — pruning +# them keeps the recursive ~/.emrg scan fast (a full Python dist under +# install/, git history, node_modules, etc. would dominate the walk). +_PRUNE_DIRS = { + "install", "updates", "logs", ".git", "node_modules", ".venv", + "__pycache__", "dist", "build", ".cache", "Cache", +} + + +def sessions_index_path() -> Path: + """Return the global index file path (~/.emrg/sessions_index.json).""" + return config_dir() / _INDEX_FILENAME + + +def _load(index_path: Path) -> dict[str, str]: + """Read the index; corrupt/missing file yields an empty dict (never raises).""" + if not index_path.exists(): + return {} + try: + data = json.loads(index_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + logger.warning("corrupt sessions index %s — resetting", index_path) + return {} + if not isinstance(data, dict): + return {} + return {str(k): str(v) for k, v in data.items()} + + +def _write(data: dict[str, str], index_path: Path) -> None: + """Atomically write the index (tmp file + os.replace); never raises.""" + index_path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_path = tempfile.mkstemp( + dir=str(index_path.parent), prefix=".sessions_index_", suffix=".tmp" + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False, indent=2) + os.replace(tmp_path, index_path) + except OSError: + logger.warning("failed to write sessions index %s", index_path, exc_info=True) + try: + os.unlink(tmp_path) + except OSError: + pass + + +def _upsert(session_id: str, session_dir: str, index_path: Path) -> None: + """Idempotently set index[session_id] = session_dir (skip if unchanged).""" + data = _load(index_path) + if data.get(session_id) == session_dir: + return # already correct — avoid a redundant rewrite on every meta save + data[session_id] = session_dir + _write(data, index_path) + + +def upsert_session_index(session_id: str, session_dir: Path) -> None: + """Record a session in the global index (write hook for Session meta saves).""" + _upsert(str(session_id), str(session_dir), sessions_index_path()) + + +def _remove(session_id: str, index_path: Path) -> None: + data = _load(index_path) + if session_id in data: + del data[session_id] + _write(data, index_path) + + +def remove_session_index(session_id: str) -> None: + """Remove a session from the global index (delete hook for Session.delete).""" + _remove(str(session_id), sessions_index_path()) + + +def _read_meta_session_id(meta_path: Path) -> str | None: + """Return the session_id from a meta.json, or None if missing/corrupt.""" + try: + meta = json.loads(meta_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError): + return None + sid = meta.get("session_id") + return str(sid) if sid else None + + +def _iter_project_sessions(project_path: Path): + """Yield (session_id, session_dir) from /.emrg/sessions/*/meta.json.""" + sessions_dir = project_path / ".emrg" / "sessions" + if not sessions_dir.is_dir(): + return + for entry in sorted(sessions_dir.iterdir()): + if not entry.is_dir(): + continue + meta_path = entry / "meta.json" + if not meta_path.exists(): + continue + sid = _read_meta_session_id(meta_path) + if sid: + yield sid, str(entry) + + +def _iter_nested_sessions(root: Path): + """Yield (session_id, session_dir) for every /.emrg/sessions under root. + + Recursively walks ``root`` (pruning heavy subtrees) so unregistered + projects under ~/.emrg (e.g. ~/.emrg itself, ~/.emrg/source) are covered, + not just paths listed in projects.yml. + """ + for dirpath, dirnames, _ in os.walk(root, followlinks=False): + dirnames[:] = [d for d in dirnames if d not in _PRUNE_DIRS] + if os.path.basename(dirpath) == "sessions" and os.path.basename( + os.path.dirname(dirpath) + ) == ".emrg": + sessions_dir = Path(dirpath) + for entry in sorted(sessions_dir.iterdir()): + if not entry.is_dir(): + continue + meta_path = entry / "meta.json" + if not meta_path.exists(): + continue + sid = _read_meta_session_id(meta_path) + if sid: + yield sid, str(entry) + + +def rebuild_sessions_index( + config_root: Path, project_paths: list[str] | None = None +) -> int: + """Backfill the index from on-disk sessions (daemon startup scan). + + Scans ``config_root`` recursively (covers ~/.emrg and anything nested under + it) plus each explicit project path (covers projects outside ~/.emrg), and + upserts every discovered session into ``config_root/sessions_index.json``. + Returns the number of distinct sessions indexed. + """ + index_path = config_root / _INDEX_FILENAME + data = _load(index_path) + + found: dict[str, str] = {} + for sid, sdir in _iter_nested_sessions(config_root): + found[sid] = sdir + for p in project_paths or []: + if not p: + continue + try: + for sid, sdir in _iter_project_sessions(Path(p)): + found[sid] = sdir + except OSError: + continue + + for sid, sdir in found.items(): + data[sid] = sdir + _write(data, index_path) + return len(data) diff --git a/tests/conftest.py b/tests/conftest.py index b59820f..b2b742a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -51,3 +51,21 @@ def guarded(data, path, **kwargs): monkeypatch.setattr(sched_mod, "atomic_write_yaml", guarded) monkeypatch.setattr(daemon_mod, "atomic_write_yaml", guarded) + + +@pytest.fixture(autouse=True) +def _redirect_sessions_index(monkeypatch, tmp_path): + """Redirect the global session index to a per-test tmp file. + + Rant 2026-08-13T16:42:22 added a global ~/.emrg/sessions_index.json that + Session._save_meta_with_title / Session.delete write to on every session + create/append/delete. Without redirection, the whole session test suite + would pollute the host's real index with pytest temp paths (same class as + the projects.yml leak guarded above). + """ + import emrg.sessions_index as sidx + + monkeypatch.setattr( + sidx, "sessions_index_path", + lambda: tmp_path / "sessions_index.json", + ) diff --git a/tests/test_sessions_index.py b/tests/test_sessions_index.py new file mode 100644 index 0000000..865e5bf --- /dev/null +++ b/tests/test_sessions_index.py @@ -0,0 +1,182 @@ +"""Tests for the global cross-project session index (rant 2026-08-13T16:42:22).""" + +from __future__ import annotations + +import json +from pathlib import Path + +from emrg.session import Session +from emrg.sessions_index import ( + _load, + _write, + rebuild_sessions_index, + remove_session_index, + sessions_index_path, + upsert_session_index, +) + + +def _index_path(tmp_path: Path) -> Path: + """The redirected index path used by the autouse conftest fixture.""" + return tmp_path / "sessions_index.json" + + +class TestUpsertRemove: + def test_upsert_writes_index(self, tmp_path): + upsert_session_index("s_abc", tmp_path / "projA" / ".emrg" / "sessions" / "s_abc") + data = _load(_index_path(tmp_path)) + assert data["s_abc"] == str(tmp_path / "projA" / ".emrg" / "sessions" / "s_abc") + + def test_upsert_multiple_sessions(self, tmp_path): + upsert_session_index("s_1", tmp_path / "a" / "s_1") + upsert_session_index("s_2", tmp_path / "b" / "s_2") + data = _load(_index_path(tmp_path)) + assert data == { + "s_1": str(tmp_path / "a" / "s_1"), + "s_2": str(tmp_path / "b" / "s_2"), + } + + def test_remove_deletes_entry(self, tmp_path): + upsert_session_index("s_1", tmp_path / "a" / "s_1") + upsert_session_index("s_2", tmp_path / "b" / "s_2") + remove_session_index("s_1") + data = _load(_index_path(tmp_path)) + assert "s_1" not in data + assert "s_2" in data + + def test_remove_unknown_session_is_noop(self, tmp_path): + upsert_session_index("s_1", tmp_path / "a" / "s_1") + remove_session_index("s_unknown") + data = _load(_index_path(tmp_path)) + assert data == {"s_1": str(tmp_path / "a" / "s_1")} + + def test_upsert_is_idempotent(self, tmp_path): + """Re-upserting the same value skips the rewrite (no mtime bump).""" + upsert_session_index("s_1", tmp_path / "a" / "s_1") + path = _index_path(tmp_path) + mtime_before = path.stat().st_mtime_ns + upsert_session_index("s_1", tmp_path / "a" / "s_1") + assert path.stat().st_mtime_ns == mtime_before + + def test_upsert_updates_changed_path(self, tmp_path): + upsert_session_index("s_1", tmp_path / "a" / "s_1") + upsert_session_index("s_1", tmp_path / "moved" / "s_1") + data = _load(_index_path(tmp_path)) + assert data["s_1"] == str(tmp_path / "moved" / "s_1") + + +class TestLoad: + def test_load_missing_returns_empty(self, tmp_path): + assert _load(tmp_path / "nope.json") == {} + + def test_load_corrupt_returns_empty(self, tmp_path): + p = tmp_path / "corrupt.json" + p.write_text("{not valid json", encoding="utf-8") + assert _load(p) == {} + + def test_load_non_dict_returns_empty(self, tmp_path): + p = tmp_path / "list.json" + p.write_text("[1, 2, 3]", encoding="utf-8") + assert _load(p) == {} + + def test_write_is_json_object(self, tmp_path): + p = tmp_path / "idx.json" + _write({"s_x": "/tmp/x"}, p) + assert json.loads(p.read_text(encoding="utf-8")) == {"s_x": "/tmp/x"} + + +class TestRebuild: + def _make_session(self, root: Path, sid: str) -> Path: + """Create a session dir with meta.json under /.emrg/sessions/.""" + sessions_dir = root / ".emrg" / "sessions" + sessions_dir.mkdir(parents=True, exist_ok=True) + sdir = sessions_dir / sid + sdir.mkdir(parents=True, exist_ok=True) + (sdir / "meta.json").write_text( + json.dumps({"session_id": sid, "message_count": 0}), encoding="utf-8" + ) + return sdir + + def test_rebuild_indexes_nested_sessions(self, tmp_path): + cfg = tmp_path / "cfg" + sdir = self._make_session(cfg, "s_nested") + count = rebuild_sessions_index(cfg) + assert count == 1 + data = _load(cfg / "sessions_index.json") + assert data["s_nested"] == str(sdir) + + def test_rebuild_scans_project_paths_outside_root(self, tmp_path): + cfg = tmp_path / "cfg" + cfg.mkdir(parents=True, exist_ok=True) + project = tmp_path / "outside_project" + sdir = self._make_session(project, "s_outside") + count = rebuild_sessions_index(cfg, project_paths=[str(project)]) + assert count == 1 + data = _load(cfg / "sessions_index.json") + assert data["s_outside"] == str(sdir) + + def test_rebuild_prunes_heavy_dirs(self, tmp_path): + """Sessions under install/ (bundled runtime) must not be indexed.""" + cfg = tmp_path / "cfg" + self._make_session(cfg, "s_real") + self._make_session(cfg / "install", "s_ignored") + count = rebuild_sessions_index(cfg) + assert count == 1 + data = _load(cfg / "sessions_index.json") + assert "s_real" in data + assert "s_ignored" not in data + + def test_rebuild_skips_meta_without_sid(self, tmp_path): + cfg = tmp_path / "cfg" + sessions_dir = cfg / ".emrg" / "sessions" + sessions_dir.mkdir(parents=True, exist_ok=True) + (sessions_dir / "orphan").mkdir() + (sessions_dir / "orphan" / "meta.json").write_text( + json.dumps({"message_count": 0}), encoding="utf-8" + ) + count = rebuild_sessions_index(cfg) + assert count == 0 + + def test_rebuild_preserves_manual_entries(self, tmp_path): + cfg = tmp_path / "cfg" + cfg.mkdir(parents=True, exist_ok=True) + (cfg / "sessions_index.json").write_text( + json.dumps({"s_manual": "/tmp/manual"}), encoding="utf-8" + ) + self._make_session(cfg, "s_scanned") + count = rebuild_sessions_index(cfg) + assert count == 2 + data = _load(cfg / "sessions_index.json") + assert data["s_manual"] == "/tmp/manual" + assert "s_scanned" in data + + +class TestSessionHooks: + def test_session_create_indexes(self, tmp_path): + session = Session.create(tmp_path) + data = _load(_index_path(tmp_path)) + assert data[session.session_id] == str(session._dir) + + def test_session_rename_preserves_index_path(self, tmp_path): + session = Session.create(tmp_path) + before = _load(_index_path(tmp_path))[session.session_id] + session.rename("Some Title") + after = _load(_index_path(tmp_path))[session.session_id] + assert after == before == str(session._dir) + + def test_session_delete_removes_index(self, tmp_path): + session = Session.create(tmp_path) + sid = session.session_id + assert sid in _load(_index_path(tmp_path)) + Session.delete(sid, tmp_path) + assert sid not in _load(_index_path(tmp_path)) + + def test_session_delete_nonexistent_no_crash(self, tmp_path): + assert Session.delete("s_never_existed", tmp_path) is False + assert _load(_index_path(tmp_path)) == {} + + +def test_sessions_index_path_is_under_config_dir(): + """The default index path lives under ~/.emrg (config_dir).""" + p = sessions_index_path() + assert p.name == "sessions_index.json"