Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions docs/adr/0011-prompt-cache-max-is-visible.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ intentional. `/debug` counted tokens; it did not say why a miss happened.
- `/btw` is a grok-build side-call: same system prompt, same tools JSON, same
`prompt_cache_key` as the parent. The side-question note is appended as a
user message. Tools are advertised for the prefix and not executed.
- Root affinity (`x-grok-conv-id` / `prompt_cache_key`) is the **git root**,
not the leaf cwd. No repo → the constant `graff-scratch`. Do not hash cwd:
sibling sandboxes and worktrees share one prefix and must share one key.

## Consequences

Expand All @@ -54,3 +57,12 @@ Live Grok 4.6 (Responses, SuperGrok OAuth, 2026-08-19): same-process
append-only turn 2 cached 3,712 of turn 1's 3,721 input tokens. `/btw` after
that turn reused **3,712** tokens (99.8% of the prior prompt). A new process
in the same folder started cold at 128 — the official first-request write.

Affinity seed (2026-08-28 rematch): the root key is the **git root**, not
the leaf cwd (`graff-scratch` when there is no repo). xAI partitions cache
by `prompt_cache_key`; a cwd-derived id made every graff-evals sandbox a
cold ~8k write (first-call cache 0–512) while grok-build's first calls
arrived already warm (11k–43k). Prefix bytes still have to match — this
only stops an identical system+tools prefix missing because the sandbox
path changed. Worktrees of one repo share; a repo CLAUDE.md still misses
another repo.
2 changes: 1 addition & 1 deletion docs/adr/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ record only when you need the evidence or the edge cases.
| [0008](0008-synthetic-evals-use-external-verifiers.md) | Synthetic coding evals promote only external-verifier passes; model judges may tiebreak correctness, never decide it. |
| [0009](0009-gpt-5-6-explicit-prompt-cache-boundary.md) | GPT-5.6 OpenAI Platform marks the stable prefix explicitly; Codex and xAI stay on their supported keyed automatic-cache paths. |
| [0010](0010-background-jobs-wait-for-exit.md) | `bash_output`/`agent_output` `wait_ms>0` blocks until exit (10h cap); do not poll every 30s. |
| [0011](0011-prompt-cache-max-is-visible.md) | Prompt-cache max is on: stable catalog by default; `/cache` is the HUD; `/btw` rides the parent prefix; children share role-lane `x-grok-conv-id` / `prompt_cache_key` (not the root id). |
| [0011](0011-prompt-cache-max-is-visible.md) | Prompt-cache max is on: stable catalog by default; `/cache` is the HUD; `/btw` rides the parent prefix; children share role-lane keys; root affinity is git-root (or `graff-scratch`), not cwd. |
| [0012](0012-overflow-handles-named-limits-extra-roots.md) | Fat tool results become `tr_N` handles; named `--context-limit` caps prefix bytes; `--add-dir` extra roots are PathConfine allow-lists, not cwd/skill/session sources. |
| [0013](0013-list-dir-lives-in-codedb.md) | Directory listing is `codedb list_dir` (in-process BFS, gitignore, 10k cap), not a new always-on catalog tool. |
| [0014](0014-session-resume-carries-the-room-cursor.md) | `/resume` restores the peer-channel byte cursor and inbox; it does not replay the room into history. |
Expand Down
153 changes: 153 additions & 0 deletions src/cache_affinity.zig
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
//! Sticky prompt-cache partition (`x-grok-conv-id` / `prompt_cache_key`).
//!
//! xAI caches prefix bytes on the server a key routes to. A cwd-derived key
//! made every sibling sandbox a cold ~8k write (rematch 2026-08-28: graff
//! first-call cache 0–512, grok 11k–43k already warm). The seed is the git
//! root when cwd sits in a repo — worktrees and eval sandboxes under that
//! tree share — else the constant `scratch_seed` so scratch `-p` shares too.
//! Prefix bytes still have to match; a repo CLAUDE.md does not hit another
//! repo. ADR 0011.

const std = @import("std");
const Io = std.Io;

pub const scratch_seed = "graff-scratch";
const salt = "graff-cache-affinity-v2";

var id_buf: [36]u8 = undefined;
var id_len: usize = 0;
var id_lock: std.atomic.Value(bool) = .init(false);

fn hasGit(io: Io, dir_abs: []const u8) bool {
var buf: [std.fs.max_path_bytes + 8]u8 = undefined;
const p = std.fmt.bufPrint(&buf, "{s}/.git", .{dir_abs}) catch return false;
Io.Dir.cwd().access(io, p, .{}) catch return false;
return true;
}

/// Directory that contains `.git` (file or dir) at or above `start_abs`.
/// Worktree `.git` is a file; `access` sees it. Does not chdir.
pub fn gitRootOf(io: Io, start_abs: []const u8, out: []u8) ?[]const u8 {
if (start_abs.len == 0 or start_abs.len > out.len) return null;
@memcpy(out[0..start_abs.len], start_abs);
var end = start_abs.len;
while (end > 1 and out[end - 1] == '/') end -= 1;
while (true) {
const dir = out[0..end];
if (hasGit(io, dir)) return dir;
if (end == 1 and dir[0] == '/') return null;
const slash = std.mem.lastIndexOfScalar(u8, dir, '/') orelse return null;
if (slash == 0) {
if (hasGit(io, out[0..1])) return out[0..1];
return null;
}
end = slash;
}
}

/// Git root of `cwd_abs`, or `scratch_seed` when the tree is not a repo.
pub fn affinitySeed(io: Io, cwd_abs: []const u8, buf: []u8) []const u8 {
return gitRootOf(io, cwd_abs, buf) orelse scratch_seed;
}

pub fn uuid5(seed: []const u8) [36]u8 {
var raw: [16]u8 = undefined;
var digest: [32]u8 = undefined;
var h = std.crypto.hash.sha2.Sha256.init(.{});
h.update(salt);
h.update(seed);
h.final(&digest);
@memcpy(&raw, digest[0..16]);
raw[6] = (raw[6] & 0x0f) | 0x50;
raw[8] = (raw[8] & 0x3f) | 0x80;
const hex = std.fmt.bytesToHex(raw, .lower);
var out: [36]u8 = undefined;
@memcpy(out[0..8], hex[0..8]);
out[8] = '-';
@memcpy(out[9..13], hex[8..12]);
out[13] = '-';
@memcpy(out[14..18], hex[12..16]);
out[18] = '-';
@memcpy(out[19..23], hex[16..20]);
out[23] = '-';
@memcpy(out[24..36], hex[20..32]);
return out;
}

/// Process-cached root partition. First cwd realpath + walk wins for the
/// process (graff does not chdir). Tests that need a fresh mint call `reset`.
pub fn rootId(io: Io) []const u8 {
while (id_lock.cmpxchgWeak(false, true, .acquire, .monotonic) != null) std.atomic.spinLoopHint();
defer id_lock.store(false, .release);
if (id_len == 0) {
var cwd_buf: [std.fs.max_path_bytes]u8 = undefined;
const n = Io.Dir.cwd().realPathFile(io, ".", &cwd_buf) catch blk: {
@memcpy(cwd_buf[0..1], ".");
break :blk 1;
};
var seed_buf: [std.fs.max_path_bytes]u8 = undefined;
const seed = affinitySeed(io, cwd_buf[0..n], &seed_buf);
id_buf = uuid5(seed);
id_len = 36;
}
return id_buf[0..id_len];
}

pub fn reset() void {
while (id_lock.cmpxchgWeak(false, true, .acquire, .monotonic) != null) std.atomic.spinLoopHint();
defer id_lock.store(false, .release);
id_len = 0;
}

fn tmpAbs(io: Io, tmp: *std.testing.TmpDir, buf: []u8) ![]const u8 {
return buf[0..try tmp.dir.realPath(io, buf)];
}

test "uuid5 is durable and version-5" {
const a = uuid5("/repo");
const b = uuid5("/repo");
try std.testing.expectEqualStrings(&a, &b);
try std.testing.expectEqual(@as(u8, '5'), a[14]);
try std.testing.expect(!std.mem.eql(u8, &a, &uuid5("/other")));
try std.testing.expect(!std.mem.eql(u8, &a, &uuid5(scratch_seed)));
}

test "nested dir under a repo shares the git root; a scratch tree does not use cwd" {
const io = std.testing.io;
var repo = std.testing.tmpDir(.{ .iterate = true });
defer repo.cleanup();
repo.dir.writeFile(io, .{ .sub_path = ".git", .data = "gitdir: /somewhere\n" }) catch unreachable;
repo.dir.createDirPath(io, "sandboxes/task-a") catch unreachable;

var cwd_buf: [std.fs.max_path_bytes]u8 = undefined;
const root_abs = try tmpAbs(io, &repo, &cwd_buf);
var nest_buf: [std.fs.max_path_bytes]u8 = undefined;
const nest = try std.fmt.bufPrint(&nest_buf, "{s}/sandboxes/task-a", .{root_abs});

var a: [std.fs.max_path_bytes]u8 = undefined;
var b: [std.fs.max_path_bytes]u8 = undefined;
try std.testing.expectEqualStrings(root_abs, gitRootOf(io, nest, &a).?);
try std.testing.expectEqualStrings(root_abs, affinitySeed(io, nest, &b));
try std.testing.expectEqualStrings(&uuid5(root_abs), &uuid5(affinitySeed(io, nest, &a)));

// zig-cache tmp dirs sit inside this repo, so a no-git case has to start
// on a path whose parents are not a checkout (`/proc/...` is enough).
const outside = "/proc/graff-cache-affinity-missing";
var seed_buf: [std.fs.max_path_bytes]u8 = undefined;
try std.testing.expect(gitRootOf(io, outside, &seed_buf) == null);
try std.testing.expectEqualStrings(scratch_seed, affinitySeed(io, outside, &seed_buf));
}

test "rootId matches uuid5 of this process's affinity seed" {
reset();
defer reset();
const io = std.testing.io;
const got = rootId(io);
try std.testing.expectEqual(@as(usize, 36), got.len);
var cwd_buf: [std.fs.max_path_bytes]u8 = undefined;
const n = Io.Dir.cwd().realPathFile(io, ".", &cwd_buf) catch return error.TestUnexpectedResult;
var seed_buf: [std.fs.max_path_bytes]u8 = undefined;
const seed = affinitySeed(io, cwd_buf[0..n], &seed_buf);
try std.testing.expectEqualStrings(&uuid5(seed), got);
try std.testing.expectEqualStrings(got, rootId(io)); // cached
}
51 changes: 7 additions & 44 deletions src/http_headers.zig
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ const Io = std.Io;
const Provider = @import("provider.zig").Provider;
const root = @import("main.zig");
const kimi_catalog = @import("kimi_catalog.zig");
const cache_affinity = @import("cache_affinity.zig");

var session_id_buf: [36]u8 = undefined;
var session_id_len: usize = 0;
Expand Down Expand Up @@ -70,51 +71,13 @@ pub fn requestCacheKey(io: Io, label: []const u8, agent: *const anyopaque, provi
return promptCacheKey(io, label, agent, buf);
}

var project_id_buf: [36]u8 = undefined;
var project_id_len: usize = 0;
var project_id_lock: std.atomic.Value(bool) = .init(false);

/// Durable per-project id (cwd-derived UUIDv5, no state to persist). A new
/// session in the same repo reuses the bucket the last session wrote, so
/// turn 1 can hit the warm system+tools prefix if the provider still has
/// it. Different models do not share a cache (the server keys by model);
/// they each get this same routing id so *their* later sessions can find
/// *their* prefix. Same-project conversation tails may evict each other;
/// the expensive prefix still hits.
/// Durable project partition (UUIDv5, no state to persist). Seed is the git
/// root when cwd is inside a repo, else `graff-scratch` — not the leaf cwd
/// (ADR 0011 / rematch 2026-08-28). Worktrees and sibling eval sandboxes
/// under one repo share the bucket so turn 1 can hit a warm system+tools
/// prefix. Different models do not share a cache (the server keys by model).
pub fn projectRootId(io: Io) []const u8 {
while (project_id_lock.cmpxchgWeak(false, true, .acquire, .monotonic) != null) std.atomic.spinLoopHint();
defer project_id_lock.store(false, .release);
if (project_id_len == 0) {
var raw: [16]u8 = undefined;
{
var cwd_buf: [4096]u8 = undefined;
const n = Io.Dir.cwd().realPathFile(io, ".", &cwd_buf) catch blk: {
@memcpy(cwd_buf[0..1], ".");
break :blk 1;
};
const cwd = cwd_buf[0..n];
var digest: [32]u8 = undefined;
var h = std.crypto.hash.sha2.Sha256.init(.{});
h.update("graff-kimi-project-cache-v1");
h.update(cwd);
h.final(&digest);
@memcpy(&raw, digest[0..16]);
raw[6] = (raw[6] & 0x0f) | 0x50; // version 5: name-derived
raw[8] = (raw[8] & 0x3f) | 0x80; // variant 1
}
const hex = std.fmt.bytesToHex(raw, .lower);
@memcpy(project_id_buf[0..8], hex[0..8]);
project_id_buf[8] = '-';
@memcpy(project_id_buf[9..13], hex[8..12]);
project_id_buf[13] = '-';
@memcpy(project_id_buf[14..18], hex[12..16]);
project_id_buf[18] = '-';
@memcpy(project_id_buf[19..23], hex[16..20]);
project_id_buf[23] = '-';
@memcpy(project_id_buf[24..36], hex[20..32]);
project_id_len = 36;
}
return project_id_buf[0..project_id_len];
return cache_affinity.rootId(io);
}

/// Side-calls that replay the parent history (`/btw`) share the root
Expand Down
3 changes: 3 additions & 0 deletions src/prompt_cache_hud.zig
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,7 @@ pub fn render(w: *Io.Writer) !void {
\\ Vercel gateway implicit cache (cached_tokens); coding-agent /v1
\\ /btw parent tools + system + cache key; note is the user message
\\ subagents role-lane x-grok-conv-id / prompt_cache_key (not the root id)
\\ affinity git-root or graff-scratch, not cwd (sibling sandboxes share)
\\
);
}
Expand Down Expand Up @@ -309,6 +310,8 @@ test "render stays content-free and names remaining levers" {
try std.testing.expect(contains(text, "x-grok-conv-id"));
try std.testing.expect(contains(text, "append-only"));
try std.testing.expect(contains(text, "subagents"));
try std.testing.expect(contains(text, "git-root"));
try std.testing.expect(contains(text, "graff-scratch"));
try std.testing.expect(!contains(text, "SECRET-PROMPT"));
try std.testing.expect(!contains(text, "/Users/me"));
try std.testing.expect(!contains(text, "bash"));
Expand Down
1 change: 1 addition & 0 deletions src/test_hooks.zig
Original file line number Diff line number Diff line change
Expand Up @@ -327,6 +327,7 @@ test {
_ = sandbox_tests;
_ = provider_tests;
_ = turn_chrome;
_ = @import("cache_affinity.zig"); // ADR 0011: git-root / scratch partition, not cwd
_ = tool_surface;
_ = agent_catalog;
_ = session_connect_tests;
Expand Down