diff --git a/scripts/test-pty-overflow.py b/scripts/test-pty-overflow.py new file mode 100644 index 00000000..aed058d3 --- /dev/null +++ b/scripts/test-pty-overflow.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +"""End-to-end real-PTY test for anthropic/openai context-overflow handling (#201-#203). + +Points graff at a local OpenAI-compatible backend (the built-in `lmstudio` provider, +http://127.0.0.1:1234) whose only reply is an injected error, and drives a real turn +through the terminal. Two scenarios prove the overflow detection is BOTH correct and +precise: + + A. error.code = context_length_exceeded with a message that matches none of the + English substrings (Dutch). graff must still detect the overflow via the + STRUCTURED code (#203/G2), pin the meter to the window, and stay responsive + rather than wedge (#201/#202). Observable: the prompt's ctx meter reads + "k/k ctx (100% ...)". + B. error.code = rate_limit_exceeded with a non-overflow message. graff must NOT + mistake it for an overflow: the meter must not pin. Proves detection is precise. + +Requires 127.0.0.1:1234 to be free (the lmstudio provider URL is fixed); skips if a +real LM Studio (or anything) already holds it. +""" + +import http.server +import json +import os +import re +import socket +import sys +import tempfile +import threading + +from pty_harness import PtySession, terminal_text + + +_arg = sys.argv[1] if len(sys.argv) > 1 else "graff" +GRAFF = os.path.abspath(_arg) if os.sep in _arg else _arg + +# "k/k ctx (% · compact@k)" — the "·" is U+00B7. +METER_RE = re.compile(r"(\d+)k/(\d+)k ctx \((\d+)% · compact@(\d+)k\)") +PINNED_RE = re.compile(r"(\d+)k/(\d+)k ctx \(100% · compact@\d+k\)") + + +class OpenAiErrorMock: + """Serves one fixed OpenAI-style error envelope for every /v1/chat/completions.""" + + def __init__(self, error_obj: dict) -> None: + self.body = json.dumps({"error": error_obj}).encode() + self.hits = 0 + parent = self + + class Handler(http.server.BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + parent.hits += 1 + length = int(self.headers.get("content-length", 0)) + if length: + self.rfile.read(length) + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(parent.body))) + self.end_headers() + self.wfile.write(parent.body) + + def do_GET(self) -> None: # noqa: N802 (e.g. a /v1/models probe) + self.send_response(404) + self.end_headers() + + def log_message(self, *_a) -> None: # silence the default stderr logging + pass + + self.httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 1234), Handler) + + def start(self) -> None: + threading.Thread(target=self.httpd.serve_forever, daemon=True).start() + + def stop(self) -> None: + self.httpd.shutdown() + self.httpd.server_close() + + +def _run(error_obj: dict, tmp: str): + """Run one turn against a mock returning error_obj; return (rendered_text, hits).""" + mock = OpenAiErrorMock(error_obj) + mock.start() + try: + env = { + "HOME": tmp, + "LMSTUDIO_API_KEY": "local-pty-test", + "GRAFF_FLEET": "off", + "GRAFF_NO_TELEMETRY": "1", + } + ambient = tuple( + k for k in os.environ + if (k.startswith("GRAFF_") or k.startswith("CODEX_") or k == "NO_COLOR") + and k not in env + ) + with PtySession( + GRAFF, + ["--model", "lmstudio", "--no-telemetry"], + cwd=tmp, + env=env, + unset_env=ambient, + timeout=20.0, + ) as session: + session.wait_for_literal("] ›") + cursor = len(session.raw) + session.send_line("hello") + # The turn ends with an api error either way; wait for it, then settle. + session.wait_for_literal("api error:", start=cursor) + session.pump_for(1.5) + rendered = terminal_text(bytes(session.raw[cursor:])) + + # Session must remain usable after the failed turn (no wedge): a local + # command still works and the REPL exits cleanly. + c2 = len(session.raw) + session.send_line("/help") + session.wait_for_literal("/models [health]", start=c2) + session.send_key("ctrl-d") + result = session.read_until_exit(5.0) + if result.timed_out or result.exit_code != 0: + raise SystemExit( + f"REPL did not exit cleanly: exit={result.exit_code} " + f"timed_out={result.timed_out}" + ) + return rendered, mock.hits + finally: + mock.stop() + + +def main() -> None: + # The lmstudio provider URL is hardcoded to :1234; bail cleanly if it's taken. + probe = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + # SO_REUSEADDR binds over a TIME_WAIT port left by a prior run, but still fails + # against a real LISTENing server — so back-to-back runs work, a live LM Studio skips. + probe.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + try: + probe.bind(("127.0.0.1", 1234)) + except OSError: + print("skip 127.0.0.1:1234 is in use (real LM Studio?) — overflow PTY test skipped") + return + finally: + probe.close() + + with tempfile.TemporaryDirectory(prefix="graff-pty-overflow-") as tmp: + # Disable the AI tab-titler so it doesn't fire an extra quiet turn. + harness = os.path.join(tmp, ".harness") + os.makedirs(harness, exist_ok=True) + with open(os.path.join(harness, "settings.json"), "w", encoding="utf-8") as fh: + json.dump({"ai_title": False}, fh) + + # Scenario A: structured overflow code, message matches NO English substring. + dutch = "de aanvraag overschrijdt het maximale vensterformaat van dit model" + rendered, hits = _run( + {"message": dutch, "type": "invalid_request_error", "code": "context_length_exceeded"}, + tmp, + ) + if hits < 1: + raise AssertionError("A: graff never reached the backend") + if "api error" not in rendered or dutch not in rendered: + raise AssertionError(f"A: overflow error was not surfaced:\n{rendered}") + pinned = PINNED_RE.search(rendered) + if not pinned: + raise AssertionError( + "A: meter did not pin to the window — the structured error.code " + f"(context_length_exceeded) was not detected as overflow (#203/G2):\n{rendered}" + ) + m = METER_RE.search(rendered) + if m.group(1) != m.group(2): + raise AssertionError(f"A: meter used != window despite pin: {m.group(0)!r}") + print(f"ok overflow-by-code detected end to end; meter pinned to {m.group(2)}k (100%)") + + # Scenario B: a non-overflow code + non-overflow message must NOT pin. + rendered, _ = _run( + {"message": "too many requests", "type": "rate_limit_error", "code": "rate_limit_exceeded"}, + tmp, + ) + if "too many requests" not in rendered: + raise AssertionError(f"B: rate-limit error was not surfaced:\n{rendered}") + stray = PINNED_RE.search(rendered) + if stray: + raise AssertionError( + "B: meter pinned on a NON-overflow error — detection is not precise " + f"({stray.group(0)!r}):\n{rendered}" + ) + print("ok non-overflow error did not pin the meter (detection is precise)") + + +if __name__ == "__main__": + main() diff --git a/src/agent.zig b/src/agent.zig index c3b63c21..dfcc684d 100644 --- a/src/agent.zig +++ b/src/agent.zig @@ -346,6 +346,7 @@ pub const Agent = struct { // unchanged. pub const request = @import("agent_request.zig").request; pub const inputOverCompactThreshold = @import("agent_request.zig").inputOverCompactThreshold; + pub const fullInputEstimateTokens = @import("agent_request.zig").fullInputEstimateTokens; pub const recordUsage = @import("agent_request.zig").recordUsage; pub const usageInt = @import("agent_request.zig").usageInt; pub const recordCost = @import("agent_request.zig").recordCost; diff --git a/src/agent_compact.zig b/src/agent_compact.zig index 1d409c1a..02b8dcb9 100644 --- a/src/agent_compact.zig +++ b/src/agent_compact.zig @@ -455,7 +455,8 @@ pub fn trimOldestToolOutputs(self: *Agent) usize { if (seen > total - keep_recent) break; // keep the most recent verbatim reclaimed += truncateToolOutput(self.arena, m, stub_cap, "[old tool output truncated to recover context (#163)]"); } - if (reclaimed > 0) self.last_context_tokens = 0; // force a re-measure next turn + // #202: reflect the trimmed size instead of blinding the meter to 0. + if (reclaimed > 0) self.last_context_tokens = self.fullInputEstimateTokens(); return reclaimed; } @@ -476,7 +477,9 @@ pub fn capOversizedToolOutputs(self: *Agent, cap: usize) usize { if (isToolOutputMsg(m.*)) reclaimed += truncateToolOutput(self.arena, m, cap, "[tool output truncated: over this model's per-result cap — read/fetch a smaller range (#193)]"); } - if (reclaimed > 0) self.last_context_tokens = 0; // force a re-measure next turn + // #202: reflect the trimmed size instead of blinding the meter to 0, so the + // between-turns gate keeps working and an overflow recover-pin isn't clobbered. + if (reclaimed > 0) self.last_context_tokens = self.fullInputEstimateTokens(); return reclaimed; } @@ -554,7 +557,9 @@ test "trimOldestToolOutputs recovers a runaway tool-loop history (#163)" { try std.testing.expect(emergencyCutIndex(agent.messages.items) == null); const reclaimed = trimOldestToolOutputs(&agent); try std.testing.expect(reclaimed > 0); // recovered instead of wedging - try std.testing.expectEqual(@as(usize, 0), agent.last_context_tokens); // forces a re-measure + // #202: re-measured to the trimmed size instead of blinding the meter to 0 + try std.testing.expect(agent.last_context_tokens > 0); + try std.testing.expectEqual(agent.fullInputEstimateTokens(), agent.last_context_tokens); var truncated: usize = 0; var full: usize = 0; for (agent.messages.items) |m| { @@ -614,7 +619,9 @@ test "capOversizedToolOutputs (#193): bounds an oversized output in every wire f const reclaimed = capOversizedToolOutputs(&agent, cap); try std.testing.expect(reclaimed > 0); - try std.testing.expectEqual(@as(usize, 0), agent.last_context_tokens); // forces a re-measure + // #202: re-measured to the trimmed size instead of blinding the meter to 0 + try std.testing.expect(agent.last_context_tokens > 0); + try std.testing.expectEqual(agent.fullInputEstimateTokens(), agent.last_context_tokens); // every oversized tool output is now within the cap, with a marker const out0 = agent.messages.items[0].object.get("output").?.string; diff --git a/src/agent_request.zig b/src/agent_request.zig index 5f9903f5..4ddeb6cc 100644 --- a/src/agent_request.zig +++ b/src/agent_request.zig @@ -78,7 +78,14 @@ test "isAuthError (#148): auth failures only, not credits/rate/other" { /// too long", "exceed context limit"). Drives the in-turn emergency-trim + retry /// recovery symmetrically for every provider (#193) — before it, only the codex /// path recovered and anthropic/openai died on an over-window turn. -fn isContextOverflow(msg: []const u8) bool { +fn isContextOverflow(msg: []const u8, code: ?[]const u8) bool { + // #203: match the structured error code first (openai/codex parity — a local or + // non-English provider whose message text differs still recovers), then fall back + // to the human-readable phrasing. + if (code) |c| { + const codes = [_][]const u8{ "context_length_exceeded", "context_window_exceeded" }; + for (codes) |k| if (std.mem.eql(u8, c, k)) return true; + } const needles = [_][]const u8{ "context window", // codex/responses: "exceeds the context window" "context length", // openai: "maximum context length is N tokens" @@ -91,6 +98,22 @@ fn isContextOverflow(msg: []const u8) bool { return false; } +/// The structured error code from a parsed error envelope, if any: openai / lmstudio / +/// deepseek put it at root.error.code; some providers use a top-level root.code (#203). +fn errorCode(root: std.json.ObjectMap) ?[]const u8 { + if (root.get("error")) |ev| { + if (ev == .object) { + if (ev.object.get("code")) |cv| { + if (cv == .string) return cv.string; + } + } + } + if (root.get("code")) |cv| { + if (cv == .string) return cv.string; + } + return null; +} + /// #193 follow-up: shared in-turn context-overflow recovery for the three /// anthropic/openai error branches (streamed error event, non-streamed /// `{"type":"error"}` envelope, and the generic apiErrorMessage path). Before @@ -103,8 +126,8 @@ fn isContextOverflow(msg: []const u8) bool { /// `context_retried` shared across every branch of a request) so a second overflow /// falls through and never loops. These wire formats send the full input each /// rebuild, so — unlike the codex branch — no closeCodexWs re-anchor is needed. -fn recoverContextOverflow(self: *Agent, msg: []const u8, retried: *bool) bool { - if (!isContextOverflow(msg)) return false; +fn recoverContextOverflow(self: *Agent, msg: []const u8, code: ?[]const u8, retried: *bool) bool { + if (!isContextOverflow(msg, code)) return false; self.last_context_tokens = self.provider.context; if (retried.* or self.emergencyTrim() == 0) return false; retried.* = true; @@ -112,18 +135,70 @@ fn recoverContextOverflow(self: *Agent, msg: []const u8, retried: *bool) bool { return true; } -test "isContextOverflow (#193): matches every provider's overflow phrasing, not unrelated errors" { +test "isContextOverflow (#193/#203): matches structured code + every provider's phrasing, not unrelated errors" { // codex/responses, openai, anthropic wire-format rejections all recover in-turn - try std.testing.expect(isContextOverflow("Your input exceeds the context window of 272000 tokens")); - try std.testing.expect(isContextOverflow("This model's maximum context length is 128000 tokens. However, you requested 130000")); - try std.testing.expect(isContextOverflow("context_length_exceeded")); - try std.testing.expect(isContextOverflow("prompt is too long: 219373 tokens > 200000 maximum")); - try std.testing.expect(isContextOverflow("input length and max_tokens exceed context limit")); - // unrelated API errors must NOT trigger a trim + retry - try std.testing.expect(!isContextOverflow("The API Key appears to be invalid or may have expired.")); - try std.testing.expect(!isContextOverflow("tool_choice is not supported")); - try std.testing.expect(!isContextOverflow("rate limit exceeded")); - try std.testing.expect(!isContextOverflow("model not found")); + try std.testing.expect(isContextOverflow("Your input exceeds the context window of 272000 tokens", null)); + try std.testing.expect(isContextOverflow("This model's maximum context length is 128000 tokens. However, you requested 130000", null)); + try std.testing.expect(isContextOverflow("context_length_exceeded", null)); + try std.testing.expect(isContextOverflow("prompt is too long: 219373 tokens > 200000 maximum", null)); + try std.testing.expect(isContextOverflow("input length and max_tokens exceed context limit", null)); + // #203: a structured error code recovers even when the message text is unfamiliar + // (a local / non-English provider whose phrasing we don't match on) + try std.testing.expect(isContextOverflow("de invoerlengte overschrijdt het venster", "context_length_exceeded")); + try std.testing.expect(isContextOverflow("", "context_window_exceeded")); + // unrelated API errors must NOT trigger a trim + retry, by message or by code + try std.testing.expect(!isContextOverflow("The API Key appears to be invalid or may have expired.", null)); + try std.testing.expect(!isContextOverflow("tool_choice is not supported", null)); + try std.testing.expect(!isContextOverflow("rate limit exceeded", null)); + try std.testing.expect(!isContextOverflow("model not found", null)); + try std.testing.expect(!isContextOverflow("some unrelated failure", "rate_limit_exceeded")); +} + +test "errorCode (#203): pulls root.error.code (openai/lmstudio), falls back to root.code, else null" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + // openai/lmstudio/deepseek shape: {"error":{"code":"context_length_exceeded",...}} + var err: std.json.ObjectMap = .empty; + try err.put(a, "code", .{ .string = "context_length_exceeded" }); + var root1: std.json.ObjectMap = .empty; + try root1.put(a, "error", .{ .object = err }); + try std.testing.expectEqualStrings("context_length_exceeded", errorCode(root1).?); + // top-level code fallback + var root2: std.json.ObjectMap = .empty; + try root2.put(a, "code", .{ .string = "context_window_exceeded" }); + try std.testing.expectEqualStrings("context_window_exceeded", errorCode(root2).?); + // neither present → null (falls back to substring detection) + var root3: std.json.ObjectMap = .empty; + try root3.put(a, "message", .{ .string = "hi" }); + try std.testing.expect(errorCode(root3) == null); +} + +test "recordUsage (#202): floors the meter from the local estimate when usage is absent" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const a = arena_state.allocator(); + + var msgs = std.json.Array.init(a); + var m: std.json.ObjectMap = .empty; + try m.put(a, "role", .{ .string = "user" }); + try m.put(a, "content", .{ .string = "the quick brown fox jumps over the lazy dog" }); + try msgs.append(.{ .object = m }); + + var agent: Agent = undefined; + agent.arena = a; + agent.messages = msgs; + agent.last_context_tokens = 0; + + // a response body with NO usage object previously froze the meter at its stale + // value; now it floors to max(full-input estimate, req_body_len/4) so the + // between-turns compaction gate can still fire. + const root: std.json.ObjectMap = .empty; + recordUsage(&agent, root, 4000); + + try std.testing.expect(agent.last_context_tokens > 0); + const expected = @max(fullInputEstimateTokens(&agent), @as(u64, 1000)); // 4000/4 + try std.testing.expectEqual(expected, agent.last_context_tokens); } test "recoverContextOverflow (#193): overflow trims + retries once; guard and non-overflow fall through" { @@ -156,15 +231,15 @@ test "recoverContextOverflow (#193): overflow trims + retries once; guard and no // overflow + trimmable history -> recovers (retry the turn), guard flips var retried = false; - try std.testing.expect(recoverContextOverflow(&agent, "prompt is too long: 999 tokens > 100 maximum", &retried)); + try std.testing.expect(recoverContextOverflow(&agent, "prompt is too long: 999 tokens > 100 maximum", null, &retried)); try std.testing.expect(retried); // a second overflow this request -> guard blocks a re-trim (no loop), but the // meter stays pinned to the window so the between-turns compaction still engages - try std.testing.expect(!recoverContextOverflow(&agent, "prompt is too long", &retried)); + try std.testing.expect(!recoverContextOverflow(&agent, "prompt is too long", null, &retried)); try std.testing.expectEqual(agent.provider.context, agent.last_context_tokens); // an unrelated error never recovers, regardless of the guard var retried2 = false; - try std.testing.expect(!recoverContextOverflow(&agent, "invalid api key", &retried2)); + try std.testing.expect(!recoverContextOverflow(&agent, "invalid api key", null, &retried2)); try std.testing.expect(!retried2); } @@ -217,7 +292,14 @@ pub fn request(self: *Agent, tools: ?[]const u8) !std.json.ObjectMap { // window past what the in-turn recovery below can reclaim (it keeps the most // recent outputs verbatim). Window-proportional, so large-context models keep // full tool results untouched. - _ = self.capOversizedToolOutputs(self.provider.perOutputCap()); + const capped = self.capOversizedToolOutputs(self.provider.perOutputCap()); + if (capped > 0) { + // #202: don't truncate silently. The model already sees an inline marker; + // surface it to the trace and (interactively) to the user too. + if (self.tracer) |tr| tr.note("context", "capped an oversized tool output before send"); + if (!main_mod.json_mode and !self.sub) + self.say("[tool output over this model's per-result cap — truncated {d} bytes before send (#193)]\n", .{capped}) catch {}; + } var context_retried = false; // #193: at most one in-turn overflow recovery per request rebuild: while (true) { const live = !self.sub and self.out != null and !self.stream_quiet; @@ -361,7 +443,7 @@ pub fn request(self: *Agent, tools: ?[]const u8) !std.json.ObjectMap { // session would wedge (every retry resends the same // oversized history). Pin the meter to the window so the // ApiError compact-and-recover path engages. - if (isContextOverflow(msg)) { + if (isContextOverflow(msg, null)) { self.last_context_tokens = self.provider.context; // #193: the local pre-send gate uses a byte/4 LOWER bound, so // the backend can still reject an input it let through. A good @@ -396,12 +478,13 @@ pub fn request(self: *Agent, tools: ?[]const u8) !std.json.ObjectMap { const eo = if (root.get("error")) |ev| (if (ev == .object) ev.object else null) else null; const etype = if (eo) |e| (if (e.get("type")) |tv| (if (tv == .string) tv.string else "error") else "error") else "error"; const emsg = if (eo) |e| (if (e.get("message")) |mv| (if (mv == .string) mv.string else "") else "") else ""; - if (recoverContextOverflow(self, emsg, &context_retried)) continue; // #193: streamed error event that is an overflow → trim + retry + const ecode = if (eo) |e| (if (e.get("code")) |cv| (if (cv == .string) cv.string else null) else null) else null; + if (recoverContextOverflow(self, emsg, ecode, &context_retried)) continue; // #193/#203: streamed error event overflow (by code or phrasing) → trim + retry if (self.tracer) |tr| tr.api(self.label, self.provider.model, ms, body.len, resp_body.len, 0, 0, true); try self.sayApiError("api error ({s}): {s}", .{ etype, emsg }); return error.ApiError; }; - self.recordUsage(root); + self.recordUsage(root, body.len); if (self.tracer) |tr| tr.api(self.label, self.provider.model, ms, body.len, resp_body.len, self.last_context_tokens, self.last_cache_read, false); if (main_mod.json_mode and !self.sub) self.emit(.{ .type = "model_call_finished", .provider = self.provider.id, .model = self.provider.model, .ok = true, .ms = ms }); return root; @@ -420,7 +503,8 @@ pub fn request(self: *Agent, tools: ?[]const u8) !std.json.ObjectMap { const eo = if (root.get("error")) |ev| (if (ev == .object) ev.object else null) else null; const etype = if (eo) |e| (if (e.get("type")) |tv| (if (tv == .string) tv.string else "error") else "error") else "error"; const emsg = if (eo) |e| (if (e.get("message")) |mv| (if (mv == .string) mv.string else "") else "") else ""; - if (recoverContextOverflow(self, emsg, &context_retried)) continue; // #193: anthropic {"type":"error"} overflow → trim + retry + const ecode = if (eo) |e| (if (e.get("code")) |cv| (if (cv == .string) cv.string else null) else null) else null; + if (recoverContextOverflow(self, emsg, ecode, &context_retried)) continue; // #193/#203: {"type":"error"} overflow (by code or phrasing) → trim + retry if (self.tracer) |tr| tr.api(self.label, self.provider.model, ms, body.len, resp_body.len, 0, 0, true); try self.sayApiError("api error ({s}): {s}", .{ etype, emsg }); return error.ApiError; @@ -458,24 +542,31 @@ pub fn request(self: *Agent, tools: ?[]const u8) !std.json.ObjectMap { // #193 follow-up: recover an anthropic/openai context-window rejection // in-turn instead of failing the turn (before this only codex recovered; // anthropic and openai died). Shared with the two error branches above. - if (recoverContextOverflow(self, msg, &context_retried)) continue; + // #203: openai-compatible errors arrive here (no top-level "type":"error"), + // so pull the structured code from root.error.code for isContextOverflow — a + // local provider whose message text we don't match on still recovers. + if (recoverContextOverflow(self, msg, errorCode(root), &context_retried)) continue; if (self.tracer) |tr| tr.api(self.label, self.provider.model, ms, body.len, resp_body.len, 0, 0, true); try self.sayApiError("api error: {s}", .{msg}); return error.ApiError; } - self.recordUsage(root); + self.recordUsage(root, body.len); if (self.tracer) |tr| tr.api(self.label, self.provider.model, ms, body.len, resp_body.len, self.last_context_tokens, self.last_cache_read, false); if (main_mod.json_mode and !self.sub) self.emit(.{ .type = "model_call_finished", .provider = self.provider.id, .model = self.provider.model, .ok = true, .ms = ms }); return root; } } -pub fn recordUsage(self: *Agent, root: std.json.ObjectMap) void { - const usage = root.get("usage") orelse return; - if (usage != .object) return; - const u = usage.object; +pub fn recordUsage(self: *Agent, root: std.json.ObjectMap, req_body_len: usize) void { self.last_cache_read = 0; + // #202: keep the context meter live when the provider omits usage — otherwise + // the between-turns compaction gate freezes at a stale value and a long session + // can wedge. Mirror the codex/.responses fallback (req_body_len/4, floored at the + // full-input estimate) that recordUsageResponses already applies (#174). + const usage = root.get("usage") orelse return floorContextTokens(self, req_body_len / 4); + if (usage != .object) return floorContextTokens(self, req_body_len / 4); + const u = usage.object; switch (self.provider.kind) { .anthropic => { var total: i64 = 0; @@ -506,6 +597,14 @@ pub fn recordUsage(self: *Agent, root: std.json.ObjectMap) void { } } +/// #202: floor the context meter at the local estimate (full-input byte/4, or the +/// request-body byte/4 when the history isn't serialized yet) when the API omits +/// usage, so auto-compaction still triggers. Never lowers an existing higher count. +fn floorContextTokens(self: *Agent, est: u64) void { + const floor = @max(self.fullInputEstimateTokens(), est); + if (floor > self.last_context_tokens) self.last_context_tokens = floor; +} + /// An integer usage field, or 0 if absent / wrong type. pub fn usageInt(obj: std.json.ObjectMap, name: []const u8) i64 { if (obj.get(name)) |v| if (v == .integer) return v.integer; @@ -629,7 +728,15 @@ pub fn fullInputEstimateTokens(self: *Agent) u64 { /// Guarded on a known window (compactAt()==0 → don't compact blindly). pub fn inputOverCompactThreshold(self: *Agent) bool { const threshold = self.provider.compactAt(); - return threshold > 0 and fullInputEstimateTokens(self) >= threshold; + if (threshold == 0) return false; + // #203: fullInputEstimateTokens omits the ever-present system prompt + tool + // schemas, so it undercounts the real input and this gate under-fires. Add a + // baseline for that fixed prefill, clamped to 1/8 of the window so it can never + // dominate a small (local) window. (Kept out of fullInputEstimateTokens itself, + // which must stay pure over self.messages for the unit tests.) + const prefill_baseline_tokens: u64 = 8000; + const prefill = @min(prefill_baseline_tokens, self.provider.context / 8); + return fullInputEstimateTokens(self) + prefill >= threshold; } test "inputOverCompactThreshold (#193): local estimate gates a pre-send compact" { diff --git a/src/pricing.zig b/src/pricing.zig index 61c86ef7..ee34b16b 100644 --- a/src/pricing.zig +++ b/src/pricing.zig @@ -303,6 +303,23 @@ pub fn contextFor(provider_id: []const u8, model: []const u8) u64 { return default_context; } +/// Whether a model has a catalogued context window (baked row or fresh overlay), +/// as opposed to falling through to default_context. Mirrors contextFor's lookup so +/// a caller can tell an unknown/local model from a known 200k one (#203) — needed +/// because contextFor returns default_context for both. +pub fn isKnownModel(provider_id: []const u8, model: []const u8) bool { + for (models()) |m| { + if (std.mem.eql(u8, m.provider, provider_id) and std.mem.eql(u8, m.name, model)) return true; + } + for (context_overlay) |m| { + if (std.mem.eql(u8, m.name, model)) return true; + } + for (models()) |m| { + if (std.mem.eql(u8, m.name, model)) return true; + } + return false; +} + /// Fuzzy model selection for `/model ` (graff-style). Exact name wins; /// otherwise case-insensitive substring, preferring a model whose provider has /// a key/login available so `/model sonnet` lands on a usable provider. Returns diff --git a/src/provider.zig b/src/provider.zig index 484ae7a9..ac865363 100644 --- a/src/provider.zig +++ b/src/provider.zig @@ -14,6 +14,26 @@ const std = @import("std"); const pricing = @import("pricing.zig"); const contextFor = pricing.contextFor; +/// #203: declare the context window (tokens) for an unknown/local model whose real +/// window graff cannot look up, replacing the conservative default. Applied only +/// when contextFor falls back to default_context (see contextWindowFor) so it can +/// never shrink a known/catalogued window. Set from GRAFF_CONTEXT / GRAFF_CONTEXT_WINDOW. +pub var g_context_override: ?u64 = null; + +/// #204: override the auto-compaction threshold as a percent of the window +/// (default 80). null → 80. Unlike codex we allow lowering AND raising (1..100). +pub var g_compact_pct_override: ?u8 = null; + +/// The context window for a provider+model, honoring g_context_override for an +/// unknown/local model — i.e. only when contextFor returns the conservative default, +/// never overriding a known window (#203). +fn contextWindowFor(provider_id: []const u8, model: []const u8) u64 { + // Only an unknown/local model (no catalogued window) takes the override, so a + // global GRAFF_CONTEXT can never shrink a known model that happens to be 200k. + if (g_context_override) |ov| if (!pricing.isKnownModel(provider_id, model)) return ov; + return contextFor(provider_id, model); +} + /// Wire format + auth style + endpoint per provider. Base URLs and env-var /// names from models.dev/api.json (snapshot 2026-06-10); the anthropic and /// openai bases are the canonical ones (models.dev lists them as null). @@ -81,22 +101,34 @@ pub const Provider = struct { pub const Kind = enum { anthropic, openai, responses }; pub const Auth = enum { x_api_key, bearer }; - /// Auto-compact past 80% of the model's context window. + /// Auto-compact past a percentage (default 80%) of the model's context window. + /// GRAFF_COMPACT_PCT overrides the percentage, clamped to 1..100 (#204). Unlike + /// codex's one-directional clamp, the override may lower OR raise the threshold. pub fn compactAt(p: Provider) u64 { - return p.context / 10 * 8; + const pct: u64 = if (g_compact_pct_override) |o| @min(o, 100) else 80; + return p.context / 100 * pct; } - /// #193 follow-up: the largest a SINGLE tool output may be, in serialized - /// bytes, before it is truncated at send time (capOversizedToolOutputs). - /// Window-proportional — ~50% of the window in estimated tokens (~4 bytes / - /// token) — so large-context models keep full tool results untouched and only - /// a result big enough to threaten the window on its own is bounded. This - /// guarantees no single output can alone overflow past what the in-turn - /// emergency-trim recovery can reclaim (it keeps the most-recent outputs - /// verbatim). 0 (unknown window) disables the cap. + /// #201: absolute ceiling for a single tool output regardless of window size, + /// so a huge-context model still bounds one pathological result. + const abs_output_cap_bytes: usize = 256 * 1024; + + /// #193 follow-up / #201: the largest a SINGLE tool output may be, in serialized + /// bytes, before it is truncated at send time (capOversizedToolOutputs). It must + /// stay small enough that `keep_recent` (=4) such outputs — which + /// trimOldestToolOutputs keeps VERBATIM during in-turn recovery — still leave + /// room for the trimmed remainder + system prompt to fit on retry. At the old + /// `context * 2` (~50% of the window each) four recent outputs pinned ~2x the + /// window, past what recovery could reclaim → wedge (#201). Now window-proportional + /// at ~1/8 of the window in estimated tokens (context/2 bytes at ~4 bytes/token), + /// so 4 recent outputs occupy ~50% of the window, plus an absolute ceiling for + /// very large windows. Large-context models still keep normal tool results + /// untouched; only a result big enough to threaten the window is bounded. + /// 0 (unknown window) disables the cap. pub fn perOutputCap(p: Provider) usize { if (p.context == 0) return 0; - return @intCast(p.context * 2); + const proportional: usize = @intCast(p.context / 2); + return @min(proportional, abs_output_cap_bytes); } }; @@ -147,7 +179,7 @@ pub const Keys = struct { .url = if (is_codex) g_codex_url_override orelse spec.url else spec.url, .api_key = key, .model = model, - .context = contextFor(spec.id, model), + .context = contextWindowFor(spec.id, model), .account = if (is_codex) keys.codex_account else "", .source = keys.source(spec.id), }; @@ -247,3 +279,40 @@ test "Keys.build: g_codex_url_override rewires only the codex endpoint" { const anthropic = try all.providerById("anthropic", "claude-opus-4-8"); try std.testing.expectEqualStrings("https://api.anthropic.com/v1/messages", anthropic.url); } + +test "perOutputCap (#201): window-proportional with an absolute ceiling, keep_recent-safe" { + var p: Provider = undefined; + // ~1/8 of the window in tokens (context/2 bytes at ~4 bytes/token) + p.context = 270_000; + try std.testing.expectEqual(@as(usize, 135_000), p.perOutputCap()); + // #201 invariant: keep_recent (=4) verbatim outputs must stay reclaimable — + // 4 * cap (in estimated tokens) < the window. + try std.testing.expect(4 * (p.perOutputCap() / 4) < p.context); + // absolute ceiling bounds a huge window so one result can't dominate + p.context = 4_000_000; + try std.testing.expectEqual(@as(usize, 256 * 1024), p.perOutputCap()); + // unknown window disables the cap + p.context = 0; + try std.testing.expectEqual(@as(usize, 0), p.perOutputCap()); +} + +test "contextWindowFor (#203): GRAFF_CONTEXT overrides only an unknown/local model" { + g_context_override = 8192; + defer g_context_override = null; + // unknown/local model (no catalogued window) → the override applies + try std.testing.expectEqual(@as(u64, 8192), contextWindowFor("lmstudio", "some-local-gguf")); + // a known, catalogued model keeps its real window even with the override set + try std.testing.expect(pricing.isKnownModel("anthropic", "claude-opus-4-8")); + try std.testing.expect(contextWindowFor("anthropic", "claude-opus-4-8") != 8192); +} + +test "compactAt (#204): GRAFF_COMPACT_PCT overrides the 80% default, both directions" { + var p: Provider = undefined; + p.context = 100_000; + try std.testing.expectEqual(@as(u64, 80_000), p.compactAt()); // default 80% + g_compact_pct_override = 70; + defer g_compact_pct_override = null; + try std.testing.expectEqual(@as(u64, 70_000), p.compactAt()); // lowered + g_compact_pct_override = 95; + try std.testing.expectEqual(@as(u64, 95_000), p.compactAt()); // raised — no codex-style clamp to 90 +} diff --git a/src/providers.zig b/src/providers.zig index 25b4c7ab..5bab6f6b 100644 --- a/src/providers.zig +++ b/src/providers.zig @@ -86,6 +86,12 @@ fn applyProviderInner(root: *Agent, arena: Allocator, p: Provider, persist: bool root.effort_rejected = false; // new model may accept reasoning_effort; relearn root.ws_off = false; // a previous Codex WS fallback must not leak across switches root.provider = p; + // #204: a provider switch changes the window; don't carry the previous model's + // absolute token count against it. Re-estimate from the (kept or translated) + // history so the meter + compaction gate stay consistent until the next response + // returns real usage. (The cross-format history-clear above already set 0; + // fullInputEstimateTokens over an empty history is 0.) + root.last_context_tokens = root.fullInputEstimateTokens(); root.fallback_active = !persist; root.fallback_blocked = false; if (persist) saveModel(root.io, root.home, p.id, p.model); diff --git a/src/session_run.zig b/src/session_run.zig index d4d6d698..2aa43485 100644 --- a/src/session_run.zig +++ b/src/session_run.zig @@ -368,6 +368,23 @@ pub fn setupSkillsAndTheme(io: Io, arena: Allocator, environ_map: anytype, out: if (secs > 0) agent_ws.codex_ws_idle_ms = @intCast(@min(secs, 86_400) * 1000); } else |_| {} } + // #203: GRAFF_CONTEXT / GRAFF_CONTEXT_WINDOW declares the context window (in + // tokens) for an unknown/local model whose real window graff can't look up, + // replacing the conservative 200k fallback so the compaction gate + per-output + // cap are sized correctly. Only affects models that fall back to the default + // (see provider.contextWindowFor). Ignored if unparseable or 0. + if (environ_map.get("GRAFF_CONTEXT") orelse environ_map.get("GRAFF_CONTEXT_WINDOW")) |v| { + if (std.fmt.parseInt(u64, std.mem.trim(u8, v, " \t"), 10)) |n| { + if (n > 0) provider_mod.g_context_override = n; + } else |_| {} + } + // #204: GRAFF_COMPACT_PCT overrides the auto-compaction threshold as a percent + // of the window (default 80). Clamped to 1..100; ignored if unparseable or 0. + if (environ_map.get("GRAFF_COMPACT_PCT")) |v| { + if (std.fmt.parseInt(u8, std.mem.trim(u8, v, " \t"), 10)) |pct| { + if (pct > 0) provider_mod.g_compact_pct_override = @min(pct, 100); + } else |_| {} + } ws.g_debug = environ_map.get("GRAFF_WS_DEBUG") != null; // GRAFF_WS_FORCE_FAIL_ONCE=1|true|on|yes arms the one-shot forced WS // connect failure (integration-test seam for the SSE fallback + ws_off