diff --git a/src/agent/posix-tool-plugins.test.ts b/src/agent/posix-tool-plugins.test.ts index 420973bd5..88c73198f 100644 --- a/src/agent/posix-tool-plugins.test.ts +++ b/src/agent/posix-tool-plugins.test.ts @@ -119,12 +119,14 @@ describe("buildCorePosixToolPlugins", () => { new AbortController().signal, ); expect(allowed.isError).not.toBe(true); - // The read-file guard caps the read before result-truncation would run, - // so a 90KB single line comes back line-truncated and bounded. - expect(String(allowed.content)).toContain("line truncated at 2000 chars"); - expect(Buffer.byteLength(String(allowed.content), "utf8")).toBeLessThan( - 4096, - ); + // The read-file guard windows the overlong line into a bounded page, so + // a 90KB single line comes back pageable: the tail stays reachable by + // path+offset continuation, not only by grep. + expect(String(allowed.content)).not.toContain("line truncated"); + expect(String(allowed.content)).toContain("to continue"); + expect( + Buffer.byteLength(String(allowed.content), "utf8"), + ).toBeLessThanOrEqual(50 * 1024); } finally { await rm(cwd, { recursive: true, force: true }); } diff --git a/src/plugins/read-file-guard-plugin.test.ts b/src/plugins/read-file-guard-plugin.test.ts index a780bb79f..db385a349 100644 --- a/src/plugins/read-file-guard-plugin.test.ts +++ b/src/plugins/read-file-guard-plugin.test.ts @@ -3,8 +3,13 @@ import { afterAll, beforeAll, describe, expect, test } from "bun:test"; import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { createSizeCapTransform } from "@intx/inference"; import { createBlobReader } from "@intx/types/runtime"; -import type { ToolCall, ToolResult } from "@intx/types/runtime"; +import type { + StrategyContext, + ToolCall, + ToolResult, +} from "@intx/types/runtime"; import { READ_FILE_DEFAULT_MAX_LINES, READ_FILE_MAX_BYTES, @@ -15,6 +20,7 @@ import { readFileBounded, readFileGuardPlugin, } from "./read-file-guard-plugin.js"; +import { resultTruncationPlugin } from "./result-truncation-plugin.js"; const neverAbort = () => new AbortController().signal; @@ -99,6 +105,8 @@ describe("readFileBounded", () => { expect(isError).toBe(true); expect(content).toContain("beyond end of file"); expect(content).toContain("(2 lines)"); + expect(content).toContain(p); + expect(content).toContain("valid offsets 0-1"); }); test("truncates an overlong single line", async () => { @@ -164,7 +172,7 @@ describe("readFileBounded", () => { expect(content).not.toContain("continue"); }); - test("a newline-less file past the scan ceiling returns content, not empty", async () => { + test("a newline-less file past the scan ceiling returns a windowed page, not empty", async () => { const giant = "a".repeat(READ_FILE_MAX_SCAN_BYTES + 1024); const p = await fixture("giant-line.txt", giant); const { content, isError } = await readFileBounded( @@ -176,7 +184,21 @@ describe("readFileBounded", () => { expect(isError).toBeUndefined(); expect(content.length).toBeGreaterThan(0); expect(content).toContain(" 1\t"); - expect(content).toContain("scan limit"); + // The scan-capped remainder is windowed like a smaller overlong line, so + // the footer offers a deliverable offset page instead of a truncated line + // under a scan notice whose tail is unreachable. + expect(content).not.toContain("line truncated"); + expect(content).not.toContain("scan limit"); + expect(content).toContain("output limit"); + expect(content).toContain("Use offset="); + const body = content.split("\n\n")[0] ?? ""; + const numbered = body.trimEnd().split("\n"); + expect(numbered.length).toBeGreaterThan(1); + for (const line of numbered) { + expect(line.replace(/^\s*\d+\t/, "").length).toBeLessThanOrEqual( + READ_FILE_MAX_LINE_LENGTH, + ); + } }); test("abort rejects with read_file timeout guidance", async () => { @@ -271,9 +293,10 @@ describe("readFileBounded", () => { ); }); - test("offset past the scan ceiling reports the scan limit, not a fake EOF", async () => { - // Many short lines totaling more than the scan ceiling; a huge offset can - // never be reached within one scan pass. + test("a dead offset on a file larger than the scan ceiling reports beyond-EOF with path and valid range", async () => { + // Skip bytes are not scanned, so an offset past true EOF on a >8MB file + // still reaches the end of the file. Report the real line count and valid + // range, not a scan-limit that would hide a reachable EOF. const line = `${"y".repeat(80)}\n`; const count = Math.ceil( (READ_FILE_MAX_SCAN_BYTES + 1_000_000) / line.length, @@ -286,8 +309,309 @@ describe("readFileBounded", () => { neverAbort(), ); expect(isError).toBe(true); - expect(content).toContain("scan limit"); - expect(content).not.toContain("beyond end of file"); + expect(content).toContain("beyond end of file"); + expect(content).toContain(`(${count} lines)`); + expect(content).toContain(p); + expect(content).toContain(`valid offsets 0-${count - 1}`); + expect(content).not.toContain("scan limit"); + }); +}); + +describe("CL-8979 large-file pagination", () => { + const BIG_LINES = 45_000; + const bigRow = (i: number): string => `L${i}-` + "p".repeat(243); + + async function bigFixture(name: string): Promise { + const rows = Array.from({ length: BIG_LINES }, (_, i) => bigRow(i)); + return fixture(name, `${rows.join("\n")}\n`); + } + + function continueOffset(content: string): number | null { + const match = /Use offset=(\d+) to continue/.exec(content); + return match === null ? null : Number(match[1]); + } + + function bodyRows(content: string): string[] { + const body = content.split("\n\n")[0] ?? ""; + return body + .split("\n") + .filter((line) => line.trim().length > 0) + .map((line) => line.replace(/^\s*\d+\t/, "")); + } + + function chainRunner(): ( + id: string, + args: Record, + ) => Promise { + const plugin = readFileGuardPlugin(dir, {}); + const middleware = plugin.middleware; + if (middleware === undefined) throw new Error("expected middleware"); + const fallback = async (call: ToolCall): Promise => ({ + callId: call.id, + content: "FALLBACK", + }); + return (id, args) => + middleware(fallback)( + { id, name: "read_file", arguments: args }, + neverAbort(), + ); + } + + function blobChainRunner( + readBlob: (key: string) => Promise, + ): (id: string, args: Record) => Promise { + const plugin = readFileGuardPlugin(dir, { + blobReader: createBlobReader({ readBlob }), + }); + const middleware = plugin.middleware; + if (middleware === undefined) throw new Error("expected middleware"); + const fallback = async (call: ToolCall): Promise => ({ + callId: call.id, + content: "FALLBACK", + }); + return (id, args) => + middleware(fallback)( + { id, name: "read_file", arguments: args }, + neverAbort(), + ); + } + + test("reads a deep page of a file larger than the scan ceiling", async () => { + const p = await bigFixture("cl8979-big.txt"); + const res = await readFileBounded(p, 43_000, 5, neverAbort()); + expect(res.isError).toBeUndefined(); + expect(String(res.content)).toContain(bigRow(43_000)); + expect(String(res.content)).not.toContain("scan limit"); + }); + + test("chains plain path+offset continuation on one path through to the last line", async () => { + const name = "cl8979-chain.txt"; + await bigFixture(name); + const run = chainRunner(); + const collected: string[] = []; + let offset = 0; + let hops = 0; + for (;;) { + const result = await run(`chain-${hops}`, { + path: name, + limit: 200, + offset, + }); + hops += 1; + const content = String(result.content); + expect(result.isError).toBeFalsy(); + expect(content).not.toContain("scan limit"); + collected.push(...bodyRows(content)); + const next = continueOffset(content); + if (next === null) break; + offset = next; + expect(hops).toBeLessThan(2000); + } + expect(hops).toBeGreaterThan(1); + expect(collected.length).toBe(BIG_LINES); + expect(collected).toEqual( + Array.from({ length: BIG_LINES }, (_, i) => bigRow(i)), + ); + }, 120_000); + + test("chains same-URI+offset continuation on a blob past the scan ceiling without re-scanning", async () => { + const rows = Array.from({ length: BIG_LINES }, (_, i) => bigRow(i)); + const bytes = new TextEncoder().encode(`${rows.join("\n")}\n`); + const run = blobChainRunner(async (key) => { + if (key === "cl8979-blob") return bytes; + throw new Error(`missing ${key}`); + }); + const collected: string[] = []; + const path = "tool-output:///cl8979-blob"; + let offset = 0; + let hops = 0; + let sawOffsetFooter = false; + for (;;) { + const result = await run(`blob-${hops}`, { path, limit: 200, offset }); + hops += 1; + const content = String(result.content); + expect(result.isError).toBeFalsy(); + expect(content).not.toContain("scan limit"); + collected.push(...bodyRows(content)); + const next = continueOffset(content); + if (next === null) break; + sawOffsetFooter = true; + expect(next).toBeGreaterThan(offset); + offset = next; + expect(hops).toBeLessThan(2000); + } + expect(hops).toBeGreaterThan(1); + expect(sawOffsetFooter).toBe(true); + expect(collected.length).toBe(BIG_LINES); + expect(collected[BIG_LINES - 1]).toBe(bigRow(BIG_LINES - 1)); + }, 120_000); + + test("windows an overlong single file line so the tail is reachable", async () => { + const payload = `HEAD-${"y".repeat(100_000)}-TAIL`; + const p = await fixture("cl8979-giant.txt", `${payload}\nEND\n`); + let offset = 0; + let hops = 0; + let collected = ""; + for (;;) { + const res = await readFileBounded(p, offset, 10, neverAbort()); + hops += 1; + expect(res.isError).toBeUndefined(); + const content = String(res.content); + expect(content).not.toContain("line truncated"); + collected += `${content}\n`; + const rows = bodyRows(content); + expect(rows.length).toBeGreaterThan(1); + const next = continueOffset(content); + if (next === null) break; + offset = next; + expect(hops).toBeLessThan(100); + } + expect(collected).toContain("HEAD-"); + expect(collected).toContain("-TAIL"); + expect(collected).toContain("END"); + }); + + test("windows a single file line past the scan ceiling with exact reassembly", async () => { + const filler = "0123456789ABCDEF".repeat( + Math.ceil((READ_FILE_MAX_SCAN_BYTES + 4096) / 16), + ); + const payload = `HEAD-${filler}-TAIL`; + expect(payload.length).toBeGreaterThan(READ_FILE_MAX_SCAN_BYTES); + const p = await fixture("cl8979-scan-giant.txt", `${payload}\nEND\n`); + const rows: string[] = []; + let offset = 0; + let hops = 0; + for (;;) { + const res = await readFileBounded(p, offset, 2000, neverAbort()); + hops += 1; + expect(res.isError).toBeUndefined(); + const content = String(res.content); + // No silent tail loss: every scanned byte is windowed, never truncated, + // and no footer promises continuation it cannot deliver. + expect(content).not.toContain("line truncated"); + expect(content).not.toContain("scan limit"); + rows.push(...bodyRows(content)); + const next = continueOffset(content); + if (next === null) break; + expect(next).toBeGreaterThan(offset); + offset = next; + expect(hops).toBeLessThan(500); + } + expect(hops).toBeGreaterThan(1); + expect(rows[rows.length - 1]).toBe("END"); + expect(rows.slice(0, -1).join("")).toBe(payload); + }, 180_000); + + test("a large-file page passes the result-truncation layer byte-identical", async () => { + const name = "cl8979-page.txt"; + const rows = Array.from( + { length: 3_000 }, + (_, i) => `cell-${i}-` + "v".repeat(50), + ); + await fixture(name, `${rows.join("\n")}\n`); + const plugin = readFileGuardPlugin(dir, {}); + const guardMiddleware = plugin.middleware; + if (guardMiddleware === undefined) throw new Error("expected middleware"); + const fallback = async (call: ToolCall): Promise => ({ + callId: call.id, + content: "FALLBACK", + }); + const guard = guardMiddleware(fallback); + const guardOnly = await guard( + { id: "page-1", name: "read_file", arguments: { path: name } }, + neverAbort(), + ); + expect(guardOnly.isError).toBeFalsy(); + expect(String(guardOnly.content)).toContain("to continue"); + const spilled = new Map(); + const truncPlugin = resultTruncationPlugin({ + getBlobWriter: () => async (key: string, payload: Uint8Array) => { + spilled.set(key, payload); + }, + }); + const truncMiddleware = truncPlugin.middleware; + if (truncMiddleware === undefined) throw new Error("expected middleware"); + const composed = truncMiddleware(guard); + const res = await composed( + { id: "page-1", name: "read_file", arguments: { path: name } }, + neverAbort(), + ); + expect(String(res.content)).toBe(String(guardOnly.content)); + expect(spilled.size).toBe(0); + }); + + test("a large-file page keeps Use offset= through leisure and the reactor 10k size-cap", async () => { + const name = "cl8979-reactor-page.txt"; + const rows = Array.from( + { length: 3_000 }, + (_, i) => `cell-${i}-` + "v".repeat(50), + ); + await fixture(name, `${rows.join("\n")}\n`); + const plugin = readFileGuardPlugin(dir, {}); + const guardMiddleware = plugin.middleware; + if (guardMiddleware === undefined) throw new Error("expected middleware"); + const fallback = async (call: ToolCall): Promise => ({ + callId: call.id, + content: "FALLBACK", + }); + const guard = guardMiddleware(fallback); + const spilled = new Map(); + const truncPlugin = resultTruncationPlugin({ + getBlobWriter: () => async (key: string, payload: Uint8Array) => { + spilled.set(key, payload); + }, + }); + const truncMiddleware = truncPlugin.middleware; + if (truncMiddleware === undefined) throw new Error("expected middleware"); + const leisure = truncMiddleware(guard); + const leisurePage = await leisure( + { id: "page-cap", name: "read_file", arguments: { path: name } }, + neverAbort(), + ); + const leisureContent = String(leisurePage.content); + expect(leisurePage.isError).toBeFalsy(); + expect(leisureContent).toContain("Use offset="); + expect(leisureContent.length).toBeGreaterThan(10_000); + + const reactorCap = createSizeCapTransform({ + maxChars: 10_000, + contextStore: { + writeBlob: async (key: string, payload: Uint8Array) => { + spilled.set(key, payload); + }, + }, + }); + const capped = await reactorCap.apply( + { + call: { id: "page-cap", name: "read_file", arguments: { path: name } }, + result: leisurePage, + }, + {} as StrategyContext, + ); + const modelFacing = String(capped.output.content); + expect(modelFacing).toContain("Use offset="); + expect(modelFacing).toBe(leisureContent); + expect(modelFacing).not.toContain("Tool output truncated"); + }); + + test("an aborted read rejects with a timeout, not a fallback page", async () => { + const p = await fixture("cl8979-abort.txt", "x".repeat(1000)); + const ctl = new AbortController(); + ctl.abort(); + const read = readFileBounded(p, 0, 2000, ctl.signal); + await expect(read).rejects.toThrow("[timed out before completing]"); + }); + + test("a binary file still surfaces a refusal instead of a fallback page", async () => { + await fixture( + "cl8979-bin.dat", + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x00, 0xff, 0x00]), + ); + const run = chainRunner(); + const result = await run("bin-1", { path: "cl8979-bin.dat" }); + expect(result.isError).toBe(true); + expect(String(result.content)).toMatch(/binary/); + expect(String(result.content)).not.toBe("FALLBACK"); }); }); diff --git a/src/plugins/read-file-guard-plugin.ts b/src/plugins/read-file-guard-plugin.ts index 1f27dfe7a..a15b68828 100644 --- a/src/plugins/read-file-guard-plugin.ts +++ b/src/plugins/read-file-guard-plugin.ts @@ -21,8 +21,11 @@ import { formatReadFileTimeoutMessage } from "./tool-time-budget.js"; export const READ_FILE_MAX_BYTES = 50 * 1024; export const READ_FILE_DEFAULT_MAX_LINES = 2000; export const READ_FILE_MAX_LINE_LENGTH = 2000; -// Absolute ceiling on bytes scanned from disk, so a deep offset into a huge file -// stays time-bounded even though memory is already bounded by the streaming read. +// Absolute ceiling on bytes scanned from disk past the requested offset, so an +// emission window stays time-bounded even though memory is already bounded by +// the streaming read. Bytes skipped to reach a nonzero offset do not count: +// continuation past the ceiling must read through to the end, not dead-end +// with a scan limit while unread content remains. export const READ_FILE_MAX_SCAN_BYTES = 8 * 1024 * 1024; /** Refuse tool-output blobs larger than this before bounded paging. */ export const READ_FILE_MAX_TOOL_OUTPUT_BYTES = READ_FILE_MAX_SCAN_BYTES; @@ -81,6 +84,13 @@ function mapFilesystemStreamError( * When `wrapLongLines` is set, overlong lines are split into successive numbered * windows instead of being truncated and dropped — so a giant JSON line can be * paged through with the same offset protocol as a multi-line file. + * When `windowHugeLines` is set instead, only single lines that on their own + * exceed the output budget are windowed; ordinary lines keep their numbers, so + * plain path+offset pagination stays line-aligned. + * The scan ceiling counts only bytes past the requested offset: bytes skipped + * to reach a nonzero offset never trip it, so continuation on a large file + * reads through to the end instead of dead-ending with a scan limit while + * unread content remains. */ function readStreamBounded( stream: Readable, @@ -91,10 +101,15 @@ function readStreamBounded( options: { mapStreamError?: (err: NodeJS.ErrnoException) => Error; wrapLongLines?: boolean; + windowHugeLines?: boolean; } = {}, ): Promise { return new Promise((resolveP, rejectP) => { - const { mapStreamError, wrapLongLines = false } = options; + const { + mapStreamError, + wrapLongLines = false, + windowHugeLines = false, + } = options; const decoder = new StringDecoder("utf8"); const contentBudget = READ_FILE_MAX_BYTES - NOTICE_RESERVE_BYTES; @@ -103,6 +118,7 @@ function readStreamBounded( let firstChunk = true; let lineNo = 0; let scanned = 0; + let skipDone = offset <= 0; let outBytes = 0; let emitted = 0; let lastEmittedLine = 0; @@ -136,6 +152,7 @@ function readStreamBounded( const handleLine = (raw: string, overflow: boolean): boolean => { lineNo++; if (lineNo <= offset) return true; + skipDone = true; if (emitted >= limit) { truncReason = "lines"; return false; @@ -174,10 +191,13 @@ function readStreamBounded( const nl = pending.indexOf("\n"); if (nl === -1) { if (wrapLongLines) return emitWrapped(pending, false); - if (pending.length > READ_FILE_MAX_LINE_LENGTH) { + if (pending.length > READ_FILE_MAX_LINE_LENGTH && !windowHugeLines) { pending = pending.slice(0, READ_FILE_MAX_LINE_LENGTH); pendingOverflow = true; } + // windowHugeLines keeps the full pending: the scan trip ends the + // stream, and flushRemainder windows the remainder so every scanned + // byte stays reachable through offset continuation. return true; } const line = pending.slice(0, nl); @@ -187,7 +207,9 @@ function readStreamBounded( } else { const overflow = pendingOverflow; pendingOverflow = false; - if (!handleLine(line, overflow)) return false; + if (!overflow && windowHugeLines && line.length > contentBudget) { + if (!emitWrapped(line, true)) return false; + } else if (!handleLine(line, overflow)) return false; } } }; @@ -198,6 +220,17 @@ function readStreamBounded( emitWrapped(pending, true); return; } + // Windowed even when scan-capped: every window burns a line number, + // so the footer's offset resumes at the next window instead of promising + // continuation that skips the unshown middle of an overlong line. + if ( + !pendingOverflow && + windowHugeLines && + pending.length > contentBudget + ) { + emitWrapped(pending, true); + return; + } handleLine(pending, pendingOverflow); }; @@ -207,6 +240,9 @@ function readStreamBounded( done({ content: "" }); return; } + // Skip bytes are not scanned, so a dead offset on a file larger than + // the ceiling still reaches EOF. Report the true range, not a scan + // limit that would hide a reachable end of file. if (endReached) { done({ content: `[offset ${offset} is beyond end of file ${displayPath} (${lineNo} lines); valid offsets 0-${lineNo - 1}]`, @@ -242,7 +278,7 @@ function readStreamBounded( return; } } - scanned += chunk.length; + if (skipDone) scanned += chunk.length; pending += decoder.write(chunk); if (!drainPending()) { finishOk(); @@ -292,6 +328,7 @@ export function readFileBounded( signal, { mapStreamError: (err) => mapFilesystemStreamError(absolutePath, err), + windowHugeLines: true, }, ); } diff --git a/src/plugins/result-truncation-plugin.ts b/src/plugins/result-truncation-plugin.ts index 5bd79403d..cac34d96f 100644 --- a/src/plugins/result-truncation-plugin.ts +++ b/src/plugins/result-truncation-plugin.ts @@ -293,10 +293,29 @@ export async function applyToolResultTruncation( return result; } +// A read_file page already carries its own continuation contract (a plain +// `Use offset=` footer). Re-cutting it at the 10k leisure cap would slice the +// footer off the page boundary and strand the pagination chain, so +// footer-bearing read_file pages pass through intact. +// Pages without a footer take the normal path. +const READ_FILE_CONTINUATION_RE = /Use offset=\d+ to continue\./; + +function isPagedReadFilePage( + toolName: string | undefined, + result: ToolResult, +): boolean { + return ( + toolName === "read_file" && + typeof result.content === "string" && + READ_FILE_CONTINUATION_RE.test(result.content) + ); +} + async function archiveThenTruncate( result: ToolResult, callId: string, options: ResultTruncationPluginOptions, + toolName?: string, ): Promise { const archive = options.getEvidenceArchive?.(); try { @@ -320,6 +339,7 @@ async function archiveThenTruncate( } const before = result.content; + if (isPagedReadFilePage(toolName, result)) return result; const truncated = await applyToolResultTruncation( result, spillOptionsForCall(callId, options), @@ -368,7 +388,12 @@ export function wrapAgentToolResultTruncation( return { ...tool, handler: async (call: ToolCall, signal: AbortSignal) => - archiveThenTruncate(await inner(call, signal), call.id, options), + archiveThenTruncate( + await inner(call, signal), + call.id, + options, + tool.definition.name, + ), }; } const inner = tool.handler; @@ -380,6 +405,7 @@ export function wrapAgentToolResultTruncation( { callId: call.id, content: await inner(call.arguments, signal) }, call.id, options, + tool.definition.name, ), }; } @@ -397,7 +423,7 @@ export function resultTruncationPlugin( return { middleware: (next) => async (call, signal) => { const result = await next(call, signal); - return archiveThenTruncate(result, call.id, options); + return archiveThenTruncate(result, call.id, options, call.name); }, }; } diff --git a/vendor/intx-inference/PATCHES.md b/vendor/intx-inference/PATCHES.md index 8397d98ca..b232462e5 100644 --- a/vendor/intx-inference/PATCHES.md +++ b/vendor/intx-inference/PATCHES.md @@ -575,6 +575,7 @@ revisit point is the next vendored sync (see `docs/VENDORING.md`). | state-ts-deep-freeze-turns-revision | Make `ReactorState.snapshot().turns` a lazy, revision-tracked getter | Alexander Guy | This ledger (#state-ts-deep-freeze-turns-revision) | Next vendored sync | | inference-ts-cl-7783-truncated-tool-call | Surface `stop_reason`/`finish_reason` on usage events; fail the turn instead of dispatching unparseable tool calls at end-of-stream finalization | Alexander Guy | This ledger (#inference-ts-cl-7783-truncated-tool-call) | Next vendored sync | | google-genai-files-ts-body-init-cast | Widen `BodyInit` to accept Node's `Uint8Array` typing so the cast can be removed | Alexander Guy | This ledger (#google-genai-files-ts-body-init-cast) | Next vendored sync | +| size-cap-ts-paged-read-file | Skip `createSizeCapTransform` for footer-bearing `read_file` pages so `Use offset=` survives the default 10k cap | Alexander Guy | This ledger (#size-cap-ts-paged-read-file) | Next vendored sync | Contact basis: identified from the read-only upstream clone (`faremeter/interchange`); Alexander Guy is the @@ -605,6 +606,21 @@ the two on next sync, and consider restoring the dropped guard suite. **Removal path:** Upstream adding `claude-fable-5-1` to its own `ADAPTIVE_THINKING_MODELS`. +## size-cap-ts-paged-read-file + +`transforms/size-cap.ts` — Footer-bearing `read_file` pages already carry a +plain `Use offset=N to continue.` contract and are sized to the 50KB page +budget. The default 10k size-cap would slice the body and drop the footer, +stranding pagination. Pass those pages through unchanged (no spill). Other +`read_file` results, and any other tool, still cap as before. + +**Disposition:** Promotion candidate. Requires upstream to skip size-cap when +a `read_file` result already names the next offset. Downstream users that +never emit that footer are unaffected. +**Removal path:** Upstream PR to `@intx/inference` exempting footer-bearing +`read_file` pages from `createSizeCapTransform`. +**Re-carry:** isolated predicate at the start of `apply`; low merge risk. + --- The `void track(p)` → `track(p)` change at three call sites in `reactor.ts` diff --git a/vendor/intx-inference/src/transforms/size-cap.test.ts b/vendor/intx-inference/src/transforms/size-cap.test.ts index 996c0c4c5..342c870b3 100644 --- a/vendor/intx-inference/src/transforms/size-cap.test.ts +++ b/vendor/intx-inference/src/transforms/size-cap.test.ts @@ -169,4 +169,56 @@ describe("createSizeCapTransform", () => { } expect(thrown?.message).toContain("positive finite"); }); + + test("passes a footer-bearing read_file page through even when over maxChars", async () => { + const { store, calls } = recordingWriteBlob(); + const transform = createSizeCapTransform({ + maxChars: 10_000, + contextStore: store, + }); + + const body = Array.from( + { length: 3000 }, + (_, i) => + `${String(i + 1).padStart(6, " ")}\tcell-${String(i)}-${"v".repeat(50)}`, + ).join("\n"); + const page = + `${body}\n\n[Showing lines 1-3000; stopped at the 50KB output limit. ` + + `Use offset=3000 to continue.]`; + expect(page.length).toBeGreaterThan(10_000); + expect(page).toContain("Use offset="); + + const result: ToolResult = { callId: "rf1", content: page }; + const out = await transform.apply( + { call: call("rf1", "read_file"), result }, + emptyContext(), + ); + + expect(out.output).toBe(result); + expect(out.output.content).toBe(page); + expect(String(out.output.content)).toContain("Use offset="); + expect(out.record.reason).toBe("paged-read-file"); + expect(out.blobs).toBeUndefined(); + expect(calls).toHaveLength(0); + }); + + test("still caps oversize read_file results that lack a continuation footer", async () => { + const { store, calls } = recordingWriteBlob(); + const transform = createSizeCapTransform({ + maxChars: 10_000, + contextStore: store, + }); + + const full = "x".repeat(12_000); + const result: ToolResult = { callId: "rf2", content: full }; + const out = await transform.apply( + { call: call("rf2", "read_file"), result }, + emptyContext(), + ); + + expect(out.record.reason).toBe("exceeded-cap"); + expect(String(out.output.content)).toContain("Tool output truncated"); + expect(String(out.output.content)).not.toContain("Use offset="); + expect(calls).toHaveLength(1); + }); }); diff --git a/vendor/intx-inference/src/transforms/size-cap.ts b/vendor/intx-inference/src/transforms/size-cap.ts index 65145eb13..7dc8fb3b5 100644 --- a/vendor/intx-inference/src/transforms/size-cap.ts +++ b/vendor/intx-inference/src/transforms/size-cap.ts @@ -8,6 +8,8 @@ // // Within-cap results pass through unchanged (no blob is written) but still // produce a `TransformRecord` so the manifest captures every invocation. +// Locally patched — see vendor/intx-inference/PATCHES.md#size-cap-ts-paged-read-file: +// footer-bearing `read_file` pages also pass through, even when over `maxChars`. import type { ContextStore, @@ -19,12 +21,22 @@ import type { const SIZE_CAP_VERSION = "1"; const SIZE_CAP_NAME = "size-cap"; +// Locally patched — see vendor/intx-inference/PATCHES.md#size-cap-ts-paged-read-file +const READ_FILE_CONTINUATION_RE = /Use offset=\d+ to continue\./; export type SizeCapTransformOptions = { maxChars: number; contextStore: Pick; }; +function isPagedReadFilePage(toolName: string, result: ToolResult): boolean { + return ( + toolName === "read_file" && + typeof result.content === "string" && + READ_FILE_CONTINUATION_RE.test(result.content) + ); +} + /** * Create a `ToolResultTransform` that caps inline tool result content at * `maxChars` characters. Oversized results are spilled to the context store @@ -54,6 +66,20 @@ export function createSizeCapTransform( ? result.content : JSON.stringify(result.content); + // Locally patched — see vendor/intx-inference/PATCHES.md#size-cap-ts-paged-read-file + if (isPagedReadFilePage(call.name, result)) { + return { + output: result, + record: { + strategy: SIZE_CAP_NAME, + version: SIZE_CAP_VERSION, + parameters: { maxChars }, + reason: "paged-read-file", + decisions: { callId: call.id, length: text.length }, + }, + }; + } + if (text.length <= maxChars) { return { output: result, diff --git a/vendor/intx-tools-posix/src/registry.ts b/vendor/intx-tools-posix/src/registry.ts index 42dca4ba1..d57aa8f82 100644 --- a/vendor/intx-tools-posix/src/registry.ts +++ b/vendor/intx-tools-posix/src/registry.ts @@ -42,7 +42,7 @@ export const TOOL_DEFINITIONS: ToolDefinition[] = [ { name: TOOL_NAMES.READ_FILE, description: - "Read a file and return its content with line numbers. The path argument accepts either a filesystem path or a tool-output URI of the form tool-output:///{callId} that references a prior tool result.", + "Read a file and return its content with line numbers. The path argument accepts either a filesystem path or a tool-output URI of the form tool-output:///{callId} that references a prior tool result. Large files are returned one page at a time: pass offset and limit to page through the file, then follow the `Use offset=N to continue.` footer to read the next page through to the end.", inputSchema: { type: "object", properties: { diff --git a/vendor/intx-tools-posix/src/tools-posix.test.ts b/vendor/intx-tools-posix/src/tools-posix.test.ts index 25b4159a4..f384ab535 100644 --- a/vendor/intx-tools-posix/src/tools-posix.test.ts +++ b/vendor/intx-tools-posix/src/tools-posix.test.ts @@ -13,6 +13,7 @@ import { realpathSync } from "node:fs"; import { createBlobReader, type BlobReader } from "@intx/types/runtime"; import { createPosixTools, composeMiddleware } from "./index"; import type { PosixTools, ToolHandler, ToolPlugin } from "./index"; +import { TOOL_DEFINITIONS } from "./registry"; import { matchGlob, shouldSkip } from "./glob-match"; let tmpDir: string; @@ -1267,3 +1268,12 @@ describe("plugin wiring", () => { ); }); }); + +describe("read_file registry contract", () => { + test("documents the offset/limit page-and-continue contract", () => { + const def = TOOL_DEFINITIONS.find((entry) => entry.name === "read_file"); + expect(def).toBeDefined(); + expect(def?.description ?? "").toMatch(/Use offset=/); + expect(def?.description ?? "").toMatch(/page/i); + }); +});