fix(openai): bound tool-call arguments by bytes, not provider chunking

extensions/ext-llm-openai/src/openai-tool-input.test.ts

@@ -0,0 +1,74 @@
+import { assertEquals } from "#veryfront/testing/assert.ts";
+import { describe, it } from "#veryfront/testing/bdd.ts";
+import {
+  appendOpenAIStreamToolArgument,
+  joinOpenAIStreamToolArguments,
+  MAX_OPENAI_STREAM_TOOL_ARGUMENT_BYTES,
+  MAX_OPENAI_STREAM_TOOL_ARGUMENT_FRAGMENTS,
+  type OpenAIStreamToolArgumentBudget,
+} from "./openai-tool-input.ts";
+
+function emptyBudget(): OpenAIStreamToolArgumentBudget {
+  return { bytes: 0, fragments: 0 };
+}
+
+describe("ext-llm-openai/openai-tool-input", () => {
+  it("accepts far more non-empty fragments than the fragment cap while under the byte budget", () => {
+    // The provider's tokenizer decides how finely arguments are chunked, so a
+    // caller cannot control fragment count. Counting non-empty fragments against
+    // the flood guard rejected ordinary tool calls at roughly 2-8% of the byte
+    // budget they were nominally allowed -- every scheduled run of one project
+    // failed on this hourly. The byte budget is what bounds content.
+    const budget = emptyBudget();
+    const chunks: string[] = [];
+    const fragment = '{"a":1},';
+    const count = MAX_OPENAI_STREAM_TOOL_ARGUMENT_FRAGMENTS * 4;
+
+    let limit: string | undefined;
+    for (let i = 0; i < count; i++) {
+      limit = appendOpenAIStreamToolArgument(budget, chunks, fragment);
+      if (limit) break;
+    }
+
+    assertEquals(limit, undefined, `expected no limit for ${count} small fragments`);
+    assertEquals(chunks.length, count, "every non-empty fragment must be kept");
+    assertEquals(
+      budget.bytes < MAX_OPENAI_STREAM_TOOL_ARGUMENT_BYTES,
+      true,
+      "the payload must still be well inside the byte budget",
+    );
+    assertEquals(joinOpenAIStreamToolArguments(chunks).length, fragment.length * count);
+  });
+
+  it("still stops an unbounded flood of zero-byte fragments", () => {
+    // Zero-byte fragments never advance the byte budget, so they are the only
+    // ones that can arrive without bound. They are what this cap guards.
+    const budget = emptyBudget();
+    const chunks: string[] = [];
+
+    let limit: string | undefined;
+    for (let i = 0; i < MAX_OPENAI_STREAM_TOOL_ARGUMENT_FRAGMENTS + 10; i++) {
+      limit = appendOpenAIStreamToolArgument(budget, chunks, "");
+      if (limit) break;
+    }
+
+    assertEquals(limit, "fragments", "empty-fragment floods must still be rejected");
+    assertEquals(chunks.length, 0, "empty fragments are never appended");
+  });
+
+  it("still enforces the byte budget", () => {
+    const budget = emptyBudget();
+    const chunks: string[] = [];
+    const big = "x".repeat(MAX_OPENAI_STREAM_TOOL_ARGUMENT_BYTES);
+
+    assertEquals(appendOpenAIStreamToolArgument(budget, chunks, big), undefined);
+    assertEquals(appendOpenAIStreamToolArgument(budget, chunks, "y"), "bytes");
+  });
+
+  it("counts multi-byte characters by UTF-8 length, not code units", () => {
+    const budget = emptyBudget();
+    const chunks: string[] = [];
+    appendOpenAIStreamToolArgument(budget, chunks, "€");
+    assertEquals(budget.bytes, 3);
+  });
+});

extensions/ext-llm-openai/src/openai-tool-input.ts

@@ -13,20 +13,35 @@ export function appendOpenAIStreamToolArgument(
   chunks: string[],
   fragment: string,
 ): "bytes" | "fragments" | undefined {
-  if (budget.fragments >= MAX_OPENAI_STREAM_TOOL_ARGUMENT_FRAGMENTS) {
-    return "fragments";
+  const fragmentBytes = TOOL_ARGUMENT_ENCODER.encode(fragment).byteLength;
+
+  // Zero-byte fragments never advance the byte budget, so they are the only
+  // ones that can arrive without bound -- and they are what the fragment cap
+  // exists to stop. Both provider streams exercise it exactly that way, by
+  // flooding empty deltas.
+  //
+  // Counting non-empty fragments here too made the cap bind long before the
+  // byte budget: at typical delta sizes, 4096 fragments is roughly 2-8% of the
+  // 1 MiB a tool call is nominally allowed, which left the byte limit
+  // unreachable. Fragment count is chosen by the provider's tokenizer rather
+  // than by the caller, so the same tool call passed or failed depending on how
+  // the stream happened to be chunked.
+  if (fragmentBytes === 0) {
+    if (budget.fragments >= MAX_OPENAI_STREAM_TOOL_ARGUMENT_FRAGMENTS) {
+      return "fragments";
+    }
+    budget.fragments++;
+    return undefined;
   }
 
-  const fragmentBytes = TOOL_ARGUMENT_ENCODER.encode(fragment).byteLength;
   if (fragmentBytes > MAX_OPENAI_STREAM_TOOL_ARGUMENT_BYTES - budget.bytes) {
     return "bytes";
   }
 
-  budget.fragments++;
+  // Content is bounded by bytes: every fragment kept here is at least one byte,
+  // so the array cannot outgrow the byte budget.
   budget.bytes += fragmentBytes;
-  if (fragment.length > 0) {
-    chunks.push(fragment);
-  }
+  chunks.push(fragment);
   return undefined;
 }