diff --git a/bun.lock b/bun.lock index bd1848922..5f08b55f8 100644 --- a/bun.lock +++ b/bun.lock @@ -59,6 +59,10 @@ "name": "@corbits/provider-opencode-go", "version": "0.1.0", }, + "packages/prompt-variance": { + "name": "@corbits/prompt-variance", + "version": "0.1.0", + }, "packages/zen": { "name": "@corbits/provider-zen", "version": "0.1.0", @@ -224,6 +228,8 @@ "@corbits/openai-responses": ["@corbits/openai-responses@github:corbitsdev/corbits-openai-responses#1d20dbb", { "dependencies": { "arktype": "2.2.3" }, "peerDependencies": { "@intx/inference": ">=0.3.0", "@intx/types": ">=0.3.0" } }, "corbitsdev-corbits-openai-responses-1d20dbb", "sha512-lSJz1KkKD8mEUYiQ57dJprD9JavdJGt3OBRa42tvXLsikt1fJ6qX2mGCwaEXzkfZNc2XY5o7ZpxK5swSJX+LnA=="], + "@corbits/prompt-variance": ["@corbits/prompt-variance@workspace:packages/prompt-variance"], + "@corbits/provider-opencode-go": ["@corbits/provider-opencode-go@workspace:packages/opencode-go"], "@corbits/provider-zen": ["@corbits/provider-zen@workspace:packages/zen"], diff --git a/packages/prompt-variance/package.json b/packages/prompt-variance/package.json new file mode 100644 index 000000000..714ecc30f --- /dev/null +++ b/packages/prompt-variance/package.json @@ -0,0 +1,13 @@ +{ + "name": "@corbits/prompt-variance", + "version": "0.1.0", + "private": true, + "type": "module", + "license": "SEE LICENSE IN LICENSE.md", + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./src/index.ts" + } + } +} diff --git a/packages/prompt-variance/src/assemble.test.ts b/packages/prompt-variance/src/assemble.test.ts new file mode 100644 index 000000000..a0c0aeeca --- /dev/null +++ b/packages/prompt-variance/src/assemble.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, test } from "bun:test"; +import { assemble } from "./assemble.js"; +import { claudeRow, defaultRow, gptRow, grokRow, museRow } from "./rows.js"; + +const SECTIONS = ["contract", "Tools (names only): read_file", "context"]; + +describe("assemble", () => { + test("joins sections with the residual as the last section", () => { + const out = assemble(SECTIONS, grokRow); + expect(out).toContain("contract"); + expect(out.trimEnd().endsWith(grokRow.residual)).toBe(true); + }); + + test("a row without residual leaves the sections unchanged", () => { + expect(assemble(SECTIONS, defaultRow)).toBe(SECTIONS.join("\n\n")); + }); + + test("muse residual closes the prompt like the shipped tail append", () => { + const out = assemble(SECTIONS, museRow); + expect(out.trimEnd().endsWith(museRow.residual)).toBe(true); + }); + + test("claude and gpt residuals close the prompt the same way", () => { + expect( + assemble(SECTIONS, claudeRow).trimEnd().endsWith(claudeRow.residual), + ).toBe(true); + expect(assemble(SECTIONS, gptRow).trimEnd().endsWith(gptRow.residual)).toBe( + true, + ); + }); + + test("tool mounting is out of scope: assemble returns a string, not tool names", () => { + const out = assemble(SECTIONS, grokRow); + expect(typeof out).toBe("string"); + }); +}); diff --git a/packages/prompt-variance/src/assemble.ts b/packages/prompt-variance/src/assemble.ts new file mode 100644 index 000000000..fd56fc1df --- /dev/null +++ b/packages/prompt-variance/src/assemble.ts @@ -0,0 +1,16 @@ +import type { PromptVarianceRow } from "./rows.js"; + +/** + * Append a family variance row's residual at the tail of the assembled + * sections (CL-8269). An empty residual leaves the sections unchanged. + * Residuals render last so they cannot disturb the cached prompt prefix. + * Tool mounting is out of scope: run.ts applies + * ModelFamilyPolicy.advertisedToolDeny at mount time. + */ +export function assemble( + sections: readonly string[], + familyRow: PromptVarianceRow, +): string { + if (familyRow.residual.length === 0) return sections.join("\n\n"); + return [...sections, familyRow.residual].join("\n\n"); +} diff --git a/packages/prompt-variance/src/index.ts b/packages/prompt-variance/src/index.ts new file mode 100644 index 000000000..43b010f03 --- /dev/null +++ b/packages/prompt-variance/src/index.ts @@ -0,0 +1,14 @@ +export { assemble } from "./assemble.js"; +export { resolvePromptVariance } from "./resolve.js"; +export { + claudeRow, + defaultRow, + FAMILY_IDS, + FAMILY_ROWS, + gptRow, + grokRow, + grokToolBudgetResidual, + museRow, + type PromptVarianceFamily, + type PromptVarianceRow, +} from "./rows.js"; diff --git a/packages/prompt-variance/src/resolve.test.ts b/packages/prompt-variance/src/resolve.test.ts new file mode 100644 index 000000000..8e071683c --- /dev/null +++ b/packages/prompt-variance/src/resolve.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "bun:test"; +import { resolvePromptVariance } from "./resolve.js"; +import { grokRow } from "./rows.js"; + +describe("resolvePromptVariance", () => { + test("resolves each family to its row", () => { + expect(resolvePromptVariance({ family: "default" }).id).toBe("default"); + expect(resolvePromptVariance({ family: "muse" }).id).toBe("muse"); + expect(resolvePromptVariance({ family: "grok" }).id).toBe("grok"); + expect(resolvePromptVariance({ family: "claude" }).id).toBe("claude"); + expect(resolvePromptVariance({ family: "gpt" }).id).toBe("gpt"); + }); + + test("grok keeps its finish-bias residual on leaves", () => { + const row = resolvePromptVariance({ family: "grok", orchestrator: false }); + expect(row.residual).toBe(grokRow.residual); + }); + + test("grok on orchestrators falls back to the default row", () => { + const row = resolvePromptVariance({ family: "grok", orchestrator: true }); + expect(row.id).toBe("default"); + expect(row.residual).toBe(""); + }); + + test("claude keeps its task_guidance block on leaves, not orchestrators", () => { + expect( + resolvePromptVariance({ family: "claude", orchestrator: false }).residual, + ).toContain(""); + const orch = resolvePromptVariance({ + family: "claude", + orchestrator: true, + }); + expect(orch.id).toBe("default"); + expect(orch.residual).toBe(""); + }); + + test("muse keeps its residual on leaves and orchestrators alike", () => { + expect( + resolvePromptVariance({ family: "muse", orchestrator: false }).residual, + ).not.toBe(""); + expect( + resolvePromptVariance({ family: "muse", orchestrator: true }).residual, + ).not.toBe(""); + }); + + test("gpt keeps its narrate-before-tools nudge on leaves and orchestrators alike", () => { + expect( + resolvePromptVariance({ family: "gpt", orchestrator: false }).residual, + ).toContain("Narrate before tools"); + expect( + resolvePromptVariance({ family: "gpt", orchestrator: true }).residual, + ).toContain("Narrate before tools"); + }); +}); diff --git a/packages/prompt-variance/src/resolve.ts b/packages/prompt-variance/src/resolve.ts new file mode 100644 index 000000000..d3e7d100d --- /dev/null +++ b/packages/prompt-variance/src/resolve.ts @@ -0,0 +1,26 @@ +import { + defaultRow, + FAMILY_ROWS, + type PromptVarianceFamily, + type PromptVarianceRow, +} from "./rows.js"; + +/** + * Resolve the variance row for a family, mirroring the leaves-only gates in + * the model family policy: the grok finish-bias residual and the claude + * task_guidance block only make sense on leaf workers, so orchestrators + * fall back to the default row. Muse keeps its residual on both — the + * constructors append the same text at the tail either way — and gpt keeps + * its narrate-before-tools nudge on primaries and leaves alike. + */ +export function resolvePromptVariance(input: { + family: PromptVarianceFamily; + orchestrator?: boolean; +}): PromptVarianceRow { + if (input.orchestrator === true) { + if (input.family === "grok" || input.family === "claude") { + return defaultRow; + } + } + return FAMILY_ROWS[input.family]; +} diff --git a/packages/prompt-variance/src/rows.test.ts b/packages/prompt-variance/src/rows.test.ts new file mode 100644 index 000000000..5376b4e2d --- /dev/null +++ b/packages/prompt-variance/src/rows.test.ts @@ -0,0 +1,115 @@ +import { describe, expect, test } from "bun:test"; +import { + claudeRow, + defaultRow, + FAMILY_IDS, + FAMILY_ROWS, + gptRow, + grokRow, + grokToolBudgetResidual, + museRow, + type PromptVarianceFamily, +} from "./rows.js"; + +const CEREMONY_LINES = [ + "Never run git add, git commit, git stash, or any other state-changing git command unless the user asks.", + "Do not narrate a plan before acting on a small task; act, then report.", + "Verify with the test command once at the end, not after every edit.", +]; + +describe("prompt-variance family rows", () => { + test("ships exactly the default/muse/grok/claude/gpt families", () => { + expect([...FAMILY_IDS]).toEqual([ + "default", + "muse", + "grok", + "claude", + "gpt", + ]); + }); + + test("has no glm row until its eval lands (CL-8265)", () => { + for (const id of FAMILY_IDS) { + expect(id).not.toBe("glm"); + } + }); + + test("every row carries the id/residual shape — residuals-only, no deny fields", () => { + for (const row of [defaultRow, museRow, grokRow, claudeRow, gptRow]) { + expect(typeof row.id).toBe("string"); + expect(typeof row.residual).toBe("string"); + expect("advertisedToolDeny" in row).toBe(false); + expect("sectionOmit" in row).toBe(false); + expect("overrides" in row).toBe(false); + } + }); + + test("row ids match their family and FAMILY_ROWS covers all five", () => { + const ids: PromptVarianceFamily[] = [ + defaultRow.id, + museRow.id, + grokRow.id, + claudeRow.id, + gptRow.id, + ]; + expect(ids).toEqual(["default", "muse", "grok", "claude", "gpt"]); + expect(Object.keys(FAMILY_ROWS).sort()).toEqual([ + "claude", + "default", + "gpt", + "grok", + "muse", + ]); + }); + + test("muse row is the shipped CL-7869 tool-discipline text", () => { + expect(museRow.residual).toContain("Tool discipline:"); + expect(museRow.residual).toContain("Batch independent tool calls"); + expect(museRow.residual).toContain("Never re-read a file"); + expect(museRow.residual).toContain("Do not narrate; act."); + }); + + test("grok row is the single finish-bias + ceremony block (CL-8296)", () => { + expect(grokRow.residual).toContain("Finish bias (xAI / Grok worker):"); + expect(grokRow.residual).toContain("prefer the structured report"); + expect(grokRow.residual).toContain("re-open paths you already read"); + expect(grokRow.residual).toContain("done-definition is met"); + expect(grokRow.residual).toContain("never run_shell"); + }); + + test("each ceremony line appears exactly once in the grok row (P2 invariant)", () => { + for (const line of CEREMONY_LINES) { + const occurrences = grokRow.residual.split(line).length - 1; + expect(occurrences).toBe(1); + } + }); + + test("grok tool-budget residual is ceremony-free and budget text appears once (P1 invariant)", () => { + expect(grokToolBudgetResidual).toContain("Tool budget:"); + for (const line of CEREMONY_LINES) { + expect(grokToolBudgetResidual).not.toContain(line); + } + const budgetOccurrences = + grokToolBudgetResidual.split("Tool budget:").length - 1; + expect(budgetOccurrences).toBe(1); + }); + + test("default row carries no residual", () => { + expect(defaultRow.residual).toBe(""); + }); + + test("claude row is the single XML task_guidance block (CL-8309)", () => { + expect(claudeRow.residual).toContain(""); + expect(claudeRow.residual).toContain(""); + expect(claudeRow.residual).toContain("Follow the dispatch brief exactly"); + expect(claudeRow.residual).toContain("Batch independent tool calls"); + }); + + test("gpt row is the narrate-before-tools nudge (CL-8310)", () => { + expect(gptRow.residual).toContain("Narrate before tools (GPT worker):"); + expect(gptRow.residual).toContain( + "one short line saying what you are doing", + ); + expect(gptRow.residual).toContain("done-definition is met"); + }); +}); diff --git a/packages/prompt-variance/src/rows.ts b/packages/prompt-variance/src/rows.ts new file mode 100644 index 000000000..d40979175 --- /dev/null +++ b/packages/prompt-variance/src/rows.ts @@ -0,0 +1,123 @@ +/** + * Versioned model-family prompt variance (CL-8269). One row per tuned + * family: the tail residual text directors append to the assembled prompt. + * Residuals-only: the package owns residual TEXT, never tool mounting — + * tool denial stays live in ModelFamilyPolicy.advertisedToolDeny + * (src/agent/model-family-policy.ts), which run.ts applies at mount time. + * Keeping deny out of this package removes the duplicate-deny footgun. + * + * Families ship here as their lanes characterize them: default/muse/grok + * first, claude (CL-8309) and gpt (CL-8310) folded in on the P5 rebase. + * Render position is always the tail so residuals cannot disturb the + * cached prompt prefix. + */ + +/** Families with a shipped variance row. */ +export type PromptVarianceFamily = + | "default" + | "muse" + | "grok" + | "claude" + | "gpt"; + +export interface PromptVarianceRow { + id: PromptVarianceFamily; + /** Tail text appended to the assembled prompt. Empty when the family needs none. */ + residual: string; +} + +export const FAMILY_IDS: readonly PromptVarianceFamily[] = [ + "default", + "muse", + "grok", + "claude", + "gpt", +]; + +export const defaultRow: PromptVarianceRow = { + id: "default", + residual: "", +}; + +// Muse Spark does not reliably stop a tool loop at medium reasoning effort: +// the same run with these three rules appended finished in 3 turns on 4.3x +// fewer input tokens (CL-7869). Byte-identical to the shipped text so the +// size table and the constructor append cannot drift apart. +export const museRow: PromptVarianceRow = { + id: "muse", + residual: + "Tool discipline:\n" + + "- Batch independent tool calls into a single turn.\n" + + "- Never re-read a file you have already read this session.\n" + + "- Do not narrate; act.", +}; + +// Single grok finish-bias + ceremony residual (CL-8296): the finish-bias +// bullets plus the three ceremony lines from the CL-7768 design (no git, no +// pre-plan, verify once), merged into one block with no line twice. The +// don't re-read idea appears exactly once (the "re-open paths" bullet). +// Grok-only: detectModelFamily has no glm family, so no GLM row ships here. +export const grokRow: PromptVarianceRow = { + id: "grok", + residual: [ + "Finish bias (xAI / Grok worker):", + "- Once you can answer the dispatch brief, prefer the structured report over another speculative tool call.", + "- If the next call would only re-open paths you already read, write the report instead.", + "- When the dispatch brief's done-definition is met, write the report envelope instead of making one more search or micro-edit.", + "- Route file and web work through the dedicated tools, never run_shell — mining showed grok reaching for shell first when a typed tool already covered the job.", + "- Never run git add, git commit, git stash, or any other state-changing git command unless the user asks.", + "- Do not narrate a plan before acting on a small task; act, then report.", + "- Verify with the test command once at the end, not after every edit.", + ].join("\n"), +}; + +// Grok tool-budget hook (CL-8297): pure tool-loop budget, deliberately free +// of ceremony lines and family-specific routing. Grok leaves carry this via +// the ModelFamilyPolicy.promptResidual field alongside the grokRow block +// above (the finish-bias note) — each once. A named export rather than a +// family row because it travels a different seam (promptResidual field, +// not the finish-bias note). +export const grokToolBudgetResidual: string = + "Tool budget:\n" + + "- Batch independent tool calls into a single turn.\n" + + "- Never re-issue a tool call whose result you already have.\n" + + "- When the next call would only repeat prior work, write the report instead."; + +// Single XML residual for Claude-family workers (CL-8309): a prose residual +// did nothing, but one block cut Sonnet tokens. The block is +// the whole residual — never a full-prompt XML renderer, never applied +// outside the claude family. Rebuilt end to end from Anthropic's prompting +// docs: rationale first, numbered approach, named output contract. +export const claudeRow: PromptVarianceRow = { + id: "claude", + residual: [ + "", + "Autonomous coding turn: finish the work in this turn on your best judgment.", + "1. Follow the dispatch brief exactly; its Success criteria are the done-definition.", + "2. Batch independent tool calls into a single turn; work from files already read this session.", + "3. Finish the task when the done-definition is met: prefer the structured report envelope over another tool call.", + "", + ].join("\n"), +}; + +// Tiny narrate-before-tools residual for GPT workers (CL-8310): GPT-5.5 runs +// showed 6–13 silent tool-only turns. Shared thrash harness + spawn contracts +// do the structural work; this is only a narrate-before-tools nudge. +// Deliberately not manage_tasks ceremony — that is CL-7769, not this text. +export const gptRow: PromptVarianceRow = { + id: "gpt", + residual: [ + "Narrate before tools (GPT worker):", + "- Before each tool call, write one short line saying what you are doing and why.", + "- Never make back-to-back tool calls with no narration between them.", + "- When the dispatch brief's done-definition is met, write the report envelope instead of making another tool call.", + ].join("\n"), +}; + +export const FAMILY_ROWS: Record = { + default: defaultRow, + muse: museRow, + grok: grokRow, + claude: claudeRow, + gpt: gptRow, +}; diff --git a/packages/prompt-variance/src/sizes.test.ts b/packages/prompt-variance/src/sizes.test.ts new file mode 100644 index 000000000..b8a54cb54 --- /dev/null +++ b/packages/prompt-variance/src/sizes.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, test } from "bun:test"; +import { claudeRow, defaultRow, gptRow, grokRow, museRow } from "./rows.js"; + +describe("prompt-variance prompt sizes per family row", () => { + test("default residual is empty", () => { + expect(defaultRow.residual.length).toBe(0); + }); + + test("every tuned residual stays under 2000 chars / 3000 bytes", () => { + for (const row of [museRow, grokRow, claudeRow, gptRow]) { + expect(row.residual.length).toBeGreaterThan(0); + expect(row.residual.length).toBeLessThan(2000); + expect(Buffer.byteLength(row.residual, "utf8")).toBeLessThan(3000); + } + }); + + test("the grok finish-bias residual is larger than the muse discipline rules", () => { + expect(grokRow.residual.length).toBeGreaterThan(museRow.residual.length); + }); +}); diff --git a/scripts/dead-export-allowlist.txt b/scripts/dead-export-allowlist.txt index 91221f98a..34948e783 100644 --- a/scripts/dead-export-allowlist.txt +++ b/scripts/dead-export-allowlist.txt @@ -49,3 +49,19 @@ src/agent/directors/identity.ts: satisfies src/agent/directors/identity.ts: Record packages/opencode-go/src/models.ts: satisfies packages/opencode-go/src/models.ts: readonly + +# Versioned prompt-variance package public surface (CL-8269 lane, +# agent-owned). Family rows data re-exported through the package entry point; +# no in-repo barrel consumer exists yet for the entries below (src/ consumes +# claudeRow, gptRow, grokRow, grokToolBudgetResidual, and museRow through the +# barrel, and the package tests import the backing modules directly). Scoped +# to the exact ts-prune flags so a new dead export in this barrel fails the +# guard instead of hiding under a whole-file exemption. Remove each entry +# with the export it names if the public surface shrinks. +packages/prompt-variance/src/index.ts: assemble +packages/prompt-variance/src/index.ts: resolvePromptVariance +packages/prompt-variance/src/index.ts: defaultRow +packages/prompt-variance/src/index.ts: FAMILY_IDS +packages/prompt-variance/src/index.ts: FAMILY_ROWS +packages/prompt-variance/src/index.ts: PromptVarianceFamily +packages/prompt-variance/src/index.ts: PromptVarianceRow diff --git a/src/agent/model-family-policy.ts b/src/agent/model-family-policy.ts index 870f7ec45..793c6f6dd 100644 --- a/src/agent/model-family-policy.ts +++ b/src/agent/model-family-policy.ts @@ -2,6 +2,13 @@ import { detectModelFamily, type ModelFamily, } from "../subagent/provider-family.js"; +import { + claudeRow, + gptRow, + grokRow, + grokToolBudgetResidual, + museRow, +} from "../../packages/prompt-variance/src/index.js"; /** * Per-model-family tuning for the shared directors (main chat director and @@ -79,12 +86,9 @@ const DEFAULT_POLICY: Omit = { // Generic 4-line tool-budget residual (CL-8297). Grok leaves get this via // promptResidual today; other families leave the seam unfilled until their // own lanes land. Deliberately free of ceremony lines and family-specific -// routing — pure tool-loop budget. -export const GROK_TOOL_BUDGET_RESIDUAL = - "Tool budget:\n" + - "- Batch independent tool calls into a single turn.\n" + - "- Never re-issue a tool call whose result you already have.\n" + - "- When the next call would only repeat prior work, write the report instead."; +// routing — pure tool-loop budget. Single-sourced from the versioned +// prompt-variance package (CL-8269); the name stays for existing importers. +export const GROK_TOOL_BUDGET_RESIDUAL = grokToolBudgetResidual; // A directly observed 14-turn pure-tool-call session for this family // previously motivated a tightened nudge/pause pair here (6/10). That pair @@ -120,11 +124,8 @@ const KIMI_POLICY: Omit = { ...DEFAULT_POLICY }; // three rules appended finished in 3 turns on 4.3x fewer input tokens. At // minimal effort it terminates either way, so the rules earn their keep exactly // at the rungs where each wasted turn is most expensive. See CL-7869. -const MUSE_TOOL_DISCIPLINE_RULES = - "Tool discipline:\n" + - "- Batch independent tool calls into a single turn.\n" + - "- Never re-read a file you have already read this session.\n" + - "- Do not narrate; act."; +// Single-sourced from the versioned prompt-variance package (CL-8269). +const MUSE_TOOL_DISCIPLINE_RULES = museRow.residual; const MUSE_POLICY: Omit = { ...DEFAULT_POLICY, @@ -141,29 +142,17 @@ const MUSE_POLICY: Omit = { // prompt carries exactly one copy. This is a different block from the CL-8297 // tool-budget hook (GROK_TOOL_BUDGET_RESIDUAL, surfaced via the // ModelFamilyPolicy.promptResidual field): grok leaves carry both, each once. -export const GROK_PROMPT_RESIDUAL = [ - "Finish bias (xAI / Grok worker):", - "- Once you can answer the dispatch brief, prefer the structured report over another speculative tool call.", - "- If the next call would only re-open paths you already read, write the report instead.", - "- When the dispatch brief's done-definition is met, write the report envelope instead of making one more search or micro-edit.", - "- Route file and web work through the dedicated tools, never run_shell — mining showed grok reaching for shell first when a typed tool already covered the job.", - "- Never run git add, git commit, git stash, or any other state-changing git command unless the user asks.", - "- Do not narrate a plan before acting on a small task; act, then report.", - "- Verify with the test command once at the end, not after every edit.", -].join("\n"); +// Single-sourced from the versioned prompt-variance package (CL-8269); the +// name stays for existing importers. +export const GROK_PROMPT_RESIDUAL = grokRow.residual; // Claude (Anthropic) ships one XML residual, not prose: a prose residual did // nothing, but a single block cut Sonnet tokens. The block is // the whole residual — never a full-prompt XML renderer. The text lives here // (policy owns data); buildClaudeTaskGuidanceNote (prompts.ts) returns it -// verbatim so the prompt carries exactly one copy. -export const CLAUDE_TASK_GUIDANCE_NOTE = [ - "", - "Autonomous coding turn: finish the work in this turn on your best judgment.", - "1. Follow the dispatch brief exactly; its Success criteria are the done-definition.", - "2. Batch independent tool calls into a single turn; work from files already read this session.", - "3. Finish the task when the done-definition is met: prefer the structured report envelope over another tool call.", - "", -].join("\n"); +// verbatim so the prompt carries exactly one copy. Single-sourced from the +// versioned prompt-variance package (CL-8269); the name stays for existing +// importers. +export const CLAUDE_TASK_GUIDANCE_NOTE = claudeRow.residual; const CLAUDE_POLICY: Omit = { ...DEFAULT_POLICY, @@ -177,13 +166,9 @@ const CLAUDE_POLICY: Omit = { // The text lives here (policy owns data); buildGptNarrateBeforeToolsNote // (prompts.ts) returns it verbatim so the prompt carries exactly one copy. // Served cells (astra/sol/terra/…) are never named here — CL-8265 -// characterizes them later. -export const GPT_NARRATE_BEFORE_TOOLS_NOTE = [ - "Narrate before tools (GPT worker):", - "- Before each tool call, write one short line saying what you are doing and why.", - "- Never make back-to-back tool calls with no narration between them.", - "- When the dispatch brief's done-definition is met, write the report envelope instead of making another tool call.", -].join("\n"); +// characterizes them later. Single-sourced from the versioned +// prompt-variance package (CL-8269); the name stays for existing importers. +export const GPT_NARRATE_BEFORE_TOOLS_NOTE = gptRow.residual; // GPT (Codex / gpt-*) thresholds are provisional: we have no eval // characterization yet for how GPT behaves under tool-only stretches or diff --git a/src/agent/prompt-sizes.test.ts b/src/agent/prompt-sizes.test.ts index 719e336ef..dcc9335d9 100644 --- a/src/agent/prompt-sizes.test.ts +++ b/src/agent/prompt-sizes.test.ts @@ -27,7 +27,7 @@ import { resolveExecDirectorOverlay } from "../exec/runner.js"; /** * Prompt size budget (CL-7664). Numeric asserts only — copy edits must not * fail this test. Baselines are a checked-in snapshot of the max measured - * sizes across both families from the canonical fixture in + * sizes across all three families from the canonical fixture in * src/agent/prompt-sizes.ts; budgets add a +2000 char / +3000 byte allowance * (ceiling to 100) in code below. Bytes get the larger headroom because * multibyte copy can shift them faster. Adding a director is a type error @@ -76,14 +76,18 @@ const PROMPT_SIZE_BASELINE: Record< /** * Deliberate budgets above baseline + allowance, with justification. - * Empty after the origin/main rebase: every main budget fits within fresh - * baseline + allowance (draper/warden included), and the entries where main - * reads higher (intern, testsmith, gauntlet, prober) are stale-measurement - * residue, not deliberate over-allowance. + * greybeard: the grok residual (tool budget + 8-line ceremony, folded into the + * canonical promptResidual seam verbatim under CL-8296) plus the upstream + * greybeard-package growth (#1121) pushed greybeard-grok to 8218 chars, + * 218 over the 8000 baseline + allowance budget. Trimming the greybeard body + * is greybeard-lane-owned, so the overage is budgeted here instead; bytes + * stay at the current budget level (measured 8248 < 9000). */ const PROMPT_SIZE_OVERRIDES: Partial< Record -> = {}; +> = { + greybeard: { chars: 8300, bytes: 9000 }, +}; const CHAR_ALLOWANCE = 2000; const BYTE_ALLOWANCE = 3000; @@ -123,10 +127,16 @@ function budgetMessage( describe("director prompt size budget", () => { const rows = directorPromptSizeTable(); - test("covers every director in both families", () => { - expect(rows.length).toBe(DIRECTOR_IDS.length * 2); + test("covers every director in all five variance families", () => { + expect(rows.length).toBe(DIRECTOR_IDS.length * 5); for (const directorId of DIRECTOR_IDS) { - for (const family of ["default", "grok"] as const) { + for (const family of [ + "default", + "muse", + "grok", + "claude", + "gpt", + ] as const) { expect( rows.some((r) => r.directorId === directorId && r.family === family), ).toBe(true); @@ -173,6 +183,53 @@ describe("director prompt size budget", () => { } }); + test("muse family appends the shipped tool-discipline rules", () => { + for (const directorId of DIRECTOR_IDS) { + const base = rows.find( + (r) => r.directorId === directorId && r.family === "default", + ); + const muse = rows.find( + (r) => r.directorId === directorId && r.family === "muse", + ); + expect(muse?.chars ?? 0).toBeGreaterThan(base?.chars ?? 0); + expect(assembleDirectorPrompt(directorId, "muse")).toContain( + "Tool discipline:", + ); + } + }); + + test("claude leaves carry the XML task_guidance block exactly once", () => { + for (const directorId of DIRECTOR_IDS) { + const prompt = assembleDirectorPrompt(directorId, "claude"); + const occurrences = prompt.split("").length - 1; + if (DIRECTOR_REGISTRY[directorId].spawn.maySpawn) { + expect(occurrences).toBe(0); + } else { + expect(occurrences).toBe(1); + expect(prompt).toContain(""); + } + } + }); + + test("gpt directors carry the narrate-before-tools nudge exactly once", () => { + for (const directorId of DIRECTOR_IDS) { + const prompt = assembleDirectorPrompt(directorId, "gpt"); + const occurrences = + prompt.split("Narrate before tools (GPT worker):").length - 1; + expect(occurrences).toBe(1); + } + }); + + test("default family carries no residual", () => { + for (const directorId of DIRECTOR_IDS) { + const prompt = assembleDirectorPrompt(directorId, "default"); + expect(prompt).not.toContain(""); + expect(prompt).not.toContain("Narrate before tools (GPT worker):"); + expect(prompt).not.toContain("Tool budget:"); + expect(prompt).not.toContain("Finish bias (xAI / Grok worker):"); + } + }); + test("measurement is deterministic", () => { const again = directorPromptSizeTable(); expect(again.map((r) => r.chars)).toEqual(rows.map((r) => r.chars)); @@ -181,7 +238,13 @@ describe("director prompt size budget", () => { test("tool names match the production mount: no dupes, no phantoms", () => { for (const directorId of DIRECTOR_IDS) { - for (const family of ["default", "grok"] as const) { + for (const family of [ + "default", + "muse", + "grok", + "claude", + "gpt", + ] as const) { const names = canonicalToolNamesForDirector( DIRECTOR_REGISTRY[directorId], family, @@ -189,7 +252,7 @@ describe("director prompt size budget", () => { expect(new Set(names).size, `${directorId} [${family}]`).toBe( names.length, ); - // Neither fixture family is Codex, so the Codex proxies + // No fixture family is Codex, so the Codex proxies // (createCodexToolProxies returns [] when !isCodex) must be absent, // as must list_dir, which no subagent mount installs. for (const phantom of [ @@ -215,17 +278,26 @@ describe("director prompt size budget", () => { test("formatPromptSizeTable renders one row per director", () => { const table = formatPromptSizeTable(rows); expect(table).toContain( - "| director | default chars (bytes) | grok chars (bytes) |", + "| director | default chars (bytes) | muse chars (bytes) | grok chars (bytes) | claude chars (bytes) | gpt chars (bytes) |", ); for (const directorId of DIRECTOR_IDS) { const base = rows.find( (r) => r.directorId === directorId && r.family === "default", ); + const muse = rows.find( + (r) => r.directorId === directorId && r.family === "muse", + ); const grok = rows.find( (r) => r.directorId === directorId && r.family === "grok", ); + const claude = rows.find( + (r) => r.directorId === directorId && r.family === "claude", + ); + const gpt = rows.find( + (r) => r.directorId === directorId && r.family === "gpt", + ); expect(table).toContain( - `| ${directorId} | ${base?.chars} (${base?.bytes}) | ${grok?.chars} (${grok?.bytes}) |`, + `| ${directorId} | ${base?.chars} (${base?.bytes}) | ${muse?.chars} (${muse?.bytes}) | ${grok?.chars} (${grok?.bytes}) | ${claude?.chars} (${claude?.bytes}) | ${gpt?.chars} (${gpt?.bytes}) |`, ); } }); diff --git a/src/agent/prompt-sizes.ts b/src/agent/prompt-sizes.ts index 8a2944656..54b55b12b 100644 --- a/src/agent/prompt-sizes.ts +++ b/src/agent/prompt-sizes.ts @@ -38,8 +38,9 @@ import { webSearchDefinition } from "../tools/web-search.js"; * appendix, with the Grok finish-bias note gated by * shouldApplyGrokAntiThrash (leaves on Grok-family providers only) and the * family promptResidual (CL-8297 tool budget for grok leaves, XML - * task_guidance block for claude leaves) resolved from the model family - * policy. + * task_guidance block for claude leaves, narrate-before-tools nudge for + * gpt leaves) resolved from the model family policy. Residual texts are + * single-sourced from the versioned prompt-variance package (CL-8269). * * The env and provider inputs are pinned here so sizes never drift with the * machine, date, or checkout — only real prompt changes move the numbers. @@ -57,16 +58,25 @@ export const CANONICAL_PROMPT_ENV: EnvironmentInfo = { }; const GROK_PROVIDER = { providerName: "xai/default", model: "grok-4.6" }; -// Default-family probe: an unrecognized provider stays on the default -// family no matter how many family rows land (claude/gpt already ship), -// so the default column carries no residual. +// Default-family probe: anthropic/claude-sonnet-4 hits the claude row and +// openai/gpt-4.1 hits the gpt row, so an unrecognized provider is the probe +// that still resolves to the default family (no residual). +const MUSE_PROVIDER = { + providerName: "opencode-go", + model: "muse-spark-1.3-contributor", +}; const DEFAULT_PROVIDER = { providerName: "unknown-provider", model: "unknown-model", }; +const CLAUDE_PROVIDER = { + providerName: "anthropic", + model: "claude-sonnet-4", +}; +const GPT_PROVIDER = { providerName: "openai", model: "gpt-4.1" }; -/** Families in the size table: default assembly vs Grok (+finish-bias note). */ -export type PromptSizeFamily = "default" | "grok"; +/** Families in the size table: default (no residual), muse, grok, claude, gpt. */ +export type PromptSizeFamily = "default" | "muse" | "grok" | "claude" | "gpt"; /** * Pinned AGENTS.md body for prefix measurement. Production reads the live @@ -120,7 +130,13 @@ export function canonicalToolNamesForDirector( const providerName = family === "grok" ? GROK_PROVIDER.providerName - : DEFAULT_PROVIDER.providerName; + : family === "muse" + ? MUSE_PROVIDER.providerName + : family === "claude" + ? CLAUDE_PROVIDER.providerName + : family === "gpt" + ? GPT_PROVIDER.providerName + : DEFAULT_PROVIDER.providerName; const filtered = [...preFilterMountNames(isCodexProviderName(providerName))]; const capabilities = packageToCapabilities(pkg); const names = @@ -162,9 +178,18 @@ export function assembleDirectorPrompt( ): string { const pkg = DIRECTOR_REGISTRY[directorId]; const orchestrator = pkg.spawn.maySpawn; - const provider = family === "grok" ? GROK_PROVIDER : DEFAULT_PROVIDER; + const provider = + family === "grok" + ? GROK_PROVIDER + : family === "muse" + ? MUSE_PROVIDER + : family === "claude" + ? CLAUDE_PROVIDER + : family === "gpt" + ? GPT_PROVIDER + : DEFAULT_PROVIDER; const policy = resolveModelFamilyPolicy({ ...provider, orchestrator }); - return buildSubAgentSystemPrompt( + const prompt = buildSubAgentSystemPrompt( [formatDirectorSystemPrompt(pkg)], CANONICAL_PROMPT_ENV, undefined, @@ -175,6 +200,12 @@ export function assembleDirectorPrompt( promptResidual: policy.promptResidual, }, ); + // Mirror the SubAgentDirector constructor (nudge-director.ts): family + // tool-discipline rules go at the tail of the prompt on the wire. + return policy.toolDisciplineRules !== undefined && + policy.toolDisciplineRules.length > 0 + ? `${prompt}\n\n${policy.toolDisciplineRules}` + : prompt; } /** @@ -234,7 +265,13 @@ export function measureSkywalkerPrefix(): SkywalkerPrefixSize { export function directorPromptSizeTable(): DirectorPromptSize[] { const rows: DirectorPromptSize[] = []; for (const directorId of DIRECTOR_IDS) { - for (const family of ["default", "grok"] as const) { + for (const family of [ + "default", + "muse", + "grok", + "claude", + "gpt", + ] as const) { rows.push(measureDirectorPrompt(directorId, family)); } } @@ -244,18 +281,27 @@ export function directorPromptSizeTable(): DirectorPromptSize[] { /** Render the size table as markdown (for PR bodies and budget updates). */ export function formatPromptSizeTable(rows: DirectorPromptSize[]): string { const lines = [ - "| director | default chars (bytes) | grok chars (bytes) |", - "| --- | --- | --- |", + "| director | default chars (bytes) | muse chars (bytes) | grok chars (bytes) | claude chars (bytes) | gpt chars (bytes) |", + "| --- | --- | --- | --- | --- | --- |", ]; for (const directorId of DIRECTOR_IDS) { const base = rows.find( (r) => r.directorId === directorId && r.family === "default", ); + const muse = rows.find( + (r) => r.directorId === directorId && r.family === "muse", + ); const grok = rows.find( (r) => r.directorId === directorId && r.family === "grok", ); + const claude = rows.find( + (r) => r.directorId === directorId && r.family === "claude", + ); + const gpt = rows.find( + (r) => r.directorId === directorId && r.family === "gpt", + ); lines.push( - `| ${directorId} | ${base?.chars} (${base?.bytes}) | ${grok?.chars} (${grok?.bytes}) |`, + `| ${directorId} | ${base?.chars} (${base?.bytes}) | ${muse?.chars} (${muse?.bytes}) | ${grok?.chars} (${grok?.bytes}) | ${claude?.chars} (${claude?.bytes}) | ${gpt?.chars} (${gpt?.bytes}) |`, ); } return lines.join("\n");