Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 78 additions & 0 deletions src/agent/grok-residual.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
import { describe, expect, it } from "bun:test";
import { GROK_PROMPT_RESIDUAL } from "./model-family-policy.js";
import {
buildGrokLeafAntiThrashNote,
buildSubAgentSystemPrompt,
} from "./prompts.js";
import { shouldApplyGrokAntiThrash } from "../subagent/provider-family.js";

// The three Grok ceremony lines (CL-7768 Design, merged by CL-8296): no git,
// no pre-plan, verify once.
const CEREMONY_LINES = [
"- Never run git add, git commit, git stash, or any other state-changing git command unless the user asks.",
"- Do not narrate a plan before acting on a small task; act, then report.",
"- Verify with the test command once at the end, not after every edit.",
] as const;

function countOccurrences(haystack: string, needle: string): number {
return haystack.split(needle).length - 1;
}

describe("grok ceremony merge (CL-8296)", () => {
it("exposes a single grok residual with each ceremony line exactly once", () => {
expect(GROK_PROMPT_RESIDUAL).toContain("Finish bias (xAI / Grok worker):");
for (const line of CEREMONY_LINES) {
expect(countOccurrences(GROK_PROMPT_RESIDUAL, line)).toBe(1);
}
});

it("keeps the don't re-read line exactly once — no duplicate", () => {
expect(
countOccurrences(GROK_PROMPT_RESIDUAL, "re-open paths you already read"),
).toBe(1);
});

it("is grok-only: the finish-bias gate fires for grok leaves alone", () => {
expect(
shouldApplyGrokAntiThrash({
providerName: "xai/default",
model: "grok-4.6",
orchestrator: false,
}),
).toBe(true);
for (const input of [
{ providerName: "anthropic", model: "claude-sonnet-4" },
{ providerName: "moonshot", model: "kimi-k2" },
{ providerName: "opencode-go", model: "muse-spark-1.3-contributor" },
{ providerName: "openai", model: "gpt-4.1" },
] as const) {
expect(shouldApplyGrokAntiThrash({ ...input, orchestrator: false })).toBe(
false,
);
}
expect(
shouldApplyGrokAntiThrash({
providerName: "xai/default",
model: "grok-4.6",
orchestrator: true,
}),
).toBe(false);
});

it("buildGrokLeafAntiThrashNote is the same single residual (one source of truth)", () => {
expect(buildGrokLeafAntiThrashNote()).toBe(GROK_PROMPT_RESIDUAL);
});

it("the assembled grok worker prompt carries the merged residual exactly once", () => {
const prompt = buildSubAgentSystemPrompt(undefined, undefined, undefined, {
orchestrator: false,
grokAntiThrash: true,
});
expect(countOccurrences(prompt, "Finish bias (xAI / Grok worker):")).toBe(
1,
);
for (const line of CEREMONY_LINES) {
expect(countOccurrences(prompt, line)).toBe(1);
}
});
});
84 changes: 80 additions & 4 deletions src/agent/model-family-policy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ import { resolveModelFamilyPolicy } from "./model-family-policy.js";
describe("resolveModelFamilyPolicy", () => {
test("defaults are permissive for an unrecognized provider", () => {
const policy = resolveModelFamilyPolicy({
providerName: "anthropic",
model: "claude-sonnet-4",
providerName: "openai",
model: "gpt-4.1",
});
expect(policy.family).toBe("default");
expect(policy.applyGrokFinishBias).toBe(false);
Expand Down Expand Up @@ -55,8 +55,8 @@ describe("resolveModelFamilyPolicy", () => {

test("advertisedToolDeny is empty by default and never contains use_skill", () => {
const leaf = resolveModelFamilyPolicy({
providerName: "anthropic",
model: "claude-opus-4-6",
providerName: "openai",
model: "gpt-4.1",
orchestrator: false,
});
expect(leaf.advertisedToolDeny).toEqual([]);
Expand Down Expand Up @@ -102,4 +102,80 @@ describe("resolveModelFamilyPolicy", () => {
expect(muse.toolDisciplineRules).toContain("Never re-read a file");
expect(base.toolDisciplineRules).toBeUndefined();
});

describe("promptResidual (CL-8297)", () => {
test("grok leaf carries the generic 4-line tool-budget residual", () => {
const leaf = resolveModelFamilyPolicy({
providerName: "xai/default",
model: "grok-4.6",
});
expect(leaf.family).toBe("grok");
expect(leaf.promptResidual).toBeDefined();
if (!leaf.promptResidual)
throw new Error("expected promptResidual to be defined");
expect(leaf.promptResidual.split("\n")).toHaveLength(4);
expect(leaf.promptResidual).toContain("Tool budget:");
});

test("grok orchestrators and default family carry no residual", () => {
const orchestrator = resolveModelFamilyPolicy({
providerName: "xai/default",
model: "grok-4.6",
orchestrator: true,
});
expect(orchestrator.promptResidual).toBeUndefined();
// Default-family probe: anthropic/claude-sonnet-4 would hit the claude
// row now, and the gpt row has NOT landed yet (#1135), so openai/gpt-4.1
// is the probe that still resolves to the default family.
const base = resolveModelFamilyPolicy({
providerName: "openai",
model: "gpt-4.1",
});
expect(base.family).toBe("default");
expect(base.promptResidual).toBeUndefined();
});
});

test("claude leaves carry the XML task_guidance residual; orchestrators do not", () => {
const leaf = resolveModelFamilyPolicy({
providerName: "anthropic",
model: "claude-sonnet-4",
orchestrator: false,
});
expect(leaf.family).toBe("claude");
expect(leaf.promptResidual).toContain("<task_guidance>");
expect(leaf.promptResidual).toContain("</task_guidance>");
expect(leaf.advertisedToolDeny).toEqual([]);
const orchestrator = resolveModelFamilyPolicy({
providerName: "anthropic",
model: "claude-sonnet-4",
orchestrator: true,
});
expect(orchestrator.promptResidual).toBeUndefined();
});

// The gpt family row has NOT landed yet (#1135): openai/gpt-4.1 and
// codex/gpt-5.1 are default-family probes here, asserting they resolve to
// the default family with no residual. Grok keeps its CL-8297 tool-budget
// residual — the "no residual" claim below is default-family-only.
test("gpt probes resolve to default with no residual; grok keeps its tool budget", () => {
for (const input of [
{ providerName: "openai", model: "gpt-4.1" },
{ providerName: "codex", model: "gpt-5.1" },
] as const) {
const policy = resolveModelFamilyPolicy({
...input,
orchestrator: false,
});
expect(policy.family).toBe("default");
expect(policy.promptResidual).toBeUndefined();
}
const grok = resolveModelFamilyPolicy({
providerName: "xai/default",
model: "grok-4.6",
orchestrator: false,
});
expect(grok.family).toBe("grok");
expect(grok.promptResidual).toContain("Tool budget:");
});
});
68 changes: 68 additions & 0 deletions src/agent/model-family-policy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,14 @@ export interface ModelFamilyPolicy {
* at the tail so it cannot disturb the cached prompt prefix.
*/
toolDisciplineRules?: string;
/**
* Provider-family residual appended once to the assembled leaf system
* prompt (CL-8297). Tool-budget text for grok, the XML task_guidance block
* for claude; the GPT seam stays unfilled until its lane lands (#1135).
* Withheld from orchestrators and appended at the tail so it cannot
* disturb the cached prompt prefix. Undefined for families that need none.
*/
promptResidual?: string | undefined;
}

const DEFAULT_WRAP_UP_NUDGE_TEXT =
Expand All @@ -58,6 +66,7 @@ const GROK_WRAP_UP_NUDGE_TEXT =
// sits comfortably above the observed healthy ceiling; the nudge is a
// check-in, not a stop, so erring high costs nothing. Tightened only for
// families with observed runaway tool-only behavior (see grok below).
/** Default policy: permissive, no finish bias, no prompt residual. */
const DEFAULT_POLICY: Omit<ModelFamilyPolicy, "family"> = {
toolOnlyTurnNudgeAt: 25,
wrapUpNudgeText: DEFAULT_WRAP_UP_NUDGE_TEXT,
Expand All @@ -66,6 +75,16 @@ const DEFAULT_POLICY: Omit<ModelFamilyPolicy, "family"> = {
advertisedToolDeny: [],
};

// Generic 4-line tool-budget residual (CL-8297). Grok leaves get this via
// promptResidual today; other families leave the seam unfilled until their
// own lanes land. Deliberately free of ceremony lines and family-specific
// routing — pure tool-loop budget.
export const GROK_TOOL_BUDGET_RESIDUAL =
"Tool budget:\n" +
"- Batch independent tool calls into a single turn.\n" +
"- Never re-issue a tool call whose result you already have.\n" +
"- When the next call would only repeat prior work, write the report instead.";

// A directly observed 14-turn pure-tool-call session for this family
// previously motivated a tightened nudge/pause pair here (6/10). That pair
// was miscalibrated: it fired on a session that was making real progress
Expand All @@ -83,6 +102,8 @@ const GROK_POLICY: Omit<ModelFamilyPolicy, "family"> = {
applyGrokFinishBias: true,
// Leaf value; the resolver clears it for orchestrators below.
advertisedToolDeny: ["skill_search"],
// Leaf value; the resolver clears it for orchestrators below.
promptResidual: GROK_TOOL_BUDGET_RESIDUAL,
};

// Kimi (Moonshot) detection ships now so callers can branch on family, but
Expand All @@ -109,6 +130,45 @@ const MUSE_POLICY: Omit<ModelFamilyPolicy, "family"> = {
toolDisciplineRules: MUSE_TOOL_DISCIPLINE_RULES,
};

// Single grok finish-bias + ceremony residual (CL-8296): the finish-bias
// bullets plus the three ceremony lines from the CL-7768 design (no git, no
// pre-plan, verify once), merged into one block with no line twice. The don't
// re-read idea appears exactly once (the "re-open paths" bullet) — it is not
// repeated. Grok-only: detectModelFamily has no glm family, so per the <30min
// rule no GLM row ships here. The text lives here once; exported for
// buildGrokLeafAntiThrashNote (prompts.ts), which returns it verbatim, so the
// prompt carries exactly one copy. This is a different block from the CL-8297
// tool-budget hook (GROK_TOOL_BUDGET_RESIDUAL, surfaced via the
// ModelFamilyPolicy.promptResidual field): grok leaves carry both, each once.
export const GROK_PROMPT_RESIDUAL = [
"Finish bias (xAI / Grok worker):",
"- Once you can answer the dispatch brief, prefer the structured report over another speculative tool call.",
"- If the next call would only re-open paths you already read, write the report instead.",
"- When the dispatch brief's done-definition is met, write the report envelope instead of making one more search or micro-edit.",
"- Route file and web work through the dedicated tools, never run_shell — mining showed grok reaching for shell first when a typed tool already covered the job.",
"- Never run git add, git commit, git stash, or any other state-changing git command unless the user asks.",
"- Do not narrate a plan before acting on a small task; act, then report.",
"- Verify with the test command once at the end, not after every edit.",
].join("\n");
// Claude (Anthropic) ships one XML residual, not prose: a prose residual did
// nothing, but a single <task_guidance> block cut Sonnet tokens. The block is
// the whole residual — never a full-prompt XML renderer. The text lives here
// (policy owns data); buildClaudeTaskGuidanceNote (prompts.ts) returns it
// verbatim so the prompt carries exactly one copy.
export const CLAUDE_TASK_GUIDANCE_NOTE = [
"<task_guidance>",
"Autonomous coding turn: finish the work in this turn on your best judgment.",
"1. Follow the dispatch brief exactly; its Success criteria are the done-definition.",
"2. Batch independent tool calls into a single turn; work from files already read this session.",
"3. Finish the task when the done-definition is met: prefer the structured report envelope over another tool call.",
"</task_guidance>",
].join("\n");

const CLAUDE_POLICY: Omit<ModelFamilyPolicy, "family"> = {
...DEFAULT_POLICY,
promptResidual: CLAUDE_TASK_GUIDANCE_NOTE,
};

export function resolveModelFamilyPolicy(input: {
providerName: string;
model?: string;
Expand All @@ -126,6 +186,7 @@ export function resolveModelFamilyPolicy(input: {
...policy,
applyGrokFinishBias: policy.applyGrokFinishBias && !orchestrator,
advertisedToolDeny: orchestrator ? [] : policy.advertisedToolDeny,
promptResidual: orchestrator ? undefined : policy.promptResidual,
};
}
case "kimi":
Expand All @@ -136,6 +197,13 @@ export function resolveModelFamilyPolicy(input: {
};
case "muse":
return { family, ...MUSE_POLICY };
case "claude":
// Like the grok finish-bias residual, the task_guidance block only makes
// sense on leaf workers — orchestrators dispatch rather than doing the
// work directly, so they resolve to the permissive default (no residual).
return orchestrator
? { family, ...DEFAULT_POLICY }
: { family, ...CLAUDE_POLICY };
default:
return { family: "default", ...DEFAULT_POLICY };
}
Expand Down
22 changes: 22 additions & 0 deletions src/agent/prompt-sizes.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -332,3 +332,25 @@ describe("skywalker grok prefix (infer envelope vs trimmed director)", () => {
}
});
});

describe("grok tool-budget residual (CL-8297)", () => {
const countOccurrences = (haystack: string, needle: string): number =>
haystack.split(needle).length - 1;

test("a grok leaf director prompt contains the tool budget exactly once", () => {
const prompt = assembleDirectorPrompt("builder", "grok");
expect(countOccurrences(prompt, "Tool budget:")).toBe(1);
});

test("default-family and orchestrator prompts carry no tool budget", () => {
const defaultPrompt = assembleDirectorPrompt("builder", "default");
expect(defaultPrompt).not.toContain("Tool budget:");
// The default probe (openai/gpt-4.1) resolves to the default family, so
// the default column carries no family residual at all — not the claude
// task_guidance block either.
expect(defaultPrompt).not.toContain("<task_guidance>");
expect(assembleDirectorPrompt("skywalker", "grok")).not.toContain(
"Tool budget:",
);
});
});
14 changes: 11 additions & 3 deletions src/agent/prompt-sizes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
type DirectorPackage,
} from "./directors/types.js";
import { buildChatSystemPrompt, buildSubAgentSystemPrompt } from "./prompts.js";
import { resolveModelFamilyPolicy } from "./model-family-policy.js";
import {
formatAgentsMdExtension,
MAX_AGENTS_MD_BYTES,
Expand All @@ -35,7 +36,10 @@ import { webSearchDefinition } from "../tools/web-search.js";
* Assembles each director prompt exactly as src/subagent/run.ts does:
* extensions=[director systemPromptRole] + environment + tools +
* appendix, with the Grok finish-bias note gated by
* shouldApplyGrokAntiThrash (leaves on Grok-family providers only).
* shouldApplyGrokAntiThrash (leaves on Grok-family providers only) and the
* family promptResidual (CL-8297 tool budget for grok leaves, XML
* task_guidance block for claude leaves) resolved from the model family
* policy.
*
* The env and provider inputs are pinned here so sizes never drift with the
* machine, date, or checkout — only real prompt changes move the numbers.
Expand All @@ -53,9 +57,11 @@ export const CANONICAL_PROMPT_ENV: EnvironmentInfo = {
};

const GROK_PROVIDER = { providerName: "xai/default", model: "grok-4.6" };
// Default-family probe: openai/gpt-4.1 resolves to the default family (the
// gpt row lands later in CL-8310), so the default column carries no residual.
const DEFAULT_PROVIDER = {
providerName: "anthropic",
model: "claude-sonnet-4",
providerName: "openai",
model: "gpt-4.1",
};

/** Families in the size table: default assembly vs Grok (+finish-bias note). */
Expand Down Expand Up @@ -156,6 +162,7 @@ export function assembleDirectorPrompt(
const pkg = DIRECTOR_REGISTRY[directorId];
const orchestrator = pkg.spawn.maySpawn;
const provider = family === "grok" ? GROK_PROVIDER : DEFAULT_PROVIDER;
const policy = resolveModelFamilyPolicy({ ...provider, orchestrator });
return buildSubAgentSystemPrompt(
[formatDirectorSystemPrompt(pkg)],
CANONICAL_PROMPT_ENV,
Expand All @@ -164,6 +171,7 @@ export function assembleDirectorPrompt(
orchestrator,
toolNames: canonicalToolNamesForDirector(pkg, family),
grokAntiThrash: shouldApplyGrokAntiThrash({ ...provider, orchestrator }),
promptResidual: policy.promptResidual,
},
);
}
Expand Down
Loading
Loading