Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 13 additions & 0 deletions packages/prompt-variance/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"name": "@corbits/prompt-variance",
"version": "0.1.0",
"private": true,
"type": "module",
"license": "SEE LICENSE IN LICENSE.md",
"exports": {
".": {
"types": "./src/index.ts",
"default": "./src/index.ts"
}
}
}
36 changes: 36 additions & 0 deletions packages/prompt-variance/src/assemble.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import { describe, expect, test } from "bun:test";
import { assemble } from "./assemble.js";
import { claudeRow, defaultRow, gptRow, grokRow, museRow } from "./rows.js";

const SECTIONS = ["contract", "Tools (names only): read_file", "context"];

describe("assemble", () => {
test("joins sections with the residual as the last section", () => {
const out = assemble(SECTIONS, grokRow);
expect(out).toContain("contract");
expect(out.trimEnd().endsWith(grokRow.residual)).toBe(true);
});

test("a row without residual leaves the sections unchanged", () => {
expect(assemble(SECTIONS, defaultRow)).toBe(SECTIONS.join("\n\n"));
});

test("muse residual closes the prompt like the shipped tail append", () => {
const out = assemble(SECTIONS, museRow);
expect(out.trimEnd().endsWith(museRow.residual)).toBe(true);
});

test("claude and gpt residuals close the prompt the same way", () => {
expect(
assemble(SECTIONS, claudeRow).trimEnd().endsWith(claudeRow.residual),
).toBe(true);
expect(assemble(SECTIONS, gptRow).trimEnd().endsWith(gptRow.residual)).toBe(
true,
);
});

test("tool mounting is out of scope: assemble returns a string, not tool names", () => {
const out = assemble(SECTIONS, grokRow);
expect(typeof out).toBe("string");
});
});
16 changes: 16 additions & 0 deletions packages/prompt-variance/src/assemble.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
import type { PromptVarianceRow } from "./rows.js";

/**
* Append a family variance row's residual at the tail of the assembled
* sections (CL-8269). An empty residual leaves the sections unchanged.
* Residuals render last so they cannot disturb the cached prompt prefix.
* Tool mounting is out of scope: run.ts applies
* ModelFamilyPolicy.advertisedToolDeny at mount time.
*/
export function assemble(
sections: readonly string[],
familyRow: PromptVarianceRow,
): string {
if (familyRow.residual.length === 0) return sections.join("\n\n");
return [...sections, familyRow.residual].join("\n\n");
}
14 changes: 14 additions & 0 deletions packages/prompt-variance/src/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
export { assemble } from "./assemble.js";
export { resolvePromptVariance } from "./resolve.js";
export {
claudeRow,
defaultRow,
FAMILY_IDS,
FAMILY_ROWS,
gptRow,
grokRow,
grokToolBudgetResidual,
museRow,
type PromptVarianceFamily,
type PromptVarianceRow,
} from "./rows.js";
54 changes: 54 additions & 0 deletions packages/prompt-variance/src/resolve.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
import { describe, expect, test } from "bun:test";
import { resolvePromptVariance } from "./resolve.js";
import { grokRow } from "./rows.js";

describe("resolvePromptVariance", () => {
test("resolves each family to its row", () => {
expect(resolvePromptVariance({ family: "default" }).id).toBe("default");
expect(resolvePromptVariance({ family: "muse" }).id).toBe("muse");
expect(resolvePromptVariance({ family: "grok" }).id).toBe("grok");
expect(resolvePromptVariance({ family: "claude" }).id).toBe("claude");
expect(resolvePromptVariance({ family: "gpt" }).id).toBe("gpt");
});

test("grok keeps its finish-bias residual on leaves", () => {
const row = resolvePromptVariance({ family: "grok", orchestrator: false });
expect(row.residual).toBe(grokRow.residual);
});

test("grok on orchestrators falls back to the default row", () => {
const row = resolvePromptVariance({ family: "grok", orchestrator: true });
expect(row.id).toBe("default");
expect(row.residual).toBe("");
});

test("claude keeps its task_guidance block on leaves, not orchestrators", () => {
expect(
resolvePromptVariance({ family: "claude", orchestrator: false }).residual,
).toContain("<task_guidance>");
const orch = resolvePromptVariance({
family: "claude",
orchestrator: true,
});
expect(orch.id).toBe("default");
expect(orch.residual).toBe("");
});

test("muse keeps its residual on leaves and orchestrators alike", () => {
expect(
resolvePromptVariance({ family: "muse", orchestrator: false }).residual,
).not.toBe("");
expect(
resolvePromptVariance({ family: "muse", orchestrator: true }).residual,
).not.toBe("");
});

test("gpt keeps its narrate-before-tools nudge on leaves and orchestrators alike", () => {
expect(
resolvePromptVariance({ family: "gpt", orchestrator: false }).residual,
).toContain("Narrate before tools");
expect(
resolvePromptVariance({ family: "gpt", orchestrator: true }).residual,
).toContain("Narrate before tools");
});
});
26 changes: 26 additions & 0 deletions packages/prompt-variance/src/resolve.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import {
defaultRow,
FAMILY_ROWS,
type PromptVarianceFamily,
type PromptVarianceRow,
} from "./rows.js";

/**
* Resolve the variance row for a family, mirroring the leaves-only gates in
* the model family policy: the grok finish-bias residual and the claude
* task_guidance block only make sense on leaf workers, so orchestrators
* fall back to the default row. Muse keeps its residual on both — the
* constructors append the same text at the tail either way — and gpt keeps
* its narrate-before-tools nudge on primaries and leaves alike.
*/
export function resolvePromptVariance(input: {
family: PromptVarianceFamily;
orchestrator?: boolean;
}): PromptVarianceRow {
if (input.orchestrator === true) {
if (input.family === "grok" || input.family === "claude") {
return defaultRow;
}
}
return FAMILY_ROWS[input.family];
}
115 changes: 115 additions & 0 deletions packages/prompt-variance/src/rows.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
import { describe, expect, test } from "bun:test";
import {
claudeRow,
defaultRow,
FAMILY_IDS,
FAMILY_ROWS,
gptRow,
grokRow,
grokToolBudgetResidual,
museRow,
type PromptVarianceFamily,
} from "./rows.js";

const CEREMONY_LINES = [
"Never run git add, git commit, git stash, or any other state-changing git command unless the user asks.",
"Do not narrate a plan before acting on a small task; act, then report.",
"Verify with the test command once at the end, not after every edit.",
];

describe("prompt-variance family rows", () => {
test("ships exactly the default/muse/grok/claude/gpt families", () => {
expect([...FAMILY_IDS]).toEqual([
"default",
"muse",
"grok",
"claude",
"gpt",
]);
});

test("has no glm row until its eval lands (CL-8265)", () => {
for (const id of FAMILY_IDS) {
expect(id).not.toBe("glm");
}
});

test("every row carries the id/residual shape — residuals-only, no deny fields", () => {
for (const row of [defaultRow, museRow, grokRow, claudeRow, gptRow]) {
expect(typeof row.id).toBe("string");
expect(typeof row.residual).toBe("string");
expect("advertisedToolDeny" in row).toBe(false);
expect("sectionOmit" in row).toBe(false);
expect("overrides" in row).toBe(false);
}
});

test("row ids match their family and FAMILY_ROWS covers all five", () => {
const ids: PromptVarianceFamily[] = [
defaultRow.id,
museRow.id,
grokRow.id,
claudeRow.id,
gptRow.id,
];
expect(ids).toEqual(["default", "muse", "grok", "claude", "gpt"]);
expect(Object.keys(FAMILY_ROWS).sort()).toEqual([
"claude",
"default",
"gpt",
"grok",
"muse",
]);
});

test("muse row is the shipped CL-7869 tool-discipline text", () => {
expect(museRow.residual).toContain("Tool discipline:");
expect(museRow.residual).toContain("Batch independent tool calls");
expect(museRow.residual).toContain("Never re-read a file");
expect(museRow.residual).toContain("Do not narrate; act.");
});

test("grok row is the single finish-bias + ceremony block (CL-8296)", () => {
expect(grokRow.residual).toContain("Finish bias (xAI / Grok worker):");
expect(grokRow.residual).toContain("prefer the structured report");
expect(grokRow.residual).toContain("re-open paths you already read");
expect(grokRow.residual).toContain("done-definition is met");
expect(grokRow.residual).toContain("never run_shell");
});

test("each ceremony line appears exactly once in the grok row (P2 invariant)", () => {
for (const line of CEREMONY_LINES) {
const occurrences = grokRow.residual.split(line).length - 1;
expect(occurrences).toBe(1);
}
});

test("grok tool-budget residual is ceremony-free and budget text appears once (P1 invariant)", () => {
expect(grokToolBudgetResidual).toContain("Tool budget:");
for (const line of CEREMONY_LINES) {
expect(grokToolBudgetResidual).not.toContain(line);
}
const budgetOccurrences =
grokToolBudgetResidual.split("Tool budget:").length - 1;
expect(budgetOccurrences).toBe(1);
});

test("default row carries no residual", () => {
expect(defaultRow.residual).toBe("");
});

test("claude row is the single XML task_guidance block (CL-8309)", () => {
expect(claudeRow.residual).toContain("<task_guidance>");
expect(claudeRow.residual).toContain("</task_guidance>");
expect(claudeRow.residual).toContain("Follow the dispatch brief exactly");
expect(claudeRow.residual).toContain("Batch independent tool calls");
});

test("gpt row is the narrate-before-tools nudge (CL-8310)", () => {
expect(gptRow.residual).toContain("Narrate before tools (GPT worker):");
expect(gptRow.residual).toContain(
"one short line saying what you are doing",
);
expect(gptRow.residual).toContain("done-definition is met");
});
});
Loading
Loading