From 6acfec8b32024e645fb01ab287df9c5dcb666798 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 24 Mar 2026 05:04:39 +0000 Subject: [PATCH 1/3] refactor(cli): retire generate command surface --- README.md | 5 +- apps/cli/README.md | 5 +- apps/cli/src/commands/generate/index.ts | 46 ---- apps/cli/src/commands/generate/rubrics.ts | 215 ------------------ apps/cli/src/index.ts | 19 +- .../test/fixtures/mock-rubric-generator.ts | 45 ---- .../generate-migration.integration.test.ts | 44 ++++ .../test/generate-rubrics.integration.test.ts | 184 --------------- .../src/content/docs/evaluation/rubrics.mdx | 10 +- apps/web/src/content/docs/tools/generate.mdx | 52 ----- .../features/rubric/evals/dataset.eval.yaml | 4 +- .../src/evaluation/evaluators/llm-grader.ts | 2 +- .../skills/agentv-eval-writer/SKILL.md | 5 +- 13 files changed, 70 insertions(+), 566 deletions(-) delete mode 100644 apps/cli/src/commands/generate/index.ts delete mode 100644 apps/cli/src/commands/generate/rubrics.ts delete mode 100644 apps/cli/test/fixtures/mock-rubric-generator.ts create mode 100644 apps/cli/test/generate-migration.integration.test.ts delete mode 100644 apps/cli/test/generate-rubrics.integration.test.ts delete mode 100644 apps/web/src/content/docs/tools/generate.mdx diff --git a/README.md b/README.md index 0062949d7..315d1efc6 100644 --- a/README.md +++ b/README.md @@ -518,10 +518,7 @@ tests: Scoring: `(satisfied weights) / (total weights)` → verdicts: `pass` (≥0.8), `borderline` (≥0.6), `fail` -Auto-generate rubrics from expected outcomes: -```bash -agentv generate rubrics evals/my-eval.yaml -``` +Author assertions directly in your eval file. When you want help choosing between simple assertions, deterministic graders, and LLM-based graders, use the `agentv-eval-writer` skill. See [rubric evaluator](https://agentv.dev/evaluation/rubrics/) for detailed patterns. diff --git a/apps/cli/README.md b/apps/cli/README.md index 0062949d7..315d1efc6 100644 --- a/apps/cli/README.md +++ b/apps/cli/README.md @@ -518,10 +518,7 @@ tests: Scoring: `(satisfied weights) / (total weights)` → verdicts: `pass` (≥0.8), `borderline` (≥0.6), `fail` -Auto-generate rubrics from expected outcomes: -```bash -agentv generate rubrics evals/my-eval.yaml -``` +Author assertions directly in your eval file. When you want help choosing between simple assertions, deterministic graders, and LLM-based graders, use the `agentv-eval-writer` skill. See [rubric evaluator](https://agentv.dev/evaluation/rubrics/) for detailed patterns. diff --git a/apps/cli/src/commands/generate/index.ts b/apps/cli/src/commands/generate/index.ts deleted file mode 100644 index 0cdf525e9..000000000 --- a/apps/cli/src/commands/generate/index.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { command, flag, option, optional, positional, string, subcommands } from 'cmd-ts'; - -import { generateRubricsCommand } from './rubrics.js'; - -const rubricsCommand = command({ - name: 'rubrics', - description: 'Generate rubrics from criteria in YAML eval file', - args: { - file: positional({ - type: string, - displayName: 'file', - description: 'Path to YAML eval file', - }), - target: option({ - type: optional(string), - long: 'target', - short: 't', - description: 'Override target for rubric generation (default: file target or openai:gpt-4o)', - }), - verbose: flag({ - long: 'verbose', - short: 'v', - description: 'Show detailed progress', - }), - }, - handler: async ({ file, target, verbose }) => { - try { - await generateRubricsCommand({ - file, - target, - verbose, - }); - } catch (error) { - console.error(`Error: ${(error as Error).message}`); - process.exit(1); - } - }, -}); - -export const generateCommand = subcommands({ - name: 'generate', - description: 'Generate evaluation artifacts', - cmds: { - rubrics: rubricsCommand, - }, -}); diff --git a/apps/cli/src/commands/generate/rubrics.ts b/apps/cli/src/commands/generate/rubrics.ts deleted file mode 100644 index f5d70d389..000000000 --- a/apps/cli/src/commands/generate/rubrics.ts +++ /dev/null @@ -1,215 +0,0 @@ -import { readFile, writeFile } from 'node:fs/promises'; -import path from 'node:path'; -import { pathToFileURL } from 'node:url'; -import { type YAMLSeq, isMap, isSeq, parseDocument } from 'yaml'; - -import { - type GenerateRubricsOptions as CoreGenerateRubricsOptions, - type JsonObject, - type JsonValue, - type RubricItem, - createProvider, - generateRubrics, -} from '@agentv/core'; -import { selectTarget } from '../eval/targets.js'; - -interface GenerateRubricsOptions { - readonly file: string; - readonly target?: string; - readonly verbose?: boolean; -} - -interface RawEvalCase { - readonly id?: string; - readonly criteria?: string; - readonly outcome?: string; - readonly question?: string; - readonly reference_answer?: string; - readonly rubrics?: JsonValue; - readonly input?: readonly unknown[]; -} - -interface RawTestSuite { - readonly tests?: readonly unknown[]; - readonly target?: string; - readonly execution?: { - readonly target?: string; - }; -} - -function isJsonObject(value: unknown): value is JsonObject { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -function asString(value: unknown): string | undefined { - return typeof value === 'string' ? value : undefined; -} - -// Allow overriding generateRubrics for testing -async function loadRubricGenerator(): Promise< - (options: CoreGenerateRubricsOptions) => Promise -> { - const customGenerator = process.env.AGENTEVO_CLI_RUBRIC_GENERATOR; - if (customGenerator) { - const generatorPath = path.resolve(customGenerator); - const generatorUrl = pathToFileURL(generatorPath).href; - const module = (await import(generatorUrl)) as { - generateRubrics: (options: CoreGenerateRubricsOptions) => Promise; - }; - return module.generateRubrics; - } - return generateRubrics; -} - -export async function generateRubricsCommand(options: GenerateRubricsOptions): Promise { - const { file, target: targetOverride, verbose } = options; - - console.log(`Generating rubrics for: ${file}`); - - // Read the YAML file - const absolutePath = path.resolve(file); - const content = await readFile(absolutePath, 'utf8'); - const doc = parseDocument(content); - const parsed = doc.toJSON() as unknown; - - if (!isJsonObject(parsed)) { - throw new Error(`Invalid YAML file format: ${file}`); - } - - const suite = parsed as RawTestSuite; - const evalcases = suite.tests; - - if (!Array.isArray(evalcases)) { - throw new Error(`No tests found in ${file}`); - } - - // Resolve target using the same logic as eval command - const targetSelection = await selectTarget({ - testFilePath: absolutePath, - repoRoot: process.cwd(), - cwd: process.cwd(), - cliTargetName: targetOverride, - dryRun: false, - dryRunDelay: 0, - dryRunDelayMin: 0, - dryRunDelayMax: 0, - env: process.env, - }); - - if (verbose) { - console.log(`Using target: ${targetSelection.targetName}`); - } - - const provider = createProvider(targetSelection.resolvedTarget); - const generateRubricsFunc = await loadRubricGenerator(); - - let updatedCount = 0; - let skippedCount = 0; - - // Get the cases node from the document for modification - const evalcasesNode = doc.getIn(['tests']); - if (!evalcasesNode || !isSeq(evalcasesNode)) { - throw new Error('tests must be a sequence'); - } - - // Process each test - for (let i = 0; i < evalcases.length; i++) { - const rawCase = evalcases[i]; - if (!isJsonObject(rawCase)) { - continue; - } - - const evalCase = rawCase as RawEvalCase; - const id = asString(evalCase.id) ?? 'unknown'; - const expectedOutcome = asString(evalCase.criteria) ?? asString(evalCase.outcome); - - // Skip if no expected outcome - if (!expectedOutcome) { - if (verbose) { - console.log(` Skipping ${id}: no criteria`); - } - skippedCount++; - continue; - } - - // Skip if rubrics already exist - if (evalCase.rubrics !== undefined) { - if (verbose) { - console.log(` Skipping ${id}: rubrics already defined`); - } - skippedCount++; - continue; - } - - // Generate rubrics - console.log(` Generating rubrics for: ${id}`); - - const question = extractQuestion(evalCase); - const referenceAnswer = asString(evalCase.reference_answer); - - const rubrics = await generateRubricsFunc({ - criteria: expectedOutcome, - question, - referenceAnswer, - provider, - }); - - // Update the test with rubrics in the YAML document - const caseNode = (evalcasesNode as YAMLSeq).items[i]; - if (caseNode && isMap(caseNode)) { - caseNode.set( - 'rubrics', - rubrics - .filter((r) => r.outcome !== undefined) - .map((r) => ({ - id: r.id, - outcome: r.outcome, - weight: r.weight, - required: r.required ?? true, - })), - ); - } - - updatedCount++; - - if (verbose) { - console.log(` Generated ${rubrics.length} rubric(s)`); - } - } - - // Write back to file - if (updatedCount > 0) { - const output = doc.toString(); - await writeFile(absolutePath, output, 'utf8'); - console.log(`\nUpdated ${updatedCount} test(s) with generated rubrics`); - if (skippedCount > 0) { - console.log(`Skipped ${skippedCount} test(s)`); - } - } else { - console.log('\nNo tests updated (all already have rubrics or missing criteria)'); - } -} - -function extractQuestion(evalCase: RawEvalCase): string | undefined { - const explicitQuestion = asString(evalCase.question); - if (explicitQuestion) { - return explicitQuestion; - } - - // Try to extract from input - const inputMessages = evalCase.input; - if (!Array.isArray(inputMessages)) { - return undefined; - } - - for (const msg of inputMessages) { - if (!isJsonObject(msg)) { - continue; - } - if (msg.role === 'user' && typeof msg.content === 'string') { - return msg.content; - } - } - - return undefined; -} diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index 9833ea825..dda795127 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -6,7 +6,6 @@ import { convertCommand } from './commands/convert/index.js'; import { createCommand } from './commands/create/index.js'; import { evalPromptCommand } from './commands/eval/commands/prompt/index.js'; import { evalCommand } from './commands/eval/index.js'; -import { generateCommand } from './commands/generate/index.js'; import { initCmdTsCommand } from './commands/init/index.js'; import { resultsCommand } from './commands/results/index.js'; import { resultsServeCommand } from './commands/results/serve.js'; @@ -28,7 +27,6 @@ export const app = subcommands({ compare: compareCommand, convert: convertCommand, create: createCommand, - generate: generateCommand, init: initCmdTsCommand, results: resultsCommand, self: selfCommand, @@ -56,7 +54,6 @@ const TOP_LEVEL_COMMANDS = new Set([ 'compare', 'convert', 'create', - 'generate', 'init', 'results', 'self', @@ -113,6 +110,18 @@ export function preprocessArgv(argv: string[]): string[] { return result; } +function isRetiredGenerateInvocation(argv: string[]): boolean { + return argv.slice(2)[0] === 'generate'; +} + +function getRetiredGenerateMessage(): string { + return [ + '`agentv generate` has been retired.', + 'Use the `agentv-eval-writer` skill for eval authoring help.', + 'Choose assertions that fit the criteria: plain assertions when they are enough, deterministic graders when possible, and LLM-based grading when judgment is needed.', + ].join('\n'); +} + export async function runCli(argv: string[] = process.argv): Promise { // Kick off update check: reads from local cache (fast), spawns a detached // child to refresh if stale. The notice is printed on process exit so it @@ -125,6 +134,10 @@ export async function runCli(argv: string[] = process.argv): Promise { updateNotice = n; }); + if (isRetiredGenerateInvocation(argv)) { + throw new Error(getRetiredGenerateMessage()); + } + const processedArgv = preprocessArgv(argv); await run(binary(app), processedArgv); } diff --git a/apps/cli/test/fixtures/mock-rubric-generator.ts b/apps/cli/test/fixtures/mock-rubric-generator.ts deleted file mode 100644 index 820771ae8..000000000 --- a/apps/cli/test/fixtures/mock-rubric-generator.ts +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Mock rubric generator for testing the generate rubrics command. - * This file is loaded by the CLI when AGENTEVO_CLI_RUBRIC_GENERATOR is set. - */ - -import type { GenerateRubricsOptions } from '@agentv/core'; -import type { RubricItem } from '@agentv/core'; - -/** - * Mock implementation of generateRubrics that returns deterministic test rubrics. - */ -export async function generateRubrics( - options: GenerateRubricsOptions, -): Promise { - // Return mock rubrics based on the criteria - const { criteria } = options; - - // Generate deterministic rubrics for testing - return [ - { - id: 'completeness', - outcome: `Answer must address all aspects of: ${criteria}`, - weight: 0.4, - required: true, - }, - { - id: 'accuracy', - outcome: 'Answer must be factually correct', - weight: 0.3, - required: true, - }, - { - id: 'clarity', - outcome: 'Answer must be clear and well-structured', - weight: 0.2, - required: false, - }, - { - id: 'conciseness', - outcome: 'Answer must be concise without unnecessary details', - weight: 0.1, - required: false, - }, - ]; -} diff --git a/apps/cli/test/generate-migration.integration.test.ts b/apps/cli/test/generate-migration.integration.test.ts new file mode 100644 index 000000000..69738b30f --- /dev/null +++ b/apps/cli/test/generate-migration.integration.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from 'bun:test'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execa } from 'execa'; +import { assertCoreBuild } from './setup-core-build.js'; + +assertCoreBuild(); + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const projectRoot = path.resolve(__dirname, '../../..'); +const CLI_ENTRY = path.join(projectRoot, 'apps/cli/src/cli.ts'); + +describe('generate command migration', () => { + it('does not list generate in top-level help', async () => { + const result = await execa('bun', [CLI_ENTRY, '--help'], { + cwd: projectRoot, + reject: false, + env: { + ...process.env, + CI: 'true', + }, + }); + + expect(result.exitCode).toBe(0); + expect(result.stdout).not.toContain('generate'); + }); + + it('prints a migration message for retired generate invocations', async () => { + const result = await execa('bun', [CLI_ENTRY, 'generate', 'rubrics', 'evals/example.yaml'], { + cwd: projectRoot, + reject: false, + env: { + ...process.env, + CI: 'true', + }, + }); + + expect(result.exitCode).toBe(1); + expect(result.stderr).toContain('`agentv generate` has been retired.'); + expect(result.stderr).toContain('Use the `agentv-eval-writer` skill for eval authoring help.'); + expect(result.stderr).toContain('Choose assertions that fit the criteria'); + }); +}); diff --git a/apps/cli/test/generate-rubrics.integration.test.ts b/apps/cli/test/generate-rubrics.integration.test.ts deleted file mode 100644 index 334e3940f..000000000 --- a/apps/cli/test/generate-rubrics.integration.test.ts +++ /dev/null @@ -1,184 +0,0 @@ -/** - * Integration tests for the `agentv generate rubrics` command. - * - * These tests verify that the command correctly updates YAML files with generated rubrics. - */ - -import { afterEach, describe, expect, it } from 'bun:test'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { execa } from 'execa'; -import { assertCoreBuild } from './setup-core-build.js'; - -assertCoreBuild(); - -interface GenerateFixture { - readonly baseDir: string; - readonly suiteDir: string; - readonly testFilePath: string; -} - -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); -const projectRoot = path.resolve(__dirname, '../../..'); -const CLI_ENTRY = path.join(projectRoot, 'apps/cli/src/cli.ts'); -const MOCK_GENERATOR = path.join(projectRoot, 'apps/cli/test/fixtures/mock-rubric-generator.ts'); - -async function createFixture(withComments = false): Promise { - const baseDir = await mkdtemp(path.join(tmpdir(), 'agentv-generate-test-')); - const suiteDir = path.join(baseDir, 'suite'); - await mkdir(suiteDir, { recursive: true }); - - const agentvDir = path.join(suiteDir, '.agentv'); - await mkdir(agentvDir, { recursive: true }); - - const targetsPath = path.join(agentvDir, 'targets.yaml'); - const targetsContent = `$schema: agentv-targets-v2.2 -targets: - - name: default - provider: mock -`; - await writeFile(targetsPath, targetsContent, 'utf8'); - - const testFilePath = path.join(suiteDir, 'test.yaml'); - - let testFileContent = `description: Generate rubrics integration test - -execution: - target: default - -tests:`; - - if (withComments) { - testFileContent += '\n # This is a test comment\n # TODO: update this test case'; - } - - testFileContent += ` - - id: case-with-outcome - criteria: System should respond politely and helpfully`; - - testFileContent += ` - input: - - role: user - content: "Hello, can you help me?" - expected_output: - - role: assistant - content: "Of course! How can I help you?"`; - - if (withComments) { - testFileContent += '\n\n # Another test case'; - } - - testFileContent += ` - - id: case-without-outcome - input: - - role: user - content: "This case has no expected outcome" -`; - - await writeFile(testFilePath, testFileContent, 'utf8'); - - return { baseDir, suiteDir, testFilePath }; -} - -async function runGenerateRubrics( - fixture: GenerateFixture, - args: readonly string[] = [], -): Promise<{ stdout: string; stderr: string }> { - try { - const result = await execa( - 'bun', - [CLI_ENTRY, 'generate', 'rubrics', fixture.testFilePath, ...args], - { - cwd: fixture.suiteDir, - env: { - ...process.env, - CI: 'true', - AGENTEVO_CLI_RUBRIC_GENERATOR: MOCK_GENERATOR, - }, - reject: false, - }, - ); - return { stdout: result.stdout, stderr: result.stderr }; - } catch (error) { - if (error && typeof error === 'object' && 'stdout' in error && 'stderr' in error) { - return { - stdout: String(error.stdout), - stderr: String(error.stderr), - }; - } - throw error; - } -} - -describe('generate rubrics integration', () => { - let fixture: GenerateFixture; - - afterEach(async () => { - if (fixture) { - await rm(fixture.baseDir, { recursive: true, force: true }); - } - }); - - it('should generate rubrics for cases with criteria', async () => { - fixture = await createFixture(); - - const { stdout } = await runGenerateRubrics(fixture); - - expect(stdout).toContain('Generating rubrics for:'); - expect(stdout).toContain('case-with-outcome'); - expect(stdout).toMatch(/Updated \d+ test\(s\) with generated rubrics/); - - // Read the updated file - const content = await readFile(fixture.testFilePath, 'utf8'); - - // Check that rubrics were added - expect(content).toContain('rubrics:'); - expect(content).toContain('id:'); - expect(content).toContain('outcome:'); - expect(content).toContain('weight:'); - expect(content).toContain('required:'); - - // Case without outcome should not have rubrics - const caseWithoutOutcome = content.split('case-without-outcome')[1]; - expect(caseWithoutOutcome).not.toContain('rubrics:'); - }, 30000); - - it('should preserve comments and structure', async () => { - fixture = await createFixture(true); - - const originalContent = await readFile(fixture.testFilePath, 'utf8'); - - // Verify comments exist in original - expect(originalContent).toContain('# This is a test comment'); - expect(originalContent).toContain('# TODO: update this test case'); - expect(originalContent).toContain('# Another test case'); - - await runGenerateRubrics(fixture); - - // Read the updated file - const updatedContent = await readFile(fixture.testFilePath, 'utf8'); - - // Check that comments are preserved - expect(updatedContent).toContain('# This is a test comment'); - expect(updatedContent).toContain('# TODO: update this test case'); - expect(updatedContent).toContain('# Another test case'); - - // Check that rubrics were added - expect(updatedContent).toContain('rubrics:'); - - // Check that structure is maintained (basic indentation check) - const lines = updatedContent.split('\n'); - const casesLine = lines.findIndex((line) => line.includes('tests:')); - const rubricsLine = lines.findIndex((line) => line.includes('rubrics:')); - - // rubrics should be indented more than tests - if (casesLine >= 0 && rubricsLine >= 0) { - const casesIndent = lines[casesLine].match(/^\s*/)?.[0].length ?? 0; - const rubricsIndent = lines[rubricsLine].match(/^\s*/)?.[0].length ?? 0; - expect(rubricsIndent).toBeGreaterThan(casesIndent); - } - }, 30000); -}); diff --git a/apps/web/src/content/docs/evaluation/rubrics.mdx b/apps/web/src/content/docs/evaluation/rubrics.mdx index 6f03d7c8c..d17bec308 100644 --- a/apps/web/src/content/docs/evaluation/rubrics.mdx +++ b/apps/web/src/content/docs/evaluation/rubrics.mdx @@ -115,15 +115,9 @@ score = sum(criterion_score / 10 * weight) / sum(total_weights) | `borderline` | ≥ 0.6 | | `fail` | < 0.6 | -## Auto-Generate Rubrics +## Authoring Rubrics -Generate rubrics from expected outcomes: - -```bash -agentv generate rubrics evals/my-eval.yaml -``` - -This analyzes each test's `criteria` and creates structured rubric criteria. +Write rubric criteria directly in `assertions`. If you want help choosing between plain assertions, deterministic evaluators, and rubric or LLM-based grading, use the `agentv-eval-writer` skill. Keep the evaluator choice driven by the criteria rather than one fixed recipe. ## Combining with Other Evaluators diff --git a/apps/web/src/content/docs/tools/generate.mdx b/apps/web/src/content/docs/tools/generate.mdx deleted file mode 100644 index f309d2e39..000000000 --- a/apps/web/src/content/docs/tools/generate.mdx +++ /dev/null @@ -1,52 +0,0 @@ ---- -title: Generate -description: Auto-generate rubrics from expected outcomes -sidebar: - order: 3 ---- - -The `generate` command creates structured evaluation criteria from your tests. - -## Generate Rubrics - -Auto-generate rubrics from expected outcomes: - -```bash -agentv generate rubrics evals/my-eval.yaml -``` - -This analyzes each test's `criteria` field and creates structured rubric criteria with appropriate weights. - -## How It Works - -1. Reads each test's `criteria` -2. Uses an LLM to decompose the criteria into individual checkable rubric items -3. Assigns weights based on importance -4. Writes rubric criteria back under `assertions: - type: rubrics` - -## Example - -Before: - -```yaml -tests: - - id: quicksort - criteria: Explains quicksort with time complexity and examples - input: Explain quicksort -``` - -After running `agentv generate rubrics`: - -```yaml -tests: - - id: quicksort - criteria: Explains quicksort with time complexity and examples - input: Explain quicksort - assertions: - - type: rubrics - criteria: - - Explains divide-and-conquer approach - - Describes partition step - - States O(n log n) average time complexity - - Provides a concrete example -``` diff --git a/examples/features/rubric/evals/dataset.eval.yaml b/examples/features/rubric/evals/dataset.eval.yaml index 9f3e8ad0e..691cf7884 100644 --- a/examples/features/rubric/evals/dataset.eval.yaml +++ b/examples/features/rubric/evals/dataset.eval.yaml @@ -150,7 +150,7 @@ tests: # ========================================== # Example 4: Using criteria without rubrics # Demonstrates: criteria as the evaluation criteria field - # Note: Use 'agentv generate rubrics' to create rubrics from criteria + # Note: Author assertions directly when you want stricter evaluation structure # ========================================== - id: summary-task @@ -176,7 +176,7 @@ tests: cuts and renewable energy adoption. # No rubrics defined - will use default llm_grader evaluator - # To generate rubrics: agentv generate rubrics evals/dataset.eval.yaml + # Add assertions directly if you want deterministic checks or explicit rubrics # ========================================== # Example 5: Multi-criteria score_ranges diff --git a/packages/core/src/evaluation/evaluators/llm-grader.ts b/packages/core/src/evaluation/evaluators/llm-grader.ts index 0f570e418..8b420e7d3 100644 --- a/packages/core/src/evaluation/evaluators/llm-grader.ts +++ b/packages/core/src/evaluation/evaluators/llm-grader.ts @@ -292,7 +292,7 @@ export class LlmGraderEvaluator implements Evaluator { ): Promise { if (!rubrics || rubrics.length === 0) { throw new Error( - `No rubrics found for evaluator "${context.evaluator?.name ?? 'llm-grader'}". Run "agentv generate rubrics" first.`, + `No rubrics found for evaluator "${context.evaluator?.name ?? 'llm-grader'}". Add rubric criteria under assertions or use the agentv-eval-writer skill for authoring help.`, ); } diff --git a/plugins/agentv-dev/skills/agentv-eval-writer/SKILL.md b/plugins/agentv-dev/skills/agentv-eval-writer/SKILL.md index 7d3ae408b..4dc8bfbd2 100644 --- a/plugins/agentv-dev/skills/agentv-eval-writer/SKILL.md +++ b/plugins/agentv-dev/skills/agentv-eval-writer/SKILL.md @@ -549,8 +549,9 @@ agentv compare --baseline # agentv compare --baseline --candidate # pairwise agentv compare # two-file pairwise -# Generate rubrics from criteria -agentv generate rubrics [--target ] +# Author assertions directly in the eval file +# Prefer simple assertions when they fit the criteria; use deterministic or LLM-based graders when needed +agentv validate ``` ## Code Judge SDK From 20ef843f3f84ac4ce79a6a8fe71f9d92105928fb Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 24 Mar 2026 07:04:30 +0000 Subject: [PATCH 2/3] refactor(cli): drop generate migration shim --- apps/cli/src/index.ts | 16 ---------------- .../test/generate-migration.integration.test.ts | 16 ---------------- 2 files changed, 32 deletions(-) diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index dda795127..9128bca24 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -110,18 +110,6 @@ export function preprocessArgv(argv: string[]): string[] { return result; } -function isRetiredGenerateInvocation(argv: string[]): boolean { - return argv.slice(2)[0] === 'generate'; -} - -function getRetiredGenerateMessage(): string { - return [ - '`agentv generate` has been retired.', - 'Use the `agentv-eval-writer` skill for eval authoring help.', - 'Choose assertions that fit the criteria: plain assertions when they are enough, deterministic graders when possible, and LLM-based grading when judgment is needed.', - ].join('\n'); -} - export async function runCli(argv: string[] = process.argv): Promise { // Kick off update check: reads from local cache (fast), spawns a detached // child to refresh if stale. The notice is printed on process exit so it @@ -134,10 +122,6 @@ export async function runCli(argv: string[] = process.argv): Promise { updateNotice = n; }); - if (isRetiredGenerateInvocation(argv)) { - throw new Error(getRetiredGenerateMessage()); - } - const processedArgv = preprocessArgv(argv); await run(binary(app), processedArgv); } diff --git a/apps/cli/test/generate-migration.integration.test.ts b/apps/cli/test/generate-migration.integration.test.ts index 69738b30f..5d1452b19 100644 --- a/apps/cli/test/generate-migration.integration.test.ts +++ b/apps/cli/test/generate-migration.integration.test.ts @@ -25,20 +25,4 @@ describe('generate command migration', () => { expect(result.exitCode).toBe(0); expect(result.stdout).not.toContain('generate'); }); - - it('prints a migration message for retired generate invocations', async () => { - const result = await execa('bun', [CLI_ENTRY, 'generate', 'rubrics', 'evals/example.yaml'], { - cwd: projectRoot, - reject: false, - env: { - ...process.env, - CI: 'true', - }, - }); - - expect(result.exitCode).toBe(1); - expect(result.stderr).toContain('`agentv generate` has been retired.'); - expect(result.stderr).toContain('Use the `agentv-eval-writer` skill for eval authoring help.'); - expect(result.stderr).toContain('Choose assertions that fit the criteria'); - }); }); From 0e1cf5ac0b9cb94deb1c06789cc3f2100b04b600 Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Tue, 24 Mar 2026 07:12:56 +0000 Subject: [PATCH 3/3] test(cli): remove retired generate cleanup test --- .../generate-migration.integration.test.ts | 28 ------------------- 1 file changed, 28 deletions(-) delete mode 100644 apps/cli/test/generate-migration.integration.test.ts diff --git a/apps/cli/test/generate-migration.integration.test.ts b/apps/cli/test/generate-migration.integration.test.ts deleted file mode 100644 index 5d1452b19..000000000 --- a/apps/cli/test/generate-migration.integration.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { describe, expect, it } from 'bun:test'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { execa } from 'execa'; -import { assertCoreBuild } from './setup-core-build.js'; - -assertCoreBuild(); - -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); -const projectRoot = path.resolve(__dirname, '../../..'); -const CLI_ENTRY = path.join(projectRoot, 'apps/cli/src/cli.ts'); - -describe('generate command migration', () => { - it('does not list generate in top-level help', async () => { - const result = await execa('bun', [CLI_ENTRY, '--help'], { - cwd: projectRoot, - reject: false, - env: { - ...process.env, - CI: 'true', - }, - }); - - expect(result.exitCode).toBe(0); - expect(result.stdout).not.toContain('generate'); - }); -});