diff --git a/README.md b/README.md index 0062949d7..315d1efc6 100644 --- a/README.md +++ b/README.md @@ -518,10 +518,7 @@ tests: Scoring: `(satisfied weights) / (total weights)` → verdicts: `pass` (≥0.8), `borderline` (≥0.6), `fail` -Auto-generate rubrics from expected outcomes: -```bash -agentv generate rubrics evals/my-eval.yaml -``` +Author assertions directly in your eval file. When you want help choosing between simple assertions, deterministic graders, and LLM-based graders, use the `agentv-eval-writer` skill. See [rubric evaluator](https://agentv.dev/evaluation/rubrics/) for detailed patterns. diff --git a/apps/cli/README.md b/apps/cli/README.md index 0062949d7..315d1efc6 100644 --- a/apps/cli/README.md +++ b/apps/cli/README.md @@ -518,10 +518,7 @@ tests: Scoring: `(satisfied weights) / (total weights)` → verdicts: `pass` (≥0.8), `borderline` (≥0.6), `fail` -Auto-generate rubrics from expected outcomes: -```bash -agentv generate rubrics evals/my-eval.yaml -``` +Author assertions directly in your eval file. When you want help choosing between simple assertions, deterministic graders, and LLM-based graders, use the `agentv-eval-writer` skill. See [rubric evaluator](https://agentv.dev/evaluation/rubrics/) for detailed patterns. diff --git a/apps/cli/src/commands/generate/index.ts b/apps/cli/src/commands/generate/index.ts deleted file mode 100644 index 0cdf525e9..000000000 --- a/apps/cli/src/commands/generate/index.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { command, flag, option, optional, positional, string, subcommands } from 'cmd-ts'; - -import { generateRubricsCommand } from './rubrics.js'; - -const rubricsCommand = command({ - name: 'rubrics', - description: 'Generate rubrics from criteria in YAML eval file', - args: { - file: positional({ - type: string, - displayName: 'file', - description: 'Path to YAML eval file', - }), - target: option({ - type: optional(string), - long: 'target', - short: 't', - description: 'Override target for rubric generation (default: file target or openai:gpt-4o)', - }), - verbose: flag({ - long: 'verbose', - short: 'v', - description: 'Show detailed progress', - }), - }, - handler: async ({ file, target, verbose }) => { - try { - await generateRubricsCommand({ - file, - target, - verbose, - }); - } catch (error) { - console.error(`Error: ${(error as Error).message}`); - process.exit(1); - } - }, -}); - -export const generateCommand = subcommands({ - name: 'generate', - description: 'Generate evaluation artifacts', - cmds: { - rubrics: rubricsCommand, - }, -}); diff --git a/apps/cli/src/commands/generate/rubrics.ts b/apps/cli/src/commands/generate/rubrics.ts deleted file mode 100644 index f5d70d389..000000000 --- a/apps/cli/src/commands/generate/rubrics.ts +++ /dev/null @@ -1,215 +0,0 @@ -import { readFile, writeFile } from 'node:fs/promises'; -import path from 'node:path'; -import { pathToFileURL } from 'node:url'; -import { type YAMLSeq, isMap, isSeq, parseDocument } from 'yaml'; - -import { - type GenerateRubricsOptions as CoreGenerateRubricsOptions, - type JsonObject, - type JsonValue, - type RubricItem, - createProvider, - generateRubrics, -} from '@agentv/core'; -import { selectTarget } from '../eval/targets.js'; - -interface GenerateRubricsOptions { - readonly file: string; - readonly target?: string; - readonly verbose?: boolean; -} - -interface RawEvalCase { - readonly id?: string; - readonly criteria?: string; - readonly outcome?: string; - readonly question?: string; - readonly reference_answer?: string; - readonly rubrics?: JsonValue; - readonly input?: readonly unknown[]; -} - -interface RawTestSuite { - readonly tests?: readonly unknown[]; - readonly target?: string; - readonly execution?: { - readonly target?: string; - }; -} - -function isJsonObject(value: unknown): value is JsonObject { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -function asString(value: unknown): string | undefined { - return typeof value === 'string' ? value : undefined; -} - -// Allow overriding generateRubrics for testing -async function loadRubricGenerator(): Promise< - (options: CoreGenerateRubricsOptions) => Promise -> { - const customGenerator = process.env.AGENTEVO_CLI_RUBRIC_GENERATOR; - if (customGenerator) { - const generatorPath = path.resolve(customGenerator); - const generatorUrl = pathToFileURL(generatorPath).href; - const module = (await import(generatorUrl)) as { - generateRubrics: (options: CoreGenerateRubricsOptions) => Promise; - }; - return module.generateRubrics; - } - return generateRubrics; -} - -export async function generateRubricsCommand(options: GenerateRubricsOptions): Promise { - const { file, target: targetOverride, verbose } = options; - - console.log(`Generating rubrics for: ${file}`); - - // Read the YAML file - const absolutePath = path.resolve(file); - const content = await readFile(absolutePath, 'utf8'); - const doc = parseDocument(content); - const parsed = doc.toJSON() as unknown; - - if (!isJsonObject(parsed)) { - throw new Error(`Invalid YAML file format: ${file}`); - } - - const suite = parsed as RawTestSuite; - const evalcases = suite.tests; - - if (!Array.isArray(evalcases)) { - throw new Error(`No tests found in ${file}`); - } - - // Resolve target using the same logic as eval command - const targetSelection = await selectTarget({ - testFilePath: absolutePath, - repoRoot: process.cwd(), - cwd: process.cwd(), - cliTargetName: targetOverride, - dryRun: false, - dryRunDelay: 0, - dryRunDelayMin: 0, - dryRunDelayMax: 0, - env: process.env, - }); - - if (verbose) { - console.log(`Using target: ${targetSelection.targetName}`); - } - - const provider = createProvider(targetSelection.resolvedTarget); - const generateRubricsFunc = await loadRubricGenerator(); - - let updatedCount = 0; - let skippedCount = 0; - - // Get the cases node from the document for modification - const evalcasesNode = doc.getIn(['tests']); - if (!evalcasesNode || !isSeq(evalcasesNode)) { - throw new Error('tests must be a sequence'); - } - - // Process each test - for (let i = 0; i < evalcases.length; i++) { - const rawCase = evalcases[i]; - if (!isJsonObject(rawCase)) { - continue; - } - - const evalCase = rawCase as RawEvalCase; - const id = asString(evalCase.id) ?? 'unknown'; - const expectedOutcome = asString(evalCase.criteria) ?? asString(evalCase.outcome); - - // Skip if no expected outcome - if (!expectedOutcome) { - if (verbose) { - console.log(` Skipping ${id}: no criteria`); - } - skippedCount++; - continue; - } - - // Skip if rubrics already exist - if (evalCase.rubrics !== undefined) { - if (verbose) { - console.log(` Skipping ${id}: rubrics already defined`); - } - skippedCount++; - continue; - } - - // Generate rubrics - console.log(` Generating rubrics for: ${id}`); - - const question = extractQuestion(evalCase); - const referenceAnswer = asString(evalCase.reference_answer); - - const rubrics = await generateRubricsFunc({ - criteria: expectedOutcome, - question, - referenceAnswer, - provider, - }); - - // Update the test with rubrics in the YAML document - const caseNode = (evalcasesNode as YAMLSeq).items[i]; - if (caseNode && isMap(caseNode)) { - caseNode.set( - 'rubrics', - rubrics - .filter((r) => r.outcome !== undefined) - .map((r) => ({ - id: r.id, - outcome: r.outcome, - weight: r.weight, - required: r.required ?? true, - })), - ); - } - - updatedCount++; - - if (verbose) { - console.log(` Generated ${rubrics.length} rubric(s)`); - } - } - - // Write back to file - if (updatedCount > 0) { - const output = doc.toString(); - await writeFile(absolutePath, output, 'utf8'); - console.log(`\nUpdated ${updatedCount} test(s) with generated rubrics`); - if (skippedCount > 0) { - console.log(`Skipped ${skippedCount} test(s)`); - } - } else { - console.log('\nNo tests updated (all already have rubrics or missing criteria)'); - } -} - -function extractQuestion(evalCase: RawEvalCase): string | undefined { - const explicitQuestion = asString(evalCase.question); - if (explicitQuestion) { - return explicitQuestion; - } - - // Try to extract from input - const inputMessages = evalCase.input; - if (!Array.isArray(inputMessages)) { - return undefined; - } - - for (const msg of inputMessages) { - if (!isJsonObject(msg)) { - continue; - } - if (msg.role === 'user' && typeof msg.content === 'string') { - return msg.content; - } - } - - return undefined; -} diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index 9833ea825..9128bca24 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -6,7 +6,6 @@ import { convertCommand } from './commands/convert/index.js'; import { createCommand } from './commands/create/index.js'; import { evalPromptCommand } from './commands/eval/commands/prompt/index.js'; import { evalCommand } from './commands/eval/index.js'; -import { generateCommand } from './commands/generate/index.js'; import { initCmdTsCommand } from './commands/init/index.js'; import { resultsCommand } from './commands/results/index.js'; import { resultsServeCommand } from './commands/results/serve.js'; @@ -28,7 +27,6 @@ export const app = subcommands({ compare: compareCommand, convert: convertCommand, create: createCommand, - generate: generateCommand, init: initCmdTsCommand, results: resultsCommand, self: selfCommand, @@ -56,7 +54,6 @@ const TOP_LEVEL_COMMANDS = new Set([ 'compare', 'convert', 'create', - 'generate', 'init', 'results', 'self', diff --git a/apps/cli/test/fixtures/mock-rubric-generator.ts b/apps/cli/test/fixtures/mock-rubric-generator.ts deleted file mode 100644 index 820771ae8..000000000 --- a/apps/cli/test/fixtures/mock-rubric-generator.ts +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Mock rubric generator for testing the generate rubrics command. - * This file is loaded by the CLI when AGENTEVO_CLI_RUBRIC_GENERATOR is set. - */ - -import type { GenerateRubricsOptions } from '@agentv/core'; -import type { RubricItem } from '@agentv/core'; - -/** - * Mock implementation of generateRubrics that returns deterministic test rubrics. - */ -export async function generateRubrics( - options: GenerateRubricsOptions, -): Promise { - // Return mock rubrics based on the criteria - const { criteria } = options; - - // Generate deterministic rubrics for testing - return [ - { - id: 'completeness', - outcome: `Answer must address all aspects of: ${criteria}`, - weight: 0.4, - required: true, - }, - { - id: 'accuracy', - outcome: 'Answer must be factually correct', - weight: 0.3, - required: true, - }, - { - id: 'clarity', - outcome: 'Answer must be clear and well-structured', - weight: 0.2, - required: false, - }, - { - id: 'conciseness', - outcome: 'Answer must be concise without unnecessary details', - weight: 0.1, - required: false, - }, - ]; -} diff --git a/apps/cli/test/generate-rubrics.integration.test.ts b/apps/cli/test/generate-rubrics.integration.test.ts deleted file mode 100644 index 334e3940f..000000000 --- a/apps/cli/test/generate-rubrics.integration.test.ts +++ /dev/null @@ -1,184 +0,0 @@ -/** - * Integration tests for the `agentv generate rubrics` command. - * - * These tests verify that the command correctly updates YAML files with generated rubrics. - */ - -import { afterEach, describe, expect, it } from 'bun:test'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { execa } from 'execa'; -import { assertCoreBuild } from './setup-core-build.js'; - -assertCoreBuild(); - -interface GenerateFixture { - readonly baseDir: string; - readonly suiteDir: string; - readonly testFilePath: string; -} - -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); -const projectRoot = path.resolve(__dirname, '../../..'); -const CLI_ENTRY = path.join(projectRoot, 'apps/cli/src/cli.ts'); -const MOCK_GENERATOR = path.join(projectRoot, 'apps/cli/test/fixtures/mock-rubric-generator.ts'); - -async function createFixture(withComments = false): Promise { - const baseDir = await mkdtemp(path.join(tmpdir(), 'agentv-generate-test-')); - const suiteDir = path.join(baseDir, 'suite'); - await mkdir(suiteDir, { recursive: true }); - - const agentvDir = path.join(suiteDir, '.agentv'); - await mkdir(agentvDir, { recursive: true }); - - const targetsPath = path.join(agentvDir, 'targets.yaml'); - const targetsContent = `$schema: agentv-targets-v2.2 -targets: - - name: default - provider: mock -`; - await writeFile(targetsPath, targetsContent, 'utf8'); - - const testFilePath = path.join(suiteDir, 'test.yaml'); - - let testFileContent = `description: Generate rubrics integration test - -execution: - target: default - -tests:`; - - if (withComments) { - testFileContent += '\n # This is a test comment\n # TODO: update this test case'; - } - - testFileContent += ` - - id: case-with-outcome - criteria: System should respond politely and helpfully`; - - testFileContent += ` - input: - - role: user - content: "Hello, can you help me?" - expected_output: - - role: assistant - content: "Of course! How can I help you?"`; - - if (withComments) { - testFileContent += '\n\n # Another test case'; - } - - testFileContent += ` - - id: case-without-outcome - input: - - role: user - content: "This case has no expected outcome" -`; - - await writeFile(testFilePath, testFileContent, 'utf8'); - - return { baseDir, suiteDir, testFilePath }; -} - -async function runGenerateRubrics( - fixture: GenerateFixture, - args: readonly string[] = [], -): Promise<{ stdout: string; stderr: string }> { - try { - const result = await execa( - 'bun', - [CLI_ENTRY, 'generate', 'rubrics', fixture.testFilePath, ...args], - { - cwd: fixture.suiteDir, - env: { - ...process.env, - CI: 'true', - AGENTEVO_CLI_RUBRIC_GENERATOR: MOCK_GENERATOR, - }, - reject: false, - }, - ); - return { stdout: result.stdout, stderr: result.stderr }; - } catch (error) { - if (error && typeof error === 'object' && 'stdout' in error && 'stderr' in error) { - return { - stdout: String(error.stdout), - stderr: String(error.stderr), - }; - } - throw error; - } -} - -describe('generate rubrics integration', () => { - let fixture: GenerateFixture; - - afterEach(async () => { - if (fixture) { - await rm(fixture.baseDir, { recursive: true, force: true }); - } - }); - - it('should generate rubrics for cases with criteria', async () => { - fixture = await createFixture(); - - const { stdout } = await runGenerateRubrics(fixture); - - expect(stdout).toContain('Generating rubrics for:'); - expect(stdout).toContain('case-with-outcome'); - expect(stdout).toMatch(/Updated \d+ test\(s\) with generated rubrics/); - - // Read the updated file - const content = await readFile(fixture.testFilePath, 'utf8'); - - // Check that rubrics were added - expect(content).toContain('rubrics:'); - expect(content).toContain('id:'); - expect(content).toContain('outcome:'); - expect(content).toContain('weight:'); - expect(content).toContain('required:'); - - // Case without outcome should not have rubrics - const caseWithoutOutcome = content.split('case-without-outcome')[1]; - expect(caseWithoutOutcome).not.toContain('rubrics:'); - }, 30000); - - it('should preserve comments and structure', async () => { - fixture = await createFixture(true); - - const originalContent = await readFile(fixture.testFilePath, 'utf8'); - - // Verify comments exist in original - expect(originalContent).toContain('# This is a test comment'); - expect(originalContent).toContain('# TODO: update this test case'); - expect(originalContent).toContain('# Another test case'); - - await runGenerateRubrics(fixture); - - // Read the updated file - const updatedContent = await readFile(fixture.testFilePath, 'utf8'); - - // Check that comments are preserved - expect(updatedContent).toContain('# This is a test comment'); - expect(updatedContent).toContain('# TODO: update this test case'); - expect(updatedContent).toContain('# Another test case'); - - // Check that rubrics were added - expect(updatedContent).toContain('rubrics:'); - - // Check that structure is maintained (basic indentation check) - const lines = updatedContent.split('\n'); - const casesLine = lines.findIndex((line) => line.includes('tests:')); - const rubricsLine = lines.findIndex((line) => line.includes('rubrics:')); - - // rubrics should be indented more than tests - if (casesLine >= 0 && rubricsLine >= 0) { - const casesIndent = lines[casesLine].match(/^\s*/)?.[0].length ?? 0; - const rubricsIndent = lines[rubricsLine].match(/^\s*/)?.[0].length ?? 0; - expect(rubricsIndent).toBeGreaterThan(casesIndent); - } - }, 30000); -}); diff --git a/apps/web/src/content/docs/evaluation/rubrics.mdx b/apps/web/src/content/docs/evaluation/rubrics.mdx index 6f03d7c8c..d17bec308 100644 --- a/apps/web/src/content/docs/evaluation/rubrics.mdx +++ b/apps/web/src/content/docs/evaluation/rubrics.mdx @@ -115,15 +115,9 @@ score = sum(criterion_score / 10 * weight) / sum(total_weights) | `borderline` | ≥ 0.6 | | `fail` | < 0.6 | -## Auto-Generate Rubrics +## Authoring Rubrics -Generate rubrics from expected outcomes: - -```bash -agentv generate rubrics evals/my-eval.yaml -``` - -This analyzes each test's `criteria` and creates structured rubric criteria. +Write rubric criteria directly in `assertions`. If you want help choosing between plain assertions, deterministic evaluators, and rubric or LLM-based grading, use the `agentv-eval-writer` skill. Keep the evaluator choice driven by the criteria rather than one fixed recipe. ## Combining with Other Evaluators diff --git a/apps/web/src/content/docs/tools/generate.mdx b/apps/web/src/content/docs/tools/generate.mdx deleted file mode 100644 index f309d2e39..000000000 --- a/apps/web/src/content/docs/tools/generate.mdx +++ /dev/null @@ -1,52 +0,0 @@ ---- -title: Generate -description: Auto-generate rubrics from expected outcomes -sidebar: - order: 3 ---- - -The `generate` command creates structured evaluation criteria from your tests. - -## Generate Rubrics - -Auto-generate rubrics from expected outcomes: - -```bash -agentv generate rubrics evals/my-eval.yaml -``` - -This analyzes each test's `criteria` field and creates structured rubric criteria with appropriate weights. - -## How It Works - -1. Reads each test's `criteria` -2. Uses an LLM to decompose the criteria into individual checkable rubric items -3. Assigns weights based on importance -4. Writes rubric criteria back under `assertions: - type: rubrics` - -## Example - -Before: - -```yaml -tests: - - id: quicksort - criteria: Explains quicksort with time complexity and examples - input: Explain quicksort -``` - -After running `agentv generate rubrics`: - -```yaml -tests: - - id: quicksort - criteria: Explains quicksort with time complexity and examples - input: Explain quicksort - assertions: - - type: rubrics - criteria: - - Explains divide-and-conquer approach - - Describes partition step - - States O(n log n) average time complexity - - Provides a concrete example -``` diff --git a/examples/features/rubric/evals/dataset.eval.yaml b/examples/features/rubric/evals/dataset.eval.yaml index 9f3e8ad0e..691cf7884 100644 --- a/examples/features/rubric/evals/dataset.eval.yaml +++ b/examples/features/rubric/evals/dataset.eval.yaml @@ -150,7 +150,7 @@ tests: # ========================================== # Example 4: Using criteria without rubrics # Demonstrates: criteria as the evaluation criteria field - # Note: Use 'agentv generate rubrics' to create rubrics from criteria + # Note: Author assertions directly when you want stricter evaluation structure # ========================================== - id: summary-task @@ -176,7 +176,7 @@ tests: cuts and renewable energy adoption. # No rubrics defined - will use default llm_grader evaluator - # To generate rubrics: agentv generate rubrics evals/dataset.eval.yaml + # Add assertions directly if you want deterministic checks or explicit rubrics # ========================================== # Example 5: Multi-criteria score_ranges diff --git a/packages/core/src/evaluation/evaluators/llm-grader.ts b/packages/core/src/evaluation/evaluators/llm-grader.ts index 0f570e418..8b420e7d3 100644 --- a/packages/core/src/evaluation/evaluators/llm-grader.ts +++ b/packages/core/src/evaluation/evaluators/llm-grader.ts @@ -292,7 +292,7 @@ export class LlmGraderEvaluator implements Evaluator { ): Promise { if (!rubrics || rubrics.length === 0) { throw new Error( - `No rubrics found for evaluator "${context.evaluator?.name ?? 'llm-grader'}". Run "agentv generate rubrics" first.`, + `No rubrics found for evaluator "${context.evaluator?.name ?? 'llm-grader'}". Add rubric criteria under assertions or use the agentv-eval-writer skill for authoring help.`, ); } diff --git a/plugins/agentv-dev/skills/agentv-eval-writer/SKILL.md b/plugins/agentv-dev/skills/agentv-eval-writer/SKILL.md index 7d3ae408b..4dc8bfbd2 100644 --- a/plugins/agentv-dev/skills/agentv-eval-writer/SKILL.md +++ b/plugins/agentv-dev/skills/agentv-eval-writer/SKILL.md @@ -549,8 +549,9 @@ agentv compare --baseline # agentv compare --baseline --candidate # pairwise agentv compare # two-file pairwise -# Generate rubrics from criteria -agentv generate rubrics [--target ] +# Author assertions directly in the eval file +# Prefer simple assertions when they fit the criteria; use deterministic or LLM-based graders when needed +agentv validate ``` ## Code Judge SDK