feat: add validation warnings for incomplete custom evaluator templates

AGENTS.md

@@ -15,13 +15,16 @@ This is a TypeScript monorepo for AgentV - an AI agent evaluation framework.
 - `packages/core/` - Evaluation engine, providers, grading
 - `apps/cli/` - Command-line interface (published as `agentv`)
 
-## Essential Commands
-- `pnpm install` - Install dependencies
-- `pnpm build` - Build all packages
-- `pnpm test` - Run tests
-- `pnpm typecheck` - Type checking
-- `pnpm lint` - Lint code
-- `pnpm format` - Format with Prettier
+## Quality Assurance Workflow
+
+After making any significant changes (refactoring, new features, bug fixes), always run the following verification steps in order:
+
+1. `pnpm build` - Ensure code compiles without errors
+2. `pnpm typecheck` - Verify TypeScript type safety across the monorepo
+3. `pnpm lint` - Check code style and catch potential issues
+4. `pnpm test` - Run all tests to verify functionality
+
+Only consider the work complete when all four steps pass successfully. This ensures code quality, prevents regressions, and maintains the integrity of the codebase.
 
 ## Functional Testing
 

apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agentv",
-  "version": "0.15.1",
+  "version": "0.16.0",
   "description": "CLI entry point for AgentV",
   "type": "module",
   "repository": {

apps/cli/src/commands/eval/run-eval.ts

@@ -2,6 +2,7 @@ import {
   runEvaluation as defaultRunEvaluation,
   type EvaluationCache,
   type EvaluationResult,
+  type EvalCase,
   type ProviderResponse,
   ensureVSCodeSubagents,
   loadEvalCases,
@@ -205,6 +206,7 @@ async function prepareFileMetadata(params: {
   readonly options: NormalizedOptions;
 }): Promise<{
   readonly evalIds: readonly string[];
+  readonly evalCases: readonly EvalCase[];
   readonly selection: TargetSelection;
   readonly inlineTargetLabel: string;
 }> {
@@ -240,7 +242,7 @@ async function prepareFileMetadata(params: {
     ? evalCases.filter((value) => value.id === options.evalId).map((value) => value.id)
     : evalCases.map((value) => value.id);
 
-  return { evalIds: filteredIds, selection, inlineTargetLabel };
+  return { evalIds: filteredIds, evalCases, selection, inlineTargetLabel };
 }
 
 async function runWithLimit<T>(
@@ -276,6 +278,7 @@ async function runSingleEvalFile(params: {
   readonly displayIdTracker: { getOrAssign(evalKey: string): number };
   readonly selection: TargetSelection;
   readonly inlineTargetLabel: string;
+  readonly evalCases: readonly EvalCase[];
 }): Promise<{ results: EvaluationResult[]; promptDumpDir?: string }> {
   const {
     testFilePath,
@@ -291,6 +294,7 @@ async function runSingleEvalFile(params: {
     displayIdTracker,
     selection,
     inlineTargetLabel,
+    evalCases,
   } =
     params;
 
@@ -363,6 +367,7 @@ async function runSingleEvalFile(params: {
     cache,
     useCache: options.cache,
     evalId: options.evalId,
+    evalCases,
     verbose: options.verbose,
     maxConcurrency: resolvedWorkers,
     onResult: async (result: EvaluationResult) => {
@@ -420,7 +425,7 @@ export async function runEvalCommand(input: RunEvalCommandInput): Promise<void>
     : undefined;
   const fileMetadata = new Map<
     string,
-    { readonly evalIds: readonly string[]; readonly selection: TargetSelection; readonly inlineTargetLabel: string }
+    { readonly evalIds: readonly string[]; readonly evalCases: readonly EvalCase[]; readonly selection: TargetSelection; readonly inlineTargetLabel: string }
   >();
   for (const testFilePath of resolvedTestFiles) {
     const meta = await prepareFileMetadata({
@@ -483,6 +488,7 @@ export async function runEvalCommand(input: RunEvalCommandInput): Promise<void>
         displayIdTracker,
         selection: targetPrep.selection,
         inlineTargetLabel: targetPrep.inlineTargetLabel,
+        evalCases: targetPrep.evalCases,
       });
 
       allResults.push(...result.results);

apps/cli/test/fixtures/mock-run-evaluation.ts

@@ -15,6 +15,8 @@ interface RunEvaluationOptionsLike {
   readonly cache?: unknown;
   readonly useCache?: boolean;
   readonly testId?: string;
+  readonly evalId?: string;
+  readonly evalCases?: ReadonlyArray<unknown>;
   readonly verbose?: boolean;
   readonly onResult?: (result: EvaluationResultLike) => Promise<void> | void;
 }

openspec/changes/add-custom-evaluator-validation/proposal.md

@@ -0,0 +1,48 @@
+# Proposal: Add Custom Evaluator Template Validation
+
+## Summary
+
+Add validation warnings when custom LLM judge evaluator templates are missing required template variables, helping users create effective evaluator prompts that have something to evaluate against.
+
+## Problem
+
+Custom evaluator templates can be created without including the essential fields needed to perform evaluation. When a template contains only variables like `{{ question }}` without `{{ candidate_answer }}` or `{{ expected_messages }}`, the LLM judge receives no content to actually evaluate, resulting in meaningless or inconsistent evaluations.
+
+## Solution
+
+Add validation logic in `resolveCustomPrompt()` that checks custom evaluator templates (both file-based and inline) for the presence of at least one required evaluation field:
+- `{{ candidate_answer }}` - to evaluate the agent's response
+- `{{ expected_messages }}` - to compare against expected output
+
+When neither field is present, display a clear warning message explaining what's missing and why it's needed.
+
+## Impact
+
+### User Impact
+- **Improved UX**: Users get immediate feedback when their evaluator templates are incomplete
+- **Better Documentation**: Clear guidance on what fields are required
+- **Prevents Silent Failures**: Catches incomplete templates before they produce poor evaluations
+
+### System Impact
+- **Minimal**: Validation runs during template loading, adds negligible overhead
+- **Non-breaking**: Warning only, doesn't block evaluation execution
+
+## Scope
+
+### In Scope
+- Validation function to check for required template variables
+- Warning messages for file-based and inline prompts
+- Unit tests covering validation scenarios
+- Documentation updates in custom-evaluators.md
+
+### Out of Scope
+- Validation of other template variables
+- Enforcement of evaluation criteria or scoring rubrics
+- Validation of code-based evaluators
+
+## Implementation Notes
+
+- Validation occurs in `packages/core/src/evaluation/orchestrator.ts`
+- Uses regex pattern matching to detect template variables
+- Warnings use ANSI color codes (yellow) for visibility
+- Permissive approach: warns but allows evaluation to continue

openspec/changes/add-custom-evaluator-validation/specs/validation/spec.md

@@ -0,0 +1,64 @@
+# Validation Capability Delta
+
+## Why
+
+Custom LLM judge evaluator templates can be created without the essential fields needed to perform evaluation. When a template contains only `{{ question }}` without `{{ candidate_answer }}` or `{{ expected_messages }}`, the LLM judge has no content to evaluate against, resulting in meaningless evaluations.
+
+This validation provides immediate feedback to help users create effective evaluator templates.
+
+## ADDED Requirements
+
+### Requirement: Custom Evaluator Template Validation
+
+The system SHALL validate custom LLM judge evaluator templates to ensure they contain fields necessary for evaluation.
+
+#### Scenario: Template missing both required fields shows warning
+
+**Given** a custom evaluator template with content `"{{ question }}"`
+**When** the template is loaded
+**Then** a warning is displayed containing:
+- Message: "Custom evaluator template at [source] is missing required fields"
+- List of required fields: `{{ candidate_answer }}` and `{{ expected_messages }}`
+- Explanation: "Without these, there is nothing to evaluate against"
+
+#### Scenario: Template with candidate_answer does not warn
+
+**Given** a custom evaluator template containing `"{{ candidate_answer }}"`
+**When** the template is loaded
+**Then** no validation warning is displayed
+
+#### Scenario: Template with expected_messages does not warn
+
+**Given** a custom evaluator template containing `"{{ expected_messages }}"`
+**When** the template is loaded
+**Then** no validation warning is displayed
+
+#### Scenario: Validation applies to file-based prompts
+
+**Given** an evaluator configured with `promptPath: "./my-eval.md"`
+**And** the file contains only `"{{ question }}"`
+**When** the custom prompt is resolved
+**Then** a warning is displayed referencing the file path
+
+#### Scenario: Invalid template variables are detected
+
+**Given** a custom evaluator template containing `"{{ candiate_answer }} for {{ invalid_var }}"`
+**When** validation runs
+**Then** a warning is displayed listing the invalid variables
+**And** the warning lists all valid template variables
+
+#### Scenario: Validation is permissive
+
+**Given** a custom evaluator template missing required fields
+**When** validation runs
+**Then** a warning is displayed
+**But** evaluation continues without blocking
+
+## Implementation Notes
+
+- Validation occurs in `validateCustomPromptContent()` function
+- Uses regex patterns to detect template variables: `/\{\{\s*(candidate_answer|expected_messages)\s*\}\}/`
+- Validates all template variables against a whitelist of valid names
+- Valid variables: `candidate_answer`, `expected_messages`, `question`, `expected_outcome`, `reference_answer`, `input_messages`, `output_messages` (legacy)
+- Warnings use ANSI yellow color codes for visibility
+- File-based (`promptPath`) evaluators are validated

openspec/changes/add-custom-evaluator-validation/tasks.md

@@ -0,0 +1,28 @@
+# Tasks: Add Custom Evaluator Template Validation
+
+## Implementation Tasks
+
+- [x] Add `validateCustomPromptContent()` function to check for required template variables
+- [x] Add validation for invalid/unknown template variables (catches typos)
+- [x] Update `resolveCustomPrompt()` to call validation for both file-based and inline prompts
+- [x] Add ANSI color constants for warning output
+- [x] Create warning message explaining required fields
+- [x] Create warning message for invalid variables
+- [x] Add unit test: warns when template is missing required fields
+- [x] Add unit test: no warning when `{{ candidate_answer }}` is present
+- [x] Add unit test: no warning when `{{ expected_messages }}` is present
+- [x] Add unit test: warns when template contains invalid variables (typos, undefined vars)
+- [x] Add unit test: validates inline prompts in addition to file-based prompts
+- [x] Fix lint errors (ESLint no-explicit-any warnings)
+- [x] Fix TypeScript compilation errors
+- [x] Verify all tests pass
+- [x] Update documentation to reflect validation behavior
+
+## Validation
+
+- [x] All unit tests pass (13 tests in orchestrator.test.ts)
+- [x] Lint passes (`pnpm lint`)
+- [x] TypeScript compilation passes (`pnpm typecheck`)
+- [x] Warning appears for incomplete templates
+- [x] Warning appears for invalid/unknown template variables
+- [x] No warnings for valid templates with required fields

openspec/changes/rename-output-messages-variable/proposal.md

@@ -0,0 +1,58 @@
+# Proposal: Rename output_messages Template Variable to expected_messages
+
+## Summary
+
+Rename the LLM judge template variable from `{{ output_messages }}` to `{{ expected_messages }}` for consistency with eval file schema and clearer semantics.
+
+## Problem
+
+The template variable name `output_messages` is inconsistent and confusing:
+- **Eval schema uses**: `expected_messages` field (not `output_messages`)
+- **Validation checks for**: `{{ expected_messages }}` template variable
+- **Documentation shows**: `{{output_messages}}` as available variable
+- **Name is ambiguous**: "output" could refer to agent output or expected output
+
+This inconsistency creates confusion when writing custom evaluator templates.
+
+## Solution
+
+Rename `output_messages` to `expected_messages` throughout the codebase:
+1. Template variable substitution in evaluators
+2. Documentation references
+3. Keep backward compatibility during transition (optional - TBD)
+
+## Impact
+
+### User Impact
+- **Breaking Change**: Existing custom evaluator templates using `{{ output_messages }}` will need updates
+- **Migration Path**: Users must update their templates to use `{{ expected_messages }}`
+- **Documentation**: All examples and guides updated to reflect new name
+
+### System Impact
+- **Code Changes**: Update variable name in evaluator template substitution
+- **Documentation**: Update custom-evaluators.md and related docs
+- **Tests**: Update test fixtures and expectations
+
+## Scope
+
+### In Scope
+- Rename template variable in `evaluators.ts` 
+- Update documentation in `.claude/skills/agentv-eval-builder/references/custom-evaluators.md`
+- Update any example evaluator templates
+- Validation already checks for `expected_messages` (no change needed)
+
+### Out of Scope
+- Backward compatibility support (decide if needed)
+- Changes to eval file schema (already uses `expected_messages`)
+- Changes to validation logic (already correct)
+
+## Questions for Decision
+
+1. **Backward Compatibility**: Should we support both `{{ output_messages }}` and `{{ expected_messages }}` temporarily?
+   - **Option A**: Breaking change - only support new name
+   - **Option B**: Support both with deprecation warning
+   - **Recommendation**: Option A (breaking change) - simpler, cleaner
+
+2. **Migration Timing**: When to release this change?
+   - Should coordinate with next major/minor version
+   - Include in release notes with migration guide

openspec/changes/rename-output-messages-variable/specs/evaluation/spec.md

@@ -0,0 +1,42 @@
+# Evaluation Capability Delta
+
+## Why
+
+The template variable name `output_messages` is inconsistent with the eval file schema field `expected_messages` and creates confusion. Renaming to `expected_messages` provides consistency across the system and clearer semantics (these are the expected messages, not just output).
+
+## MODIFIED Requirements
+
+### Requirement: LLM Judge Template Variables
+
+The system SHALL provide template variables for LLM judge custom evaluator prompts to reference evaluation context.
+
+**Change**: Renamed `{{ output_messages }}` variable to `{{ expected_messages }}` for consistency with eval schema.
+
+#### Scenario: expected_messages variable available
+
+**Given** a custom LLM judge evaluator template
+**When** template substitution occurs
+**Then** the `{{ expected_messages }}` variable contains JSON stringified output segments
+
+**Note**: Previously named `{{ output_messages }}` - renamed for consistency with eval schema
+
+#### Scenario: Template variable substitution
+
+**Given** an eval case with `output_segments` data
+**And** a custom evaluator template containing `{{ expected_messages }}`
+**When** the template is rendered
+**Then** `{{ expected_messages }}` is replaced with `JSON.stringify(output_segments, null, 2)`
+
+#### Scenario: Backward compatibility not supported
+
+**Given** a custom evaluator template containing `{{ output_messages }}`
+**When** template substitution occurs  
+**Then** the variable is NOT substituted (remains as literal text)
+**And** validation may warn about missing required fields
+
+## Implementation Notes
+
+- Update variable name in `evaluators.ts` template substitution logic
+- Update documentation to reflect new variable name
+- This is a breaking change - existing templates must update to use `{{ expected_messages }}`
+- Validation already checks for `{{ expected_messages }}` (no changes needed)

openspec/changes/rename-output-messages-variable/tasks.md

@@ -0,0 +1,31 @@
+# Tasks: Rename output_messages Template Variable
+
+## Implementation Tasks
+
+- [ ] Update variable name in `packages/core/src/evaluation/evaluators.ts` (line ~105)
+  - Change `output_messages` to `expected_messages` in template variable substitution
+- [ ] Update documentation in `.claude/skills/agentv-eval-builder/references/custom-evaluators.md`
+  - Change `{{output_messages}}` to `{{expected_messages}}` in template variables list
+- [ ] Search for any other references to `output_messages` in documentation
+  - Check README files
+  - Check example evaluator templates
+  - Check inline code comments
+- [ ] Update tests if they reference the old variable name
+  - Check test fixtures
+  - Check test assertions
+- [ ] Verify validation still works correctly (already checks for `expected_messages`)
+
+## Validation
+
+- [ ] All unit tests pass
+- [ ] Lint passes (`pnpm lint`)
+- [ ] TypeScript compilation passes (`pnpm typecheck`)
+- [ ] Custom evaluator templates work with `{{ expected_messages }}`
+- [ ] Documentation accurately reflects the change
+- [ ] No references to `output_messages` remain in user-facing content
+
+## Migration Notes
+
+- **Breaking Change**: Existing custom evaluator templates using `{{ output_messages }}` will need to be updated
+- Include in release notes with clear migration instructions
+- Consider version bump (minor or major depending on project versioning policy)

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentv/core",
-  "version": "0.15.1",
+  "version": "0.16.0",
   "description": "Primitive runtime components for AgentV",
   "type": "module",
   "repository": {