diff --git a/CLAUDE.md b/CLAUDE.md index 07ae07c37..f73242530 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,20 +1 @@ -Always follow instructions in `@AGENTS.md`. - - -# OpenSpec Instructions - -These instructions are for AI assistants working in this project. - -Always open `@/openspec/AGENTS.md` when the request: -- Mentions planning or proposals (words like proposal, spec, change, plan) -- Introduces new capabilities, breaking changes, architecture shifts, or big performance/security work -- Sounds ambiguous and you need the authoritative spec before coding - -Use `@/openspec/AGENTS.md` to learn: -- How to create and apply change proposals -- Spec format and conventions -- Project structure and guidelines - -Keep this managed block so 'openspec update' can refresh the instructions. - - +Always follow instructions in `@AGENTS.md`. \ No newline at end of file diff --git a/apps/cli/package.json b/apps/cli/package.json index d76760654..8a2eff5ca 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -1,6 +1,6 @@ { "name": "agentevo", - "version": "0.1.4", + "version": "0.1.6", "description": "CLI entry point for AgentEvo", "type": "module", "repository": { @@ -32,6 +32,7 @@ "@agentevo/core": "workspace:*", "commander": "^12.1.0", "dotenv": "^16.4.5", + "log-update": "^7.0.1", "yaml": "^2.6.1" }, "devDependencies": { diff --git a/apps/cli/src/commands/eval/index.ts b/apps/cli/src/commands/eval/index.ts index 8d895eca9..febf83d61 100644 --- a/apps/cli/src/commands/eval/index.ts +++ b/apps/cli/src/commands/eval/index.ts @@ -18,9 +18,32 @@ export function registerEvalCommand(program: Command): Command { .option("--target ", "Override target name from targets.yaml", "default") .option("--targets ", "Path to targets.yaml (overrides discovery)") .option("--test-id ", "Run only the test case with this identifier") + .option( + "--workers ", + "Number of parallel workers (default: 1, max: 50). Can also be set per-target in targets.yaml", + (value) => parseInteger(value, 1), + ) .option("--out ", "Write results to the specified path") .option("--format ", "Output format: 'jsonl' or 'yaml' (default: jsonl)", "jsonl") .option("--dry-run", "Use mock provider responses instead of real LLM calls", false) + .option( + "--dry-run-delay ", + "Fixed delay in milliseconds for dry-run mode (overridden by delay range if specified)", + (value) => parseInteger(value, 0), + 0, + ) + .option( + "--dry-run-delay-min ", + "Minimum delay in milliseconds for dry-run mode (requires --dry-run-delay-max)", + (value) => parseInteger(value, 0), + 0, + ) + .option( + "--dry-run-delay-max ", + "Maximum delay in milliseconds for dry-run mode (requires --dry-run-delay-min)", + (value) => parseInteger(value, 0), + 0, + ) .option( "--agent-timeout ", "Timeout in seconds for provider responses (default: 120)", diff --git a/apps/cli/src/commands/eval/jsonl-writer.ts b/apps/cli/src/commands/eval/jsonl-writer.ts index ea2d82dfc..486450e5b 100644 --- a/apps/cli/src/commands/eval/jsonl-writer.ts +++ b/apps/cli/src/commands/eval/jsonl-writer.ts @@ -1,3 +1,4 @@ +import { Mutex } from "async-mutex"; import { createWriteStream } from "node:fs"; import { mkdir } from "node:fs/promises"; import path from "node:path"; @@ -5,6 +6,7 @@ import { finished } from "node:stream/promises"; export class JsonlWriter { private readonly stream: ReturnType; + private readonly mutex = new Mutex(); private closed = false; private constructor(stream: ReturnType) { @@ -18,16 +20,18 @@ export class JsonlWriter { } async append(record: unknown): Promise { - if (this.closed) { - throw new Error("Cannot write to closed JSONL writer"); - } - const line = `${JSON.stringify(record)}\n`; - if (!this.stream.write(line)) { - await new Promise((resolve, reject) => { - this.stream.once("drain", resolve); - this.stream.once("error", reject); - }); - } + await this.mutex.runExclusive(async () => { + if (this.closed) { + throw new Error("Cannot write to closed JSONL writer"); + } + const line = `${JSON.stringify(record)}\n`; + if (!this.stream.write(line)) { + await new Promise((resolve, reject) => { + this.stream.once("drain", resolve); + this.stream.once("error", reject); + }); + } + }); } async close(): Promise { diff --git a/apps/cli/src/commands/eval/progress-display.ts b/apps/cli/src/commands/eval/progress-display.ts new file mode 100644 index 000000000..e7aa1403b --- /dev/null +++ b/apps/cli/src/commands/eval/progress-display.ts @@ -0,0 +1,163 @@ +import logUpdate from "log-update"; + +export interface WorkerProgress { + workerId: number; + testId: string; + status: "pending" | "running" | "completed" | "failed"; + startedAt?: number; + completedAt?: number; + error?: string; +} + +export class ProgressDisplay { + private readonly workers: Map = new Map(); + private readonly maxWorkers: number; + private totalTests = 0; + private completedTests = 0; + private renderTimer?: NodeJS.Timeout; + private isInteractive: boolean; + + constructor(maxWorkers: number) { + this.maxWorkers = maxWorkers; + this.isInteractive = process.stderr.isTTY && !process.env.CI; + } + + start(): void { + if (this.isInteractive) { + // Print initial empty line for visual separation + console.log(""); + } + } + + setTotalTests(count: number): void { + this.totalTests = count; + } + + updateWorker(progress: WorkerProgress): void { + this.workers.set(progress.workerId, progress); + + if (progress.status === "completed" || progress.status === "failed") { + this.completedTests++; + } + + if (this.isInteractive) { + this.scheduleRender(); + } else { + // In non-interactive mode, just print completion events + if (progress.status === "completed") { + console.log(`✓ Test ${progress.testId} completed`); + } else if (progress.status === "failed") { + console.log(`✗ Test ${progress.testId} failed${progress.error ? `: ${progress.error}` : ""}`); + } + } + } + + private scheduleRender(): void { + if (this.renderTimer) { + return; + } + this.renderTimer = setTimeout(() => { + this.renderTimer = undefined; + this.render(); + }, 100); // Debounce renders to 100ms + } + + private render(): void { + if (!this.isInteractive) { + return; + } + + const lines: string[] = []; + + // Empty line above progress display + //lines.push(""); + + // Header with overall progress + const progressBar = this.buildProgressBar(this.completedTests, this.totalTests); + lines.push(`${progressBar} ${this.completedTests}/${this.totalTests} tests`); + + // Empty line between progress and workers + lines.push(""); + + // Worker status lines + const sortedWorkers = Array.from(this.workers.values()).sort((a, b) => a.workerId - b.workerId); + for (const worker of sortedWorkers) { + const line = this.formatWorkerLine(worker); + lines.push(line); + } + + // Use log-update to handle all cursor positioning + logUpdate(lines.join("\n")); + } + + private formatWorkerLine(worker: WorkerProgress): string { + const workerLabel = `Worker ${worker.workerId}`.padEnd(10); + const statusIcon = this.getStatusIcon(worker.status); + const elapsed = worker.startedAt ? this.formatElapsed(Date.now() - worker.startedAt) : ""; + const timeLabel = elapsed ? ` (${elapsed})` : ""; + + let testLabel = worker.testId; + if (testLabel.length > 50) { + testLabel = testLabel.substring(0, 47) + "..."; + } + + return `${workerLabel} ${statusIcon} ${testLabel}${timeLabel}`; + } + + private getStatusIcon(status: WorkerProgress["status"]): string { + switch (status) { + case "pending": + return "⏳"; + case "running": + return "🔄"; + case "completed": + return "✅"; + case "failed": + return "❌"; + default: + return " "; + } + } + + private formatElapsed(ms: number): string { + const seconds = Math.floor(ms / 1000); + if (seconds < 60) { + return `${seconds}s`; + } + const minutes = Math.floor(seconds / 60); + const remainingSeconds = seconds % 60; + return `${minutes}m ${remainingSeconds}s`; + } + + private buildProgressBar(current: number, total: number): string { + if (total === 0) { + return "[ ]"; + } + + const width = 20; + const filled = Math.floor((current / total) * width); + const empty = width - filled; + const bar = "█".repeat(filled) + "░".repeat(empty); + const percentage = Math.floor((current / total) * 100); + + return `[${bar}] ${percentage}%`; + } + + finish(): void { + if (this.renderTimer) { + clearTimeout(this.renderTimer); + this.renderTimer = undefined; + } + + if (this.isInteractive) { + this.render(); + logUpdate.done(); // Persist the final output + } + } + + clear(): void { + if (this.isInteractive) { + logUpdate.clear(); + } + } +} \ No newline at end of file diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 16a957ffa..f09b14aa0 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -3,6 +3,7 @@ import { type EvaluationCache, type EvaluationResult, type ProviderResponse, + ensureVSCodeSubagents, } from "@agentevo/core"; import { constants } from "node:fs"; import { access, mkdir } from "node:fs/promises"; @@ -15,6 +16,7 @@ import { getDefaultExtension, type OutputFormat, } from "./output-writer.js"; +import { ProgressDisplay } from "./progress-display.js"; import { calculateEvaluationSummary, formatEvaluationSummary } from "./statistics.js"; import { selectTarget } from "./targets.js"; @@ -27,9 +29,13 @@ interface NormalizedOptions { readonly target?: string; readonly targetsPath?: string; readonly testId?: string; + readonly workers?: number; readonly outPath?: string; readonly format: OutputFormat; readonly dryRun: boolean; + readonly dryRunDelay: number; + readonly dryRunDelayMin: number; + readonly dryRunDelayMax: number; readonly agentTimeoutSeconds: number; readonly maxRetries: number; readonly cache: boolean; @@ -66,13 +72,19 @@ function normalizeOptions(rawOptions: Record): NormalizedOption const formatStr = normalizeString(rawOptions.format) ?? "jsonl"; const format: OutputFormat = formatStr === "yaml" ? "yaml" : "jsonl"; + const workers = normalizeNumber(rawOptions.workers, 0); + return { target: normalizeString(rawOptions.target), targetsPath: normalizeString(rawOptions.targets), testId: normalizeString(rawOptions.testId), + workers: workers > 0 ? workers : undefined, outPath: normalizeString(rawOptions.out), format, dryRun: normalizeBoolean(rawOptions.dryRun), + dryRunDelay: normalizeNumber(rawOptions.dryRunDelay, 0), + dryRunDelayMin: normalizeNumber(rawOptions.dryRunDelayMin, 0), + dryRunDelayMax: normalizeNumber(rawOptions.dryRunDelayMax, 0), agentTimeoutSeconds: normalizeNumber(rawOptions.agentTimeout, 120), maxRetries: normalizeNumber(rawOptions.maxRetries, 2), cache: normalizeBoolean(rawOptions.cache), @@ -167,6 +179,9 @@ export async function runEvalCommand(input: RunEvalCommandInput): Promise explicitTargetsPath: options.targetsPath, cliTargetName: options.target, dryRun: options.dryRun, + dryRunDelay: options.dryRunDelay, + dryRunDelayMin: options.dryRunDelayMin, + dryRunDelayMax: options.dryRunDelayMax, env: process.env, }); @@ -190,8 +205,46 @@ export async function runEvalCommand(input: RunEvalCommandInput): Promise const cache = options.cache ? createEvaluationCache() : undefined; const agentTimeoutMs = Math.max(0, options.agentTimeoutSeconds) * 1000; + // Resolve workers: CLI flag > target setting > default (1) + let resolvedWorkers = options.workers ?? targetSelection.resolvedTarget.workers ?? 1; + if (resolvedWorkers < 1 || resolvedWorkers > 50) { + throw new Error(`Workers must be between 1 and 50, got: ${resolvedWorkers}`); + } + + // VSCode providers require window focus, so only 1 worker is allowed + const isVSCodeProvider = ["vscode", "vscode-insiders"].includes( + targetSelection.resolvedTarget.kind + ); + if (isVSCodeProvider && resolvedWorkers > 1) { + console.warn(`Warning: VSCode providers require window focus. Limiting workers from ${resolvedWorkers} to 1 to prevent race conditions.`); + resolvedWorkers = 1; + } + + if (options.verbose) { + const workersSource = options.workers + ? "CLI flag" + : targetSelection.resolvedTarget.workers + ? "target setting" + : "default"; + console.log(`Using ${resolvedWorkers} worker(s) (source: ${workersSource})`); + } + + // Auto-provision subagents for VSCode targets + if (isVSCodeProvider && !options.dryRun) { + await ensureVSCodeSubagents({ + kind: targetSelection.resolvedTarget.kind as "vscode" | "vscode-insiders", + count: resolvedWorkers, + verbose: options.verbose, + }); + } + const evaluationRunner = await resolveEvaluationRunner(); + // Initialize progress display + const progressDisplay = new ProgressDisplay(resolvedWorkers); + progressDisplay.start(); + const pendingTests = new Set(); + try { const results = await evaluationRunner({ testFilePath, @@ -206,11 +259,29 @@ export async function runEvalCommand(input: RunEvalCommandInput): Promise useCache: options.cache, testId: options.testId, verbose: options.verbose, + maxConcurrency: resolvedWorkers, onResult: async (result: EvaluationResult) => { await outputWriter.append(result); }, + onProgress: async (event) => { + // Track pending events to determine total test count + if (event.status === "pending") { + pendingTests.add(event.workerId); + progressDisplay.setTotalTests(pendingTests.size); + } + progressDisplay.updateWorker({ + workerId: event.workerId, + testId: event.testId, + status: event.status, + startedAt: event.startedAt, + completedAt: event.completedAt, + error: event.error, + }); + }, }); + progressDisplay.finish(); + await outputWriter.close(); const summary = calculateEvaluationSummary(results); diff --git a/apps/cli/src/commands/eval/targets.ts b/apps/cli/src/commands/eval/targets.ts index 349250b79..e261b639b 100644 --- a/apps/cli/src/commands/eval/targets.ts +++ b/apps/cli/src/commands/eval/targets.ts @@ -131,6 +131,9 @@ export interface TargetSelectionOptions { readonly explicitTargetsPath?: string; readonly cliTargetName?: string; readonly dryRun: boolean; + readonly dryRunDelay: number; + readonly dryRunDelayMin: number; + readonly dryRunDelayMax: number; readonly env: NodeJS.ProcessEnv; } @@ -152,7 +155,7 @@ function pickTargetName(options: { } export async function selectTarget(options: TargetSelectionOptions): Promise { - const { testFilePath, repoRoot, cwd, explicitTargetsPath, cliTargetName, dryRun, env } = options; + const { testFilePath, repoRoot, cwd, explicitTargetsPath, cliTargetName, dryRun, dryRunDelay, dryRunDelayMin, dryRunDelayMax, env } = options; const targetsFilePath = await discoverTargetsFile({ explicitPath: explicitTargetsPath, @@ -178,7 +181,12 @@ export async function selectTarget(options: TargetSelectionOptions): Promise; + private readonly mutex = new Mutex(); private closed = false; private isFirst = true; @@ -20,30 +22,32 @@ export class YamlWriter { } async append(record: unknown): Promise { - if (this.closed) { - throw new Error("Cannot write to closed YAML writer"); - } - - // Convert to YAML with proper multi-line string handling - const yamlDoc = stringifyYaml(record, { - indent: 2, - lineWidth: 0, // Disable line wrapping - defaultStringType: "PLAIN", - defaultKeyType: "PLAIN", - }); + await this.mutex.runExclusive(async () => { + if (this.closed) { + throw new Error("Cannot write to closed YAML writer"); + } + + // Convert to YAML with proper multi-line string handling + const yamlDoc = stringifyYaml(record, { + indent: 2, + lineWidth: 0, // Disable line wrapping + defaultStringType: "PLAIN", + defaultKeyType: "PLAIN", + }); - // Add YAML document separator (---) between records - const separator = this.isFirst ? "---\n" : "\n---\n"; - this.isFirst = false; + // Add YAML document separator (---) between records + const separator = this.isFirst ? "---\n" : "\n---\n"; + this.isFirst = false; - const content = `${separator}${yamlDoc}`; + const content = `${separator}${yamlDoc}`; - if (!this.stream.write(content)) { - await new Promise((resolve, reject) => { - this.stream.once("drain", resolve); - this.stream.once("error", reject); - }); - } + if (!this.stream.write(content)) { + await new Promise((resolve, reject) => { + this.stream.once("drain", resolve); + this.stream.once("error", reject); + }); + } + }); } async close(): Promise { diff --git a/apps/cli/test/commands/eval/vscode-worker-limit.test.ts b/apps/cli/test/commands/eval/vscode-worker-limit.test.ts new file mode 100644 index 000000000..f94dfb896 --- /dev/null +++ b/apps/cli/test/commands/eval/vscode-worker-limit.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, it } from "vitest"; + +describe("VSCode worker limit validation", () => { + it("should limit workers to 1 for vscode provider when workers > 1", () => { + // This test verifies that when using vscode or vscode-insiders providers, + // the workers count is automatically limited to 1 to prevent race conditions + // caused by window focus requirements. + + const targetSelection = { + resolvedTarget: { + kind: "vscode" as const, + name: "test-vscode", + workers: undefined, + config: { + command: "code", + waitForResponse: true, + dryRun: false, + }, + }, + }; + + const options = { + workers: 3, + }; + + // Simulate the logic from run-eval.ts + let resolvedWorkers = options.workers ?? targetSelection.resolvedTarget.workers ?? 1; + const isVSCodeProvider = ["vscode", "vscode-insiders"].includes( + targetSelection.resolvedTarget.kind + ); + + if (isVSCodeProvider && resolvedWorkers > 1) { + resolvedWorkers = 1; + } + + expect(resolvedWorkers).toBe(1); + }); + + it("should limit workers to 1 for vscode-insiders provider when workers > 1", () => { + const targetSelection = { + resolvedTarget: { + kind: "vscode-insiders" as const, + name: "test-vscode-insiders", + workers: undefined, + config: { + command: "code-insiders", + waitForResponse: true, + dryRun: false, + }, + }, + }; + + const options = { + workers: 5, + }; + + let resolvedWorkers = options.workers ?? targetSelection.resolvedTarget.workers ?? 1; + const isVSCodeProvider = ["vscode", "vscode-insiders"].includes( + targetSelection.resolvedTarget.kind + ); + + if (isVSCodeProvider && resolvedWorkers > 1) { + resolvedWorkers = 1; + } + + expect(resolvedWorkers).toBe(1); + }); + + it("should allow multiple workers for non-vscode providers", () => { + const targetSelection = { + resolvedTarget: { + kind: "azure" as const, + name: "test-azure", + workers: undefined, + config: { + resourceName: "test", + deploymentName: "test", + apiKey: "test", + }, + }, + }; + + const options = { + workers: 5, + }; + + let resolvedWorkers = options.workers ?? targetSelection.resolvedTarget.workers ?? 1; + const isVSCodeProvider = ["vscode", "vscode-insiders"].includes( + targetSelection.resolvedTarget.kind + ); + + if (isVSCodeProvider && resolvedWorkers > 1) { + resolvedWorkers = 1; + } + + expect(resolvedWorkers).toBe(5); + }); + + it("should not apply limit when workers is already 1", () => { + const targetSelection = { + resolvedTarget: { + kind: "vscode" as const, + name: "test-vscode", + workers: undefined, + config: { + command: "code", + waitForResponse: true, + dryRun: false, + }, + }, + }; + + const options = { + workers: 1, + }; + + let resolvedWorkers = options.workers ?? targetSelection.resolvedTarget.workers ?? 1; + const isVSCodeProvider = ["vscode", "vscode-insiders"].includes( + targetSelection.resolvedTarget.kind + ); + + if (isVSCodeProvider && resolvedWorkers > 1) { + resolvedWorkers = 1; + } + + expect(resolvedWorkers).toBe(1); + }); +}); diff --git a/apps/cli/test/eval.integration.test.ts b/apps/cli/test/eval.integration.test.ts index c97a0c375..2c61cf3ee 100644 --- a/apps/cli/test/eval.integration.test.ts +++ b/apps/cli/test/eval.integration.test.ts @@ -16,7 +16,7 @@ interface EvalFixture { const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const projectRoot = path.resolve(__dirname, "../../.."); -const CLI_ENTRY = path.join(projectRoot, "apps/cli/src/index.ts"); +const CLI_ENTRY = path.join(projectRoot, "apps/cli/src/cli.ts"); const MOCK_RUNNER = path.join(projectRoot, "apps/cli/test/fixtures/mock-run-evaluation.ts"); const require = createRequire(import.meta.url); const TSX_LOADER = pathToFileURL(require.resolve("tsx")).href; @@ -64,18 +64,24 @@ async function runCli( const baseEnv: Record = { ...process.env } as Record; delete baseEnv.CLI_ENV_SAMPLE; - const result = await execaNode(CLI_ENTRY, args, { - cwd: fixture.suiteDir, - env: { - ...baseEnv, - AGENTEVO_CLI_EVAL_RUNNER: MOCK_RUNNER, - AGENTEVO_CLI_EVAL_RUNNER_OUTPUT: fixture.diagnosticsPath, - ...extraEnv, - }, - nodeOptions: ["--import", TSX_LOADER], - }); + try { + const result = await execaNode(CLI_ENTRY, args, { + cwd: fixture.suiteDir, + env: { + ...baseEnv, + AGENTEVO_CLI_EVAL_RUNNER: MOCK_RUNNER, + AGENTEVO_CLI_EVAL_RUNNER_OUTPUT: fixture.diagnosticsPath, + ...extraEnv, + }, + nodeOptions: ["--import", TSX_LOADER], + reject: false, + }); - return { stdout: result.stdout, stderr: result.stderr }; + return { stdout: result.stdout, stderr: result.stderr }; + } catch (error) { + console.error("CLI execution failed:", error); + throw error; + } } function extractOutputPath(stdout: string): string { @@ -125,7 +131,7 @@ describe("agentevo eval CLI", () => { ]); expect(stderr).toBe(""); - expect(stdout).toContain("Using target (test-file): file-target"); + expect(stdout).toContain("Using target (test-file): file-target [provider=mock]"); expect(stdout).toContain("Mean score: 0.750"); expect(stdout).toContain("Std deviation: 0.212"); @@ -183,7 +189,7 @@ describe("agentevo eval CLI", () => { "cli-target", ]); - expect(stdout).toContain("Using target (cli): cli-target"); + expect(stdout).toContain("Using target (cli): cli-target [provider=mock]"); const diagnostics = await readDiagnostics(fixture); expect(diagnostics.target).toBe("cli-target"); @@ -199,7 +205,7 @@ describe("agentevo eval CLI", () => { const { stdout } = await runCli(fixture, ["eval", fixture.testFilePath, "--verbose"]); - expect(stdout).toContain("Using target (default): default"); + expect(stdout).toContain("Using target (default): default [provider=mock]"); const diagnostics = await readDiagnostics(fixture); expect(diagnostics.target).toBe("default"); diff --git a/docs/features/progress-display.md b/docs/features/progress-display.md new file mode 100644 index 000000000..d10daf6d9 --- /dev/null +++ b/docs/features/progress-display.md @@ -0,0 +1,198 @@ +# Worker Progress Display + +When running evaluations with parallel workers, AgentEvo displays real-time progress similar to Docker Compose or pnpm's parallel package installations. + +## Quick Start + +```bash +# Run with 4 parallel workers - shows progress display +pnpm agentevo eval tests/example.test.yaml --workers 4 + +# Sequential execution - no progress display +pnpm agentevo eval tests/example.test.yaml --workers 1 +``` + +## Display Modes + +### Interactive Mode (Terminal) + +When running in a TTY terminal, you'll see a live-updating display: + +``` +[████████████░░░░░░░░] 60% 12/20 tests + +Worker 1 🔄 simple-text-conversation (45s) +Worker 2 ✅ multi-turn-debugging-conversation +Worker 3 🔄 code-generation-with-constraints (12s) +Worker 4 ⏳ api-integration-test +``` + +The display updates automatically as workers pick up new tests and complete them. + +### Non-Interactive Mode (CI/Logs) + +When running in CI or with redirected output, you'll see simple line-by-line output: + +``` +✓ Test simple-text-conversation completed +✓ Test multi-turn-debugging-conversation completed +✗ Test code-generation-with-constraints failed: Timeout exceeded +✓ Test api-integration-test completed +``` + +## Status Indicators + +| Icon | Status | Description | +|------|--------|-------------| +| ⏳ | Pending | Test is queued, waiting for an available worker | +| 🔄 | Running | Test is actively being executed | +| ✅ | Completed | Test finished successfully | +| ❌ | Failed | Test encountered an error | + +## Configuration + +### Enable Progress Display + +Progress display is **automatically enabled** when: +- Using `--workers > 1` (parallel execution) +- Running in an interactive terminal (TTY) + +### Disable Progress Display + +Progress display is **automatically disabled** when: +- Using `--workers 1` (sequential execution - no need for parallel tracking) +- Running in CI environment (detected via `CI=true` environment variable) +- Output is redirected to a file or pipe (non-TTY) + +## Examples + +### Parallel Execution with Progress + +```bash +# Run evaluation with 4 workers - shows live progress +pnpm agentevo eval docs/examples/simple/evals/example.test.yaml --workers 4 +``` + +Output: +``` +Using target: azure_base [provider=azure-openai] +Output path: .agentevo/results/example_2025-11-15T10-30-45.jsonl + +[████████████████████] 100% 3/3 tests + +Worker 1 ✅ simple-text-conversation +Worker 2 ✅ multi-turn-debugging-conversation +Worker 3 ✅ code-generation-with-constraints + +Evaluation Summary: + Total Tests: 3 + Passed: 3 + Failed: 0 + Success Rate: 100.0% + Avg Score: 95.3% + +Results written to: .agentevo/results/example_2025-11-15T10-30-45.jsonl +``` + +### Sequential Execution (No Progress Display) + +```bash +# Run evaluation sequentially - no progress display needed +pnpm agentevo eval docs/examples/simple/evals/example.test.yaml --workers 1 +``` + +Output: +``` +Using target: azure_base [provider=azure-openai] +Output path: .agentevo/results/example_2025-11-15T10-30-45.jsonl + +Evaluation Summary: + Total Tests: 3 + Passed: 3 + Failed: 0 + Success Rate: 100.0% + Avg Score: 95.3% + +Results written to: .agentevo/results/example_2025-11-15T10-30-45.jsonl +``` + +### CI Environment (Non-Interactive) + +```bash +# In CI, you get simple line-by-line output even with parallel workers +CI=true pnpm agentevo eval docs/examples/simple/evals/example.test.yaml --workers 4 +``` + +Output: +``` +Using target: azure_base [provider=azure-openai] +Output path: .agentevo/results/example_2025-11-15T10-30-45.jsonl +✓ Test simple-text-conversation completed +✓ Test multi-turn-debugging-conversation completed +✓ Test code-generation-with-constraints completed + +Evaluation Summary: + Total Tests: 3 + Passed: 3 + Failed: 0 + Success Rate: 100.0% + Avg Score: 95.3% + +Results written to: .agentevo/results/example_2025-11-15T10-30-45.jsonl +``` + +## Technical Details + +### Implementation + +The progress display is implemented in `apps/cli/src/commands/eval/progress-display.ts` and uses: + +- **ANSI escape codes** for cursor manipulation in TTY mode +- **Debounced rendering** (100ms) to prevent flicker +- **Automatic environment detection** (TTY vs non-TTY) +- **Worker tracking** via unique IDs assigned to each test + +### Performance + +- **Minimal overhead:** Progress updates are debounced and rendered at most 10 times per second +- **Non-blocking:** All updates are asynchronous and don't slow down test execution +- **Memory efficient:** Only tracks status for active workers, not historical data + +### Customization + +Currently, the progress display automatically adapts to your environment. Future versions may include: + +- `--no-progress` flag to force disable display +- `--progress-style=