diff --git a/README.md b/README.md index 61b827c3e..81ee7cc70 100644 --- a/README.md +++ b/README.md @@ -347,7 +347,6 @@ targets: - name: vscode_dev provider: vscode - workspace_template: ${{ WORKSPACE_PATH }} judge_target: azure-base - name: local_agent @@ -358,6 +357,8 @@ targets: Supports: `azure`, `anthropic`, `gemini`, `codex`, `copilot`, `pi-coding-agent`, `claude`, `vscode`, `vscode-insiders`, `cli`, and `mock`. +Workspace templates are configured at eval-level under `workspace.template` (not per-target `workspace_template`). + Use `${{ VARIABLE_NAME }}` syntax to reference your `.env` file. See `.agentv/targets.yaml` after `agentv init` for detailed examples and all provider-specific fields. ## Evaluation Features diff --git a/apps/cli/README.md b/apps/cli/README.md index 61b827c3e..81ee7cc70 100644 --- a/apps/cli/README.md +++ b/apps/cli/README.md @@ -347,7 +347,6 @@ targets: - name: vscode_dev provider: vscode - workspace_template: ${{ WORKSPACE_PATH }} judge_target: azure-base - name: local_agent @@ -358,6 +357,8 @@ targets: Supports: `azure`, `anthropic`, `gemini`, `codex`, `copilot`, `pi-coding-agent`, `claude`, `vscode`, `vscode-insiders`, `cli`, and `mock`. +Workspace templates are configured at eval-level under `workspace.template` (not per-target `workspace_template`). + Use `${{ VARIABLE_NAME }}` syntax to reference your `.env` file. See `.agentv/targets.yaml` after `agentv init` for detailed examples and all provider-specific fields. ## Evaluation Features diff --git a/apps/cli/src/commands/eval/commands/run.ts b/apps/cli/src/commands/eval/commands/run.ts index 5a9f3ca60..91dd72a68 100644 --- a/apps/cli/src/commands/eval/commands/run.ts +++ b/apps/cli/src/commands/eval/commands/run.ts @@ -105,53 +105,16 @@ export const evalRunCommand = command({ long: 'verbose', description: 'Enable verbose logging', }), - keepWorkspaces: flag({ - long: 'keep-workspaces', - description: - 'Always keep temporary workspaces after evaluation (default: keep on failure only)', - }), - cleanupWorkspaces: flag({ - long: 'cleanup-workspaces', - description: 'Always cleanup temporary workspaces, even on failure', - }), - poolWorkspaces: flag({ - long: 'pool-workspaces', - description: 'Enable workspace pooling (default for shared workspaces with repos)', - }), - noPool: flag({ - long: 'no-pool', - description: 'Disable workspace pooling (clone fresh each run)', - }), - workspace: option({ - type: optional(string), - long: 'workspace', - description: 'Use an existing directory as the workspace directly (skips clone/copy/pool)', - }), workspaceMode: option({ type: optional(string), long: 'workspace-mode', - description: "Workspace mode: 'pooled', 'ephemeral', or 'static'", + description: "Workspace mode: 'pooled', 'temp', or 'static'", }), workspacePath: option({ type: optional(string), long: 'workspace-path', description: 'Static workspace directory path (used when workspace mode is static)', }), - workspaceClean: option({ - type: optional(string), - long: 'workspace-clean', - description: "Pooled reset clean mode: 'standard' or 'full'", - }), - retainOnSuccess: option({ - type: optional(string), - long: 'retain-on-success', - description: "Workspace retention on success: 'keep' or 'cleanup'", - }), - retainOnFailure: option({ - type: optional(string), - long: 'retain-on-failure', - description: "Workspace retention on failure: 'keep' or 'cleanup'", - }), otelFile: option({ type: optional(string), long: 'otel-file', @@ -216,16 +179,8 @@ export const evalRunCommand = command({ cache: args.cache, noCache: args.noCache, verbose: args.verbose, - keepWorkspaces: args.keepWorkspaces, - cleanupWorkspaces: args.cleanupWorkspaces, - poolWorkspaces: args.poolWorkspaces, - noPool: args.noPool, - workspace: args.workspace, workspaceMode: args.workspaceMode, workspacePath: args.workspacePath, - workspaceClean: args.workspaceClean, - retainOnSuccess: args.retainOnSuccess, - retainOnFailure: args.retainOnFailure, trace: false, otelFile: args.otelFile, traceFile: args.traceFile, diff --git a/apps/cli/src/commands/eval/run-eval.ts b/apps/cli/src/commands/eval/run-eval.ts index 4a1fa753f..d887f5fcd 100644 --- a/apps/cli/src/commands/eval/run-eval.ts +++ b/apps/cli/src/commands/eval/run-eval.ts @@ -69,8 +69,6 @@ interface NormalizedOptions { readonly cache: boolean; readonly noCache: boolean; readonly verbose: boolean; - readonly keepWorkspaces: boolean; - readonly cleanupWorkspaces: boolean; readonly otelFile?: string; readonly traceFile?: string; readonly exportOtel: boolean; @@ -78,14 +76,8 @@ interface NormalizedOptions { readonly otelCaptureContent: boolean; readonly otelGroupTurns: boolean; readonly retryErrors?: string; - readonly poolWorkspaces?: boolean; - readonly poolMaxSlots?: number; - readonly workspace?: string; - readonly workspaceMode?: 'pooled' | 'ephemeral' | 'static'; + readonly workspaceMode?: 'pooled' | 'temp' | 'static'; readonly workspacePath?: string; - readonly workspaceClean?: 'standard' | 'full'; - readonly retainOnSuccess?: 'keep' | 'cleanup'; - readonly retainOnFailure?: 'keep' | 'cleanup'; } function normalizeBoolean(value: unknown): boolean { @@ -134,16 +126,8 @@ function normalizeOptionalNumber(value: unknown): number | undefined { return undefined; } -function normalizeWorkspaceMode(value: unknown): 'pooled' | 'ephemeral' | 'static' | undefined { - return value === 'pooled' || value === 'ephemeral' || value === 'static' ? value : undefined; -} - -function normalizeWorkspaceClean(value: unknown): 'standard' | 'full' | undefined { - return value === 'standard' || value === 'full' ? value : undefined; -} - -function normalizeRetention(value: unknown): 'keep' | 'cleanup' | undefined { - return value === 'keep' || value === 'cleanup' ? value : undefined; +function normalizeWorkspaceMode(value: unknown): 'pooled' | 'temp' | 'static' | undefined { + return value === 'pooled' || value === 'temp' || value === 'static' ? value : undefined; } function normalizeOptions( @@ -198,6 +182,18 @@ function normalizeOptions( // Output dir: CLI --out > config output.dir > auto-generated const cliOut = normalizeString(rawOptions.out); const configOut = config?.output?.dir; + const cliWorkspacePath = normalizeString(rawOptions.workspacePath); + const cliWorkspaceModeRaw = normalizeString(rawOptions.workspaceMode); + const cliWorkspaceMode = normalizeWorkspaceMode(rawOptions.workspaceMode); + if (cliWorkspacePath && cliWorkspaceModeRaw && cliWorkspaceMode !== 'static') { + throw new Error('--workspace-path requires --workspace-mode=static (or omit --workspace-mode)'); + } + + const yamlExecutionRecord = yamlExecution as Record | undefined; + const yamlWorkspaceMode = normalizeWorkspaceMode(yamlExecutionRecord?.workspace_mode); + const yamlWorkspacePath = normalizeString(yamlExecutionRecord?.workspace_path); + const workspacePath = cliWorkspacePath ?? yamlWorkspacePath; + const workspaceMode = cliWorkspacePath ? 'static' : (cliWorkspaceMode ?? yamlWorkspaceMode); return { target: singleTarget, @@ -223,11 +219,6 @@ function normalizeOptions( normalizeBoolean(rawOptions.verbose) || yamlExecution?.verbose === true || config?.execution?.verbose === true, - keepWorkspaces: - normalizeBoolean(rawOptions.keepWorkspaces) || - yamlExecution?.keep_workspaces === true || - config?.execution?.keepWorkspaces === true, - cleanupWorkspaces: normalizeBoolean(rawOptions.cleanupWorkspaces), // Precedence: CLI > YAML config > TS config otelFile: normalizeString(rawOptions.otelFile) ?? @@ -250,23 +241,8 @@ function normalizeOptions( otelCaptureContent: normalizeBoolean(rawOptions.otelCaptureContent), otelGroupTurns: normalizeBoolean(rawOptions.otelGroupTurns), retryErrors: normalizeString(rawOptions.retryErrors), - // Pool: --no-pool explicitly disables; --pool-workspaces explicitly enables; YAML config; default undefined (orchestrator defaults to true) - poolWorkspaces: normalizeBoolean(rawOptions.noPool) - ? false - : normalizeBoolean(rawOptions.poolWorkspaces) - ? true - : yamlExecution?.pool_workspaces, - poolMaxSlots: yamlExecution?.pool_slots, - workspace: normalizeString(rawOptions.workspace), - workspaceMode: normalizeWorkspaceMode(rawOptions.workspaceMode), - workspacePath: normalizeString(rawOptions.workspacePath), - workspaceClean: normalizeWorkspaceClean(rawOptions.workspaceClean), - retainOnSuccess: - normalizeRetention(rawOptions.retainOnSuccess) ?? - (normalizeBoolean(rawOptions.keepWorkspaces) ? 'keep' : undefined), - retainOnFailure: - normalizeRetention(rawOptions.retainOnFailure) ?? - (normalizeBoolean(rawOptions.cleanupWorkspaces) ? 'cleanup' : undefined), + workspaceMode, + workspacePath, } satisfies NormalizedOptions; } @@ -578,7 +554,6 @@ async function runSingleEvalFile(params: { // Create streaming observer for real-time OTel span export const streamingObserver = otelExporter?.createStreamingObserver() ?? null; - const results = await evaluationRunner({ testFilePath, repoRoot, @@ -604,16 +579,8 @@ async function runSingleEvalFile(params: { evalCases, verbose: options.verbose, maxConcurrency: resolvedWorkers, - keepWorkspaces: options.keepWorkspaces, - cleanupWorkspaces: options.cleanupWorkspaces, - poolWorkspaces: options.poolWorkspaces, - poolMaxSlots: options.poolMaxSlots, - workspace: options.workspace, workspaceMode: options.workspaceMode, workspacePath: options.workspacePath, - workspaceClean: options.workspaceClean, - retainOnSuccess: options.retainOnSuccess, - retainOnFailure: options.retainOnFailure, trials: trialsConfig, totalBudgetUsd, failOnError, @@ -717,30 +684,22 @@ export async function runEvalCommand(input: RunEvalCommandInput): Promise retryNonErrorResults = await loadNonErrorResults(retryPath); } - if (options.keepWorkspaces && options.cleanupWorkspaces) { - console.warn( - 'Warning: Both --keep-workspaces and --cleanup-workspaces specified. --cleanup-workspaces takes precedence.', - ); - } - // Validate static workspace path exists and is a directory - const explicitWorkspacePath = options.workspacePath ?? options.workspace; - if (explicitWorkspacePath) { - const resolvedWorkspace = path.resolve(explicitWorkspacePath); + if (options.workspacePath) { + const resolvedWorkspace = path.resolve(options.workspacePath); try { const { stat } = await import('node:fs/promises'); const stats = await stat(resolvedWorkspace); if (!stats.isDirectory()) { - throw new Error(`--workspace path is not a directory: ${resolvedWorkspace}`); + throw new Error(`--workspace-path is not a directory: ${resolvedWorkspace}`); } } catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') { - throw new Error(`--workspace path does not exist: ${resolvedWorkspace}`); + throw new Error(`--workspace-path does not exist: ${resolvedWorkspace}`); } throw err; } - // Update workspace paths to resolved absolute path - options = { ...options, workspace: resolvedWorkspace, workspacePath: resolvedWorkspace }; + options = { ...options, workspacePath: resolvedWorkspace }; } if (options.verbose) { diff --git a/examples/features/workspace-setup-script/README.md b/examples/features/workspace-setup-script/README.md index 71046d8b7..d2fdccfd4 100644 --- a/examples/features/workspace-setup-script/README.md +++ b/examples/features/workspace-setup-script/README.md @@ -1,6 +1,6 @@ # Workspace Setup Script -Demonstrates using a `before_all` lifecycle hook to clean and re-initialize an allagents workspace before evaluation runs, including sourcing plugin files (like `AGENTS.md` and prompt files) into the workspace. +Demonstrates using a `before_all` lifecycle hook to clean and re-initialize an allagents workspace before evaluation runs, then register a project-scoped marketplace and sync plugin content (including prompt files). ## Problem @@ -8,18 +8,21 @@ Demonstrates using a `before_all` lifecycle hook to clean and re-initialize an a ## Solution -A generic Node.js script that any eval can reuse. It reads `workspace_path` from AgentV's stdin JSON, removes the stale `.allagents/` directory, and runs `allagents workspace init` with the template path passed via `--from`. +A generic Node.js script that any eval can reuse. It reads `workspace_path` from AgentV's stdin JSON, removes stale `.allagents/` state, runs `allagents workspace init --from`, registers a project-scoped marketplace, then runs `allagents workspace sync`. ``` workspace-setup-script/ ├── evals/ │ └── dataset.eval.yaml # Eval with before_all hook ├── plugins/ -│ └── my-plugin/ # External plugin (sourced by allagents) +│ └── my-plugin/ # Plugin content (AGENTS + prompt) │ ├── AGENTS.md # Agent guidelines │ └── .github/ │ └── prompts/ │ └── summarize-repo.prompt.md +├── marketplace/ +│ └── .claude-plugin/ +│ └── marketplace.json # Local marketplace manifest ├── scripts/ │ └── workspace-setup.mjs # Generic setup script (reusable across evals) └── workspace-template/ @@ -27,23 +30,27 @@ workspace-setup-script/ └── workspace.yaml # Template for allagents init ``` -## Sourcing plugin files with allagents +## Plugin Installation via Project Marketplace -The `plugins/` directory lives outside `workspace-template/` as an external source. The `.allagents/workspace.yaml` uses `workspace.source` and `workspace.files` to tell allagents to copy specific files to the workspace root: +The `.allagents/workspace.yaml` installs a plugin from a named marketplace: ```yaml # .allagents/workspace.yaml -workspace: - source: ../plugins/my-plugin/ - files: - - AGENTS.md +plugins: + - my-plugin@workspace-setup-script-marketplace ``` -The source path is relative to the workspace template root (parent of `.allagents/`). When `npx allagents workspace init --from` runs, it resolves `../plugins/my-plugin/` from the template location and copies `AGENTS.md` to the workspace root. +The setup script registers that marketplace using project scope: + +```bash +npx --yes allagents plugin marketplace add ../marketplace --scope project +``` + +This matches the project-scoped marketplace flow introduced in `allagents` (PR #224). ## Eval YAML -The template path is passed as an argument. Use `--require` to validate that expected artifacts exist in the workspace after initialization: +The template path and local marketplace path are passed as arguments. Use `--require` to validate expected artifacts after sync: ```yaml workspace: @@ -55,8 +62,12 @@ workspace: - ../scripts/workspace-setup.mjs - --from - ../workspace-template/.allagents/workspace.yaml + - --marketplace-source + - ../marketplace - --require - AGENTS.md + - --require + - .github/prompts/summarize-repo.prompt.md ``` The `--require` flag accepts one or more file paths (relative to the workspace root). If any required file is missing after `allagents workspace init`, the script exits with an error listing the missing files. @@ -83,9 +94,10 @@ The `type: file` path is resolved from the eval file's directory up to the repo 1. AgentV copies `workspace-template/` to a pooled workspace 2. The setup script removes stale `.allagents/` config and runs `npx allagents workspace init` -3. Allagents reads `workspace.source`/`workspace.files` and copies `AGENTS.md` from the plugin to the workspace root -4. The `--require AGENTS.md` check validates the artifact exists -5. AgentV clones repos and runs tests against the initialized workspace +3. The setup script registers the local marketplace with `--scope project` +4. `allagents workspace sync` installs `my-plugin@workspace-setup-script-marketplace` +5. `--require` checks verify `AGENTS.md` and `.github/prompts/summarize-repo.prompt.md` exist +6. AgentV clones repos and runs tests against the initialized workspace ## Cross-platform diff --git a/examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml b/examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml index d4f1c43f7..4e6872983 100644 --- a/examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml +++ b/examples/features/workspace-setup-script/evals/dataset-vscode.eval.yaml @@ -11,6 +11,12 @@ workspace: - ../scripts/workspace-setup.mjs - --from - ../workspace-template/.allagents/workspace.yaml + - --marketplace-source + - ../marketplace + - --require + - AGENTS.md + - --require + - .github/prompts/summarize-repo.prompt.md after_each: reset: fast repos: diff --git a/examples/features/workspace-setup-script/evals/dataset.eval.yaml b/examples/features/workspace-setup-script/evals/dataset.eval.yaml index 7d2b81d80..406b6372a 100644 --- a/examples/features/workspace-setup-script/evals/dataset.eval.yaml +++ b/examples/features/workspace-setup-script/evals/dataset.eval.yaml @@ -11,8 +11,12 @@ workspace: - ../scripts/workspace-setup.mjs - --from - ../workspace-template/.allagents/workspace.yaml + - --marketplace-source + - ../marketplace - --require - AGENTS.md + - --require + - .github/prompts/summarize-repo.prompt.md repos: - path: ./my-repo source: diff --git a/examples/features/workspace-setup-script/marketplace/.claude-plugin/marketplace.json b/examples/features/workspace-setup-script/marketplace/.claude-plugin/marketplace.json new file mode 100644 index 000000000..d7cef0488 --- /dev/null +++ b/examples/features/workspace-setup-script/marketplace/.claude-plugin/marketplace.json @@ -0,0 +1,11 @@ +{ + "name": "workspace-setup-script-marketplace", + "description": "Local marketplace used by the workspace setup script example", + "plugins": [ + { + "name": "my-plugin", + "description": "Demo plugin containing AGENTS.md and summarize prompt", + "source": "../plugins/my-plugin" + } + ] +} diff --git a/examples/features/workspace-setup-script/scripts/workspace-setup.mjs b/examples/features/workspace-setup-script/scripts/workspace-setup.mjs index 3f825f558..7c45ee286 100644 --- a/examples/features/workspace-setup-script/scripts/workspace-setup.mjs +++ b/examples/features/workspace-setup-script/scripts/workspace-setup.mjs @@ -22,13 +22,13 @@ import { spawnSync } from 'node:child_process'; import { cpSync, existsSync, readFileSync, rmSync } from 'node:fs'; -import { basename, join } from 'node:path'; +import { basename, isAbsolute, join, resolve } from 'node:path'; // --- parse arguments --- const fromIndex = process.argv.indexOf('--from'); if (fromIndex === -1 || !process.argv[fromIndex + 1]) { console.error( - 'Usage: workspace-setup.mjs --from [--source ...] [--require ...]', + 'Usage: workspace-setup.mjs --from [--source ...] [--marketplace-source ] [--marketplace-name ] [--require ...]', ); process.exit(1); } @@ -52,6 +52,14 @@ for (let i = 0; i < process.argv.length; i++) { } } +// Optional project-scoped marketplace source to register after init. +const marketplaceSourceIndex = process.argv.indexOf('--marketplace-source'); +const marketplaceSource = + marketplaceSourceIndex !== -1 ? process.argv[marketplaceSourceIndex + 1] : undefined; +const marketplaceNameIndex = process.argv.indexOf('--marketplace-name'); +const marketplaceName = + marketplaceNameIndex !== -1 ? process.argv[marketplaceNameIndex + 1] : undefined; + // --- stdin context from AgentV --- const { workspace_path } = JSON.parse(readFileSync(0, 'utf8')); if (!workspace_path) { @@ -89,6 +97,45 @@ if (result.status !== 0) { process.exit(result.status ?? 1); } +// --- optionally register project-scoped marketplace and resync --- +if (marketplaceSource) { + const resolvedMarketplaceSource = isAbsolute(marketplaceSource) + ? marketplaceSource + : resolve(process.cwd(), marketplaceSource); + + const addMarketplaceArgs = [ + '--yes', + 'allagents', + 'plugin', + 'marketplace', + 'add', + resolvedMarketplaceSource, + '--scope', + 'project', + ]; + if (marketplaceName) { + addMarketplaceArgs.push('--name', marketplaceName); + } + + const addMarketplaceResult = spawnSync(npx, addMarketplaceArgs, { + stdio: ['ignore', 'inherit', 'inherit'], + shell: process.platform === 'win32', + cwd: workspace_path, + }); + if (addMarketplaceResult.status !== 0) { + process.exit(addMarketplaceResult.status ?? 1); + } + + const syncResult = spawnSync(npx, ['--yes', 'allagents', 'workspace', 'sync'], { + stdio: ['ignore', 'inherit', 'inherit'], + shell: process.platform === 'win32', + cwd: workspace_path, + }); + if (syncResult.status !== 0) { + process.exit(syncResult.status ?? 1); + } +} + // --- validate required artifacts exist in workspace --- const missing = requiredFiles.filter((file) => !existsSync(join(workspace_path, file))); if (missing.length > 0) { diff --git a/examples/features/workspace-setup-script/workspace-template/.allagents/workspace.yaml b/examples/features/workspace-setup-script/workspace-template/.allagents/workspace.yaml index a27617f47..9da75a4fe 100644 --- a/examples/features/workspace-setup-script/workspace-template/.allagents/workspace.yaml +++ b/examples/features/workspace-setup-script/workspace-template/.allagents/workspace.yaml @@ -4,12 +4,8 @@ repositories: repo: agentv description: Example repository for workspace setup demo -plugins: [] +plugins: + - my-plugin@workspace-setup-script-marketplace clients: - copilot - -workspace: - source: ../plugins/my-plugin/ - files: - - AGENTS.md diff --git a/packages/core/src/evaluation/orchestrator.ts b/packages/core/src/evaluation/orchestrator.ts index f63c17be7..4b6c959ed 100644 --- a/packages/core/src/evaluation/orchestrator.ts +++ b/packages/core/src/evaluation/orchestrator.ts @@ -215,7 +215,7 @@ export interface RunEvaluationOptions { /** Pre-existing workspace directory to use directly (skips clone/copy/pool) */ readonly workspace?: string; /** Workspace materialization mode override */ - readonly workspaceMode?: 'pooled' | 'ephemeral' | 'static'; + readonly workspaceMode?: 'pooled' | 'temp' | 'static'; /** Static workspace path override (used when workspaceMode=static) */ readonly workspacePath?: string; /** Workspace clean policy override for pooled reset */ @@ -443,10 +443,16 @@ export async function runEvaluation( // Resolve worker count and pool mode const isPerTestIsolation = suiteWorkspace?.isolation === 'per_test'; - const configuredMode = suiteWorkspace?.mode ?? workspaceMode; - const configuredStaticPath = suiteWorkspace?.static_path ?? workspacePath ?? legacyWorkspacePath; - const useStaticWorkspace = - configuredMode === 'static' || (!!configuredStaticPath && !configuredMode); + const cliWorkspacePath = workspacePath ?? legacyWorkspacePath; + const yamlWorkspacePath = suiteWorkspace?.path; + if (cliWorkspacePath && workspaceMode && workspaceMode !== 'static') { + throw new Error('--workspace-path requires --workspace-mode static when both are provided'); + } + const configuredMode = cliWorkspacePath + ? 'static' + : (workspaceMode ?? suiteWorkspace?.mode ?? (yamlWorkspacePath ? 'static' : 'pooled')); + const configuredStaticPath = cliWorkspacePath ?? yamlWorkspacePath; + const useStaticWorkspace = configuredMode === 'static'; // static workspace is incompatible with per_test isolation if (useStaticWorkspace && isPerTestIsolation) { @@ -455,7 +461,10 @@ export async function runEvaluation( ); } if (configuredMode === 'static' && !configuredStaticPath) { - throw new Error('workspace.mode=static requires workspace.static_path or --workspace-path'); + throw new Error('workspace.mode=static requires workspace.path or --workspace-path'); + } + if (configuredMode !== 'static' && configuredStaticPath) { + throw new Error('workspace.path requires workspace.mode=static'); } const hasSharedWorkspace = !!( @@ -465,13 +474,8 @@ export async function runEvaluation( (suiteWorkspace?.repos?.length && !isPerTestIsolation) ); - // Pool support: mode-based first, then legacy pool booleans, then default. - const poolEnabled = - configuredMode === 'pooled' - ? true - : configuredMode === 'ephemeral' || useStaticWorkspace - ? false - : (suiteWorkspace?.pool ?? poolWorkspaces ?? true); + // Pool support is mode-based: pooled enables, temp/static disable. + const poolEnabled = configuredMode === 'pooled'; const usePool = poolEnabled !== false && !!suiteWorkspace?.repos?.length && diff --git a/packages/core/src/evaluation/providers/targets.ts b/packages/core/src/evaluation/providers/targets.ts index 15115d30d..1fb331d6c 100644 --- a/packages/core/src/evaluation/providers/targets.ts +++ b/packages/core/src/evaluation/providers/targets.ts @@ -696,6 +696,11 @@ export function resolveTargetDefinition( evalFilePath?: string, ): ResolvedTarget { const parsed = BASE_TARGET_SCHEMA.parse(definition); + if (parsed.workspace_template !== undefined || parsed.workspaceTemplate !== undefined) { + throw new Error( + `${parsed.name}: target-level workspace_template has been removed. Use eval-level workspace.template.`, + ); + } const provider = parsed.provider.toLowerCase(); const providerBatching = resolveOptionalBoolean( parsed.provider_batching ?? parsed.providerBatching, diff --git a/packages/core/src/evaluation/types.ts b/packages/core/src/evaluation/types.ts index 9c25d1219..95e1e49e8 100644 --- a/packages/core/src/evaluation/types.ts +++ b/packages/core/src/evaluation/types.ts @@ -277,11 +277,9 @@ export type WorkspaceConfig = { /** Workspace lifecycle hooks */ readonly hooks?: WorkspaceHooksConfig; /** Workspace materialization mode */ - readonly mode?: 'pooled' | 'ephemeral' | 'static'; + readonly mode?: 'pooled' | 'temp' | 'static'; /** Required when mode=static: use this existing directory directly */ - readonly static_path?: string; - /** @deprecated Use mode=pooled|ephemeral|static */ - readonly pool?: boolean; + readonly path?: string; }; export type CodeEvaluatorConfig = { diff --git a/packages/core/src/evaluation/validation/eval-file.schema.ts b/packages/core/src/evaluation/validation/eval-file.schema.ts index 4d980e5bf..281ce8663 100644 --- a/packages/core/src/evaluation/validation/eval-file.schema.ts +++ b/packages/core/src/evaluation/validation/eval-file.schema.ts @@ -294,15 +294,16 @@ const WorkspaceHooksSchema = z.object({ after_all: WorkspaceHookSchema.optional(), }); -const WorkspaceSchema = z.object({ - template: z.string().optional(), - isolation: z.enum(['shared', 'per_test']).optional(), - repos: z.array(RepoSchema).optional(), - hooks: WorkspaceHooksSchema.optional(), - mode: z.enum(['pooled', 'ephemeral', 'static']).optional(), - static_path: z.string().optional(), - pool: z.boolean().optional(), -}); +const WorkspaceSchema = z + .object({ + template: z.string().optional(), + isolation: z.enum(['shared', 'per_test']).optional(), + repos: z.array(RepoSchema).optional(), + hooks: WorkspaceHooksSchema.optional(), + mode: z.enum(['pooled', 'temp', 'static']).optional(), + path: z.string().optional(), + }) + .strict(); // --------------------------------------------------------------------------- // Execution block diff --git a/packages/core/src/evaluation/validation/targets-validator.ts b/packages/core/src/evaluation/validation/targets-validator.ts index ef5fee2c9..c507308e9 100644 --- a/packages/core/src/evaluation/validation/targets-validator.ts +++ b/packages/core/src/evaluation/validation/targets-validator.ts @@ -240,6 +240,7 @@ function validateUnknownSettings( location: string, errors: ValidationError[], ): void { + const removedTargetFields = new Set(['workspace_template', 'workspaceTemplate']); const knownSettings = getKnownSettings(provider); if (!knownSettings) { // Unknown provider, skip settings validation @@ -250,6 +251,17 @@ function validateUnknownSettings( const baseFields = new Set(['name', 'provider', 'judge_target', 'workers', '$schema', 'targets']); for (const key of Object.keys(target)) { + if (removedTargetFields.has(key)) { + errors.push({ + severity: 'error', + filePath: absolutePath, + location: `${location}.${key}`, + message: + 'target-level workspace_template has been removed. Use eval-level workspace.template.', + }); + continue; + } + if (!baseFields.has(key) && !knownSettings.has(key)) { errors.push({ severity: 'warning', diff --git a/packages/core/src/evaluation/yaml-parser.ts b/packages/core/src/evaluation/yaml-parser.ts index 96cd47eea..6e388bb3d 100644 --- a/packages/core/src/evaluation/yaml-parser.ts +++ b/packages/core/src/evaluation/yaml-parser.ts @@ -676,6 +676,17 @@ async function resolveWorkspaceConfig( function parseWorkspaceConfig(raw: unknown, evalFileDir: string): WorkspaceConfig | undefined { if (!isJsonObject(raw)) return undefined; const obj = raw as Record; + if ('static_path' in obj) { + throw new Error( + 'workspace.static_path has been removed. Use workspace.path with workspace.mode=static.', + ); + } + if ('pool' in obj) { + throw new Error("workspace.pool has been removed. Use workspace.mode='pooled' or 'temp'."); + } + if ('static' in obj) { + throw new Error("workspace.static has been removed. Use workspace.mode='static'."); + } let template = typeof obj.template === 'string' ? obj.template : undefined; if (template && !path.isAbsolute(template)) { @@ -692,15 +703,12 @@ function parseWorkspaceConfig(raw: unknown, evalFileDir: string): WorkspaceConfi : undefined; const hooks = parseWorkspaceHooksConfig(obj.hooks, evalFileDir); - const mode = - obj.mode === 'pooled' || obj.mode === 'ephemeral' || obj.mode === 'static' - ? obj.mode - : undefined; - const staticPath = typeof obj.static_path === 'string' ? obj.static_path : undefined; - const pool = typeof obj.pool === 'boolean' ? obj.pool : undefined; + const explicitMode = + obj.mode === 'pooled' || obj.mode === 'temp' || obj.mode === 'static' ? obj.mode : undefined; + const workspacePath = typeof obj.path === 'string' ? obj.path : undefined; + const mode = explicitMode ?? (workspacePath ? 'static' : undefined); - if (!template && !isolation && !repos && !hooks && !mode && !staticPath && pool === undefined) - return undefined; + if (!template && !isolation && !repos && !hooks && !mode && !workspacePath) return undefined; return { ...(template !== undefined && { template }), @@ -708,8 +716,7 @@ function parseWorkspaceConfig(raw: unknown, evalFileDir: string): WorkspaceConfi ...(repos !== undefined && { repos }), ...(hooks !== undefined && { hooks }), ...(mode !== undefined && { mode }), - ...(staticPath !== undefined && { static_path: staticPath }), - ...(pool !== undefined && { pool }), + ...(workspacePath !== undefined && { path: workspacePath }), }; } @@ -749,8 +756,7 @@ function mergeWorkspaceConfigs( repos: caseLevel.repos ?? suiteLevel.repos, ...(hasHooks && { hooks: mergedHooks as WorkspaceHooksConfig }), mode: caseLevel.mode ?? suiteLevel.mode, - static_path: caseLevel.static_path ?? suiteLevel.static_path, - pool: caseLevel.pool ?? suiteLevel.pool, + path: caseLevel.path ?? suiteLevel.path, }; } diff --git a/packages/core/test/evaluation/orchestrator.test.ts b/packages/core/test/evaluation/orchestrator.test.ts index f48a8b711..20be90450 100644 --- a/packages/core/test/evaluation/orchestrator.test.ts +++ b/packages/core/test/evaluation/orchestrator.test.ts @@ -2648,6 +2648,45 @@ describe('--workspace flag', () => { expect(results[0].error).toBeUndefined(); }); + it('errors when workspaceMode is static without workspace path', async () => { + const provider = new SequenceProvider('mock', { + responses: [{ output: [{ role: 'assistant', content: [{ type: 'text', text: 'answer' }] }] }], + }); + + await expect( + runEvaluation({ + testFilePath: 'in-memory.yaml', + repoRoot: 'in-memory', + target: baseTarget, + providerFactory: () => provider, + evaluators: evaluatorRegistry, + evalCases: [baseTestCase], + workspaceMode: 'static', + }), + ).rejects.toThrow('workspace.mode=static requires workspace.path or --workspace-path'); + }); + + it('errors when workspace path is combined with non-static workspaceMode', async () => { + const { mkdtemp } = await import('node:fs/promises'); + testDir = await mkdtemp(path.join(tmpdir(), 'agentv-ws-flag-')); + const provider = new SequenceProvider('mock', { + responses: [{ output: [{ role: 'assistant', content: [{ type: 'text', text: 'answer' }] }] }], + }); + + await expect( + runEvaluation({ + testFilePath: 'in-memory.yaml', + repoRoot: 'in-memory', + target: baseTarget, + providerFactory: () => provider, + evaluators: evaluatorRegistry, + evalCases: [baseTestCase], + workspacePath: testDir, + workspaceMode: 'temp', + }), + ).rejects.toThrow('--workspace-path requires --workspace-mode static when both are provided'); + }); + it('includes per-judge timing in scores', async () => { const provider = new SequenceProvider('mock', { responses: [ diff --git a/packages/core/test/evaluation/providers/targets.test.ts b/packages/core/test/evaluation/providers/targets.test.ts index 6727c0e91..eacd573b2 100644 --- a/packages/core/test/evaluation/providers/targets.test.ts +++ b/packages/core/test/evaluation/providers/targets.test.ts @@ -211,11 +211,7 @@ describe('resolveTargetDefinition', () => { ).toThrow(/AZURE_OPENAI_API_KEY/i); }); - it('supports vscode configuration with optional workspace template from env var', () => { - const env = { - WORKSPACE_TEMPLATE_PATH: '/path/to/workspace.code-workspace', - } satisfies Record; - + it('supports vscode configuration with executable/wait/dry_run', () => { const target = resolveTargetDefinition( { name: 'editor', @@ -223,9 +219,8 @@ describe('resolveTargetDefinition', () => { executable: 'code-insiders', wait: false, dry_run: true, - workspace_template: '${{ WORKSPACE_TEMPLATE_PATH }}', }, - env, + {}, ); expect(target.kind).toBe('vscode'); @@ -236,7 +231,6 @@ describe('resolveTargetDefinition', () => { expect(target.config.executable).toBe('code-insiders'); expect(target.config.waitForResponse).toBe(false); expect(target.config.dryRun).toBe(true); - expect(target.config.workspaceTemplate).toBe('/path/to/workspace.code-workspace'); }); it('resolves vscode executable from env var', () => { @@ -488,159 +482,18 @@ describe('resolveTargetDefinition', () => { expect(target.config.args).toEqual(['--profile', 'default', '--model', 'gpt-4']); }); - it('resolves cli workspace_template with literal path', () => { - const target = resolveTargetDefinition( - { - name: 'cli-with-template', - provider: 'cli', - command: 'echo {PROMPT}', - workspace_template: '/templates/my-workspace', - }, - {}, - ); - - expect(target.kind).toBe('cli'); - if (target.kind !== 'cli') { - throw new Error('expected cli target'); - } - - expect(target.config.workspaceTemplate).toBe('/templates/my-workspace'); - expect(target.config.cwd).toBeUndefined(); - }); - - it('resolves cli workspace_template from environment variable', () => { - const env = { - WORKSPACE_DIR: '/path/to/workspace-template', - } satisfies Record; - - const target = resolveTargetDefinition( - { - name: 'cli-with-env-template', - provider: 'cli', - command: 'echo {PROMPT}', - workspace_template: '${{ WORKSPACE_DIR }}', - }, - env, - ); - - expect(target.kind).toBe('cli'); - if (target.kind !== 'cli') { - throw new Error('expected cli target'); - } - - expect(target.config.workspaceTemplate).toBe('/path/to/workspace-template'); - }); - - it('throws when both cwd and workspace_template are specified for cli', () => { + it('rejects removed target-level workspace_template field', () => { expect(() => resolveTargetDefinition( { - name: 'cli-both', + name: 'cli-with-template', provider: 'cli', command: 'echo {PROMPT}', - cwd: '/some/path', workspace_template: '/templates/my-workspace', }, {}, ), - ).toThrow(/mutually exclusive/i); - }); - - it('resolves claude workspace_template', () => { - const target = resolveTargetDefinition( - { - name: 'claude-with-template', - provider: 'claude', - workspace_template: '/templates/claude-workspace', - }, - {}, - ); - - expect(target.kind).toBe('claude'); - if (target.kind !== 'claude') { - throw new Error('expected claude target'); - } - - expect(target.config.workspaceTemplate).toBe('/templates/claude-workspace'); - expect(target.config.cwd).toBeUndefined(); - }); - - it('throws when both cwd and workspace_template are specified for claude', () => { - expect(() => - resolveTargetDefinition( - { - name: 'claude-both', - provider: 'claude', - cwd: '/some/path', - workspace_template: '/templates/workspace', - }, - {}, - ), - ).toThrow(/mutually exclusive/i); - }); - - it('resolves codex workspace_template', () => { - const target = resolveTargetDefinition( - { - name: 'codex-with-template', - provider: 'codex', - workspace_template: '/templates/codex-workspace', - }, - {}, - ); - - expect(target.kind).toBe('codex'); - if (target.kind !== 'codex') { - throw new Error('expected codex target'); - } - - expect(target.config.workspaceTemplate).toBe('/templates/codex-workspace'); - }); - - it('throws when both cwd and workspace_template are specified for codex', () => { - expect(() => - resolveTargetDefinition( - { - name: 'codex-both', - provider: 'codex', - cwd: '/some/path', - workspace_template: '/templates/workspace', - }, - {}, - ), - ).toThrow(/mutually exclusive/i); - }); - - it('resolves copilot-sdk workspace_template', () => { - const target = resolveTargetDefinition( - { - name: 'copilot-with-template', - provider: 'copilot-sdk', - workspace_template: '/templates/copilot-workspace', - }, - {}, - ); - - expect(target.kind).toBe('copilot-sdk'); - if (target.kind !== 'copilot-sdk') { - throw new Error('expected copilot-sdk target'); - } - - expect(target.config.workspaceTemplate).toBe('/templates/copilot-workspace'); - }); - - it('throws when both cwd and workspace_template are specified for copilot-sdk', () => { - expect(() => - resolveTargetDefinition( - { - name: 'copilot-both', - provider: 'copilot-sdk', - cwd: '/some/path', - workspace_template: '/templates/workspace', - }, - {}, - ), - ).toThrow(/mutually exclusive/i); + ).toThrow(/workspace_template has been removed/i); }); it('resolves copilot alias to copilot-cli', () => { @@ -693,87 +546,18 @@ describe('resolveTargetDefinition', () => { expect(target.config.executable).toBe('copilot'); }); - it('resolves copilot-cli workspace_template', () => { - const target = resolveTargetDefinition( - { - name: 'copilot-cli-with-template', - provider: 'copilot-cli', - workspace_template: '/templates/copilot-cli-workspace', - }, - {}, - ); - - expect(target.kind).toBe('copilot-cli'); - if (target.kind !== 'copilot-cli') { - throw new Error('expected copilot-cli target'); - } - - expect(target.config.workspaceTemplate).toBe('/templates/copilot-cli-workspace'); - }); - - it('throws when both cwd and workspace_template are specified for copilot-cli', () => { + it('rejects removed target-level workspaceTemplate camelCase field', () => { expect(() => resolveTargetDefinition( { - name: 'copilot-cli-both', - provider: 'copilot-cli', - cwd: '/some/path', - workspace_template: '/templates/workspace', - }, - {}, - ), - ).toThrow(/mutually exclusive/i); - }); - - it('resolves pi-coding-agent workspace_template', () => { - const target = resolveTargetDefinition( - { - name: 'pi-with-template', - provider: 'pi-coding-agent', - workspace_template: '/templates/pi-workspace', - }, - {}, - ); - - expect(target.kind).toBe('pi-coding-agent'); - if (target.kind !== 'pi-coding-agent') { - throw new Error('expected pi-coding-agent target'); - } - - expect(target.config.workspaceTemplate).toBe('/templates/pi-workspace'); - }); - - it('throws when both cwd and workspace_template are specified for pi-coding-agent', () => { - expect(() => - resolveTargetDefinition( - { - name: 'pi-both', - provider: 'pi-coding-agent', - cwd: '/some/path', - workspace_template: '/templates/workspace', + name: 'cli-camel-case', + provider: 'cli', + command: 'echo {PROMPT}', + workspaceTemplate: '/templates/camel-case-workspace', }, {}, ), - ).toThrow(/mutually exclusive/i); - }); - - it('accepts workspaceTemplate camelCase variant', () => { - const target = resolveTargetDefinition( - { - name: 'cli-camel-case', - provider: 'cli', - command: 'echo {PROMPT}', - workspaceTemplate: '/templates/camel-case-workspace', - }, - {}, - ); - - expect(target.kind).toBe('cli'); - if (target.kind !== 'cli') { - throw new Error('expected cli target'); - } - - expect(target.config.workspaceTemplate).toBe('/templates/camel-case-workspace'); + ).toThrow(/workspace_template has been removed/i); }); }); diff --git a/packages/core/test/evaluation/repo-schema-validation.test.ts b/packages/core/test/evaluation/repo-schema-validation.test.ts index 1af9e5714..6b0a83766 100644 --- a/packages/core/test/evaluation/repo-schema-validation.test.ts +++ b/packages/core/test/evaluation/repo-schema-validation.test.ts @@ -91,6 +91,27 @@ describe('repo lifecycle schema validation', () => { expect(result.success).toBe(true); }); + it('accepts workspace.mode=temp', () => { + const result = EvalFileSchema.safeParse({ + ...baseEval, + workspace: { + mode: 'temp', + }, + }); + expect(result.success).toBe(true); + }); + + it('accepts workspace.path for static mode', () => { + const result = EvalFileSchema.safeParse({ + ...baseEval, + workspace: { + mode: 'static', + path: '/tmp/my-workspace', + }, + }); + expect(result.success).toBe(true); + }); + it('rejects invalid source type', () => { const result = EvalFileSchema.safeParse({ ...baseEval, @@ -148,6 +169,27 @@ describe('repo lifecycle schema validation', () => { expect(result.success).toBe(false); }); + it('rejects removed workspace.static_path field', () => { + const result = EvalFileSchema.safeParse({ + ...baseEval, + workspace: { + mode: 'static', + static_path: '/tmp/my-workspace', + }, + }); + expect(result.success).toBe(false); + }); + + it('rejects removed workspace.pool field', () => { + const result = EvalFileSchema.safeParse({ + ...baseEval, + workspace: { + pool: true, + }, + }); + expect(result.success).toBe(false); + }); + it('preserves existing workspace fields (template, hooks)', () => { const result = EvalFileSchema.safeParse({ ...baseEval, diff --git a/packages/core/test/evaluation/validation/targets-validator.test.ts b/packages/core/test/evaluation/validation/targets-validator.test.ts new file mode 100644 index 000000000..f42ae8078 --- /dev/null +++ b/packages/core/test/evaluation/validation/targets-validator.test.ts @@ -0,0 +1,67 @@ +import { afterAll, beforeAll, describe, expect, it } from 'bun:test'; +import { mkdir, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { validateTargetsFile } from '../../../src/evaluation/validation/targets-validator.js'; + +describe('validateTargetsFile', () => { + let tempDir: string; + + beforeAll(async () => { + tempDir = path.join(os.tmpdir(), `agentv-targets-validator-test-${Date.now()}`); + await mkdir(tempDir, { recursive: true }); + }); + + afterAll(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('rejects removed target-level workspace_template field', async () => { + const filePath = path.join(tempDir, 'removed-workspace-template.yaml'); + await writeFile( + filePath, + `targets: + - name: test-target + provider: codex-cli + workspace_template: ./template +`, + ); + + const result = await validateTargetsFile(filePath); + + expect(result.valid).toBe(false); + expect( + result.errors.some( + (error) => + error.severity === 'error' && + error.location === 'targets[0].workspace_template' && + error.message.includes('workspace_template has been removed'), + ), + ).toBe(true); + }); + + it('rejects removed target-level workspaceTemplate camelCase field', async () => { + const filePath = path.join(tempDir, 'removed-workspace-template-camel.yaml'); + await writeFile( + filePath, + `targets: + - name: test-target + provider: codex-cli + workspaceTemplate: ./template +`, + ); + + const result = await validateTargetsFile(filePath); + + expect(result.valid).toBe(false); + expect( + result.errors.some( + (error) => + error.severity === 'error' && + error.location === 'targets[0].workspaceTemplate' && + error.message.includes('workspace_template has been removed'), + ), + ).toBe(true); + }); +}); diff --git a/packages/core/test/evaluation/workspace-config-parsing.test.ts b/packages/core/test/evaluation/workspace-config-parsing.test.ts index c9bb56c64..6f6960f73 100644 --- a/packages/core/test/evaluation/workspace-config-parsing.test.ts +++ b/packages/core/test/evaluation/workspace-config-parsing.test.ts @@ -277,6 +277,66 @@ tests: expect(cases[0].workspace?.isolation).toBe('per_test'); }); + it('infers workspace.mode=static when workspace.path is provided without mode', async () => { + const evalFile = path.join(testDir, 'workspace-path-implies-static.yaml'); + await writeFile( + evalFile, + ` +workspace: + path: /tmp/shared-workspace + +tests: + - id: case-1 + input: "Hello" + criteria: "Should parse" +`, + ); + + const cases = await loadTests(evalFile, testDir); + expect(cases).toHaveLength(1); + expect(cases[0].workspace?.mode).toBe('static'); + expect(cases[0].workspace?.path).toBe('/tmp/shared-workspace'); + }); + + it('rejects removed workspace.static_path field', async () => { + const evalFile = path.join(testDir, 'workspace-static-path-removed.yaml'); + await writeFile( + evalFile, + ` +workspace: + mode: static + static_path: /tmp/shared-workspace + +tests: + - id: case-1 + input: "Hello" + criteria: "Should parse" +`, + ); + + await expect(loadTests(evalFile, testDir)).rejects.toThrow( + /workspace\.static_path has been removed/i, + ); + }); + + it('rejects removed workspace.pool field', async () => { + const evalFile = path.join(testDir, 'workspace-pool-removed.yaml'); + await writeFile( + evalFile, + ` +workspace: + pool: true + +tests: + - id: case-1 + input: "Hello" + criteria: "Should parse" +`, + ); + + await expect(loadTests(evalFile, testDir)).rejects.toThrow(/workspace\.pool has been removed/i); + }); + it('should handle case with no workspace config', async () => { const evalFile = path.join(testDir, 'no-workspace.yaml'); await writeFile( diff --git a/plugins/agentv-dev/skills/agentv-eval-builder/references/eval-schema.json b/plugins/agentv-dev/skills/agentv-eval-builder/references/eval-schema.json index 7779fb277..d18defe6a 100644 --- a/plugins/agentv-dev/skills/agentv-eval-builder/references/eval-schema.json +++ b/plugins/agentv-dev/skills/agentv-eval-builder/references/eval-schema.json @@ -4747,13 +4747,10 @@ }, "mode": { "type": "string", - "enum": ["pooled", "ephemeral", "static"] + "enum": ["pooled", "temp", "static"] }, - "static_path": { + "path": { "type": "string" - }, - "pool": { - "type": "boolean" } }, "additionalProperties": false @@ -9442,13 +9439,10 @@ }, "mode": { "type": "string", - "enum": ["pooled", "ephemeral", "static"] + "enum": ["pooled", "temp", "static"] }, - "static_path": { + "path": { "type": "string" - }, - "pool": { - "type": "boolean" } }, "additionalProperties": false @@ -12964,13 +12958,10 @@ }, "mode": { "type": "string", - "enum": ["pooled", "ephemeral", "static"] + "enum": ["pooled", "temp", "static"] }, - "static_path": { + "path": { "type": "string" - }, - "pool": { - "type": "boolean" } }, "additionalProperties": false